"""
    Diskmargin

The notation follows "An Introduction to Disk Margins", Peter Seiler, Andrew Packard, and Pascal Gahinet

# Fields:
`α`: The disk margin
`ω0`: The worst-case frequency
`f0`: The destabilizing perturbation `f0` is a complex number with simultaneous gain and phase variation. This critical perturbation causes an instability with closed-loop pole on the imaginary axis at the critical frequency ω0
`δ0`: The uncertain element generating f0.
`γmin`: The lower real-axis intercept of the disk (analogous to classical lower gain margin).
`γmax`: The upper real-axis intercept of the disk (analogous to classical upper gain margin).
`ϕm`: is analogous to the classical phase margin.
`σ`: The skew parameter that was used to calculate the margin

Note, `γmax` and `ϕm` are in smaller than the classical gain and phase margins sicne the classical margins do not consider simultaneous perturbations in gain and phase.

The "disk" margin becomes a half plane for `α = 2` and an inverted circle for `α > 2`. In this case, the upper gain margin is infinite. See the paper for more details, in particular figure 6.
"""
struct Diskmargin
    α
    ω0
    f0
    δ0
    γmin
    γmax
    σ
    ϕm
    L
end

function Diskmargin(α, σ=0; ω0=mising, f0=missing, δ0=missing, L=nothing)
    d = Disk(; α, σ)
    γmin = d.γmin
    γmax = d.γmax
    ϕm = d.ϕm
    Diskmargin(α, ω0, f0, δ0, γmin, γmax, σ, ϕm, L)
end

function Base.getproperty(dm::Diskmargin, s::Symbol)
    s ∈ fieldnames(typeof(dm)) && return getfield(dm, s)
    if s ∈ (:margin, :alpha, :diskmargin)
        return dm.α
    elseif s ∈ (:gm, :gainmargin)
        return [dm.γmin, dm.γmax]
    elseif s === :phasemargin
        return dm.ϕm
    else
        throw(ArgumentError("$(typeof(dm)) has no property named $s"))
    end
end

Base.propertynames(dm::Diskmargin) = (fieldnames(typeof(dm))..., :margin, :gainmargin, :phasemargin)

function Base.show(io::IO, dm::Diskmargin)
    println(io, "Disk margin with:")
    println(io, "Margin: ", dm.α)
    println(io, "Frequency: ", dm.ω0)
    if dm.γmax < dm.γmin # In this case, we have an "inverted circle"
        println(io, "Gain margins: [$(dm.γmin), Inf]")
    else
        println(io, "Gain margins: [$(dm.γmin), $(dm.γmax)]")
    end
    println(io, "Phase margin: ", dm.ϕm)
    println(io, "Skew: ", dm.σ)
    println(io, "Worst-case perturbation: ", dm.f0)
end

"""
    Disk

Represents a perturbation disc in the complex plane. `Disk(0.5, 2)` represents all perturbations in the circle centered at 1.25 with radius 0.75, or in other words, a gain margin of 2 and a pahse margin of 36.9 degrees.

A disk can be converted to a Nyquist exclusion disk by `nyquist(disk)` and plotted using `plot(disk)`.

# Arguments:
- `γmin`: Lower intercept
- `γmax`: Upper intercept
- `c`: Center
- `r`: Radius
- `ϕm`: Angle of tangent line through origin.

If γmax < γmin the disk is inverted.
See [`diskmargin`](@ref) for disk margin computations.
"""
struct Disk{T}
    γmin::T
    γmax::T
    c::T
    r::T
    ϕm::T
end

Disk(args...) = Disk(promote(args...)...)

center_radius(γmin, γmax) = 1/2 * (γmax + γmin), 1/2 * (γmax - γmin)

function Disk(γmin, γmax)
    c, r = center_radius(γmin, γmax)
    Disk(γmin, γmax, c, r)
end

function Disk(γmin, γmax, c, r)
    if !isfinite(γmax)
        ϕm = 90.0
    else
        ϕm = (1 + γmin*γmax) / (γmin + γmax)
        ϕm = ϕm >= 1 ? 0 : ϕm <= -1 ? Inf : rad2deg(acos(ϕm))
    end
    Disk(γmin, γmax, c, r, ϕm)
end

function Disk(; α, σ)
    γmin = (2 - α*(1-σ)) / (2 + α*(1+σ))
    γmax = (2 + α*(1-σ)) / (2 - α*(1+σ))
    Disk(γmin, γmax)
end

Disk(dm::Diskmargin) = Disk(dm.γmin, dm.γmax)

"""
    nyquist(d::Disk)

Transform a `Disk` representing a diskmargin to a exclusion disk in the Nyquist plane. This can be useful for visualizing a diskmargin in the Nyquist plane.
"""
ControlSystems.nyquist(d::Disk) = Disk(-inv(d.γmin), -inv(d.γmax)) # translate the disk to a nyquist exclusion disk

"""
    diskmargin(L, σ = 0)
    diskmargin(L, σ::Real, ω)

Calculate the disk margin of LTI system `L`. `L` is supposed to be a loop-transfer function, i.e., it should be square. If `L = PC` the disk margin for output perturbations is computed, whereas if `L = CP`, input perturbations are considered. If the method `diskmargin(P, C, args...)` is used, both are computed.

The implementation and notation follows
"An Introduction to Disk Margins", Peter Seiler, Andrew Packard, and Pascal Gahinet
https://arxiv.org/abs/2003.04771

The margins are aviable as fields of the returned objects, see [`Diskmargin`](@ref).

# Arguments:
- `L`: A loop-transfer function.
- `σ`: If little is known about the distribution of gain variations then σ = 0 is a reasonable choice as it allows for a gain increase or decrease by the same relative amount. *The choice σ < 0* is justified if the gain can decrease by a larger factor than it can increase. Similarly, *the choice σ > 0* is justified when the gain can increase by a larger factor than it can decrease. *If σ = −1* then the disk margin condition is αmax = inv(MT). This margin is related to the robust stability condition for models with multiplicative uncertainty of the form P (1 + δ). If σ = +1 then the disk margin condition is αmax = inv(MS)
- `kwargs`: Are sent to the [`hinfnorm`](@ref) calculation
- `ω`: If a vector of frequencies is supplied, the frequency-dependent disk margin will be computed, see example below.

# Example:
```
L = tf(25, [1,10,10,10])
dm = diskmargin(L, 0)
plot(dm) # Plot the disk margin to illustrate maximum allowed simultaneous gain and phase variations.

nyquistplot(L)
plot!(dm, nyquist=true) # plot a nyquist exclusion disk. The Nyquist curve will be tangent to this disk at `dm.ω0`
nyquistplot!(dm.f0*L) # If we perturb the system with the worst-case perturbation `f0`, the curve will pass through the critical point -1.

## Frequency-dependent margin
w = exp10.(LinRange(-2, 2, 500))
dms = diskmargin(L, 0, w)
plot(w, dms)
```

See also [`ncfmargin`](@ref).
"""
function diskmargin(L::LTISystem, σ::Real=0; kwargs...)
    issiso(L) || return sim_diskmargin(L, σ)
    M = 1/(1 + L) + (σ-1)/2
    n,ω0 = hinfnorm(M; kwargs...)
    diskmargin(L, σ, ω0)
end

"""
    diskmargin(L::LTISystem, σ::Real, ω)

Calculate the diskmargin at a particular frequency or vector of frequencies. If `ω` is a vector, you get a frequency-dependent diskmargin plot if you plot the returned value.
See also [`ncfmargin`](@ref).
"""
diskmargin(L::LTISystem, σ::Real, ω::AbstractArray) = map(w->diskmargin(L, σ, w), ω)

function diskmargin(L::LTISystem, σ::Real, ω0::Real)
    issiso(L) || return sim_diskmargin(L, σ, [ω0])[]
    M = 1/(1 + L) + (σ-1)/2
    freq = isdiscrete(L) ? cis(ω0*L.Ts) : complex(0, ω0)
    Sω = evalfr(M, freq)[]
    αmax = 1/abs(Sω)
    δ0 = inv(Sω)
    dp = Disk(; α = αmax, σ)
    if δ0 == 2/(σ+1)  # S = 1, L = 0
        Diskmargin(αmax, ω0, Inf, δ0, 0, Inf, σ, dp.ϕm, L)
    else
        f0 = (2 + δ0*(1-σ)) / (2 - δ0*(1+σ))
        Diskmargin(αmax, ω0, f0, δ0, dp.γmin, dp.γmax, σ, dp.ϕm, L)
    end
end

## Plotting ====================================================================

γϕcurve(dm::Diskmargin; kwargs...) = γϕcurve(dm.α, dm.σ; kwargs...)

"""
    γϕcurve(α, σ; N = 200)

Internal function. Get the curve of extremal stable gain and phase perturbations. This function is called with a single diskmargin object is plotted.

- `N`: Number of points on the curve.
"""
function γϕcurve(α, σ; N = 200)
    θ = LinRange(0, π, N)
    f = @. (2 - α*cis(θ)*(1-σ)) / (2 + α*cis(θ)*(1+σ))
    @. (abs(f), abs(rad2deg(angle(f))))
end

@recipe function plot(d::Disk; nyquist = false)
    θ = LinRange(0, 2pi, 200)
    re, im = @. cos(θ), sin(θ)
    if nyquist
        d = ControlSystems.nyquist(d)
    end
    c,r = d.c, d.r
    @series begin
        fill --> true
        fillalpha --> 0.5
        @. r*re+c, r*im
    end
end

@recipe function plot(dm::Diskmargin; nyquist=false)
    if nyquist
        @series begin
            label --> "σ = $(dm.σ)"
            ControlSystems.nyquist(Disk(dm))
        end
    else
        γ, ϕ = γϕcurve(dm)
        @series begin
            title --> "Stable region for combined gain and phase variation"
            xguide --> "Gain variation"
            yguide --> "Phase variation"
            label --> "σ = $(dm.σ)"
            fill --> true
            fillalpha --> 0.5
            γ, ϕ
        end
    end
end

@recipe function plot(dm::AbstractVector{<:Diskmargin}; lower=true, phase=true)
    w = [dm.ω0 for dm in dm]
    layout --> (phase ? 2 : 1, 1)
    link --> :x
    @series begin
        subplot --> 1
        title --> "Gain margin"
        label --> (lower ? ["Lower" "Upper"] : "Upper")
        # xguide --> "Frequency"
        xscale --> :log10
        yscale --> :log10
        gma = getfield.(dm, :γmax)
        gmi = getfield.(dm, :γmin)
        # ylims --> (0, min(10, maximum(gma)))
        data = (lower ? [gmi gma] : gma)
        data = max.(0, data)
        replace!(data, 0 => -Inf)
        w, data
    end
    phase || return
    @series begin
        subplot --> 2
        title --> "Phase margin"
        xguide --> "Frequency"
        xscale --> :log10
        label --> ""
        ϕm = getfield.(dm, :ϕm)
        w, ϕm
    end
end

@recipe function plot(dm::AbstractVector{<:AbstractVector{Diskmargin}})
    reduce(hcat, dm)
end

@recipe function plot(dm::AbstractMatrix{Diskmargin}; lower=true, phase=true)
    w = getfield.(dm[:,1], :ω0)
    ny = size(dm, 2)
    length(w) == size(dm, 1) || throw(ArgumentError("Frequency vector and diskmargin vector must have the same lengths."))
    layout --> (phase ? 2 : 1, 1)
    link --> :x
    neg2inf(x) = x <= 0 ? Inf : x
    for i = 1:ny
        is = ny == 1 ? "" : "$i"
        @series begin
            subplot --> 1
            title --> "Gain margin"
            # label --> permutedims([["Lower $i" for i = 1:ny]; ["Upper $i" for i = 1:ny]])
            label --> (lower ? ["Upper"*is "Lower"*is] : "Upper"*is)
            # xguide --> "Frequency"
            xscale --> :log10
            yscale --> :log10
            gma = getfield.(dm[:, i], :γmax) .|> neg2inf
            gmi = getfield.(dm[:, i], :γmin) .|> neg2inf
            replace!(gmi, 0 => -Inf)
            # ylims --> (0, min(10, maximum(gma)))
            w, (lower ? [gmi gma] : gma)
        end
        phase || continue
        @series begin
            subplot --> 2
            title --> "Phase margin"
            xguide --> "Frequency"
            xscale --> :log10
            label --> is#permutedims(["$i" for i = 1:ny])
            ϕm = getfield.(dm[:, i], :ϕm)
            w, ϕm
        end
    end
end


using SubsetSelection

####################
## OLS
####################

# FUNCTION inner_op
# """Computes the exact regression error for a given selection s of features.
#
# INPUT
#   ℓ           - LossFunction to use
#   Y           - Vector of outputs. For classification, use ±1 labels
#   Z           - Array of inputs
#   s           - Sparsity pattern. 0/1 vector
#   γ           - Regularization paramter
#
# OUTPUT
#   α           - Dual variables
#   c           - function value c(s)
#   ∇c          - subgradient of c at s"""
# function inner_op(ℓ::OLS, Y, X, s, γ)
#
#   indices = find(s->s>0.5, s); k = length(indices)
#   n,p = size(X)
#
#   # Compute optimal dual parameter
#   α = sparse_inverse(ℓ, Y, X[:, indices], γ)
#
#   c = SubsetSelection.dual(ℓ, Y, X, α, indices, k, γ)
#
#   ∇c = zeros(p)
#   for j in 1:p
#     ∇c[j] = -γ/2*dot(X[:,j],α)^2
#   end
#
#   return c, ∇c
# end

# FUNCTION sparse_inverse
"""Computes the optimal dual variables
          α = (∑ γ X_j X_j' + I)^(-1) Y
using the Matrix Inversion Lemma.

INPUT
  ℓ           - LossFunction to use
  Y           - Vector of observed responses
  X           - Matrix of observed features
  γ           - Regularization parameter

OUTPUT
  α           -Optimal dual variables"""
function sparse_inverse(ℓ::LossFunction, Y, X, γ)

  n = size(X, 1)
  k = size(X, 2)

  CM = Matrix{Float64}(I,k,k)/γ + X'*X      # The capacitance matrix
  α = -Y + X*(CM\(X'*Y))       # Matrix Inversion Lemma

  return α
end

# # FUNCTION recover_primal
# """Computes the Ridge regressor
#
# INPUT
#   ℓ           - LossFunction to use
#   Y           - Vector of observed responses
#   Z           - Matrix of observed features
#   γ           - Regularization parameter
#
# OUTPUT
#   w           - Optimal regressor"""
# function recover_primal(ℓ::OLS, Y, Z, γ)
#   α = sparse_inverse(ℓ, Y, Z, γ)           # Optimal dual variable α
#   return -γ*Z'*α                            # Regressor
# end

####################
## CLASSIFICATION
####################

# FUNCTION inner_op
"""Computes the exact regression error for a given selection s of features.

INPUT
  ℓ           - LossFunction to use
  Y           - Vector of outputs. For classification, use ±1 labels
  X           - Array of inputs
  s           - Sparsity pattern. 0/1 vector
  γ           - Regularization paramter

OUTPUT
  α           - Dual variables
  c           - function value c(s)
  ∇c          - subgradient of c at s"""
function inner_op(ℓ::LossFunction, Y, X, s, γ; stochastic=false)
    if stochastic
        inner_op_stochastic(ℓ, Y, X, s, γ)
    else
        inner_op_plain(ℓ, Y, X, s, γ)
    end
end

function inner_op_plain(ℓ::LossFunction, Y, X, s, γ)
  indices = findall(s .> .5); k = length(indices)
  n,p = size(X)

  # Compute optimal dual parameter
  α = sparse_inverse(ℓ, Y, X[:, indices], γ)
  c = SubsetSelection.value_dual(ℓ, Y, X, α, indices, k, γ)

  ∇c = zeros(p)
  for j in 1:p
    ∇c[j] = -γ/2*dot(X[:,j],α)^2
  end
  return c, ∇c
end

function inner_op_stochastic(ℓ::LossFunction, Y, X, s, γ;
      B=10, bSize=max(0.1,2*sum(s)/size(X,1)) )
  indices = findall(s .> .5); k = length(indices)
  n,p = size(X)

  # Compute optimal dual parameter
  w = zeros(k)
  for b in 1:B
    subset = rand(n) .< bSize
    w .+= SubsetSelection.recover_primal(ℓ, Y[subset], X[subset,indices], γ)
  end
  w ./= B
  α = start_primal(ℓ, Y, X[:,indices], γ)
  c = SubsetSelection.value_dual(ℓ, Y, X, α, indices, k, γ)

  ∇c = zeros(p)
  for j in 1:p
    ∇c[j] = -γ/2*dot(X[:,j],α)^2
  end
  return c, ∇c

  # Compute optimal dual parameter
  # w = zeros(k)
  # c_s = 0; ∇c_s = zeros(p)
  # for b in 1:B
  #   subset = rand(n) .< bSize
  #   c, ∇c = inner_op_plain(ℓ, Y[subset], X[subset,:], s, γ)
  #   c_s += c; ∇c_s += ∇c
  # end
  # c_s /= B; ∇c_s ./= B
  # return c_s, ∇c_s
end
# using LIBLINEAR

# FUNCTION sparse_inverse
"""Computes the optimal dual variables
          α = (∑ γ X_j X_j' + I)^(-1) Y
using the Matrix Inversion Lemma.

INPUT
  ℓ           - LossFunction to use
  Y           - Vector of observed responses
  X           - Matrix of observed features
  γ           - Regularization parameter

OUTPUT
  α           -Optimal dual variables"""
function sparse_inverse(ℓ::Classification, Y, X, γ;
                                    valueThreshold=1e-8, maxIter=1e3)
    n,k = size(X)
    indices = collect(1:k); n_indices = k
    cache = SubsetSelection.Cache(n, k)

    α = start_primal(ℓ, Y, X, γ)
    value = SubsetSelection.value_dual(ℓ, Y, X, α, indices, k, γ)

    for iter in 1:maxIter
        ∇ = SubsetSelection.grad_dual(ℓ, Y, X, α, indices, n_indices, γ, cache) #Compute gradient

        if norm(∇, 1) <= 1e-14
          break
        end
        if norm(∇, 1) == Inf
            α[findall(∇ .== Inf)] .= -Y[findall(∇ .== Inf)]*1e-14
            α[findall(∇ .== -Inf)] .= -Y[findall(∇ .== -Inf)]*(1-1e-14)

            ∇[findall(∇ .== Inf)] .= 0.
            ∇[findall(∇ .== -Inf)] .= 0.
        end

        learningRate = 2/norm(∇, 1)
        α1 = α[:]
        newValue = value - 1.

        while newValue < value  #Divide step sie by two as long as f decreases
          learningRate /= 2
          α1[:] = α .+ learningRate*∇       #Compute new alpha
          SubsetSelection.proj_dual!(ℓ, Y, α1)    #Project
          newValue = SubsetSelection.value_dual(ℓ, Y, X, α1, indices, k, γ)  #Compute new f(alpha, s)
        end

        value_gap = 2*(newValue-value)/(value+newValue)
        α = α1[:]; value = newValue
        if abs(value_gap) < valueThreshold
          break
        end
    end

    return α
end

# FUNCTION start_primal
function start_primal(ℓ::Regression, Y::Array, X::Array, γ::Real)
  CM = Matrix(I, size(X,2), size(X,2))/γ + X'*X     # The capacitance matrix
  α = -Y + X*(CM\(X'*Y))            # Matrix Inversion Lemma
  return α
end

function start_primal(ℓ::Classification, Y::Array, X::Array, γ::Real)
  n,k = size(X)
  w = SubsetSelection.recover_primal(ℓ, Y, X, γ)
  α = primal2dual(ℓ, Y, X, w)
  return α
end

function primal2dual(ℓ::LogReg, Y, X, w)
  n = size(X, 1)
  return [-Y[i]*exp(-Y[i]*dot(X[i,:], w))/(1+exp(-Y[i]*dot(X[i,:], w))) for i in 1:n] #LR
end

function primal2dual(ℓ::L1SVM, Y, X, w)
  n = size(X, 1)
  return [-Y[i]*(1-Y[i]*dot(X[i,:], w) > 0) - Y[i]/2*(1-Y[i]*dot(X[i,:], w) == 0) for i in 1:n] #SVM
end

# # FUNCTION recover_primal
# """Computes the Ridge regressor
#
# INPUT
#   ℓ           - LossFunction to use
#   Y           - Vector of observed responses
#   Z           - Matrix of observed features
#   γ           - Regularization parameter
#
# OUTPUT
#   w           - Optimal regressor"""
# function recover_primal(ℓ::Classification, Y, Z, γ)
#   solverNumber = LibLinearSolver(ℓ)
#   model = linear_train(Y, Z'; verbose=false, C=γ, solver_type=Cint(solverNumber))
#   return Y[1]*model.w
# end
#
# function LibLinearSolver(ℓ::LogReg)
#   return 7
# end
# function LibLinearSolver(ℓ::L1SVM)
#   return 3
# end


# create an index for id6
# building an index is too time consuming
import Base.ht_keyindex
import Base.ht_keyindex2

function buildindex{T}(val::Vector{T})
  index = Dict{T,Vector{Int64}}()
  for (i,v) in enumerate(val)
    dindex = ht_keyindex(index, v)
    if dindex > 0
      push!(index.vals[dindex], i)
    else
      index[v] = [i]
    end
  end
  return index
end

function sumby_index{T,S}(index::Dict{T, Vector{Int64}}, val::Vector{S})
  return Dict{T,S}(kv[1] => sum(val[kv[2]]) for kv in index)
end

@time iid6 = buildindex(id6);  #259,290
@time sumby_index(iid6, v1); # 30.93
@time sumby_index(iid6, v3); #26.7

@time iid4 = buildindex(id4); #9.96
@time sumby_index(iid4, v1); #8.88
@time sumby_index(iid4, v1); #8.63

using SortingAlgorithms
function buildindex2{T}(val::Vector{T})
  index = Dict{T,Vector{Int64}}()
  for (i,v) in enumerate(val)
    dindex = ht_keyindex2(index, v)
    if dindex > 0
      push!(index.vals[dindex], i)
    else
      Base._setindex!(index, [i], v, -dindex)
    end
  end
  return index
end

@time iid42 = buildindex2(id4); #10.94


"""
    discreteDiff{T}(X::Array{T, 1})

Returns the discrete difference between adjacent elements of a time series.  So,
for instance, if one has a time series ``y_{1},y_{2},\ldots,y_{N}`` this will return
a set of ``δ`` such that ``δ_{i} = y_{i+1} - y_{i}``.  The first element of the returned
array will be a `NaN`.
"""
function discreteDiff{T}(X::Array{T, 1})
    if !(T <: AbstractFloat)
        error("Array elements must be of type with NaN.")
    end
    o = Array{T, 1}(length(X))
    o[1] = NaN
    for i in 2:length(X)
        o[i] = X[i] - X[i-1]
    end
    return o
end
export discreteDiff


export non0hist, renyi
using StatsBase

"""
```julia
non0hist(ε, dataset)
non0hist(ε, vectors...)
```
Partition a data-set into tabulated intervals (boxes) of
size `ε` and return the *sum-normalized* histogram in a 1D form, discarding all
non-zero elements. Use the `fit(Histogram, ...)` from package `StatsBase` if you
wish to keep information about the edges of the binning as well as the zero elements.
"""
function non0hist end
@inbounds function non0hist(ε, vectors::Vararg{AbstractVector{T}}) where {T<:Real}
  # Initialize:
  D = length(vectors); L = length(vectors[1])
  for i in 1:D
  length(vectors[i]) == L ||
  throw(ArgumentError("All vectors given to `non0hist` must be of equal length!"))
  end

  if T == BigFloat || T == BigInt
    ranges = StepRangeLen{T, T, T}[]
  else
    ranges = StepRangeLen{T, Base.TwicePrecision{T},Base.TwicePrecision{T}}[]
  end
  # Create ranges:
  for v in vectors
    push!(ranges, minimum(v):ε:maximum(v)+ε)  #be sure to have that +ε at the end!
  end
  # Perform Histogram:
  # Write my OWN Version, which would be many times over faster,
  # since I don't need the 0 elements.
  pks = fit(Histogram, vectors, (ranges...), closed=:left).weights/L
  return T[x for x in pks if x != 0]
end

"""
```julia
renyi(α, ε, dataset)
renyi(α, ε, vectors...)
```
Compute the `α` order generalized Rényi entropy [1] of a dataset,
by first partitioning them into boxes of of size `ε` (log base-e is used).

```julia
renyi(α, p::AbstractArray)
```
Compute the `α` order generalized Rényi entropy of an Array `p` directly,
assuming that `p` is sum-normalized.

The Rényi entropy `R_α(p) = 1/(1-α) * sum_i(p_i^α)` generalizes other known entropies,
like e.g. the Shannon entropy
(α = 1, see *the* Shannon paper [2]), the Hartley entropy (α = 0, also known as
maximum entropy), or the Collision (or correlation) entropy (α = 2).

[1] : A. Rényi, Proceedings of the fourth Berkeley Symposium on Mathematics,
Statistics and Probability, pp 547 (1960)

[2] : C. E. Shannon, Bell System Technical Journal **27**, pp 379 (1948)
"""
function renyi(α, ε, vectors::Vararg{AbstractVector{T}}) where {T<:Real}
  p = non0hist(ε, vectors...)
  renyi(α, p)
end
function renyi{T<:Real}(α::Real, p::AbstractArray{T})
  α < 0 && throw(ArgumentError("Order of Rényi entropy must be ≥ 0."))

  if α ≈ 0
    return log(length(p)) #Hartley entropy, max-entropy
  elseif α ≈ 1
    return -sum( x*log(x) for x in p ) #Shannon entropy, information to locate with ε
  elseif (isinf(α))
    return -log(maximum(p)) #Min entropy
  else
    return (1/(1-α))*log( sum(x^α for x in p) )
  end
end

# # Linear fit on dimension:
# path = "C:\\Users\\datseris\\PowerFolders\\MyScience\\Lectures\\Introduction to Physics of Complex Systems\\Exercises 2016\\10 - Generilized Dimensions And Synchronization\\data1.txt"
# f = readdlm(path)
# f1 = view(f, :, 1); f2 = view(f, :, 2)
#
# es = logspace(-1, -3, 6)
# D = zeros(es)
# for (i, ε) in enumerate(es)
#   println("ε = $ε")
#   D[i] = renyi(1, ε, f1, f2)
# end
#
# using LsqFit
# model(x, p) = p[1].*x .+ p[2]
#
# cf = curve_fit(model, -ln.(es), D, rand(2))
# plot(lnes, -cf.param[1] .* ln.(es) .+ cf.param[2])


# +
include("epidemiology.jl")

using OrdinaryDiffEq
using Catlab.Graphics.Graphviz: run_graphviz

# +
save_wd(ex, fname::AbstractString) = begin
    g = display_wd(ex)
    open(fname, "w") do io
        run_graphviz(io, g, format="svg")
    end
end
save_graph(g, fname::AbstractString) = begin
    open(fname, "w") do io
        run_graphviz(io, g, format="svg")
    end
end

splot(soln, fname) = begin
    p = plot(soln, linewidth=5, legend=false)
    savefig(p, fname)
end

splot(soln, t, fname) = begin
    p = plot(t, soln, linewidth=5, legend=false)
    savefig(p, fname)
end

splotchannels(sol, dir) = begin
    mkpath(dir)
    jp(p) = joinpath(dir, p)
    splot(sol, jp("solution.svg"))
    pltdims = [3,8,13]
    splot(sol[pltdims,:]', sol.t, jp("infecteds.svg"))
    splot(sum(sol(sol.t, idxs=pltdims), dims=1)', sol.t, jp("infecteds_sum.svg"))
    pltdims = [3,8,13] .+ 1
    splot(sol[pltdims,:]', sol.t, jp( "recovered.svg" ))
    splot(sum(sol(sol.t, idxs=pltdims), dims=1)', sol.t, jp( "recovered_sum.svg" ))
end
# -

save_wd(sir, "sir.svg")
save_graph(Graph(p_sir), "sir_graph.svg")

# define initial states and transition rates
u0 = [10.0, 1, 0]
p = [0.4, 0.4]
# create and solve ODE problem
prob = ODEProblem(p_sir,u0,(0.0,7.5),p)
sol = OrdinaryDiffEq.solve(prob,Tsit5())
# visualize the solution
splot(sol, "sir_soln.svg")

save_wd(seir, "seir.svg")
save_graph(Graph(p_seir), "seir_graph.svg")

# define initial states and transition rates
u0 = [10.0, 1, 0, 0]
p = [0.9, 0.2, 0.5]
# create and solve ODE problem
prob = ODEProblem(p_seir,u0,(0.0,15.0),p)
sol = OrdinaryDiffEq.solve(prob,Tsit5())
# visualize the solution
splot(sol, "seir_soln.svg")

save_wd(seird, "seird.svg")
save_graph(Graph(p_seird), "seird_graph.svg")

# define initial states and transition rates
u0 = [10.0, 1, 0, 0, 0]
p = [0.9, 0.2, 0.5, 0.1]
# create and solve ODE problem
prob = ODEProblem(p_seird,u0,(0.0,15.0),p)
sol = OrdinaryDiffEq.solve(prob,Tsit5())
# visualize the solution
splot(sol, "seird_soln.svg")

include("covid.jl")

[save_graph(Graph(decoration(p)), pname) for (p,pname) in [(spontaneous_petri, "spont.svg"),
                                               (transmission_petri, "transmission.svg"),
                                               (exposure_petri, "exposure.svg"),
                                               (travel_petri, "travel.svg")]]

save_wd(seird_city, "seird_city.svg")
save_graph(Graph(decoration(F(seird_city))), "seird_graph.svg")

save_wd(seird_3, "seird_3.svg")
save_graph(Graph(p_seird_3), "seird_3_graph.svg")

pltdims = [4,9, 14]
# Define time frame, 3 months
tspan = (0.0,90.0)
# Define initial states
u0 = zeros(Float64, base(pc_seird_3).n)
u0[1]  = 10000
u0[6]  = 10000
u0[11] = 10000
u0[2]  = 1
# Define transition rates
params = seirdparams(3, 5)
# Generate and solve resulting ODE
prob = ODEProblem(p_seird_3,u0,tspan,params)
sol = OrdinaryDiffEq.solve(prob,Tsit5())
splotchannels(sol, "seird")

function drawsol(f, a, b, p, dir)
    tspan = (0, 240.0)
    println("INFO: Processing $dir")
    dynparams = waveparams(f, a/sum(u0), b/sum(u0),p)
    mkpath(dir)
    prob = ODEProblem(p_seird_3,u0,tspan, dynparams)
    sol = OrdinaryDiffEq.solve(prob,Tsit5(), saveat=1:1:tspan[2])
    splotchannels(sol, dir)
    savefig(plot(sol.t, f(a,b,p), ylim=(0,12), legend=false, linewidth=5), "$dir/transmissability.svg")
end

a,b = 10, 1
dir = "dynamic/mitigation/"
drawsol(asymptotic,  a,b,1/6, "$dir/asymptotic")
drawsol(triangleasm, a,b,25 , "$dir/trianglesm")
drawsol(coswave,     a,b,25 , "$dir/coswave")
drawsol(sincwave,    a,b,100, "$dir/sincwave")
drawsol(modsincwave, a,b,100, "$dir/modsincwave")

a,b = 10, 1/2
dir = "dynamic/containment"
drawsol(asymptotic,  a,b,1/6, "$dir/asymptotic")
drawsol(triangleasm, a,b, 25, "$dir/trianglesm")
drawsol(coswave,     a,b, 25, "$dir/coswave")
drawsol(sincwave,    a,b,100, "$dir/sincwave")
drawsol(modsincwave, a,b,100, "$dir/modsincwave")


function is_horizontal_or_vertical(startx, starty, endx, endy)
    return (startx==endx) || (starty == endy)
end

function test_is_horizontal_or_vertical()
    @assert is_horizontal_or_vertical(3,4,3,7)
    @assert is_horizontal_or_vertical(2,4,3,4)
    @assert !is_horizontal_or_vertical(1,2,3,4)
end

test_is_horizontal_or_vertical()

function draw_grid(startxs, startys, endxs, endys)
    xmax = maximum([maximum(startxs), maximum(endxs)])
    ymax = maximum([maximum(startys), maximum(endys)])
    grid =  zeros(Int, xmax, ymax)
    # assume all arrays are same length
    for i in 1:length(startxs)
        if ! is_horizontal_or_vertical(startxs[i],
                                       startys[i],
                                       endxs[i],
                                       endys[i])
            continue
        end

        for ix in minimum([startxs[i], endxs[i]]):maximum([startxs[i], endxs[i]])
            for iy in minimum([startys[i], endys[i]]):maximum([startys[i], endys[i]])
                grid[ix,iy] += 1
            end
        end
    end
    return grid
end

inputs = readlines("input.txt")

starts = map(x->split(x)[1], inputs)
ends = map(x->split(x)[3], inputs)

startxs = map(x->parse(Int,split(x,",")[1]), starts)
startys = map(x->parse(Int, split(x,",")[2]), starts)
endxs = map(x->parse(Int, split(x,",")[1]), ends)
endys = map(x->parse(Int, split(x,",")[2]), ends)

grid = draw_grid(startxs, startys, endxs, endys)

overlaps = count(x->(x>1), grid)
print("Count of overlaps ", overlaps)


# This file is part of the IntervalArithmetic.jl package; MIT licensed

"""
    hull(::Type{T}, a::ValidInterval)

    Returns the hull of the interval `a` with boundary type `T`.

    Note that hull() is the fundamental conversion operation in IEEE 1788-2015.
"""
hull(::Type{T}, a::ValidInterval{T}) where T = a

hull(::Type{T}, x::Interval) where T <: AbstractFloat =
    unsafe_interval( T(x.lo, RoundDown), T(x.hi, RoundUp) )
hull(::Type{T}, ::EmptyInterval) where T = emptyinterval(T)

promote_rule(::Type{Interval{T}}, ::Type{Interval{S}}) where {T, S} =
    Interval{promote_type(T, S)}

promote_rule(::Type{Interval{T}}, ::Type{S}) where {T, S} =
    Interval{promote_type(T, S)}

# convert numbers to intervals
convert(::Type{Interval{T}}, x::T) where T = interval(x)
convert(::Type{Interval}, x::T) where T = convert(Interval{float(T)}, x)

# convert between interval subtypes
convert(::Type{Interval{T}}, x::Interval) where T = hull(T, x)
convert(::Type{Interval}, x::Interval) = x

real(a::ValidInterval) = a
float(x::ValidInterval) = x
big(x::ValidInterval) = hull(BigFloat, x)

"""
    unsafe_atomic(::Type{<:AbstractFloat}, x)

Construct the tightest interval of a given type that contains the value `x`.

# Examples

Construct an `Interval{Float64}` containing a given `BigFloat`:

```julia
julia> x = big"0.1"
1.000000000000000000000000000000000000000000000000000000000000000000000000000002e-01

julia> i = IntervalArithmetic.unsafe_atomic(Float64, x)
[0.0999999, 0.100001]

julia> i isa Interval{Float64}
true

julia> i.lo <= x <= i.hi
true

julia> i.hi === nextfloat(i.lo)
true
```

Construct an `Interval{Float32}` containing a the real number 0.1 in two ways:

```julia
julia> i2 = IntervalArithmetic.unsafe_atomic(Float32, 1//10)
[0.0999999, 0.100001]

julia> i1 === i2
true

julia> i.lo <= 1//10 <= i.hi
true
```
"""
function unsafe_atomic end

unsafe_atomic(::Type{T}, x::T) where T <: AbstractFloat =
    isfinite(x) ? unsafe_interval(x, x) : signed_infinity(x)

unsafe_atomic(::Type{T}, x::S) where {T<:AbstractFloat, S<:Real} =
    isfinite(x) ?
        unsafe_interval(T(x, RoundDown), T(x, RoundUp)) :
        signed_infinity(T(x))


mutable struct MobiusTransform{T <: Real} <: AbstractConformalTransform
    a::Complex{T}
    b::Complex{T}
    c::Complex{T}
    d::Complex{T}
end

function addequals(mobius::MobiusTransform{T}, sv::SArray{Tuple{4},Complex{Float64},1,4}) where {T <: Real}
    mobius.a += sv[1]
    mobius.b += sv[2]
    mobius.c += sv[3]
    mobius.d += sv[4]
    mobius
end

function identity_mobius()
    MobiusTransform{Float64}(1, 0, 0, 1)
end

@inline function transform(mobius::MobiusTransform{T}, point::Complex{T})::Complex{T} where {T <: Real}
    (point * mobius.a + mobius.b) / (point * mobius.c + mobius.d)
end

@inline function inversetransform(mobius::MobiusTransform{T}, point::Complex{T})::Complex{T} where {T <: Real}
    (point * mobius.d - mobius.b) / (-point * mobius.c + mobius.a)
end

struct ComplexRational{T <: Real, N} <: AbstractConformalTransform
    numerator::SArray{Tuple{N},Complex{T},1,N}
    denominator::SArray{Tuple{N},Complex{T},1,N}
end

function identity_rational()
    num = @SVector ComplexF64[0, 1]
    den = @SVector ComplexF64[1, 0]
    ComplexRational{Float64, 2}(num, den)
end

function basic_rational()
    num = @SVector ComplexF64[0, 1, 1 + 0.1im, 0.1im]
    den = @SVector ComplexF64[1, 0.1 - 0.1im, 0.2, 0.01]
    ComplexRational(num, den)
end

function basic_rational(n)
    num = @SVector ComplexF64[0:n]
    den = @SVector ComplexF64[1, 0.1 - 0.1im, 0.2, 0.01]
    ComplexRational(num, den)
end

function addequals(rational::ComplexRational{T, N}, num::SArray{Tuple{N},Complex{T},1,N}, den::SArray{Tuple{N},Complex{T},1,N}) where {T <: Real, N}
    mobius.a += sv[1]
    mobius.b += sv[2]
    mobius.c += sv[3]
    mobius.d += sv[4]
    mobius
end

@inline function transform(rational::ComplexRational{T, N}, point::Complex{T})::Complex{T} where {T <: Real, N}
    num = evalpoly(point, rational.numerator)
    den = evalpoly(point, rational.denominator)
    num / den
end

@inline function inversetransform(rational::ComplexRational{T, N}, point::Complex{T}, preimage_start::Complex{T})::Complex{T} where {T <: Real, N}
    # need to do the math in vector-of-real space instead of complex space for Optim's sake
    preimage_arr = [real(preimage_start), imag(preimage_start)]
    distinvfunc = preimage -> complexdistsq(transform(rational, Complex(preimage[1], preimage[2])), point)
    graddistinv = preimage -> gradient(distinvfunc, preimage)[1]
    optim_results = optimize(distinvfunc, graddistinv, preimage_arr, LBFGS(); inplace=false)
    preimage_final_complex = Complex(Optim.minimizer(optim_results)...)
end


"""
    PolyaGammaPSWSampler(b::Int, z::Real)

PSW sampler ([1]) for a Polya-Gamma distribution with parameters `b` and `z`,
and Laplace transform

```math
\\mathcal{L}(t) = \\cosh^b(z) \\cosh^{-b}(\\sqrt{2t + z^2})
```

References

* [1] <https://doi.org/10.1080/01621459.2013.829001>
"""
struct PolyaGammaPSWSampler{T <: Real} <: Sampleable{Univariate, Continuous}
    b::Int
    z::T
end

function Base.rand(rng::AbstractRNG, s::PolyaGammaPSWSampler)
    out = 0.0
    for _ in 1:s.b
        out += rpg_devroye_1(rng, s.z)
    end
    return out
end

function rpg_devroye_1(rng::AbstractRNG, z::Real)
    t = 0.64
    z = abs(z) * 0.5
    yz = π^2 / 8 + z^2 / 2
    while true
        if rand(rng) < mass_texpon(z, t)
            x = t + randexp(rng) / yz
        else
            x = rtigauss(rng, z, t)
        end
        s = acoef(0, x, t)
        y = s * rand(rng)
        n = 0
        while true
            n += 1
            if n % 2 == 1
                s -= acoef(n, x, t)
                y <= s && return x / 4
            else
                s += acoef(n, x, t)
                y > s && break
            end
        end
    end
end

function mass_texpon(z::Real, t::Real)
    yz = π^2 / 8 + z^2 / 2
    b = inv(sqrt(t)) * (t * z - 1)
    a = -inv(sqrt(t)) * (t * z + 1)
    t0 = log(yz) + yz * t
    tb = t0 - z + normlogcdf(b)
    ta = t0 + z + normlogcdf(a)
    qdivp = fourinvπ * (exp(tb) + exp(ta))
    return 1.0 / (1.0 + qdivp)
end

function rtigauss(rng::AbstractRNG, z::Real, t::Real)
    z = abs(z)
    μ = inv(z)
    x = t + 1.0
    if μ > t
        α = 0.0
        while rand(rng) > α
            e1 = randexp(rng)
            e2 = randexp(rng)
            while e1^2 > (2 * e2 / t)
                e1 = randexp(rng)
                e2 = randexp(rng)
            end
            x = t / (1 + t * e1)^2
            α = exp(-0.5 * z^2 * x)
        end
    else
        while x > t
            y = randn(rng)^2
            x = μ + μ^2 * y / 2 - μ * √(4 * μ * y + (μ * y)^2) / 2
            if rand(rng) > μ / (μ + x)
                x = μ^2 / x
            end
        end
    end
    return x
end

function acoef(n::Int, x::Real, t::Real)
    if x > t
        π * (n + 0.5) * exp(-(n + 0.5)^2 * π^2 * x / 2)
    else
        2 / x / √x / sqrthalfπ * (n + 0.5) * exp(-2 * (n + 0.5)^2 / x)
    end
end

function Distributions.mean(s::PolyaGammaPSWSampler)
    s.b * inv(2.0 * s.z) * tanh(s.z / 2.0)
end

function Distributions.var(s::PolyaGammaPSWSampler)
    s.b * inv(4 * s.z^3) * (sinh(s.z) - s.z) * (sech(s.z / 2)^2)
end


module SimpleConstants

export c₀, μ₀, ε₀, η₀, ℎ, ℏ, e⁻, e⁺, Å, °
export πc₀, π²c₀, iπ, i2π, ℎc₀
export eV2meter, meter2eV
export yotta, zetta, exa, peta, tera, giga, mega, kilo, hecto, deca
export deci, centi, milli, micro, nano, pico, femto, atto, zepto, yocto

## Physical and mathematical constants
# Basic constants
const c₀ = 2.99_792_458e8  # speed of light in vacuum in m/s
const μ₀ = 1.256_637_062_12e-6  # vacuum permeability in H/m; previously 4π×10⁻⁷ H/m; see 2019 redefinition of SI base units
const ε₀ = 1 / (c₀^2 * μ₀)  # vacuum permittivity in F/m
const η₀ = √(μ₀ /ε₀)  # vacuum impedance in ohm (Ω)
const ℎ = 6.626_070_15e-34  # Planck constant in joule⋅sec (J⋅s); unicode character U+210E (entered by \planck tab)
const ℏ = ℎ / 2π  # reduced Planck constant in joule⋅sec (J·s); unicode character  U+210F
const e⁻ = -1.602_176_634e-19  # charge of single electron in coulomb (C); negative value
const e⁺ = -e⁻  # charge of single proton in coulomb (C); positive value
const Å = 1e-10  # angstrom in meter (m); unicode character U+212B (entered by \Angstrom tab)
const ° = π/180  # 180° = π

# Compound constants
const πc₀ = π * c₀  # e.g., angular frequency ω = 2πc₀ / λ
const π²c₀ = π^2 * c₀
const iπ = im * π  # e.g., exp(iπ/3)
const i2π = 2iπ  # e.g., exp(i2π/3)
const ℎc₀ = ℎ * c₀  # e.g., photon energy E = ℎc₀ / λ

eV2meter(E_ineV::Real) = (E_inJ = E_ineV * e⁺; λ_inmeter = ℎc₀ / E_inJ)  # λ = c₀ / ν = hc₀ / hν
meter2eV(λ_inmeter::Real) = (E_inJ = ℎc₀ / λ_inmeter; E_ineV = E_inJ / e⁺)

## SI prefixes
# Prefixes > 1
const yotta = 1e24
const zetta = 1e21
const exa = 1e18
const peta = 1e15
const tera = 1e12
const giga = 1e9
const mega = 1e6
const kilo = 1e3
const hecto = 1e2
const deca = 1e1

# Prefixes < 1
const deci = 1e-1
const centi = 1e-2
const milli = 1e-3
const micro = 1e-6
const nano = 1e-9
const pico = 1e-12
const femto = 1e-15
const atto = 1e-18
const zepto = 1e-21
const yocto = 1e-24

end


println("loading LowRankModels")
@time @everywhere using LowRankModels

function fit_pca(m,n,k)
	# matrix to encode
	Random.seed!(1)
	A = randn(m,k)*randn(k,n)
	X=randn(k,m)
	Y=randn(k,n)
	losses = fill(QuadLoss(),n)
	r = QuadReg()
	glrm = GLRM(A,losses,r,r,k, X=X, Y=Y)
	glrm = share(glrm)
	p = Params()
	p.max_iter = 10
	X,Y,ch = fit!(glrm)
	println("Convergence history:",ch.objective)
	println("Time/iter:",ch.times[end]/10)
	return A,X,Y,ch
end

@everywhere Random.seed!(1)
fit_pca(100,100,50)


""" Conjugate gradient algorithm using a line search supporting stopping
    Generic version receiving a conjugacy formula
    kwargs may specify options for the line search.
"""
function CG_generic(nlp       :: AbstractNLPModel;
                    x         :: Vector{T}=copy(nlp.meta.x0),
                    stp       :: NLPStopping = NLPStopping(nlp,
                                                           NLPAtX(nlp.meta.x0)),
                    scaling   :: Bool = true,
                    LS_algo   :: Function = bracket{T},
                    LS_logger :: AbstractLogger = Logging.NullLogger(),
                    CG_formula :: Function = formula_HZ,
                    kwargs...      # eventually options for the line search
                    ) where T

    @info log_header([:iter, :f, :dual, :step, :slope], [Int, T, T, T, T],
                     hdr_override=Dict(:f=>"f(x)", :dual=>"‖∇f‖", :slope=>"IterLS"))

    n = nlp.meta.nvar

    f = obj(nlp,x)
    ∇f = grad(nlp, x)

    xt = similar(x)
    ∇ft = similar(∇f)

    τ₀ = 0.05
    τ₁ = 0.16

#    ϕ, ϕstp = prepare_LS(stp, x, ∇f, τ₀, f, ∇f)

    OK = update_and_start!(stp, x = x, fx = f, gx = ∇f)
    #update_and_stop!(stp,  x = x, fx = f, gx = ∇f)
    ϕ, ϕstp = prepare_LS(stp, x, ∇f, τ₀, f, ∇f)
    @info log_row(Any[0, f, norm(∇f)])

    β = 0.0
    d = zeros(n)
    scale = 1.0

    while !OK
        d = - ∇f + β*d

        slope = ∇f⋅d
        if slope > 0.0  # restart with negative gradient
            d = - ∇f
          slope =  ∇f⋅d
        end

        # Simple line search call
        # returns  t, xt = x + t*d,  ft=f(xt), ∇ft = ∇f(xt)

        t, xt, ft, ∇ft = linesearch(ϕ, ϕstp, x, d*scale, f, ∇f, τ₀, τ₁, logger = LS_logger, algo = LS_algo; kwargs...)
        fail_sub_pb = ~ϕstp.meta.optimal

        if fail_sub_pb
            OK = true
            stp.meta.fail_sub_pb = true
        else
            #move on
            s = xt - x
            y = ∇ft - ∇f
            β = 0.0
            #if (∇ft⋅∇f) < 0.75 * (∇ft⋅∇ft)   # Powell restart
            β = CG_formula(∇f,∇ft,s,d)
            #end
            if scaling
                scale = (y⋅s) / (y⋅y)
            end
            if scale <= 0.0
                scale = 1.0
            end
            x .= xt
            f = ft
            ∇f .= ∇ft
            OK = update_and_stop!(stp, x = x, gx = ∇f, fx = f)
        end

        @info log_row(Any[stp.meta.nb_of_stop, f, norm(∇f), t, ϕ.counters.neval_obj])
    end

    if !stp.meta.optimal
        @warn status(stp,list=true)
        @debug x
    end

    return stp
end


export SG_Filter, maxDerivativeOrder, polynomialOrder, apply_SG_filter, apply_SG_filter2D

import Base: filter, length

function _Vandermonde(T::DataType=Float64;halfWidth::Int=5,degree::Int=2)::Array{T,2}

    @assert halfWidth>=0
    @assert degree>=0

    x=T[i for i in -halfWidth:halfWidth]

    n = degree+1
    m = length(x)

    V = Array{T}(undef,m, n)

    for i = 1:m
        V[i,1] = T(1)
    end
    for j = 2:n
        for i = 1:m
            V[i,j] = x[i] * V[i,j-1]
        end
    end

    return V
end

#+SG_Filters
#
# A structure to store Savitzky-Golay filters.
#
struct SG_Filter{T<:AbstractFloat,N}
    _filter_set::Array{LinearFilter_DefaultCentered{T,N},1}
end
#+SG_Filters
#
# Returns the filter to be used to compute the  smoothed derivatives of order *derivativeOrder*.
#
function filter(sg::SG_Filter{T,N};derivativeOrder::Int=0) where {T<:AbstractFloat,N}
    @assert 0<= derivativeOrder <= maxDerivativeOrder(sg)
    return sg._filter_set[derivativeOrder+1]
end
#+SG_Filters
#
# Returns filter length, this is an odd number, see [[SG_Filters_Constructor][]]
length(sg::SG_Filter{T,N}) where {T<:AbstractFloat,N} = length(filter(sg))
#+SG_Filters
#
# Maximum order of the smoothed derivatives we can compute with *sg*
#
maxDerivativeOrder(sg::SG_Filter{T,N}) where {T<:AbstractFloat,N} = size(sg._filter_set,1)-1
#+SG_Filters
# Returns the degree of the polynomial used to construct the Savitzky-Golay filters, see [[SG_Filters_Constructor][]].
polynomialOrder(sg::SG_Filter{T,N}) where {T<:AbstractFloat,N} = maxDerivativeOrder(sg)

#+SG_Filters L:SG_Filters_Constructor
#
# Creates a set of Savitzky-Golay filters
#
# - filter length is 2*halfWidth+1
# - polynomial degree is degree
#
function SG_Filter(T::DataType=Float64;halfWidth::Int=5,degree::Int=2)::SG_Filter
    @assert degree>=0
    @assert halfWidth>=1
    @assert 2*halfWidth>degree

    V=_Vandermonde(T,halfWidth=halfWidth,degree=degree)
    Q,R=qr(V)
    # breaking change in Julia V1.0,
    # see https://github.com/JuliaLang/julia/issues/27397
    #
    # before Q was a "plain" matrix, now stored in compact form
    #
    # SG=R\Q'
    #
    # must be replaced by
    #
    # Q=Q*Matrix(I, size(V))
    # SG=R\Q'
    #
    Q=Q*Matrix(I, size(V))
    SG=R\Q'

    n_filter,n_coef = size(SG)

    buffer=Array{LinearFilter_DefaultCentered{T,n_coef},1}()

    for i in 1:n_filter
        SG[i,:]*=factorial(i-1)
        push!(buffer,LinearFilter(SG[i,:]))
    end

# Returns filters set
    return SG_Filter{T,n_coef}(buffer)
end

# +SG_Filters
#
# Applies SG filter to 1D signal
#
# *Returns:*
# - the smoothed signal
#
function apply_SG_filter(signal::Array{T,1},
                         sg::SG_Filter{T};
                         derivativeOrder::Int=0,
                         left_BE::Type{<:BoundaryExtension}=ConstantBE,
                         right_BE::Type{<:BoundaryExtension}=ConstantBE) where {T<:AbstractFloat}

    return directCrossCorrelation(filter(sg,derivativeOrder=derivativeOrder),
                                  signal,
                                  left_BE,
                                  right_BE)
end

# +SG_Filters
#
# Applies SG filter to 2D signal
#
# *Returns:*
# - the smoothed signal
#
function apply_SG_filter2D(signal::Array{T,2},
                           sg_I::SG_Filter{T},
                           sg_J::SG_Filter{T};
                           derivativeOrder_I::Int=0,
                           derivativeOrder_J::Int=0,
                           min_I_BE::Type{<:BoundaryExtension}=ConstantBE,
                           max_I_BE::Type{<:BoundaryExtension}=ConstantBE,
                           min_J_BE::Type{<:BoundaryExtension}=ConstantBE,
                           max_J_BE::Type{<:BoundaryExtension}=ConstantBE) where {T<:AbstractFloat}

    return directCrossCorrelation2D(filter(sg_I,derivativeOrder=derivativeOrder_I),
                                    filter(sg_J,derivativeOrder=derivativeOrder_J),
                                    signal,
                                    min_I_BE,
                                    max_I_BE,
                                    min_J_BE,
                                    max_J_BE)
end


function triangularity(g::GEQDSKFile)
    Rmin, Rmax = extrema(filter(!iszero,g.rbbbs))
    Rgeo = (Rmin + Rmax)/2
    a = (Rmax - Rmin)/2
    Rupper = g.rbbbs[argmax(g.zbbbs)]
    Rlower = g.rbbbs[argmin(g.zbbbs)]
    delta_upper = (Rgeo - Rupper)/a
    delta_lower = (Rgeo - Rlower)/a

    return delta_lower, delta_upper
end

function ellipticity(g::GEQDSKFile)
    Rmin, Rmax = extrema(filter(!iszero,g.rbbbs))
    a = (Rmax - Rmin)/2
    Zmin, Zmax = extrema(g.zbbbs)

    return (Zmax - Zmin)/(2a)
end

function elongation(g::GEQDSKFile)
    ellipticity(g)
end

function major_radius(g::GEQDSKFile)
    0.5*(+(extrema(filter(!iszero,g.rbbbs))...))
end

function minor_radius(g::GEQDSKFile)
    -0.5*(-(extrema(filter(!iszero,g.rbbbs))...))
end

function aspect_ratio(g::GEQDSKFile)
    major_radius(g)/minor_radius(g)
end


# Conjugate models for multivariate normal

using Distributions
using ConjugatePriors

using ConjugatePriors:
    posterior,
    posterior_randmodel,
    NormalWishart,
    NormalInverseWishart

ConjugatePriors.NormalInverseWishart(nix2::NormalInverseChisq) =
    NormalInverseWishart([nix2.μ], nix2.κ, nix2.ν*reshape([nix2.σ2], 1, 1), nix2.ν)

@testset "Conjugate models for multivariate normal" begin

    @testset "MvNormal -- Normal (known covariance)" begin

        n = 3
        p = 4
        X = reshape(Float64[1:12;], p, n)
        w = rand(n)
        Xw = X * Diagonal(w)

        Sigma = 0.75I + fill(0.25, p, p)
        ss = suffstats(MvNormalKnownCov(Sigma), X)
        ssw = suffstats(MvNormalKnownCov(Sigma), X, w)

        s_t = sum(X, dims=2)
        ws_t = sum(Xw, dims=2)
        tw_t = length(w)
        wtw_t = sum(w)

        @test ss.sx ≈ s_t
        @test ss.tw ≈ tw_t

        @test ssw.sx ≈ ws_t
        @test ssw.tw ≈ wtw_t

        # Posterior
        n = 10
        # n = 100
        mu_true = [2., 3.]
        Sig_true = Matrix(1.0I, 2, 2)
        Sig_true[1,2] = Sig_true[2,1] = 0.25
        mu0 = [2.5, 2.5]
        Sig0 = Matrix(1.0I, 2, 2)
        Sig0[1,2] = Sig0[2,1] = 0.5
        X = rand(MultivariateNormal(mu_true, Sig_true), n)
        pri = MultivariateNormal(mu0, Sig0)

        post = posterior((pri, Sig_true), MvNormal, X)
        @test isa(post, FullNormal)

        @test post.μ ≈ inv(inv(Sig0) + n*inv(Sig_true))*(n*inv(Sig_true)*mean(X,dims=2) + inv(Sig0)*mu0)
        @test post.Σ.mat ≈ inv(inv(Sig0) + n*inv(Sig_true))

        # posterior_sample

        ps = posterior_randmodel((pri, Sig_true), MvNormal, X)
        @test isa(ps, FullNormal)
        @test insupport(ps, ps.μ)
        @test insupport(InverseWishart, ps.Σ.mat)

    end

    @testset "NormalInverseWishart - MvNormal" begin

        mu_true = [2., 2.]
        Sig_true = Matrix(1.0I, 2, 2)
        Sig_true[1,2] = Sig_true[2,1] = 0.25

        X = rand(MultivariateNormal(mu_true, Sig_true), n)
        Xbar = mean(X,dims=2)
        Xm = X .- mean(X,dims=2)

        mu0 = [2., 3.]
        kappa0 = 3.
        nu0 = 4.
        T0 = Matrix(1.0I, 2, 2)
        T0[1,2] = T0[2,1] = .5
        pri = NormalInverseWishart(mu0, kappa0, T0, nu0)

        post = posterior(pri, MvNormal, X)

        @test post.mu ≈ (kappa0*mu0 + n*Xbar)./(kappa0 + n)
        @test post.kappa ≈ kappa0 + n
        @test post.nu ≈ nu0 + n
        @test Matrix(post.Lamchol) ≈ T0 + Xm*transpose(Xm) + kappa0*n/(kappa0+n)*(Xbar-mu0)*(Xbar-mu0)'

        ps = posterior_randmodel(pri, MultivariateNormal, X)

        @test isa(ps, MultivariateNormal)
        @test insupport(ps, ps.μ) && insupport(InverseWishart, ps.Σ.mat)

        @testset "Equivalence with Normal-Inverse Chi-squared" begin
            nix2 = NormalInverseChisq(1., 2., 3., 4.)
            niw = NormalInverseWishart(nix2)

            x = rand(Normal(3, 2), 100)

            ss = suffstats(Normal, x)
            ss_mv = suffstats(MvNormal, reshape(x, 1, :))

            post_nix2 = posterior(nix2, ss)
            post_niw = posterior(niw, ss_mv)

            @test all(post_nix2.μ .≈ post_niw.mu)
            @test post_nix2.κ ≈ post_niw.kappa
            @test post_nix2.ν ≈ post_niw.nu
            @test all(post_nix2.σ2 .≈ Matrix(post_niw.Lamchol)[1] ./ post_niw.nu)

            μ, σ2 = rand(post_nix2)
            @test logpdf(post_nix2, μ, σ2) ≈ logpdf(post_niw, [μ], reshape([σ2], 1, 1))
            @test logpdf(nix2, μ, σ2) ≈ logpdf(niw, [μ], reshape([σ2], 1, 1))

        end

    end

    @testset "NormalWishart - MvNormal" begin

        mu_true = [2., 2.]
        Lam_true = Matrix(1.0I, 2, 2)
        Lam_true[1,2] = Lam_true[2,1] = 0.25

        X = rand(MvNormal(mu_true, inv(Lam_true)), n)
        Xbar = mean(X,dims=2)
        Xm = X .- Xbar

        mu0 = [2., 3.]
        kappa0 = 3.
        nu0 = 4.
        T0 = Matrix(1.0I, 2, 2)
        T0[1,2] = T0[2,1] = .5
        pri = NormalWishart(mu0, kappa0, T0, nu0)

        post = posterior(pri, MvNormal, X)

        @test post.mu ≈ (kappa0*mu0 + n*Xbar)./(kappa0 + n)
        @test post.kappa ≈ kappa0 + n
        @test post.nu ≈ nu0 + n
        @test Matrix(post.Tchol) ≈ T0 + Xm*transpose(Xm) + kappa0*n/(kappa0+n)*(Xbar-mu0)*(Xbar-mu0)'

        ps = posterior_randmodel(pri, MvNormal, X)

        @test isa(ps, MultivariateNormal)
        @test insupport(ps, ps.μ)
        @test insupport(InverseWishart, ps.Σ.mat)  # InverseWishart on purpose

    end
end


# Example from JuliaOpt/SumOfSquares.jl
# COSMO supports MOI/JuMP and therefore also works with JuMP extensions
# like SumOfSquares.jl

using SumOfSquares, COSMO
using DynamicPolynomials
@polyvar x y
motzkin = x^4*y^2 + x^2*y^4 + 1 - 3x^2*y^2

model = SOSModel(with_optimizer(COSMO.Optimizer, rho = 1e-5);)
@constraint(model, (x^2 + y^2) * motzkin >= 0)
optimize!(model)

primal_status(model)

# The full example can be found at:
# https://github.com/JuliaOpt/SumOfSquares.jl/blob/master/examples/motzkin.ipynb


function ψ₀(x::AbstractArray{T,1}, a) where {T <: Number}
    x[1]^4 / 8 + a * (x[1]^2 * log(x[1]) / 2 - x[1]^4 / 8 )
end

function ψ₁(x::AbstractArray{T,1}) where {T <: Number}
    1
end

function ψ₂(x::AbstractArray{T,1}) where {T <: Number}
    x[1]^2
end

function ψ₃(x::AbstractArray{T,1}) where {T <: Number}
    x[2]^2 - x[1]^2 * log(x[1])
end

function ψ₄(x::AbstractArray{T,1}) where {T <: Number}
    x[1]^4 - 4 * x[1]^2 * x[2]^2
end

function ψ₅(x::AbstractArray{T,1}) where {T <: Number}
    2 * x[2]^4 - 9 * x[2]^2 * x[1]^2 + 3 * x[1]^4 * log(x[1]) - 12 * x[1]^2 * x[2]^2 * log(x[1])
end

function ψ₆(x::AbstractArray{T,1}) where {T <: Number}
    x[1]^6 - 12 * x[1]^4 * x[2]^2 + 8 * x[1]^2 * x[2]^4
end

function ψ₇(x::AbstractArray{T,1}) where {T <: Number}
    8 * x[2]^6 - 140 * x[2]^4 * x[1]^2 + 75 * x[2]^2 * x[1]^4 - 15 * x[1]^6 * log(x[1]) +
                 180 * x[1]^4 * x[2]^2 * log(x[1]) - 120 * x[1]^2 * x[2]^4 * log(x[1])
end

function ψ₈(x::AbstractArray{T,1}) where {T <: Number}
    x[2]
end

function ψ₉(x::AbstractArray{T,1}) where {T <: Number}
    x[2] * x[1]^2
end

function ψ₁₀(x::AbstractArray{T,1}) where {T <: Number}
    x[2]^3 - 3 * x[2] * x[1]^2 * log(x[1])
end

function ψ₁₁(x::AbstractArray{T,1}) where {T <: Number}
    3 * x[2] * x[1]^4 - 4 * x[2]^3 * x[1]^2
end

function ψ₁₂(x::AbstractArray{T,1}) where {T <: Number}
    8 * x[2]^5 - 45 * x[2] * x[1]^4 - 80 * x[2]^3 * x[1]^2 * log(x[1]) + 60 * x[2] * x[1]^4 * log(x[1])
end


# This is model 13.4 from Statistical Rethinking Edition 1.

# \toc

# ## Data

import CSV

using DataFrames
using Turing
using TuringModels

data_path = joinpath(TuringModels.project_root, "data", "UCBadmit.csv")
df = CSV.read(data_path, DataFrame; delim=';')

dept_map = Dict(key => idx for (idx, key) in enumerate(unique(df.dept)))
df.male = [g == "male" ? 1 : 0 for g in df.gender]
df.dept_id = [dept_map[de] for de in df.dept]
df

# ## Model

@model function m13_4(applications, dept_id, male, admit)
    sigma_dept ~ truncated(Cauchy(0, 2), 0, Inf)
    a ~ Normal(0, 10)
    a_dept ~ filldist(Normal(a, sigma_dept), 6)

    logit_p = a_dept[dept_id]

    admit .~ BinomialLogit.(applications, logit_p)
end;

# ## Output

chains = sample(
    m13_4(df.applications, df.dept_id, df.male, df.admit),
    Turing.NUTS(0.65),
    5000
)

# \defaultoutput{}

# ## Original output

m_13_4_rethinking = """
Inference for Stan model: 0c5c36512c20c41995d1bd25be525138.
    3 chains, each with iter=4500; warmup=500; thin=1;
    post-warmup draws per chain=4000, total post-warmup draws=12000.

                   mean se_mean   sd     2.5%      25%      50%      75%    97.5%
    a_dept[1]      0.59    0.00 0.07     0.46     0.54     0.59     0.64     0.73
    a_dept[2]      0.54    0.00 0.09     0.37     0.48     0.54     0.60     0.71
    a_dept[3]     -0.62    0.00 0.07    -0.75    -0.66    -0.62    -0.57    -0.49
    a_dept[4]     -0.67    0.00 0.08    -0.81    -0.72    -0.67    -0.62    -0.52
    a_dept[5]     -1.09    0.00 0.10    -1.28    -1.15    -1.09    -1.03    -0.90
    a_dept[6]     -2.65    0.00 0.15    -2.96    -2.75    -2.65    -2.55    -2.37
    a             -0.63    0.01 0.63    -1.89    -1.01    -0.64    -0.26     0.66
    sigma_dept     1.45    0.01 0.57     0.77     1.08     1.33     1.68     2.90
    lp__       -2602.37    0.03 2.08 -2607.45 -2603.49 -2602.01 -2600.86 -2599.35
               n_eff Rhat
    a_dept[1]  11285    1
    a_dept[2]  11943    1
    a_dept[3]  12707    1
    a_dept[4]  11592    1
    a_dept[5]  14598    1
    a_dept[6]  13363    1
    a           6965    1
    sigma_dept  5643    1
    lp__        5322    1
""";


using Turing
using DelimitedFiles

using BenchmarkHelper

if !haskey(BenchmarkSuite, "nuts")
    BenchmarkSuite["nuts"] = BenchmarkGroup(["nuts"])
end

fname = joinpath(dirname(@__FILE__), "sv_nuts.data")
y, header = readdlm(fname, ',', header=true)

# Stochastic volatility (SV)
@model function sv_nuts(y, dy, ::Type{T}=Vector{Float64}) where {T}
    N = size(y,1)

    τ ~ Exponential(1/100)
    ν ~ Exponential(1/100)
    s = T(undef, N)

    s[1] ~ Exponential(1/100)
    for n in 2:N
        s[n] ~ Normal(log(s[n-1]), τ)
        s[n] = exp(s[n])
        dy = log(y[n] / y[n-1]) / s[n]
        dy ~ TDist(ν)
    end
end

# Sampling parameter settings
n_samples = 10_000

# Sampling
BenchmarkSuite["nuts"]["sv"] = @benchmarkable sample(sv_nuts(y, NaN), NUTS(0.65), n_samples)


f = open("input05.txt", "r")
    l = readlines(f)
close(f)

coordinates = Array{Int}(undef, length(l), 4)

for (i, line) in enumerate(l)
    (s1, s2) = split(line, "->")
    x1 = parse(Int, split(s1, ",")[1])
    y1 = parse(Int, split(s1, ",")[2])
    x2 = parse(Int, split(s2, ",")[1])
    y2 = parse(Int, split(s2, ",")[2])
    coordinates[i,:] = [x1, y1, x2, y2]
end

max_coord = maximum(coordinates)

vent_map = zeros((max_coord+1, max_coord+1))

coordinates += ones(size(coordinates))
coordinates = trunc.(Int, coordinates)
for row in eachrow(coordinates)
    if row[1] == row[3]
        if row[2] > row[4]
            plus = ones(size(vent_map[row[4]:row[2],row[1]:row[3]]))
            vent_map[row[4]:row[2],row[1]:row[3]] += plus
        elseif row[4] > row[2]
            plus = ones(size(vent_map[row[2]:row[4],row[1]:row[3]]))
            vent_map[row[2]:row[4],row[1]:row[3]] += plus
        end
    elseif row[2] == row[4]
        if row[3] > row[1]
            plus = ones(size(vent_map[row[2]:row[4],row[1]:row[3]]))
            vent_map[row[2]:row[4],row[1]:row[3]] += plus
        elseif row[1] > row[3]
            plus = ones(size(vent_map[row[2]:row[4],row[3]:row[1]]))
            vent_map[row[2]:row[4],row[3]:row[1]] += plus
        end
    end
end
counter = 0
for i in vent_map
    if i >= 2
        global counter += 1
    end
end
println("Result 1: $counter")

for row in eachrow(coordinates)
    if row[1] != row[3] && row[2] != row[4]
        if row[1] > row[3]
            x = collect(row[1]:-1:row[3])
        else
            x = collect(row[1]:row[3])
        end
        if row[2] > row[4]
            y = collect(row[2]:-1:row[4])
        else
            y = collect(row[2]:row[4])
        end
        for (i, j) in zip(y, x)
            vent_map[i, j] += 1
        end
    end
end
counter = 0
for i in vent_map
    if i >= 2
        global counter += 1
    end
end
println("Result 2: $counter")


function loadCircleData(num_data)
    center = [5.0, 5.0]
    radiu_inner = 2
    radiu_outer = 4
    num_inner = floor(num_data / 3)
    num_outer = num_data - num_inner

    data = []
    theta = 0.0
    print(1)
    for i in 1:num_inner
        pho = (theta % 360) * pi / 180
        tmp = zeros(1,2)
        tmp[1] = radiu_inner * cos(pho) + rand() + center[1]
        tmp[2] = radiu_inner * sin(pho) + rand() + center[2]
        push!(data,temp)
        theta += 2
    end

    theta = 0.0
    for i in 1:num_outer
        pho = (theta % 360) * pi / 180
        tmp = zeros(1,2)
        tmp[1] = radiu_outer * cos(pho) + rand() + center[1]
        tmp[2] = radiu_outer * sin(pho) + rand() + center[2]
        push!(data,tmp)
        theta += 1
    end

    Mat_Label = zeros(2, 2)
    Mat_Label[1,:] = center + [ 0.5 - radiu_inner , 0]
    Mat_Label[2,:] = center + [ 0.5 - radiu_outer , 0]
    labels = [1, 2]
    Mat_Unlabel = reduce(hcat,data)
    return Mat_Label, labels, Mat_Unlabel
end


immutable Weibull <: ContinuousUnivariateDistribution
    shape::Float64
    scale::Float64
    function Weibull(sh::Real, sc::Real)
    	zero(sh) < sh && zero(sc) < sc || error("Both shape and scale must be positive")
    	new(float64(sh), float64(sc))
    end
end

Weibull(sh::Real) = Weibull(sh, 1.0)

## Support
@continuous_distr_support Weibull 0.0 Inf

## Properties
mean(d::Weibull) = d.scale * gamma(1.0 + 1.0 / d.shape)
median(d::Weibull) = d.scale * log(2.0)^(1.0 / d.shape)

mode(d::Weibull) = d.shape > 1.0 ? (ik = 1.0/d.shape; d.scale * (1.0-ik)^ik) : 0.0

var(d::Weibull) = d.scale^2 * gamma(1.0 + 2.0 / d.shape) - mean(d)^2

function skewness(d::Weibull)
    tmp = gamma(1.0 + 3.0 / d.shape) * d.scale^3
    tmp -= 3.0 * mean(d) * var(d)
    tmp -= mean(d)^3
    return tmp / std(d)^3
end

function kurtosis(d::Weibull)
    λ, k = d.scale, d.shape
    μ = mean(d)
    σ = std(d)
    γ = skewness(d)
    den = λ^4 * gamma(1.0 + 4.0 / k) -
          4.0 * γ * σ^3 * μ -
          6.0 * μ^2 * σ^2 - μ^4
    num = σ^4
    return den / num - 3.0
end

function entropy(d::Weibull)
    λ, k = d.scale, d.shape
    return ((k - 1.0) / k) * -digamma(1.0) + log(λ / k) + 1.0
end

## Functions
function pdf(d::Weibull, x::Real)
    x < zero(x) && return 0.0
    a = x/d.scale
    d.shape/d.scale * a^(d.shape-1.0) * exp(-a^d.shape)
end
function logpdf(d::Weibull, x::Real)
    x < zero(x) && return -Inf
    a = x/d.scale
    log(d.shape/d.scale) + (d.shape-1.0)*log(a) - a^d.shape
end

cdf(d::Weibull, x::Real) = x <= zero(x) ? 0.0 : -expm1(-((x / d.scale)^d.shape))
ccdf(d::Weibull, x::Real) = x <= zero(x) ? 1.0 : exp(-((x / d.scale)^d.shape))
logcdf(d::Weibull, x::Real) = x <= zero(x) ? -Inf : log1mexp(-((x / d.scale)^d.shape))
logccdf(d::Weibull, x::Real) = x <= zero(x) ? 0.0 : -(x / d.scale)^d.shape

quantile(d::Weibull, p::Real) = @checkquantile p d.scale*(-log1p(-p))^(1/d.shape)
cquantile(d::Weibull, p::Real) = @checkquantile p d.scale*(-log(p))^(1/d.shape)
invlogcdf(d::Weibull, lp::Real) = lp > zero(lp) ? NaN : d.scale*(-log1mexp(lp))^(1/d.shape)
invlogccdf(d::Weibull, lp::Real) = lp > zero(lp) ? NaN : d.scale*(-lp)^(1/d.shape)

function gradlogpdf(d::Weibull, x::Real)
  insupport(Weibull, x) ? (d.shape - 1.0) / x - d.shape * x^(d.shape - 1.0) / (d.scale^d.shape) : 0.0
end

## Sampling
rand(d::Weibull) = d.scale*Base.Random.randmtzig_exprnd()^(1/d.shape)


@testset "cxsc.intervaladdsub" begin

    @test +(interval(10.0, 20.0), interval(13.0, 17.0)) === Interval(23.0, 37.0)

    @test +(interval(13.0, 17.0), interval(10.0, 20.0)) === Interval(23.0, 37.0)

    @test -(interval(10.0, 20.0), interval(13.0, 16.0)) === Interval(-6.0, 7.0)

    @test -(interval(13.0, 16.0), interval(10.0, 20.0)) === Interval(-7.0, 6.0)

    @test -(interval(10.0, 20.0)) === Interval(-20.0, -10.0)

    @test +(interval(10.0, 20.0)) === Interval(10.0, 20.0)

end

@testset "cxsc.intervalmuldiv" begin

    @test *(interval(1.0, 2.0), interval(3.0, 4.0)) === Interval(3.0, 8.0)

    @test *(interval(-1.0, 2.0), interval(3.0, 4.0)) === Interval(-4.0, 8.0)

    @test *(interval(-2.0, 1.0), interval(3.0, 4.0)) === Interval(-8.0, 4.0)

    @test *(interval(-2.0, -1.0), interval(3.0, 4.0)) === Interval(-8.0, -3.0)

    @test *(interval(1.0, 2.0), interval(-3.0, 4.0)) === Interval(-6.0, 8.0)

    @test *(interval(-1.0, 2.0), interval(-3.0, 4.0)) === Interval(-6.0, 8.0)

    @test *(interval(-2.0, 1.0), interval(-3.0, 4.0)) === Interval(-8.0, 6.0)

    @test *(interval(-2.0, -1.0), interval(-3.0, 4.0)) === Interval(-8.0, 6.0)

    @test *(interval(1.0, 2.0), interval(-4.0, 3.0)) === Interval(-8.0, 6.0)

    @test *(interval(-1.0, 2.0), interval(-4.0, 3.0)) === Interval(-8.0, 6.0)

    @test *(interval(-2.0, 1.0), interval(-4.0, 3.0)) === Interval(-6.0, 8.0)

    @test *(interval(-2.0, -1.0), interval(-4.0, 3.0)) === Interval(-6.0, 8.0)

    @test *(interval(1.0, 2.0), interval(-4.0, -3.0)) === Interval(-8.0, -3.0)

    @test *(interval(-1.0, 2.0), interval(-4.0, -3.0)) === Interval(-8.0, 4.0)

    @test *(interval(-2.0, -1.0), interval(-4.0, -3.0)) === Interval(3.0, 8.0)

    @test /(interval(1.0, 2.0), interval(4.0, 8.0)) === Interval(0.125, 0.5)

    @test /(interval(-1.0, 2.0), interval(4.0, 8.0)) === Interval(-0.25, 0.5)

    @test /(interval(-2.0, 1.0), interval(4.0, 8.0)) === Interval(-0.5, 0.25)

    @test /(interval(-2.0, -1.0), interval(4.0, 8.0)) === Interval(-0.5, -0.125)

    @test /(interval(1.0, 2.0), interval(-4.0, 8.0)) === entireinterval()

    @test /(interval(-1.0, 2.0), interval(-4.0, 8.0)) === entireinterval()

    @test /(interval(-2.0, 1.0), interval(-4.0, 8.0)) === entireinterval()

    @test /(interval(-2.0, -1.0), interval(-4.0, 8.0)) === entireinterval()

    @test /(interval(1.0, 2.0), interval(-8.0, 4.0)) === entireinterval()

    @test /(interval(-1.0, 2.0), interval(-8.0, 4.0)) === entireinterval()

    @test /(interval(-2.0, 1.0), interval(-8.0, 4.0)) === entireinterval()

    @test /(interval(-2.0, -1.0), interval(-8.0, 4.0)) === entireinterval()

    @test /(interval(1.0, 2.0), interval(-8.0, -4.0)) === Interval(-0.5, -0.125)

    @test /(interval(-1.0, 2.0), interval(-8.0, -4.0)) === Interval(-0.5, 0.25)

    @test /(interval(-2.0, 1.0), interval(-8.0, -4.0)) === Interval(-0.25, 0.5)

    @test /(interval(-2.0, -1.0), interval(-8.0, -4.0)) === Interval(0.125, 0.5)

end

@testset "cxsc.intervalsetops" begin

    @test hull(interval(-2.0, 2.0), interval(-4.0, -3.0)) === Interval(-4.0, 2.0)

    @test hull(interval(-2.0, 2.0), interval(-4.0, -1.0)) === Interval(-4.0, 2.0)

    @test hull(interval(-2.0, 2.0), interval(-4.0, 4.0)) === Interval(-4.0, 4.0)

    @test hull(interval(-2.0, 2.0), interval(-1.0, 1.0)) === Interval(-2.0, 2.0)

    @test hull(interval(-2.0, 2.0), interval(1.0, 4.0)) === Interval(-2.0, 4.0)

    @test hull(interval(-2.0, 2.0), interval(3.0, 4.0)) === Interval(-2.0, 4.0)

    @test hull(interval(-4.0, -3.0), interval(-2.0, 2.0)) === Interval(-4.0, 2.0)

    @test hull(interval(-4.0, -1.0), interval(-2.0, 2.0)) === Interval(-4.0, 2.0)

    @test hull(interval(-4.0, 4.0), interval(-2.0, 2.0)) === Interval(-4.0, 4.0)

    @test hull(interval(-1.0, 1.0), interval(-2.0, 2.0)) === Interval(-2.0, 2.0)

    @test hull(interval(1.0, 4.0), interval(-2.0, 2.0)) === Interval(-2.0, 4.0)

    @test hull(interval(3.0, 4.0), interval(-2.0, 2.0)) === Interval(-2.0, 4.0)

    @test intersect(interval(-2.0, 2.0), interval(-4.0, -3.0)) === emptyinterval()

    @test intersect(interval(-2.0, 2.0), interval(-4.0, -1.0)) === Interval(-2.0, -1.0)

    @test intersect(interval(-2.0, 2.0), interval(-4.0, 4.0)) === Interval(-2.0, 2.0)

    @test intersect(interval(-2.0, 2.0), interval(-1.0, 1.0)) === Interval(-1.0, 1.0)

    @test intersect(interval(-2.0, 2.0), interval(1.0, 4.0)) === Interval(1.0, 2.0)

    @test intersect(interval(-2.0, 2.0), interval(3.0, 4.0)) === emptyinterval()

    @test intersect(interval(-4.0, -3.0), interval(-2.0, 2.0)) === emptyinterval()

    @test intersect(interval(-4.0, -1.0), interval(-2.0, 2.0)) === Interval(-2.0, -1.0)

    @test intersect(interval(-4.0, 4.0), interval(-2.0, 2.0)) === Interval(-2.0, 2.0)

    @test intersect(interval(-1.0, 1.0), interval(-2.0, 2.0)) === Interval(-1.0, 1.0)

    @test intersect(interval(1.0, 4.0), interval(-2.0, 2.0)) === Interval(1.0, 2.0)

    @test intersect(interval(3.0, 4.0), interval(-2.0, 2.0)) === emptyinterval()

end

@testset "cxsc.intervalmixsetops" begin

    @test hull(interval(-2.0, 2.0), interval(-4.0, -4.0)) === Interval(-4.0, 2.0)

    @test hull(interval(-2.0, 2.0), interval(1.0, 1.0)) === Interval(-2.0, 2.0)

    @test hull(interval(-2.0, 2.0), interval(4.0, 4.0)) === Interval(-2.0, 4.0)

    @test hull(interval(-4.0, -4.0), interval(-2.0, 2.0)) === Interval(-4.0, 2.0)

    @test hull(interval(1.0, 1.0), interval(-2.0, 2.0)) === Interval(-2.0, 2.0)

    @test hull(interval(4.0, 4.0), interval(-2.0, 2.0)) === Interval(-2.0, 4.0)

    @test intersect(interval(-2.0, 2.0), interval(-4.0, -4.0)) === emptyinterval()

    @test intersect(interval(-2.0, 2.0), interval(1.0, 1.0)) === Interval(1.0, 1.0)

    @test intersect(interval(-2.0, 2.0), interval(4.0, 4.0)) === emptyinterval()

    @test intersect(interval(-4.0, -4.0), interval(-2.0, 2.0)) === emptyinterval()

    @test intersect(interval(1.0, 1.0), interval(-2.0, 2.0)) === Interval(1.0, 1.0)

    @test intersect(interval(4.0, 4.0), interval(-2.0, 2.0)) === emptyinterval()

end

@testset "cxsc.scalarmixsetops" begin

    @test hull(interval(-2.0, -2.0), interval(-4.0, -4.0)) === Interval(-4.0, -2.0)

    @test hull(interval(-2.0, -2.0), interval(-2.0, -2.0)) === Interval(-2.0, -2.0)

    @test hull(interval(-2.0, -2.0), interval(2.0, 2.0)) === Interval(-2.0, 2.0)

    @test hull(interval(-4.0, -4.0), interval(-2.0, -2.0)) === Interval(-4.0, -2.0)

    @test hull(interval(-2.0, -2.0), interval(-2.0, -2.0)) === Interval(-2.0, -2.0)

    @test hull(interval(2.0, 2.0), interval(-2.0, -2.0)) === Interval(-2.0, 2.0)

end

@testset "cxsc.intervalsetcompops" begin

    @test isinterior(interval(-1.0, 2.0), interval(-1.0, 2.0)) === false

    @test isinterior(interval(-2.0, 1.0), interval(-3.0, 2.0)) === true

    @test isinterior(interval(-2.0, 2.0), interval(-1.0, 1.0)) === false

    @test isinterior(interval(-2.0, 2.0), interval(-1.0, 2.0)) === false

    @test isinterior(interval(-2.0, 2.0), interval(-2.0, 1.0)) === false

    @test isinterior(interval(-2.0, 2.0), interval(-2.0, 3.0)) === false

    @test isinterior(interval(-2.0, 2.0), interval(-3.0, 2.0)) === false

    @test isinterior(interval(-1.0, 2.0), interval(-1.0, 2.0)) === false

    @test isinterior(interval(-3.0, 2.0), interval(-2.0, 1.0)) === false

    @test isinterior(interval(-1.0, 1.0), interval(-2.0, 2.0)) === true

    @test isinterior(interval(-1.0, 2.0), interval(-2.0, 2.0)) === false

    @test isinterior(interval(-2.0, 1.0), interval(-2.0, 2.0)) === false

    @test isinterior(interval(-2.0, 3.0), interval(-2.0, 2.0)) === false

    @test isinterior(interval(-3.0, 2.0), interval(-2.0, 2.0)) === false

    @test issubset(interval(-1.0, 2.0), interval(-1.0, 2.0)) === true

    @test issubset(interval(-2.0, 1.0), interval(-3.0, 2.0)) === true

    @test issubset(interval(-2.0, 2.0), interval(-1.0, 1.0)) === false

    @test issubset(interval(-2.0, 2.0), interval(-1.0, 2.0)) === false

    @test issubset(interval(-2.0, 2.0), interval(-2.0, 1.0)) === false

    @test issubset(interval(-2.0, 2.0), interval(-2.0, 3.0)) === true

    @test issubset(interval(-2.0, 2.0), interval(-3.0, 2.0)) === true

    @test issubset(interval(-3.0, 2.0), interval(-2.0, 1.0)) === false

    @test issubset(interval(-1.0, 1.0), interval(-2.0, 2.0)) === true

    @test issubset(interval(-1.0, 2.0), interval(-2.0, 2.0)) === true

    @test issubset(interval(-2.0, 1.0), interval(-2.0, 2.0)) === true

    @test issubset(interval(-2.0, 3.0), interval(-2.0, 2.0)) === false

    @test issubset(interval(-3.0, 2.0), interval(-2.0, 2.0)) === false

    @test ==(interval(-1.0, 2.0), interval(-1.0, 2.0)) === true

    @test ==(interval(-2.0, 1.0), interval(-3.0, 2.0)) === false

    @test ==(interval(-2.0, 2.0), interval(-1.0, 1.0)) === false

    @test ==(interval(-2.0, 2.0), interval(-1.0, 2.0)) === false

    @test ==(interval(-2.0, 2.0), interval(-2.0, 1.0)) === false

    @test ==(interval(-2.0, 2.0), interval(-2.0, 3.0)) === false

    @test ==(interval(-2.0, 2.0), interval(-3.0, 2.0)) === false

end

@testset "cxsc.intervalscalarsetcompops" begin

    @test isinterior(interval(-1.0, 2.0), interval(-2.0, -2.0)) === false

    @test isinterior(interval(-2.0, 2.0), interval(-2.0, -2.0)) === false

    @test isinterior(interval(-2.0, 2.0), interval(0.0, 0.0)) === false

    @test isinterior(interval(-2.0, 2.0), interval(2.0, 2.0)) === false

    @test isinterior(interval(-2.0, 2.0), interval(3.0, 3.0)) === false

    @test isinterior(interval(-1.0, -1.0), interval(1.0, 1.0)) === false

    @test isinterior(interval(-1.0, -1.0), interval(-1.0, -1.0)) === false

    @test isinterior(interval(-2.0, -2.0), interval(-1.0, 2.0)) === false

    @test isinterior(interval(-2.0, -2.0), interval(-2.0, 2.0)) === false

    @test isinterior(interval(0.0, 0.0), interval(-2.0, 2.0)) === true

    @test isinterior(interval(2.0, 2.0), interval(-2.0, 2.0)) === false

    @test isinterior(interval(3.0, 3.0), interval(-2.0, 2.0)) === false

    @test isinterior(interval(1.0, 1.0), interval(-1.0, -1.0)) === false

    @test isinterior(interval(-1.0, -1.0), interval(-1.0, -1.0)) === false

    @test issubset(interval(-1.0, 2.0), interval(-2.0, -2.0)) === false

    @test issubset(interval(-2.0, 2.0), interval(-2.0, -2.0)) === false

    @test issubset(interval(-2.0, 2.0), interval(0.0, 0.0)) === false

    @test issubset(interval(-2.0, 2.0), interval(2.0, 2.0)) === false

    @test issubset(interval(-2.0, 2.0), interval(3.0, 3.0)) === false

    @test issubset(interval(-1.0, -1.0), interval(1.0, 1.0)) === false

    @test issubset(interval(-1.0, -1.0), interval(-1.0, -1.0)) === true

    @test issubset(interval(-2.0, -2.0), interval(-1.0, 2.0)) === false

    @test issubset(interval(-2.0, -2.0), interval(-2.0, 2.0)) === true

    @test issubset(interval(0.0, 0.0), interval(-2.0, 2.0)) === true

    @test issubset(interval(2.0, 2.0), interval(-2.0, 2.0)) === true

    @test issubset(interval(3.0, 3.0), interval(-2.0, 2.0)) === false

    @test issubset(interval(1.0, 1.0), interval(-1.0, -1.0)) === false

    @test issubset(interval(-1.0, -1.0), interval(-1.0, -1.0)) === true

    @test ==(interval(-1.0, 2.0), interval(-2.0, -2.0)) === false

    @test ==(interval(-2.0, 2.0), interval(-2.0, -2.0)) === false

    @test ==(interval(-2.0, 2.0), interval(0.0, 0.0)) === false

    @test ==(interval(-2.0, 2.0), interval(2.0, 2.0)) === false

    @test ==(interval(-2.0, 2.0), interval(3.0, 3.0)) === false

    @test ==(interval(-1.0, -1.0), interval(1.0, 1.0)) === false

    @test ==(interval(-1.0, -1.0), interval(-1.0, -1.0)) === true

end

@testset "cxsc.intervalstdfunc" begin

    @test interval(11.0, 11.0)^2 === Interval(121.0, 121.0)

    @test interval(0.0, 0.0)^2 === Interval(0.0, 0.0)

    @test interval(-9.0, -9.0)^2 === Interval(81.0, 81.0)

    @test sqrt(interval(121.0, 121.0)) === Interval(11.0, 11.0)

    @test sqrt(interval(0.0, 0.0)) === Interval(0.0, 0.0)

    @test sqrt(interval(81.0, 81.0)) === Interval(9.0, 9.0)

    @test nthroot(interval(27.0, 27.0), 3) === Interval(3.0, 3.0)

    @test nthroot(interval(0.0, 0.0), 4) === Interval(0.0, 0.0)

    @test nthroot(interval(1024.0, 1024.0), 10) === Interval(2.0, 2.0)

    @test ^(interval(2.0, 2.0), interval(2.0, 2.0)) === Interval(4.0, 4.0)

    @test ^(interval(4.0, 4.0), interval(5.0, 5.0)) === Interval(1024.0, 1024.0)

    @test ^(interval(2.0, 2.0), interval(3.0, 3.0)) === Interval(8.0, 8.0)

end


module Optimizers

export AbstractOptimizer, GD, MomentumGD, astep

include("./tensor_dt.jl")
using ..TensorDT: Tensor

include("./abstract_layers.jl")
using ..AbstractLayers: AbstractLayer, AL, forward, backward, parms, ∇parms

include("./abstract_optimizers.jl")
using ..AbstractOptimizers: AbstractOptimizer, AOpt, NAOpt

## ======================================================================
## Optimization
## ======================================================================

#### (Vanilla) Gradient Descent (GD)

struct GD <: AbstractOptimizer
  η::Float64
end

function astep(self::GD, sl::AbstractLayer)
  ## sl ≡ seq. layer
  for (_parms, _∇parms) ∈ zip(parms(sl), ∇parms(sl))
    _parms[:] = _parms - self.η .* _∇parms
  end
end

#### GD with momentum

mutable struct MomentumGD <: AbstractOptimizer
  η::Float64  ## learning rate
  α::Float64  ## momentum rate
  updates::Vector{Tensor}
  #
  function MomentumGD(;η=0.1, α=0.9, updates=Vector{Tensor}[])
    new(η, α, updates)
  end
end

function astep(self::MomentumGD, sl::AbstractLayer)
  ## sl ≡ seq. layer
  length(self.updates) == 0 &&
  (self.updates = [zeros(eltype(∇p), size(∇p)) for ∇p ∈ ∇parms(sl)])
  #
  for (_upd, _parms, _∇parms) ∈ zip(self.updates, parms(sl), ∇parms(sl))
    ## apply momentum
    _upd[:] = self.α * _upd .+ (1. - self.α) * _∇parms
    ## take a step
    _parms[:] = _parms - self.η .* _upd
  end
end

end  ## Module

Data Science From Scratch/src/optimizers.jl


export clip, nnorm, adiag, minv, maxv

function clip(M::Matrix)
    v = zeros(Float64, size(M))
    for i in 1:size(M)[1]
        for j in 1:size(M)[1]
            if M[i,j] < 0
                v[i, j] = 0
            elseif M[i,j] > 1
                v[i, j] = 1
            else
                v[i, j] = M[i, j]
            end
        end
    end
    return v
end

function nnorm(M::Matrix)
    v = zeros(Float64, size(M))
    mv = minimum(M)
    Mv = maximum(M)
    for i in 1:size(M)[1]
        for j in 1:size(M)[1]
            v[i, j] = (M[i, j] - mv) * ((1 - 0)/(Mv - mv)) + 0
        end
    end
    return v
end

function adiag(M::Matrix)
	v = zeros(size(M)[1])
	for i in 1:(size(M)[1])
		v[i] = M[(size(M)[1] + 1) - i , i]
	end
	return v
end

xax(M::Matrix) = range(1, size(M)[1], step = 1)

minv(M::Matrix) = minimum(adiag(M))
maxv(M::Matrix) = maximum(adiag(M))


using Match

function isLeapYear(y)
	y%4==0 && !(y%100==0 && !(y%400==0))
	# if !(y%4==0)
	# 	false
	# elseif !(y%100==0)
	# 	true
	# elseif !(y%400==0)
	# 	false
	# else
	# 	true
	# end
end

function nextday(currentdate)
	y = currentdate["year"]
	leapy = isLeapYear(y)
	m = currentdate["month"]
	nDaysInMonth = @match m begin
        1  => 31
        2  => leapy ? 29 : 28
        3  => 31
        4  => 30
        5  => 31
        6  => 30
        7  => 31
        8  => 31
        9  => 30
        10 => 31
        11 => 30
        12 => 31
        _  => error("bug")
    end
    n = currentdate["number"]
    isNewMonth = (n == nDaysInMonth)
    newn = isNewMonth ? 1 : n+1
    newm = isNewMonth ? (m==12 ? 1 : m+1) : m
    newy = (isNewMonth && m==12) ? y+1 : y

	d = currentdate["day"]
    newd = ((d+1 -1) % 7) +1
    #println("$d $n $m $y  $leapy $nDaysInMonth $isNewMonth")
    date = ["year"=>newy,"month"=>newm,"number"=>newn,"day"=>newd]
    #println("$date ")
    return date
end

function nextdayuntil(startdate, enddate, countpred)
	c = 0

	date = startdate
	while date["year"] < enddate["year"] ||
			date["month"] != enddate["month"] ||
			date["number"] != enddate["number"]

		if countpred(date)
			c = c+1
		end
		date = nextday(date)
	end
	return (c, date)
end

function main()
	# from http://kalender-365.de/leap-years.php
	listLeapYear = [1804,1808,1812,1816,1820,1824,1828,1832,1836,1840,1844,1848,1852,1856,1860,1864,1868,1872,1876,1880,1884,1888,1892,1896,
	1904,1908,1912,1916,1920,1924,1928,1932,1936,1940,1944,1948,1952,1956,1960,1964,1968,1972,1976,1980,1984,1988,1992,1996,2000,
	2004,2008,2012,2016,2020,2024,2028,2032,2036,2040,2044,2048,2052,2056,2060,2064,2068,2072,2076,2080,2084,2088,2092,2096]
	assert(all(map(isLeapYear, filter(x -> in(x,listLeapYear), 1800:2100))))
	assert(!(any(map(isLeapYear, filter(x -> !in(x,listLeapYear), 1800:2100)))))

	initdate  = ["year"=>1900,"month"=>1,"number"=>1,"day"=>1]
	startdate = ["year"=>1901,"month"=>1,"number"=>1]
	# augment startdate with day information
	(_,startdate) = nextdayuntil(initdate, startdate, x -> false)

	enddate   = ["year"=>2000,"month"=>12,"number"=>31]
	nextdayuntil(startdate, enddate, x -> x["number"] == 1 && x["day"] == 7)
end

main()


using monty

function main(args)
    N = parse(Int64, first(args))
    strategy = monty.switch
    wins = monty.run(N, strategy)
    println("Using the switch strategy for $N simulations")
    println("\tNumber of wins: $wins")
    println("\tOdds: $(wins / N * 100)%")
    println("")

    strategy = monty.stay
    wins = monty.run(N, strategy)
    println("Using the stay strategy for $N simulations")
    println("\tNumber of wins: $wins")
    println("\tOdds: $(wins / N * 100)%")
    println("")
end

main(ARGS)


function recpow(X, n)
	#%UNTITLED2 Summary of this function goes here
	#%   Detailed explanation goes here

	if round(n)!= n
	    error(string("The power has to be an integer", n,"\n"));
	end

	if n==Inf || isnan(n)
		Y = X.^n;
		return Y
	end

	if n==0
		Y = ones(size(X));
	else
		if n==1
			Y = X;
		else
			if mod(n,2) == 0
			Z = recpow(X, n/2);
			Y = Z .* Z;
			else
				Z = recpow(X, (n-1)/2);
				Y = Z .* Z .* X;
			end
		end
	end
	Y
end


@testset "rational.jl" begin
    rng = MersenneTwister(123456)
    x = rand(rng) * 2
    v1 = rand(rng, 3)
    v2 = rand(rng, 3)

    @testset "RationalKernel" begin
        α = rand()
        k = RationalKernel(; α=α)

        @testset "RationalKernel ≈ Exponential for large α" begin
            @test isapprox(
                RationalKernel(; α=1e9)(v1, v2),
                ExponentialKernel()(v1, v2);
                atol=1e-6,
                rtol=1e-6,
            )
        end

        @test metric(RationalKernel()) == Euclidean()
        @test metric(RationalKernel(; α=α)) == Euclidean()
        @test repr(k) == "Rational Kernel (α = $(α), metric = Euclidean(0.0))"

        k2 = RationalKernel(; α=α, metric=WeightedEuclidean(ones(3)))
        @test metric(k2) isa WeightedEuclidean
        @test k2(v1, v2) ≈ k(v1, v2)

        # Standardised tests.
        TestUtils.test_interface(k, Float64)
        test_ADs(x -> RationalKernel(; alpha=exp(x[1])), [α])
        test_params(k, ([α],))
    end

    @testset "RationalQuadraticKernel" begin
        α = rand()
        k = RationalQuadraticKernel(; α=α)

        @testset "RQ ≈ EQ for large α" begin
            @test isapprox(
                RationalQuadraticKernel(; α=1e9)(v1, v2),
                SqExponentialKernel()(v1, v2);
                atol=1e-6,
                rtol=1e-6,
            )
        end

        @test metric(RationalQuadraticKernel()) == SqEuclidean()
        @test metric(RationalQuadraticKernel(; α=α)) == SqEuclidean()
        @test repr(k) == "Rational Quadratic Kernel (α = $(α), metric = Euclidean(0.0))"

        k2 = RationalQuadraticKernel(; α=α, metric=WeightedEuclidean(ones(3)))
        @test metric(k2) isa WeightedEuclidean
        @test k2(v1, v2) ≈ k(v1, v2)

        # Standardised tests.
        TestUtils.test_interface(k, Float64)
        test_ADs(x -> RationalQuadraticKernel(; alpha=exp(x[1])), [α])
        test_params(k, ([α],))
    end

    @testset "GammaRationalKernel" begin
        k = GammaRationalKernel()

        @test repr(k) == "Gamma Rational Kernel (α = 2.0, γ = 1.0, metric = Euclidean(0.0))"

        @testset "GammaRational (γ=2) ≈ RQ with rescaled inputs" begin
            @test isapprox(
                GammaRationalKernel(; γ=2)(v1 ./ sqrt(2), v2 ./ sqrt(2)),
                RationalQuadraticKernel()(v1, v2),
            )
            a = 1 + rand()
            @test isapprox(
                GammaRationalKernel(; α=a, γ=2)(v1 ./ sqrt(2), v2 ./ sqrt(2)),
                RationalQuadraticKernel(; α=a)(v1, v2),
            )
        end

        @testset "GammaRational (γ=2) ≈ EQ for large α with rescaled inputs" begin
            v1 = randn(2)
            v2 = randn(2)
            @test isapprox(
                GammaRationalKernel(; α=1e9, γ=2)(v1 ./ sqrt(2), v2 ./ sqrt(2)),
                SqExponentialKernel()(v1, v2);
                atol=1e-6,
                rtol=1e-6,
            )
        end

        @testset "Default GammaRational ≈ Rational" begin
            @test isapprox(GammaRationalKernel()(v1, v2), RationalKernel()(v1, v2))
            a = 1 + rand()
            @test isapprox(
                GammaRationalKernel(; α=a)(v1, v2), RationalKernel(; α=a)(v1, v2)
            )
        end

        @testset "Default GammaRational ≈ Exponential for large α" begin
            v1 = randn(4)
            v2 = randn(4)
            @test isapprox(
                GammaRationalKernel(; α=1e9)(v1, v2),
                ExponentialKernel()(v1, v2);
                atol=1e-6,
                rtol=1e-6,
            )
        end

        @testset "GammaRational ≈ GammaExponential for same γ and large α" begin
            v1 = randn(3)
            v2 = randn(3)
            γ = rand() + 0.5
            @test isapprox(
                GammaRationalKernel(; α=1e9, γ=γ)(v1, v2),
                GammaExponentialKernel(; γ=γ)(v1, v2);
                atol=1e-6,
                rtol=1e-6,
            )
        end

        @test metric(GammaRationalKernel()) == Euclidean()
        @test metric(GammaRationalKernel(; γ=2.0)) == Euclidean()
        @test metric(GammaRationalKernel(; γ=2.0, α=3.0)) == Euclidean()

        k2 = GammaRationalKernel(; metric=WeightedEuclidean(ones(3)))
        @test metric(k2) isa WeightedEuclidean
        @test k2(v1, v2) ≈ k(v1, v2)

        # Standardised tests.
        TestUtils.test_interface(k, Float64)
        a = 1.0 + rand()
        test_ADs(x -> GammaRationalKernel(; α=x[1], γ=x[2]), [a, 1 + 0.5 * rand()])
        test_params(GammaRationalKernel(; α=a, γ=x), ([a], [x]))
    end
end


struct Differential <: Function
    x::Expression
    order::Int
end
Differential(x) = Differential(x,1)

Base.show(io::IO, D::Differential) = print(io,"($(D.x),$(D.order))")
Base.convert(::Type{Expr}, D::Differential) = D

(D::Differential)(x::Operation) = Operation(D, Expression[x])
function (D::Differential)(x::Variable)
    D.x === x             && return Constant(1)
    has_dependent(x, D.x) || return Constant(0)

    x′ = copy(x)
    x′.diff = D
    return x′
end
Base.:(==)(D1::Differential, D2::Differential) = D1.order == D2.order && D1.x == D2.x

function expand_derivatives(O::Operation)
    @. O.args = expand_derivatives(O.args)

    if O.op isa Differential
        D = O.op
        o = O.args[1]
        return simplify_constants(sum(i->Derivative(o,i)*expand_derivatives(D(o.args[i])),1:length(o.args)))
    end

    return O
end
expand_derivatives(x) = x

# Don't specialize on the function here
Derivative(O::Operation, idx) = Derivative(O.op, (O.args...,), Val(idx))

# Pre-defined derivatives
import DiffRules, SpecialFunctions, NaNMath
for (modu, fun, arity) ∈ DiffRules.diffrules()
    for i ∈ 1:arity
        @eval function Derivative(::typeof($modu.$fun), args::NTuple{$arity,Any}, ::Val{$i})
            M, f = $(modu, fun)
            partials = DiffRules.diffrule(M, f, args...)
            dx = @static $arity == 1 ? partials : partials[$i]
            convert(Expression, dx)
        end
    end
end

function count_order(x)
    @assert !(x isa Symbol) "The variable $x must have an order of differentiation that is greater or equal to 1!"
    n = 1
    while !(x.args[1] isa Symbol)
        n = n+1
        x = x.args[1]
    end
    n, x.args[1]
end

function _differential_macro(x)
    ex = Expr(:block)
    lhss = Symbol[]
    x = flatten_expr!(x)
    for di in x
        @assert di isa Expr && di.args[1] == :~ "@Deriv expects a form that looks like `@Deriv D''~t E'~t`"
        lhs = di.args[2]
        rhs = di.args[3]
        order, lhs = count_order(lhs)
        push!(lhss, lhs)
        expr = :($lhs = Differential($rhs, $order))
        push!(ex.args,  expr)
    end
    push!(ex.args, Expr(:tuple, lhss...))
    ex
end

macro Deriv(x...)
    esc(_differential_macro(x))
end

function calculate_jacobian(eqs,vars)
    Expression[Differential(vars[j])(eqs[i]) for i in 1:length(eqs), j in 1:length(vars)]
end

export Differential, Derivative, expand_derivatives, @Deriv, calculate_jacobian


using StructuralCausalModels, Test

ProjDir = @__DIR__
cd(ProjDir) #do

N = 100
b_AU = 0.5
b_AC = 3
b_UX = -1
b_UB = 2
b_CB = -1.5
b_CY = 1
b_XY = 5

df = DataFrame(
  :a => rand(Normal(), N)
);
df[!, :u] = rand(Normal(1, 2), N) + b_AU * df[:, :a]
df[!, :c] = rand(Normal(0, 1), N) + b_AC * df[:, :a]
df[!, :b] = rand(Normal(1, 1), N) + b_UB * df[:, :u]
df[!, :x] = rand(Normal(-2, 1), N) + b_UX * df[:, :u]
df[!, :y] = rand(Normal(1, 2), N) + b_XY * df[:, :x] + b_CY * df[:, :c]

d = OrderedDict(
  :y => [:x],
  [:c, :u] => :a,
  [:y, :b] => :c,
  [:x, :b] => :u
);
u = [:u]

dag = DAG("sr6_4_2a", d; df=df);
show(dag)

fn = joinpath(mktempdir(), "sr6_4_2a.dot")
to_graphviz(dag, fn)
Sys.isapple() && run(`open -a GraphViz.app $(fn)`)

adjustmentsets = adjustment_sets(dag, :x, :y)
println("Adjustment sets:")
adjustmentsets |> display


using Revise
using Dice
using Dice: num_flips, num_nodes, ifelse

# Number of nodes BWH needs to model a distribution on n bits
bwh_num_nodes(n) = 2^(n + 1) - n

function generate_code_bwh(p::Vector{Float64})
    @dice begin
        @assert ispow2(length(p)) "Distribution length should be power of 2"
        prefix_sums = accumulate(+, p)
        @assert last(prefix_sums) ≈ 1 "Distribution should sum to 1"
        function helper(i, j)
            if i == j
                DistInt(dicecontext(), i - 1)
            else
                first_half_end = (i + j - 1) ÷ 2
                first_half_p = prefix_sums[first_half_end] - prefix_sums[i] + p[i]
                total_p = prefix_sums[j] - prefix_sums[i] + p[i]
                ifelse(flip(first_half_p/total_p),
                    helper(i, first_half_end),
                    helper(first_half_end+1, j))
            end
        end
        helper(1, length(p))
    end
end

function test_bwh_num_nodes(p::Vector{Float64})
    code = generate_code_bwh(p)
    bdd = compile(code)
    @assert infer(code, :bdd) ≈ p  # Verify correctness

    bdd_nodes = num_nodes(bdd)
    num_bits = round(Int, log2(length(p)))
    @assert bdd_nodes == bwh_num_nodes(num_bits)  # Verify num bits

    println("BWH for a $(num_bits)-bit distribution: $(bdd_nodes) nodes, $(num_flips(bdd)) flips")
end

dist_2 = [0.5, 0.5]
dist_4 = [0.1, 0.2, 0.3, 0.4]
dist_8 = [0.02, 0.04, 0.06, 0.08, 0.1, 0.12, 0.14, 0.44]
dist_16 = [0.005, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.001, 0.11, 0.12, 0.13, 0.14, 0.022, 0.032]

for dist in [dist_2, dist_4, dist_8, dist_16]
    test_bwh_num_nodes(dist)
end


using Statistics

Statistics.varm(A::AbstractGPUArray{<:Real},m::AbstractArray{<:Real}; dims, corrected::Bool=true) =
    sum((A .- m).^2, dims=dims)/(prod(size(A)[[dims...]])::Int-corrected)

Statistics.stdm(A::AbstractGPUArray{<:Real},m::AbstractArray{<:Real}, dim::Int; corrected::Bool=true) =
    sqrt.(varm(A,m;dims=dim,corrected=corrected))

Statistics._std(A::AbstractGPUArray, corrected::Bool, mean, dims) =
    sqrt.(Statistics.var(A; corrected=corrected, mean=mean, dims=dims))

Statistics._std(A::AbstractGPUArray, corrected::Bool, mean, ::Colon) =
    sqrt.(Statistics.var(A; corrected=corrected, mean=mean, dims=:))

# Revert https://github.com/JuliaLang/Statistics.jl/pull/25
Statistics._mean(A::AbstractGPUArray, ::Colon)    = sum(A) / length(A)
Statistics._mean(f, A::AbstractGPUArray, ::Colon) = sum(f, A) / length(A)
Statistics._mean(A::AbstractGPUArray, dims)    = mean!(Base.reducedim_init(t -> t/2, +, A, dims), A)
Statistics._mean(f, A::AbstractGPUArray, dims) = sum(f, A, dims=dims) / mapreduce(i -> size(A, i), *, unique(dims); init=1)

function Statistics.covzm(x::AbstractGPUMatrix, vardim::Int=1; corrected::Bool=true)
    C = Statistics.unscaled_covzm(x, vardim)
    T = promote_type(typeof(one(eltype(C)) / 1), eltype(C))
    A = convert(AbstractArray{T}, C)
    b = 1//(size(x, vardim) - corrected)
    A .*= b
    return A
end

function Statistics.cov2cor!(C::AbstractGPUMatrix{T}, xsd::AbstractGPUArray) where T
    nx = length(xsd)
    size(C) == (nx, nx) || throw(DimensionMismatch("inconsistent dimensions"))
    tril!(C, -1)
    C += adjoint(C)
    C = Statistics.clampcor.(C ./ (xsd * xsd'))
    C[diagind(C)] .= oneunit(T)
    return C
end

function Statistics.corzm(x::AbstractGPUMatrix, vardim::Int=1)
    c = Statistics.unscaled_covzm(x, vardim)
    return Statistics.cov2cor!(c, sqrt.(diag(c)))
end


module PoissonTests

using Test
using Gridap
import Gridap: ∇
#using LinearAlgebra

domain = (0,1,0,1)
partition = (4,4)
model = CartesianDiscreteModel(domain,partition)
order = 2

const h = (domain[2]-domain[1]) / partition[1]
const γ = 10

labels = get_face_labeling(model)
add_tag_from_tags!(labels,"dirichlet",[1,2,5])
add_tag_from_tags!(labels,"neumann",[7,8])
add_tag_from_tags!(labels,"nitsche",6)

Ω = Triangulation(model)
Γn = BoundaryTriangulation(model,labels,tags="neumann")
Γd = BoundaryTriangulation(model,labels,tags="nitsche")

degree = order
dΩ = Measure(Ω,degree)
dΓn = Measure(Γn,degree)
dΓd = Measure(Γd,degree)

nn = get_normal_vector(Γn)
nd = get_normal_vector(Γd)

# Using automatic differentiation
u_scal(x) = x[1]^2 + x[2]
f_scal(x) = - Δ(u_scal)(x)

#u_scal(x) = x[1]^2 + x[2]
#∇u_scal(x) = VectorValue( 2*x[1], one(x[2]) )
#Δu_scal(x) = 2
#f_scal(x) = - Δu_scal(x)
#∇(::typeof(u_scal)) = ∇u_scal

scalar_data = Dict{Symbol,Any}()
scalar_data[:valuetype] = Float64
scalar_data[:u] = u_scal
scalar_data[:f] = f_scal

# Using automatic differentiation
u_vec(x) = VectorValue( x[1]^2 + x[2], 4*x[1] - x[2]^2 )
f_vec(x) = - Δ(u_vec)(x)

#u_vec(x) = VectorValue( x[1]^2 + x[2], 4*x[1] - x[2]^2 )
#∇u_vec(x) = TensorValue( 2*x[1], one(x[2]), 4*one(x[1]), - 2*x[2] )
#Δu_vec(x) = VectorValue( 2, -2 )
#f_vec(x) = - Δu_vec(x)
#∇(::typeof(u_vec)) = ∇u_vec

vector_data = Dict{Symbol,Any}()
vector_data[:valuetype] = VectorValue{2,Float64}
vector_data[:u] = u_vec
vector_data[:f] = f_vec

for data in [ vector_data, scalar_data ]

  T = data[:valuetype]
  u = data[:u]
  f = data[:f]

  for domain_style in (ReferenceDomain(),PhysicalDomain())

    cell_fe = FiniteElements(domain_style,model,lagrangian,T,order)
    V = TestFESpace(Ω,cell_fe,dirichlet_tags="dirichlet",labels=labels)
    U = TrialFESpace(V,u)

    uh = interpolate(u, U)

    a(u,v) =
      ∫( ∇(v)⊙∇(u) )*dΩ +
      ∫( (γ/h)*v⊙u  - v⊙(nd⋅∇(u)) - (nd⋅∇(v))⊙u )*dΓd

    l(v) =
      ∫( v⊙f )*dΩ +
      ∫( v⊙(nn⋅∇(uh)) )*dΓn +
      ∫( (γ/h)*v⊙uh - (nd⋅∇(v))⊙u )*dΓd

    op = AffineFEOperator(a,l,U,V)
    uh = solve(op)

    e = u - uh

    l2(u) = sqrt(sum( ∫( u⊙u )*dΩ ))
    h1(u) = sqrt(sum( ∫( u⊙u + ∇(u)⊙∇(u) )*dΩ ))

    el2 = l2(e)
    eh1 = h1(e)
    ul2 = l2(uh)
    uh1 = h1(uh)

    @test el2/ul2 < 1.e-8
    @test eh1/uh1 < 1.e-7

  end
end

end # module


# Load modules
using Plots, LaTeXStrings
pyplot()
using ThinFilmsTools

function main()
    # Define beam
    λ = LinRange(310,1000,500) # wavelength range [nm]
    θ = [0.] # angle of incidence [degrees]
    beam = PlaneWave(λ,θ)
    # Find wavelength closest to 405.0 nm
    aux1 = Utils.findClosest(vec(beam.λ),405.0)
    # Define layers
    layers = [
        LayerTMMO(RIdb.air(beam.λ)),
        LayerTMMO(RIdb.sno2f(beam.λ); d=150., nλ0=RIdb.sno2f([beam.λ[aux1]])),
        LayerTMMO(RIdb.silicon(beam.λ)),
    ]
    # call main script
    sol = tmm_optics(beam, layers)
    return sol
end

sol = main()

# plot the R, T and A spectra
plot(Spectrum1D(),
     sol.beam.λ, [sol.Spectra.Rp, sol.Spectra.Tp, 1.0.-(sol.Spectra.Rp.+sol.Spectra.Tp)],
     label=["Reflectance" "Transmittance" "Absorbance"],
     line=([:solid :dash :dashdot]),
     ylims=(0.0,1.0),
     xlims=(sol.beam.λ[1], sol.beam.λ[end]),
)
gui()

# plot the refractive index profile
plot(RIprofile(), sol)
gui()


struct QBosonicBasis <: AbstractQBasis
  	Dim::Int
	N::Int
	Np::Int
	OccRep::Vector{Vector{Int}}
end

function QBosonicBasis(N::Int, Np::Int)
	I = 1
	n = zeros(Int,N)
	n[1] = Np

	vectors = Vector{Vector{Int}}([])
	while true
		#@show n
		push!(vectors,n[:])
		I += 1
		if n[1] > 0
			n[1] -= 1
			n[2] += 1
		elseif n[1] == 0
			j = findfirst(!iszero, n)

			if j == N break end

			n[1] = n[j] - 1
			n[j] = 0
			n[j+1] += 1

		end
		#@show n
	end

	return QBosonicBasis(I-1, N, Np, vectors)
end

function bosonicHashfunc(Np::Int, col::Vector{Int})
	hashkey = 0
	particlei = 1
	p_i = findfirst(!iszero,col)
	col = col[p_i:end]
	while true

		#@show col

		if col[1] > 0
			col[1] -= 1
		else
 			orbital = findfirst(!iszero,col)
			p_i += orbital-1
			col = col[orbital:end]
			col[1] -= 1
		end

		hashkey += binomial((p_i-1) + particlei -1 , particlei)

		if particlei == Np break end
 		particlei += 1
	end
	return  hashkey
end

function addParticleBosonic(singleCoef::AbstractVector{<:Number}, preState::QState)
 	N = preState.QBasis.N
	Np = preState.QBasis.Np

	N >= Np+1 || throw(DomainError("you cannot creat more particles $(Np+1) than sites $N"))

	newQBasis = QBosonicBasis(N,Np+1)
	newCoef = zeros(ComplexF64,newQBasis.Dim)
	preQBasis = preState.QBasis

	for i in 1:N
	 	for (colIdx, colOccRep) in enumerate(preQBasis.OccRep)
			#@show colOccRep
	 		colOccRep[i] += 1

			# actualy this bosonicHashfunc gives directly idx in
			# OccRep vector so no dict needed
			rowIdx = bosonicHashfunc(Np+1, colOccRep)+1
			#@show i,rowIdx
			newCoef[rowIdx] +=  singleCoef[i]*preState.Coef[colIdx]*sqrt(colOccRep[i])

			#@show newCoef[rowIdx]
			colOccRep[i] -= 1
		end
		#@show newCoef
	end

	return QState(newCoef, newQBasis)
end

function constructBosonicState(blochWaveFunc::AbstractVector{<:Any})
	#@show size(blochWaveFunc[1,:,1])
	State = QState(blochWaveFunc[1][:,1], QBosonicBasis(size(blochWaveFunc[1])[1],1))
	for i in 2:size(blochWaveFunc)[1]
		State = addParticleBosonic(blochWaveFunc[i][:,1], State)
	end

	return State
end


struct McCraryBinwidth <: BinwidthSelector end

function binwidth(::McCraryBinwidth, Zs::RunningVariable)
    2 * std(Zs) / sqrt(length(Zs))
end

function StatsDiscretizations.samplehull_discretizer(Zs::RunningVariable, b)
    b = binwidth(b, Zs)
    rmin, rmax = extrema(Zs)
    cutoff = Zs.cutoff

    leftend = floor(rmin/b - eps(rmin/b))*b
    leftgrid_length =  round(Int,(cutoff-leftend)/b) + 1

    rightend = (1+ceil(rmax/b + eps(rmax/b)))*b
    rightgrid_length =  round(Int,(rightend-cutoff)/b)

    if Zs.treated ∈ (:≥, :<)
        L, R = :open, :closed
    elseif Zs.treated ∈ (:≤, :>)
        L, R = :closed, :open
    end

    discr_left =  BoundedIntervalDiscretizer{L,R}(
        range(leftend; stop=cutoff, length=leftgrid_length)
    )
    discr_right = BoundedIntervalDiscretizer{L,R}(
        range(cutoff; stop=rightend, length=rightgrid_length + 1)
    )

    if Zs.treated ∈ (:≥, :>)
        discr = (untreated = discr_left, treated = discr_right)
    elseif Zs.treated ∈ (:<, :≤)
        discr = (untreated = discr_rightt, treated = discr_left)
    end
    discr
end