# Note: agreement = output * target
#       Agreement is high when output and target are the same sign and |output| is large.
#       It is an indication that the output represents the correct class in a margin-based model.

# ============================================================

@doc doc"""
    ZeroOneLoss <: MarginLoss

The classical classification loss. It penalizes every misclassified
observation with a loss of `1` while every correctly classified
observation has a loss of `0`.
It is not convex nor continuous and thus seldom used directly.
Instead one usually works with some classification-calibrated
surrogate loss, such as [L1HingeLoss](@ref).

```math
L(a) = \begin{cases} 1 & \quad \text{if } a < 0 \\ 0 & \quad \text{if } a >= 0\\ \end{cases}
```

---
```
              Lossfunction                     Derivative
      ┌────────────┬────────────┐      ┌────────────┬────────────┐
    1 │------------┐            │    1 │                         │
      │            |            │      │                         │
      │            |            │      │                         │
      │            |            │      │_________________________│
      │            |            │      │                         │
      │            |            │      │                         │
      │            |            │      │                         │
    0 │            └------------│   -1 │                         │
      └────────────┴────────────┘      └────────────┴────────────┘
      -2                        2      -2                        2
                y * h(x)                         y * h(x)
```
"""
struct ZeroOneLoss <: MarginLoss end

deriv(loss::ZeroOneLoss, target::Number, output::Number) = zero(output)
deriv2(loss::ZeroOneLoss, target::Number, output::Number) = zero(output)

value(loss::ZeroOneLoss, agreement::T) where {T<:Number} = sign(agreement) < 0 ? one(T) : zero(T)
deriv(loss::ZeroOneLoss, agreement::T) where {T<:Number} = zero(T)
deriv2(loss::ZeroOneLoss, agreement::T) where {T<:Number} = zero(T)
value_deriv(loss::ZeroOneLoss, agreement::T) where {T<:Number} = sign(agreement) < 0 ? (one(T), zero(T)) : (zero(T), zero(T))

isminimizable(::ZeroOneLoss) = true
isdifferentiable(::ZeroOneLoss) = false
isdifferentiable(::ZeroOneLoss, at) = at != 0
istwicedifferentiable(::ZeroOneLoss) = false
istwicedifferentiable(::ZeroOneLoss, at) = at != 0
isnemitski(::ZeroOneLoss) = true
islipschitzcont(::ZeroOneLoss) = true
isconvex(::ZeroOneLoss) = false
isclasscalibrated(loss::ZeroOneLoss) = true
isclipable(::ZeroOneLoss) = true

# ============================================================

@doc doc"""
    PerceptronLoss <: MarginLoss

The perceptron loss linearly penalizes every prediction where the
resulting `agreement <= 0`.
It is Lipschitz continuous and convex, but not strictly convex.

```math
L(a) = \max \{ 0, -a \}
```

---
```
              Lossfunction                     Derivative
      ┌────────────┬────────────┐      ┌────────────┬────────────┐
    2 │\.                       │    0 │            ┌------------│
      │ '..                     │      │            |            │
      │   \.                    │      │            |            │
      │     '.                  │      │            |            │
    L │      '.                 │   L' │            |            │
      │        \.               │      │            |            │
      │         '.              │      │            |            │
    0 │           \.____________│   -1 │------------┘            │
      └────────────┴────────────┘      └────────────┴────────────┘
      -2                        2      -2                        2
                 y ⋅ ŷ                            y ⋅ ŷ
```
"""
struct PerceptronLoss <: MarginLoss end

value(loss::PerceptronLoss, agreement::T) where {T<:Number} = max(zero(T), -agreement)
deriv(loss::PerceptronLoss, agreement::T) where {T<:Number} = agreement >= 0 ? zero(T) : -one(T)
deriv2(loss::PerceptronLoss, agreement::T) where {T<:Number} = zero(T)
value_deriv(loss::PerceptronLoss, agreement::T) where {T<:Number} = agreement >= 0 ? (zero(T), zero(T)) : (-agreement, -one(T))

isdifferentiable(::PerceptronLoss) = false
isdifferentiable(::PerceptronLoss, at) = at != 0
istwicedifferentiable(::PerceptronLoss) = false
istwicedifferentiable(::PerceptronLoss, at) = at != 0
islipschitzcont(::PerceptronLoss) = true
isconvex(::PerceptronLoss) = true
isstrictlyconvex(::PerceptronLoss) = false
isstronglyconvex(::PerceptronLoss) = false
isclipable(::PerceptronLoss) = true

# ============================================================

@doc doc"""
    LogitMarginLoss <: MarginLoss

The margin version of the logistic loss. It is infinitely many
times differentiable, strictly convex, and Lipschitz continuous.

```math
L(a) = \ln (1 + e^{-a})
```

---
```
              Lossfunction                     Derivative
      ┌────────────┬────────────┐      ┌────────────┬────────────┐
    2 │ \.                      │    0 │                  ._--/""│
      │   \.                    │      │               ../'      │
      │     \.                  │      │              ./         │
      │       \..               │      │            ./'          │
    L │         '-_             │   L' │          .,'            │
      │            '-_          │      │         ./              │
      │               '\-._     │      │      .,/'               │
    0 │                    '""*-│   -1 │__.--''                  │
      └────────────┴────────────┘      └────────────┴────────────┘
      -2                        2      -4                        4
                 y ⋅ ŷ                            y ⋅ ŷ
```
"""
struct LogitMarginLoss <: MarginLoss end
value(loss::LogitMarginLoss, agreement::Number) = log1p(exp(-agreement))
deriv(loss::LogitMarginLoss, agreement::Number) = -one(agreement) / (one(agreement) + exp(agreement))
deriv2(loss::LogitMarginLoss, agreement::Number) = (eᵗ = exp(agreement); eᵗ / abs2(one(eᵗ) + eᵗ))
value_deriv(loss::LogitMarginLoss, agreement::Number) = (eᵗ = exp(-agreement); (log1p(eᵗ), -eᵗ / (one(eᵗ) + eᵗ)))

isunivfishercons(::LogitMarginLoss) = true
isdifferentiable(::LogitMarginLoss) = true
isdifferentiable(::LogitMarginLoss, at) = true
istwicedifferentiable(::LogitMarginLoss) = true
istwicedifferentiable(::LogitMarginLoss, at) = true
islipschitzcont(::LogitMarginLoss) = true
isconvex(::LogitMarginLoss) = true
isstrictlyconvex(::LogitMarginLoss) = true
isstronglyconvex(::LogitMarginLoss) = false
isclipable(::LogitMarginLoss) = false

# ============================================================

@doc doc"""
    L1HingeLoss <: MarginLoss

The hinge loss linearly penalizes every predicition where the
resulting `agreement < 1` .
It is Lipschitz continuous and convex, but not strictly convex.

```math
L(a) = \max \{ 0, 1 - a \}
```

---
```
              Lossfunction                     Derivative
      ┌────────────┬────────────┐      ┌────────────┬────────────┐
    3 │'\.                      │    0 │                  ┌------│
      │  ''_                    │      │                  |      │
      │     \.                  │      │                  |      │
      │       '.                │      │                  |      │
    L │         ''_             │   L' │                  |      │
      │            \.           │      │                  |      │
      │              '.         │      │                  |      │
    0 │                ''_______│   -1 │------------------┘      │
      └────────────┴────────────┘      └────────────┴────────────┘
      -2                        2      -2                        2
                 y ⋅ ŷ                            y ⋅ ŷ
```
"""
struct L1HingeLoss <: MarginLoss end
const HingeLoss = L1HingeLoss

value(loss::L1HingeLoss, agreement::T) where {T<:Number} = max(zero(T), one(T) - agreement)
deriv(loss::L1HingeLoss, agreement::T) where {T<:Number} = agreement >= 1 ? zero(T) : -one(T)
deriv2(loss::L1HingeLoss, agreement::T) where {T<:Number} = zero(T)
value_deriv(loss::L1HingeLoss, agreement::T) where {T<:Number} = agreement >= 1 ? (zero(T), zero(T)) : (one(T) - agreement, -one(T))

isfishercons(::L1HingeLoss) = true
isdifferentiable(::L1HingeLoss) = false
isdifferentiable(::L1HingeLoss, at) = at != 1
istwicedifferentiable(::L1HingeLoss) = false
istwicedifferentiable(::L1HingeLoss, at) = at != 1
islipschitzcont(::L1HingeLoss) = true
isconvex(::L1HingeLoss) = true
isstrictlyconvex(::L1HingeLoss) = false
isstronglyconvex(::L1HingeLoss) = false
isclipable(::L1HingeLoss) = true

# ============================================================

@doc doc"""
    L2HingeLoss <: MarginLoss

The truncated least squares loss quadratically penalizes every
predicition where the resulting `agreement < 1`.
It is locally Lipschitz continuous and convex,
but not strictly convex.

```math
L(a) = \max \{ 0, 1 - a \}^2
```

---
```
              Lossfunction                     Derivative
      ┌────────────┬────────────┐      ┌────────────┬────────────┐
    5 │     .                   │    0 │                 ,r------│
      │     '.                  │      │               ,/        │
      │      '\                 │      │             ,/          │
      │        \                │      │           ,/            │
    L │         '.              │   L' │         ./              │
      │          '.             │      │       ./                │
      │            \.           │      │     ./                  │
    0 │              '-.________│   -5 │   ./                    │
      └────────────┴────────────┘      └────────────┴────────────┘
      -2                        2      -2                        2
                 y ⋅ ŷ                            y ⋅ ŷ
```
"""
struct L2HingeLoss <: MarginLoss end

value(loss::L2HingeLoss, agreement::T) where {T<:Number} = agreement >= 1 ? zero(T) : abs2(one(T) - agreement)
deriv(loss::L2HingeLoss, agreement::T) where {T<:Number} = agreement >= 1 ? zero(T) : convert(T,2) * (agreement - one(T))
deriv2(loss::L2HingeLoss, agreement::T) where {T<:Number} = agreement >= 1 ? zero(T) : convert(T,2)
value_deriv(loss::L2HingeLoss, agreement::T) where {T<:Number} = agreement >= 1 ? (zero(T), zero(T)) : (abs2(one(T) - agreement), convert(T,2) * (agreement - one(T)))

isunivfishercons(::L2HingeLoss) = true
isdifferentiable(::L2HingeLoss) = true
isdifferentiable(::L2HingeLoss, at) = true
istwicedifferentiable(::L2HingeLoss) = false
istwicedifferentiable(::L2HingeLoss, at) = at != 1
islocallylipschitzcont(::L2HingeLoss) = true
islipschitzcont(::L2HingeLoss) = false
isconvex(::L2HingeLoss) = true
isstrictlyconvex(::L2HingeLoss) = false
isstronglyconvex(::L2HingeLoss) = false
isclipable(::L2HingeLoss) = true

# ============================================================

@doc doc"""
    SmoothedL1HingeLoss <: MarginLoss

As the name suggests a smoothed version of the L1 hinge loss.
It is Lipschitz continuous and convex, but not strictly convex.

```math
L(a) = \begin{cases} \frac{0.5}{\gamma} \cdot \max \{ 0, 1 - a \} ^2 & \quad \text{if } a \ge 1 - \gamma \\ 1 - \frac{\gamma}{2} - a & \quad \text{otherwise}\\ \end{cases}
```

---
```
              Lossfunction (γ=2)               Derivative
      ┌────────────┬────────────┐      ┌────────────┬────────────┐
    2 │\.                       │    0 │                 ,r------│
      │ '.                      │      │               ./'       │
      │   \.                    │      │              ,/         │
      │     '.                  │      │            ./'          │
    L │      '.                 │   L' │           ,'            │
      │        \.               │      │         ,/              │
      │          ',             │      │       ./'               │
    0 │            '*-._________│   -1 │______./                 │
      └────────────┴────────────┘      └────────────┴────────────┘
      -2                        2      -2                        2
                 y ⋅ ŷ                            y ⋅ ŷ
```
"""
struct SmoothedL1HingeLoss{T<:AbstractFloat} <: MarginLoss
    gamma::T

    function SmoothedL1HingeLoss{T}(γ::T) where T
        γ > 0 || error("γ must be strictly positive")
        new{T}(γ)
    end
end
SmoothedL1HingeLoss(γ::T) where {T<:AbstractFloat} = SmoothedL1HingeLoss{T}(γ)
SmoothedL1HingeLoss(γ) = SmoothedL1HingeLoss(Float64(γ))

function value(loss::SmoothedL1HingeLoss{R}, agreement::T)::promote_type(R,T) where {R,T<:Number}
    if agreement >= 1 - loss.gamma
        R(0.5) / loss.gamma * abs2(max(zero(T), one(T) - agreement))
    else
        one(T) - loss.gamma / R(2) - agreement
    end
end
function deriv(loss::SmoothedL1HingeLoss{R}, agreement::T)::promote_type(R,T) where {R,T<:Number}
    if agreement >= 1 - loss.gamma
        agreement >= 1 ? zero(T) : (agreement - one(T)) / loss.gamma
    else
        -one(T)
    end
end
function deriv2(loss::SmoothedL1HingeLoss{R}, agreement::T)::promote_type(R,T) where {R,T<:Number}
    agreement < 1 - loss.gamma || agreement > 1 ? zero(T) : one(T) / loss.gamma
end

isdifferentiable(::SmoothedL1HingeLoss) = true
isdifferentiable(::SmoothedL1HingeLoss, at) = true
istwicedifferentiable(::SmoothedL1HingeLoss) = false
istwicedifferentiable(loss::SmoothedL1HingeLoss, at) = at != 1 && at != 1 - loss.gamma
islocallylipschitzcont(::SmoothedL1HingeLoss) = true
islipschitzcont(::SmoothedL1HingeLoss) = true
isconvex(::SmoothedL1HingeLoss) = true
isstrictlyconvex(::SmoothedL1HingeLoss) = false
isstronglyconvex(::SmoothedL1HingeLoss) = false
isclipable(::SmoothedL1HingeLoss) = true

# ============================================================

@doc doc"""
    ModifiedHuberLoss <: MarginLoss

A special (4 times scaled) case of the [`SmoothedL1HingeLoss`](@ref)
with `γ=2`. It is Lipschitz continuous and convex,
but not strictly convex.

```math
L(a) = \begin{cases} \max \{ 0, 1 - a \} ^2 & \quad \text{if } a \ge -1 \\ - 4 a & \quad \text{otherwise}\\ \end{cases}
```

---
```
              Lossfunction                     Derivative
      ┌────────────┬────────────┐      ┌────────────┬────────────┐
    5 │    '.                   │    0 │                .+-------│
      │     '.                  │      │              ./'        │
      │      '\                 │      │             ,/          │
      │        \                │      │           ,/            │
    L │         '.              │   L' │         ./              │
      │          '.             │      │       ./'               │
      │            \.           │      │______/'                 │
    0 │              '-.________│   -5 │                         │
      └────────────┴────────────┘      └────────────┴────────────┘
      -2                        2      -2                        2
                 y ⋅ ŷ                            y ⋅ ŷ
```
"""
struct ModifiedHuberLoss <: MarginLoss end

function value(loss::ModifiedHuberLoss, agreement::T) where T<:Number
    agreement >= -1 ? abs2(max(zero(T), one(agreement) - agreement)) : -convert(T,4) * agreement
end
function deriv(loss::ModifiedHuberLoss, agreement::T) where T<:Number
    if agreement >= -1
        agreement > 1 ? zero(T) : convert(T,2)*agreement - convert(T,2)
    else
        -convert(T,4)
    end
end
function deriv2(loss::ModifiedHuberLoss, agreement::T) where T<:Number
    agreement < -1 || agreement > 1 ? zero(T) : convert(T,2)
end

isdifferentiable(::ModifiedHuberLoss) = true
isdifferentiable(::ModifiedHuberLoss, at) = true
istwicedifferentiable(::ModifiedHuberLoss) = false
istwicedifferentiable(loss::ModifiedHuberLoss, at) = at != 1 && at != -1
islocallylipschitzcont(::ModifiedHuberLoss) = true
islipschitzcont(::ModifiedHuberLoss) = true
isconvex(::ModifiedHuberLoss) = true
isstrictlyconvex(::ModifiedHuberLoss) = false
isstronglyconvex(::ModifiedHuberLoss) = false
isclipable(::ModifiedHuberLoss) = true

# ============================================================

@doc doc"""
    L2MarginLoss <: MarginLoss

The margin-based least-squares loss for classification,
which penalizes every prediction where `agreement != 1` quadratically.
It is locally Lipschitz continuous and strongly convex.

```math
L(a) = {\left( 1 - a \right)}^2
```

---
```
              Lossfunction                     Derivative
      ┌────────────┬────────────┐      ┌────────────┬────────────┐
    5 │     .                   │    2 │                       ,r│
      │     '.                  │      │                     ,/  │
      │      '\                 │      │                   ,/    │
      │        \                │      ├                 ,/      ┤
    L │         '.              │   L' │               ./        │
      │          '.             │      │             ./          │
      │            \.          .│      │           ./            │
    0 │              '-.____.-' │   -3 │         ./              │
      └────────────┴────────────┘      └────────────┴────────────┘
      -2                        2      -2                        2
                 y ⋅ ŷ                            y ⋅ ŷ
```
"""
struct L2MarginLoss <: MarginLoss end

value(loss::L2MarginLoss, agreement::T) where {T<:Number} = abs2(one(T) - agreement)
deriv(loss::L2MarginLoss, agreement::T) where {T<:Number} = convert(T,2) * (agreement - one(T))
deriv2(loss::L2MarginLoss, agreement::T) where {T<:Number} = convert(T,2)

isunivfishercons(::L2MarginLoss) = true
isdifferentiable(::L2MarginLoss) = true
isdifferentiable(::L2MarginLoss, at) = true
istwicedifferentiable(::L2MarginLoss) = true
istwicedifferentiable(::L2MarginLoss, at) = true
islocallylipschitzcont(::L2MarginLoss) = true
islipschitzcont(::L2MarginLoss) = false
isconvex(::L2MarginLoss) = true
isstrictlyconvex(::L2MarginLoss) = true
isstronglyconvex(::L2MarginLoss) = true
isclipable(::L2MarginLoss) = true

# ============================================================

@doc doc"""
    ExpLoss <: MarginLoss

The margin-based exponential loss for classification, which
penalizes every prediction exponentially. It is infinitely many
times differentiable, locally Lipschitz continuous and strictly
convex, but not clipable.

```math
L(a) = e^{-a}
```

---
```
              Lossfunction                     Derivative
      ┌────────────┬────────────┐      ┌────────────┬────────────┐
    5 │  \.                     │    0 │               _,,---:'""│
      │   l                     │      │           _r/"'         │
      │    l.                   │      │        .r/'             │
      │     ":                  │      │      .r'                │
    L │       \.                │   L' │     ./                  │
      │        "\..             │      │    .'                   │
      │           '":,_         │      │   ,'                    │
    0 │                ""---:.__│   -5 │  ./                     │
      └────────────┴────────────┘      └────────────┴────────────┘
      -2                        2      -2                        2
                 y ⋅ ŷ                            y ⋅ ŷ
```
"""
struct ExpLoss <: MarginLoss end

value(loss::ExpLoss, agreement::Number) = exp(-agreement)
deriv(loss::ExpLoss, agreement::Number) = -exp(-agreement)
deriv2(loss::ExpLoss, agreement::Number) = exp(-agreement)
value_deriv(loss::ExpLoss, agreement::Number) = (eᵗ = exp(-agreement); (eᵗ, -eᵗ))

isunivfishercons(::ExpLoss) = true
isdifferentiable(::ExpLoss) = true
isdifferentiable(::ExpLoss, at) = true
istwicedifferentiable(::ExpLoss) = true
istwicedifferentiable(::ExpLoss, at) = true
islocallylipschitzcont(::ExpLoss) = true
islipschitzcont(::ExpLoss) = false
isconvex(::ExpLoss) = true
isstrictlyconvex(::ExpLoss) = true
isstronglyconvex(::ExpLoss) = false
isclipable(::ExpLoss) = false

# ============================================================

@doc doc"""
    SigmoidLoss <: MarginLoss

Continuous loss which penalizes every prediction with a loss
within in the range (0,2). It is infinitely many times
differentiable, Lipschitz continuous but nonconvex.

```math
L(a) = 1 - \tanh(a)
```

---
```
              Lossfunction                     Derivative
      ┌────────────┬────────────┐      ┌────────────┬────────────┐
    2 │""'--,.                  │    0 │..                     ..│
      │      '\.                │      │ "\.                 ./" │
      │         '.              │      │    ',             ,'    │
      │           \.            │      │      \           /      │
    L │            "\.          │   L' │       \         /       │
      │              \.         │      │        \.     ./        │
      │                \,       │      │         \.   ./         │
    0 │                  '"-:.__│   -1 │          ',_,'          │
      └────────────┴────────────┘      └────────────┴────────────┘
      -2                        2      -2                        2
                 y ⋅ ŷ                            y ⋅ ŷ
```
"""
struct SigmoidLoss <: MarginLoss end

value(loss::SigmoidLoss, agreement::Number) = one(agreement) - tanh(agreement)
deriv(loss::SigmoidLoss, agreement::Number) = -abs2(sech(agreement))
deriv2(loss::SigmoidLoss, agreement::T) where {T<:Number} = convert(T,2) * tanh(agreement) * abs2(sech(agreement))

isunivfishercons(::SigmoidLoss) = true
isdifferentiable(::SigmoidLoss) = true
isdifferentiable(::SigmoidLoss, at) = true
istwicedifferentiable(::SigmoidLoss) = true
istwicedifferentiable(::SigmoidLoss, at) = true
islocallylipschitzcont(::SigmoidLoss) = true
islipschitzcont(::SigmoidLoss) = true
isclasscalibrated(::SigmoidLoss) = true
isconvex(::SigmoidLoss) = false
isstrictlyconvex(::SigmoidLoss) = false
isstronglyconvex(::SigmoidLoss) = false
isclipable(::SigmoidLoss) = false

# ============================================================

@doc doc"""
    DWDMarginLoss <: MarginLoss

The distance weighted discrimination margin loss. It is a
differentiable generalization of the [L1HingeLoss](@ref) that is
different than the [SmoothedL1HingeLoss](@ref). It is Lipschitz
continuous and convex, but not strictly convex.

```math
L(a) = \begin{cases} 1 - a & \quad \text{if } a \ge \frac{q}{q+1} \\ \frac{1}{a^q} \frac{q^q}{(q+1)^{q+1}} & \quad \text{otherwise}\\ \end{cases}
```

---
```
              Lossfunction (q=1)               Derivative
      ┌────────────┬────────────┐      ┌────────────┬────────────┐
    2 │      ".                 │    0 │                     ._r-│
      │        \.               │      │                   ./    │
      │         ',              │      │                 ./      │
      │           \.            │      │                 /       │
    L │            "\.          │   L' │                .        │
      │              \.         │      │                /        │
      │               ":__      │      │               ;         │
    0 │                   '""---│   -1 │---------------┘         │
      └────────────┴────────────┘      └────────────┴────────────┘
      -2                        2      -2                        2
                 y ⋅ ŷ                            y ⋅ ŷ
```
"""
struct DWDMarginLoss{T<:AbstractFloat} <: MarginLoss
    q::T
    function DWDMarginLoss{T}(q::T) where T
        q > 0 || error("q must be strictly positive")
        new{T}(q)
    end
end
DWDMarginLoss(q::T) where {T<:AbstractFloat} = DWDMarginLoss{T}(q)
DWDMarginLoss(q) = DWDMarginLoss(Float64(q))

function value(loss::DWDMarginLoss{R}, agreement::T)::promote_type(R, T) where {R,T<:Number}
    q = loss.q
    if agreement <= q/(q+1)
        R(1) - agreement
    else
        (q^q/(q+1)^(q+1)) / agreement^q
    end
end

function deriv(loss::DWDMarginLoss{R}, agreement::T)::promote_type(R, T) where {R,T<:Number}
    q = loss.q
    agreement <= q/(q+1) ? -one(T) : -(q/(q+1))^(q+1) / agreement^(q+1)
end

function deriv2(loss::DWDMarginLoss{R}, agreement::T)::promote_type(R, T) where {R,T<:Number}
    q = loss.q
    agreement <= q/(q+1) ? zero(T) : ( (q^(q+1))/((q+1)^q) ) / agreement^(q+2)
end

isdifferentiable(::DWDMarginLoss) = true
isdifferentiable(::DWDMarginLoss, at) = true
istwicedifferentiable(::DWDMarginLoss) = true
istwicedifferentiable(loss::DWDMarginLoss, at) = true
islocallylipschitzcont(::DWDMarginLoss) = true
islipschitzcont(::DWDMarginLoss) = true
isconvex(::DWDMarginLoss) = true
isstrictlyconvex(::DWDMarginLoss) = false
isstronglyconvex(::DWDMarginLoss) = false
isfishercons(::DWDMarginLoss) = true
isunivfishercons(::DWDMarginLoss) = true
isclipable(::DWDMarginLoss) = false


using HOODESolver
using Test

include("henon_heiles.jl")

function testprepare()
    n_tau=32
    A = [0 0 1 0; 0 0 0 0;-1 0 0 0; 0 0 0 0]
    u =rand(BigFloat, 4)
    for i=1:5
        prec = 256*2^(i-1)
        setprecision(256*2^(i-1))
        for i_eps = 1:40
            epsilon = big"1.25"^(-i_eps)
            println("prec=$prec epsilon=$epsilon")
            for order=3:10
                @time @testset "test PrepareU0 prec=$prec order=$order epsilon=$epsilon" begin
                    newprec = prec + 32 + div(-exponent(epsilon)*order^2, 3)
                    precref = newprec + 10
                    parphi = PreparePhi(epsilon, n_tau, A, henon_heiles)
                    pref=PrepareU0(parphi, order, u, precref)
                    pnew=PrepareU0(parphi, order, u, newprec)
                    @test pref.ut0 == pnew.ut0
                end
            end
        end
    end
end
testprepare()


# Load modules
using Plots, LaTeXStrings
pyplot()
using ThinFilmsTools
using Optim

# Wavelength range [nm]
λ = 200:1100
# Angle of incidence [degrees]
θ = [0.]
# Polarisation (1.0 = p, 0.0 = s, between 0.0 and 1.0 = average)
pol = 0.5
beam = PlaneWave(λ, θ; p=pol)

# Refractive indices of incident (0) and substrate (2)
incident = RIdb.air(beam.λ)
emergent = RIdb.fused_silica_uv(beam.λ)

# Define the RI model to use
layers = [
    LayerTMMO(incident),
    ModelFit(:forouhibloomer),
    LayerTMMO(emergent),
]

# Measured absolute transmittance
Texp = SpectraDB.scandia_spectrum(beam.λ)
plot(beam.λ, Texp)
gui()

options = Optim.Options(
    g_abstol=1e-8, g_reltol=1e-8, iterations=10^3, show_trace=true, store_trace=false,
);

seed = [vcat(90.0, # thickness
        [1.6, 5.0, # ϵinf, Eg
         0.15, 16, 65, # A1, B1, C1
         ],
    ),
]

solOptim = fit_tmm_optics(
    Transmittance(Texp), seed, beam, layers;
    options=options, alg=LBFGS(),
)

plot(FitSpectrum(), solOptim.beam.λ, solOptim.spectrumExp, solOptim.spectrumFit)
gui()


@symbol_func function G_CR(cur_reactor::AbstractReactor)
  cur_epsilon = epsilon_b(cur_reactor)

  ( isa(cur_epsilon, AbstractFloat) && cur_epsilon >= 1 ) && return NaN

  cur_left_part = safe_symbol(cur_reactor, :B_0) ^ 2

  cur_left_part /= (8/5) * cur_reactor.pi

  cur_left_part /= cur_reactor.sigma_TF

  cur_left_part /= ( 1 - cur_epsilon )

  cur_right_part = 4 * cur_epsilon

  cur_right_part /= 1 + cur_epsilon

  cur_right_part += log(
    ( 1 + cur_epsilon ) / ( 1 - cur_epsilon )
  )

  cur_G = cur_left_part

  cur_G *= cur_right_part

  cur_G
end


# ------------------------------------------------------------------
# Licensed under the MIT License. See LICENSE in the project root.
# ------------------------------------------------------------------

"""
    Line(a, b)

A line passing through points `a` and `b`.

See also [`Segment`](@ref).
"""
struct Line{Dim,T} <: Primitive{Dim,T}
  a::Point{Dim,T}
  b::Point{Dim,T}
end

Line(a::Tuple, b::Tuple) = Line(Point(a), Point(b))

paramdim(::Type{<:Line}) = 1

(l::Line)(t) = l.a + t * (l.b - l.a)

function Base.in(p::Point, l::Line)
  w = norm(l.b-l.a)
  d = evaluate(Euclidean(), p, l)
  # d ≈ 0.0 will be too precise, and d < atol{T} can't scale.
  d+w ≈ w
end

==(l1::Line, l2::Line) = l1.a ∈ l2 && l1.b ∈ l2 && l2.a ∈ l1 && l2.b ∈ l1


module TestMean
    using StreamStats
    using Distributions
    using Base.Test

    # Means of uniform draws
    for n in rand(1:1_000_000, 100)
        xs = rand(n)
        stat = StreamStats.Mean()
        for x in xs
            update!(stat, x)
        end
        online_m = mean(stat)
        online_ms = state(stat)
        online_n = nobs(stat)
        batch_m = mean(xs)
        @test_approx_eq(online_m, batch_m)
        @test_approx_eq(online_ms, batch_m)
        @test online_n == n
    end

    # Means of non-uniform draws
    ds = [Normal(0, 1), Normal(1e5, 1e8), LogNormal(10, 10), Cauchy(0.0, 1e-8)]
    for d in ds
        for n in rand(1:1_000_000, 10)
            xs = rand(d, n)
            stat = StreamStats.Mean()
            for x in xs
                update!(stat, x)
            end
            online_m = mean(stat)
            online_n = nobs(stat)
            batch_m = mean(xs)
            @test abs(online_m - batch_m) < 1e-8 * abs(batch_m)
            @test online_n == n
        end
    end
end


using Test, LinearAlgebra
using DynamicPolynomials
using SetProg
using Polyhedra

@testset "L1 heuristic" begin
    @polyvar x y
    p = 2x^2*y + 3x + 4y^2 - 5x^4*y^2 + 6x^2
    @test SetProg.rectangle_integrate(p, [2, 3]) ≈ -672

    Q = [1 2 4
         2 3 5
         4 5 6]
    q = SetProg.GramMatrix(Q, SetProg.monomials([x, y], 2))
    @test SetProg.rectangle_integrate(polynomial(q), [2, 3]) ≈ 3465.6

    @test SetProg.rectangle_integrate(polynomial(q), [1, 1]) ≈ 472/45

    subset = SetProg.Sets.PolySet(4, q)
    v_rep(a, b) = polyhedron(convexhull([a, b], [-a, -b], [-a, b], [a, -b]))
    h_rep(a, b) = polyhedron(HalfSpace([1, 0], a) ∩ HalfSpace([-1,  0], a) ∩
                             HalfSpace([0, 1], b) ∩ HalfSpace([ 0, -1], b))
    v_square = v_rep(1.0, 1.0)
    h_square = h_rep(1.0, 1.0)
    for square in [v_square, h_square]
        set = SetProg.Sets.Piecewise([subset, subset, subset, subset], square)
        @test SetProg.l1_integral(set, nothing) ≈ 472/45
    end

    Q = [1 2
         2 3]
    @test SetProg.rectangle_integrate(polynomial(Q, [x, y]), [2, 3]) ≈ 248

    @test SetProg.rectangle_integrate(polynomial(Q, [x, y]), [1, 1]) ≈ 16/3

    ell = SetProg.Sets.Ellipsoid(Symmetric(Q))
    for square in [v_square, h_square]
        set = SetProg.Sets.Piecewise([ell, ell, ell, ell], square)
        @test SetProg.l1_integral(set, nothing) ≈ 16/3
    end

    Δ = polyhedron(convexhull([0.0, 0.0], [1.0, 0.0], [1.0, 0.5]))
    Q = [0.0 0.5
         0.5 0.0]
    ell = SetProg.Sets.Ellipsoid(Symmetric(Q))
    @test SetProg.l1_integral(ell, Δ) ≈ 1/32
    Q = [1.0 0.0
         0.0 0.0]
    ell = SetProg.Sets.Ellipsoid(Symmetric(Q))
    @test SetProg.l1_integral(ell, Δ) ≈ 1/8
    Q = [0.0 0.0
         0.0 1.0]
    ell = SetProg.Sets.Ellipsoid(Symmetric(Q))
    @test SetProg.l1_integral(ell, Δ) ≈ 1/96
end


  module utils

  using Reexport

  export isinside,intersection,clipSH,tuple2intpoint,isoverlap

  import Luxor: Point
using Clipper

isinside(p, a, b) = (b.x - a.x) * (p.y - a.y) > (b.y - a.y) * (p.x - a.x)

function intersection(a, b, s, f)
    dc = [a.x - b.x, a.y - b.y]
    dp = [s.x - f.x, s.y - f.y]
    n1 = a.x * b.y - a.y * b.x
    n2 = s.x * f.y - s.y * f.x
    n3 = 1.0 / (dc[1] * dp[2] - dc[2] * dp[1])
    Point((n1 * dp[1] - n2 * dc[1]) * n3, (n1 * dp[2] - n2 * dc[2]) * n3)
end

function clipSH(spoly, cpoly)
    outarr = spoly
    q = cpoly[end]
    for p in cpoly
        # @show p
        inarr = outarr
        outarr = Point[]
        # DEBUG if inarr is empty, I assume the two poly does not overlap
        # if isempty(inarr) return Point[] end
        s = inarr[end]
        for vtx in inarr
            if isinside(vtx, q, p)
                if !isinside(s, q, p)
                    push!(outarr, intersection(q, p, s, vtx))
                end
                push!(outarr, vtx)
            elseif isinside(s, q, p)
                push!(outarr, intersection(q, p, s, vtx))
            end
            s = vtx
        end
        q = p
    end
    outarr
end

function isoverlap_old(spoly, cpoly)
    if isempty(clipSH(spoly, cpoly)) return false
    else return true
    end
end

function isoverlap_clip(poly0, polys...)
    path1 = Vector{IntPoint}()
    for p in poly0
        push!(path1, p)
    end
    c = Clip()
    add_path!(c, path1, PolyTypeSubject, true)

    for poly in polys
        path2 = Vector{IntPoint}()
        for p in poly
            push!(path2, p)
        end
        add_path!(c, path2, PolyTypeClip, true)
    end

    result, polys = execute(c,
                            # ClipTypeUnion,
                            # ClipTypeDifference,
                            ClipTypeIntersection,
                            PolyFillTypeEvenOdd,
                            PolyFillTypeEvenOdd)
    # @show polys
    # FIXME isempty(polys) does not work
    return length(polys) > 0
end

function tuple2intpoint(t)
    map(t) do (x,y)
        IntPoint(round(x*1e6),round(y*1e6))
    end
end

function isoverlap(poly0, polys...)
    isoverlap_clip(tuple2intpoint(poly0), map(tuple2intpoint, polys)...)
end

  end


using PyPlot

using TransitionalMCMC

# Prior Bounds
lb, ub  = -5, 5

# Prior Density and sampler
priorDen(x) = pdf(Uniform(lb, ub), x[1,:]) .* pdf(Uniform(lb, ub), x[2,:])
priorRnd(Nsamples) = rand(Uniform(lb, ub), Nsamples, 2)

# Log Likelihood
function logLik(x)
    sleep(0.005)
    return -1 .* ((x[1,:].^2 .+ x[2,:] .- 11).^2 .+ (x[1,:] .+ x[2,:].^2 .- 7).^2)
end

Nsamples = 200

samps, acc = tmcmc(logLik, priorDen, priorRnd, Nsamples, 5, 2)

plt.scatter(samps[:,1], samps[:,2])


using ElasticFDA
using Distributions
using JLD2
using LinearAlgebra
using Test

@load "simu_data.jld2"
timet = collect(LinRange(0,1,size(f,1)));
f1 = f[:,1];
f2 = f[:,2];

# test smooth_data
g = smooth_data(f1,1);
@test sum(g)-sum(f1)<1e-6
smooth_data!(f1,1);
@test sum(f1)-sum(f[:,1])<1e-6

# test trapz
d = trapz(timet, timet);
@test d ≈ 0.5

# test f_to_srsf
q1 = f_to_srsf(f1, timet);
f1a = srsf_to_f(q1,timet,f1[1]);

# test optimum reparam
gam = optimum_reparam(q1,timet,q1,f1o=f1[1],f2o=f2[1]);
@test norm(gam-LinRange(0,1,101)) < 1e-10

# test warping functions
qw = warp_q_gamma(timet, q1, gam);
fw = warp_f_gamma(timet, f1, gam);
@test norm(q1-qw) < 1e-10
@test norm(f1-fw) < 1e-10

# test srsf_align
out = srsf_align(f, timet);

# test align_fPCA
out1 = align_fPCA(f, timet, MaxItr=2);

# test vert_fPCA
out1 = vert_fPCA(out, no=3);

# test horiz_fPCA
out1 = horiz_fPCA(out, no=3);

# test gauss_model
out1 = gauss_model(out);

# test pair_warping_bayes
# out1 = pair_warping_bayes(f1[1:100], f2[1:100], iter=2);

# test group_warping_bayes
# out1 = group_warping_bayes(f, iter=2);

# test elastic_distance
da, dp = elastic_distance(f1, f1, timet);
@test da < 1e-10
@test dp < 1e-6

# test rgam
gam = rgam(101,.1,10);

# test elastic_regression
include("test_warp_regress.jl")
timet = collect(timet);
out = elastic_regression(f, y_orig, timet, max_itr=1);
out1 = elastic_prediction(f, timet, out, y=y_orig);

# test elastic logistic regression
include("test_warp_logistic.jl")
timet = collect(timet);
out = elastic_logistic(f, y_orig, timet, max_itr=1);
out1 = elastic_prediction(f, timet, out, y=y_orig);

# test elastic m-logistic regression
# include("test_warp_mlogistic.jl")
# timet = collect(timet);
# out = elastic_mlogistic(f, y_orig, timet, max_itr=1);
# out1 = elastic_prediction(f, timet, out, y=y_orig);

@load "beta.jld2";

# test curve_to_q
q1 = curve_to_q(beta[:,:,1]);
beta1 = q_to_curve(q1);

# test calc_shape_dist
beta1 = beta[:,:,1];
d1 = calc_shape_dist(beta1,beta1);

# test curve_pair_align
beta2n, q2n, gam, q1 = curve_pair_align(beta1,beta1);

# test curve_geodesic
geod, geod_q = curve_geodesic(beta1,beta[:,:,2]);

# test curve functions 'O'
mu, betamean, v, q = curve_karcher_mean(beta, maxit=1);
betan, qn, betamean1, q_mu = curve_srvf_align(beta, maxit=1);
K = curve_karcher_cov(betamean, beta);
pd = curve_principal_directions(betamean, mu, K);
s = sample_shapes(mu, K);

# test curve functions 'C'
mu, betamean, v, q = curve_karcher_mean(beta, mode='C', maxit=1);
betan, qn, betamean1, q_mu = curve_srvf_align(beta, mode='C', maxit=1);
K = curve_karcher_cov(betamean, beta, mode='C');
pd = curve_principal_directions(betamean, mu, K, mode='C');
s = sample_shapes(mu, K, mode='C');

# test image functions
# vars = load("image.jld");
# I1 = vars["I1"];
# I2 = vars["I2"];
# Inew, gam = pair_align_image(I1,I2,itermax=1);


using MomentClosure
using MomentClosure: define_M, define_μ
using Test
using Catalyst

@parameters t, k_on, k_off, k_p, γ_p, b
@variables p(t), g(t)

vars = [g, p]
ps = [k_on, k_off, k_p, γ_p, b]
S = [1 -1 0 0;
     0 0 b -1]

as = [k_on*(1-g),    # 0 -> g
      k_off*g*(p^2), # g -> 0
      k_p*g,         # 0 -> mP, m ~ Geometric(mean=b)
      γ_p*p]         # p -> 0

binary_vars = [1]

rn = ReactionSystemMod(t, vars, ps, as, S)

μ = define_μ(2,4)
M = define_M(2,4)
sys = generate_central_moment_eqs(rn, 2)

closed_eqs = moment_closure(sys, "zero", binary_vars)
@test length(closed_eqs.odes.states) == 4 && isequal(closed_eqs.closure[M[0,4]], 0)

closed_eqs = moment_closure(sys, "normal", binary_vars)
@test length(closed_eqs.odes.states) == 4 && isequal(closed_eqs.closure[M[0,4]], 3*M[0,2]^2)

closed_eqs= moment_closure(sys, "log-normal", binary_vars)
expr1 = closed_eqs.closure[M[0,3]]
expr2 = μ[0,1]^3*(1+M[0,2]*μ[0,1]^-2)^3 - μ[0,1]^3 - 3*M[0,2]*μ[0,1]
@test length(closed_eqs.odes.states) == 4 && isequal(expand(expr1), expr2)

closed_eqs = moment_closure(sys, "poisson", binary_vars)
@test length(closed_eqs.odes.states) == 4 && isequal(closed_eqs.closure[M[0,3]], μ[0,1])

closed_eqs = moment_closure(sys, "gamma", binary_vars)
expr1 = closed_eqs.closure[M[0,3]]
expr2 = 2*M[0,2]^2*μ[0,1]^-1
@test length(closed_eqs.odes.states) == 4 && isequal(expr1, expr2)

closed_eqs = moment_closure(sys, "derivative matching", binary_vars)
expr1 = closed_eqs.closure[sys.M[0,3]]
expr2 = μ[0,1]^-3 * (M[0,2]+μ[0,1]^2)^3 - μ[0,1]^3 - 3*M[0,2]*μ[0,1]
@test length(closed_eqs.odes.states) == 4 && isequal(expr1, expr2)

sys = generate_raw_moment_eqs(rn, 2)

closed_eqs = moment_closure(sys, "zero", binary_vars)
@test length(closed_eqs.odes.states) == 4

closed_eqs = moment_closure(sys, "normal", binary_vars)
@test length(closed_eqs.odes.states) == 4

closed_eqs = moment_closure(sys, "log-normal", binary_vars)
@test length(closed_eqs.odes.states) == 4

closed_eqs = moment_closure(sys, "poisson", binary_vars)
@test length(closed_eqs.odes.states) == 4

closed_eqs = moment_closure(sys, "gamma", binary_vars)
@test isequal(expr1, expr2)

closed_eqs = moment_closure(sys, "derivative matching", binary_vars)
@test isequal(expr1, expr2)


### Linear Algebra

# Import functions from Base

import Base:dot,
            transpose,
            ctranspose,
            transpose!,
            ctranspose!,
            det,
            inv,
            norm,
            rank,
            *,
            A_mul_Bt,
            At_mul_B,
            At_mul_Bt,
            Ac_mul_B,
            A_mul_Bc,
            Ac_mul_Bc,
            chol, lu,
            lufact!,
            qr,
            qrfact!,
            svd,
            svdfact!,
            \,
            diag

# Export Functions

export  isLAPACKAvailable,
        chol!,
        solveLU,
        upper,
        lower

# Export constants

export  AF_MAT_NONE,
        AF_MAT_TRANS,
        AF_MAT_CTRANS,
        AF_MAT_CONJ ,
        AF_MAT_UPPER,
        AF_MAT_LOWER,
        AF_MAT_DIAG_UNIT,
        AF_MAT_SYM,
        AF_MAT_POSDEF,
        AF_MAT_ORTHOG,
        AF_MAT_TRI_DIAG,
        AF_MAT_BLOCK_DIAG,
        AF_NORM_VECTOR_1,
        AF_NORM_VECTOR_INF,
        AF_NORM_VECTOR_2,
        AF_NORM_VECTOR_P,
        AF_NORM_MATRIX_1,
        AF_NORM_MATRIX_INF,
        AF_NORM_MATRIX_2,
        AF_NORM_MATRIX_L_PQ,
        AF_NORM_EUCLID

# Constants

AF_MAT_NONE       = 0
AF_MAT_TRANS      = 1
AF_MAT_CTRANS     = 2
AF_MAT_CONJ       = 4
AF_MAT_UPPER      = 32
AF_MAT_LOWER      = 64
AF_MAT_DIAG_UNIT  = 128
AF_MAT_SYM        = 512
AF_MAT_POSDEF     = 1024
AF_MAT_ORTHOG     = 2048
AF_MAT_TRI_DIAG   = 4096
AF_MAT_BLOCK_DIAG = 8192

AF_NORM_VECTOR_1 = 0
AF_NORM_VECTOR_INF = 1
AF_NORM_VECTOR_2 = 2
AF_NORM_VECTOR_P = 3
AF_NORM_MATRIX_1 = 4
AF_NORM_MATRIX_INF = 5
AF_NORM_MATRIX_2 = 6
AF_NORM_MATRIX_L_PQ = 7
AF_NORM_EUCLID = AF_NORM_VECTOR_2

# Dot

function dot{T,S}(a::AFVector{T}, b::AFVector{S}, lhsprop = AF_MAT_NONE, rhsprop = AF_MAT_NONE)
    out = new_ptr()
    af_dot(out, a, b, lhsprop, rhsprop)
    AFVector{af_promote(T,S)}(out[])
end

# Transpose

function transpose{T}(a::AFArray{T})
    out = new_ptr()
    af_transpose(out, a, false)
    AFArray{T}(out[])
end

function ctranspose{T}(a::AFArray{T})
    out = new_ptr()
    af_transpose(out, a, true)
    AFArray{T}(out[])
end

function transpose!{T}(a::AFArray{T})
    af_transpose_inplace(a, false)
    a
end

function ctranspose!{T}(a::AFArray{T})
    af_transpose_inplace(a, true)
    a
end

# Matrix Operations

function det{T<:Real}(a::AFArray{T})
    real = Base.Ref{Cdouble}(0)
    imag = Base.Ref{Cdouble}(0)
    af_det(real, imag, a)
    real[]
end

function det{T<:Complex}(a::AFArray{T})
    real = Base.Ref{Cdouble}(0)
    imag = Base.Ref{Cdouble}(0)
    af_det(real, imag, a)
    complex(real[], imag[])
end

function inv(a::AFArray, options::Int = AF_MAT_NONE)
    out = new_ptr()
    af_inverse(out, a, options)
    AFArray{backend_eltype(out[])}(out[])
end

function norm{T}(a::AFArray{T}; options = AF_NORM_EUCLID, p::Real = 1, q::Real = 1)
    out = Base.Ref{Cdouble}(0)
    af_norm(out, a, options, p, q)
    if T == Float32
        T(out[])
    else
        out[]
    end
end

function rank(a::AFArray; tol::Cdouble = 1e-5)
    out = Base.Ref{Cuint}(0)
    af_rank(out, a, tol)
    Int(out[])
end

# Matrix Multiply

function *{T,S}(a::AFArray{T}, b::AFArray{S}; lhsprop = AF_MAT_NONE, rhsprop = AF_MAT_NONE)
    out = new_ptr()
    af_matmul(out, a, b, lhsprop, rhsprop)
    AFMatrix{af_promote(T,S)}(out[])
end

A_mul_Bt{T,S}(a::AFMatrix{T}, b::AFMatrix{S}) = *(a, b, rhsprop = AF_MAT_TRANS)

At_mul_B{T,S}(a::AFMatrix{T}, b::AFMatrix{S}) = *(a, b, lhsprop = AF_MAT_TRANS)

At_mul_Bt{T,S}(a::AFMatrix{T}, b::AFMatrix{S}) = *(a, b, lhsprop = AF_MAT_TRANS, rhsprop = AF_MAT_TRANS)

A_mul_Bc{T,S}(a::AFMatrix{T}, b::AFMatrix{S}) = *(a, b, rhsprop=AF_MAT_CTRANS)

Ac_mul_B{T,S}(a::AFMatrix{T}, b::AFMatrix{S}) = *(a, b, lhsprop=AF_MAT_CTRANS)

Ac_mul_Bc{T,S}(a::AFMatrix{T}, b::AFMatrix{S}) = *(a, b, lhsprop=AF_MAT_CTRANS, rhsprop=AF_MAT_CTRANS)

# LAPACK

function isLAPACKAvailable()
    out = Base.Ref{Bool}(0)
    af_is_lapack_available(out)
    out[]
end

# Matrix Factorizations

function _chol{T}(a::AFMatrix{T}, is_upper::Bool)
    out = new_ptr()
    info = Base.Ref{Cint}(0)
    af_cholesky(out, info, a, is_upper)
    info[] > 0 && throw(PosDefException(info))
    AFArray{T}(out[])
end

function _chol!{T}(a::AFMatrix{T}, is_upper::Bool)
    info = Base.Ref{Cint}(0)
    af_cholesky_inplace(info, a, is_upper)
    info[] > 0 && throw(PosDefException(info))
    a
end

chol(a::AFMatrix, ::Type{Val{:U}}) = _chol(a, true)
chol(a::AFMatrix, ::Type{Val{:L}}) = _chol(a, false)
chol(a::AFMatrix) = chol(a,Val{:U})

chol!(a::AFMatrix, ::Type{Val{:U}}) = _chol!(a, true)
chol!(a::AFMatrix, ::Type{Val{:L}}) = _chol!(a, false)
chol!(a::AFMatrix) = chol!(a,Val{:U})

function lu(a::AFMatrix)
    l = new_ptr()
    u = new_ptr()
    p = new_ptr()
    af_lu(l, u, p, a)
    AFArray{backend_eltype(l[])}(l[]), AFArray{backend_eltype(u[])}(u[]),
    (AFArray{backend_eltype(p[])}(p[]) + 1)
end

function lufact!(a::AFMatrix)
    p = new_ptr()
    af_lu_inplace(p, a, true)
    a, AFArray{backend_eltype(p[])}(p[])
end

function qr(a::AFMatrix)
    q = new_ptr()
    r = new_ptr()
    tau = new_ptr()
    af_qr(q, r, tau, a)
    AFArray{backend_eltype(q[])}(q[]), AFArray{backend_eltype(r[])}(r[]),
    AFArray{backend_eltype(tau[])}(tau[])
end

function qrfact!(a::AFMatrix)
    tau = new_ptr()
    af_qr_inplace(tau, a)
    a, AFArray{backend_eltype(tau[])}(tau[])
end

function svd(a::AFArray)
    u = new_ptr()
    s = new_ptr()
    vt = new_ptr()
    af_svd(u, s, vt, a)
    AFArray{backend_eltype(u[])}(u[]),
    AFArray{backend_eltype(s[])}(s[]),
    AFArray{backend_eltype(vt[])}(vt[])
end

function svdfact!(a::AFArray)
    u = new_ptr()
    s = new_ptr()
    vt = new_ptr()
    af_svd_inplace(u, s, vt, a)
    AFArray{backend_eltype(u[])}(u[]),
    AFArray{backend_eltype(s[])}(s[]),
    AFArray{backend_eltype(vt[])}(vt[])
end

function \(a::AFArray, b::AFArray; options = AF_MAT_NONE)
    out = new_ptr()
    af_solve(out, a, b, options)
    AFArray{backend_eltype(out[])}(out[])
end

function solveLU(a::AFArray, p::AFArray, b::AFArray; options = AF_MAT_NONE)
    out = new_ptr()
    af_solve_lu(out, a, p, b, options)
    AFArray{backend_eltype(out[])}(out[])
end

function diag{T}(a::AFMatrix{T}, k::Integer = 1)
    out = new_ptr()
    af_diag_extract(out, a, k - 1)
    AFVector{T}(out[])
end

function upper{T}(a::AFMatrix{T}; is_unit_diag = false)
    out = new_ptr()
    af_upper(out, a, is_unit_diag)
    AFMatrix{T}(out[])
end

function lower{T}(a::AFMatrix{T}; is_unit_diag = false)
    out = new_ptr()
    af_lower(out, a, is_unit_diag)
    AFMatrix{T}(out[])
end


"""
    IdentityMultiple{T} < AbstractMatrix{T} where T

A scalar multiple of the identity matrix of given order and numeric type.

### Fields

- `M` -- uniform scaling operator of type `T`
- `n` -- size of the identity matrix

### Notes

This type can be used to create multiples of the identity of given size. Since
only the multiple and the order are stored, the allocations are minimal.

Internally, the type wraps Julia's lazy multiple of the identity operator,
`UniformScaling`. `IdentityMultiple` subtypes `AbstractMatix`, hence it can be
used in usual matrix arithmetic and for dispatch on `AbstractMatrix`.

The difference between `UniformScaling` and `IdentityMultiple` is that while the
size of the former is generic, the size of the latter is fixed.

### Examples

The easiest way to create an identity multiple is to use the callable version
of `LinearAlgebra.I`:

```jldoctest identitymultiple
julia> using MathematicalSystems: IdentityMultiple

julia> I2 = I(2)
IdentityMultiple{Float64} of value 1.0 and order 2

julia> I2 + I2
IdentityMultiple{Float64} of value 2.0 and order 2

julia> 4*I2
IdentityMultiple{Float64} of value 4.0 and order 2
```

The numeric type (default `Float64`) can be passed as a second argument:

```jldoctest identitymultiple
julia> I2r = I(2, Rational{Int})
IdentityMultiple{Rational{Int64}} of value 1//1 and order 2

julia> I2r + I2r
IdentityMultiple{Rational{Int64}} of value 2//1 and order 2

julia> 4*I2r
IdentityMultiple{Rational{Int64}} of value 4//1 and order 2
```

To create the matrix with a value different from the default (`1.0`), there are
two ways. Either pass the value through the callable `I`, as in

```jldoctest identitymultiple
julia> I2 = I(2.0, 2)
IdentityMultiple{Float64} of value 2.0 and order 2

julia> I2r = I(2//1, 2)
IdentityMultiple{Rational{Int64}} of value 2//1 and order 2
```

Or use the lower level constructor passing the `UniformScaling` (`I`):

```jldoctest identitymultiple
julia> I2 = IdentityMultiple(2.0*I, 2)
IdentityMultiple{Float64} of value 2.0 and order 2

julia> I2r = IdentityMultiple(2//1*I, 2)
IdentityMultiple{Rational{Int64}} of value 2//1 and order 2
```
"""
struct IdentityMultiple{T} <: AbstractMatrix{T}
    M::UniformScaling{T}
    n::Int
end

Base.IndexStyle(::Type{<:IdentityMultiple}) = IndexLinear()
Base.size(𝐼::IdentityMultiple) = (𝐼.n, 𝐼.n)
Base.getindex(𝐼::IdentityMultiple, inds...) = getindex(𝐼.M, inds...)
Base.getindex(𝐼::IdentityMultiple{T}, ind) where {T} =
    rem(ind-1, 𝐼.n+1) == 0 ? 𝐼.M.λ : zero(T)
Base.setindex!(𝐼::IdentityMultiple, X, inds...) = error("cannot store a value in an `Identity`")

Base.:(*)(x::Number, 𝐼::IdentityMultiple) = IdentityMultiple(x * 𝐼.M, 𝐼.n)

function Base.:(*)(𝐼::IdentityMultiple, v::AbstractVector)
    @assert 𝐼.n == length(v)
    return 𝐼.M.λ * v
end

function Base.:(*)(𝐼::IdentityMultiple, A::AbstractMatrix)
    @assert 𝐼.n == size(A, 1)
    return 𝐼.M.λ * A
end

function Base.:(*)(A::AbstractMatrix, 𝐼::IdentityMultiple)
    @assert size(A, 2) == 𝐼.n
    return A * 𝐼.M.λ
end

function Base.:(+)(𝐼1::IdentityMultiple, 𝐼2::IdentityMultiple)
    @assert 𝐼1.n == 𝐼2.n
    return IdentityMultiple(𝐼1.M + 𝐼2.M, 𝐼1.n)
end

function Base.:(*)(𝐼1::IdentityMultiple, 𝐼2::IdentityMultiple)
    @assert 𝐼1.n == 𝐼2.n
    return IdentityMultiple(𝐼1.M * 𝐼2.M, 𝐼1.n)
end

function Base.show(io::IO, ::MIME"text/plain", 𝐼::IdentityMultiple{T}) where T
    print(io, "$(typeof(𝐼)) of value $(𝐼.M.λ) and order $(𝐼.n)")
end

# callable identity matrix
LinearAlgebra.I(n::Int, N=Float64) = IdentityMultiple(one(N)*I, n)

LinearAlgebra.I(λ::Number, n::Int) = IdentityMultiple(λ*I, n)


## Save the small wikipedia graph
using SparseArrays
A = [0 1 1 1 1 0 1 0 0 0
     1 0 0 0 0 0 0 0 0 0
     0 0 0 0 1 1 1 0 0 0
     1 1 0 0 1 1 0 1 0 0
     1 1 1 1 0 0 0 0 1 0
     0 0 1 0 0 0 1 0 1 1
     0 0 0 0 0 1 0 0 0 1
     0 0 0 0 0 0 0 0 1 0
     0 0 0 0 0 0 0 1 0 0
     0 0 0 0 0 0 0 0 0 0]
A = sparse(A)
pages = ["PageRank", "Google", "Adjacency matrix",
"Markov chain", "Eigenvector", "Directed graph",
"Graph", "Linear system", "Vector space", "Multiset"]
ei,ej = findnz(A)
xy = [1.5 6.5;0 5.5; 0 2.25; -1.5 6.5; 0 4; -1.5 0.85; 1.5 0.85; -1 4.5; -1 3; 0 0.25]
##
using DelimitedFiles
writedlm("wiki-simple.edges", [ei ej])
writedlm("wiki-simple.xy", xy)
writedlm("wiki-simple.nodes", pages)
##
using GraphRecipes
#graphplot(ei,ej, x=xy[:,1], y=xy[:,2])
B = A.*A'
U = A - B # we need A' so that the ei, ej plots

graphplot(findnz(triu(B,1))[1:2]..., x = xy[:,1], y = xy[:,2],
  markercolor=nothing, markerstrokecolor=:white, curves=false,
  markersize=0,
  linecolor=1, linealpha=0.8, linewidth=0.7, arrow=:both, shorten=0.85,
  axis_buffer=0.5, background=nothing)

graphplot!(findnz(U)[1:2]..., x = xy[:,1], y = xy[:,2],
  markercolor=nothing, markerstrokecolor=:white, curves=false, curvature_scalar = -0.4,
  markersize=0,
   linecolor=2, linealpha=0.8, linewidth=0.7, arrow = :tail, shorten=0.85,
  axis_buffer=0.5, background=nothing)

xlims!(xlims().*1.2)
for i=1:10
  annotate!(xy[i,1],xy[i,2],text(pages[i],9,:white))
end
plot!(size=(300,300))
savefig("wiki-simple.pdf")

##
using GraphRecipes
#graphplot(ei,ej, x=xy[:,1], y=xy[:,2])
B = A.*A'
U = A - B # we need A' so that the ei, ej plots

graphplot(findnz(triu(B,1))[1:2]..., x = xy[:,1], y = xy[:,2],
  markercolor=nothing, markerstrokecolor=:white, curves=false,
  markersize=0, markerstrokewidth=0,
  linecolor=1, linealpha=0.8, linewidth=0.7, arrow=:both, shorten=0.85,
  axis_buffer=0.5, background=nothing)

graphplot!(findnz(U)[1:2]..., x = xy[:,1], y = xy[:,2],
  markercolor=nothing, markerstrokecolor=:white, curves=false, curvature_scalar = -0.4,
  markersize=0, markerstrokewidth=0,
   linecolor=2, linealpha=0.8, linewidth=0.7, arrow = :tail, shorten=0.85,
  axis_buffer=0.5, background=nothing)

xlims!(xlims().*1.2)
for i=1:10
  annotate!(xy[i,1],xy[i,2],text("$i",9,:white))
end
plot!(size=(300,300))
savefig("wiki-simple-numbers.pdf")

##
#= Tikz graph
\Vertex[L={PageRank},x=1.5,y=6.5]{1}
\Vertex[L={Google},x=0,y=5.5]{2}
\Vertex[L={Adjacency matrix},x=0,y=2.25]{3}
\Vertex[L={Markov chain},x=-1.5,y=6.5]{4}
\Vertex[L={Eigenvector},x=0,y=4]{5}
\Vertex[L={Directed graph},x=-1.5,y=0.85]{6}
\Vertex[L={Graph},x=1.5,y=0.85]{7}
\Vertex[L={Linear system},x=-1,y=4.5]{8}
\Vertex[L={Vector space},x=-1,y=3]{9}
\Vertex[L={Multiset},x=0,y=0.25]{10}

\Edge(4)(2)
\Edge(5)(2)
\Edge(4)(6)
\Edge(1)(7)
\Edge(3)(7)
\Edge(4)(8)
\Edge(5)(9)
\Edge(6)(9)
\Edge(6)(10)
\Edge(7)(10)
\Edge(2)(1)
\Edge(4)(1)
\Edge(5)(1)
\Edge(1)(2)
\Edge(5)(3)
\Edge(6)(3)
\Edge(1)(4)
\Edge(5)(4)
\Edge(1)(5)
\Edge(3)(5)
\Edge(4)(5)
\Edge(3)(6)
\Edge(7)(6)
\Edge(6)(7)
\Edge(9)(8)
\Edge(8)(9)
\Edge(1)(3)
=#


# # Data Transformations
#
# ## Histogram
#
# ```@docs
# histogram
# ```

using AlgebraOfGraphics, CairoMakie
set_aog_theme!()

df = (x=randn(1000), y=randn(1000), z=rand(["a", "b", "c"], 1000))
specs = data(df) * mapping(:x, layout=:z) * histogram(bins=range(-2, 2, length=15))
draw(specs)

#

specs = data(df) * mapping(:x, dodge=:z, color=:z) * histogram(bins=range(-2, 2, length=15))
draw(specs)

#

specs = data(df) * mapping(:x, stack=:z, color=:z) * histogram(bins=range(-2, 2, length=15))
draw(specs)

#

data(df) * mapping(:x, :y, layout=:z) * histogram(bins=15) |> draw

# ## Density
#
# ```@docs
# AlgebraOfGraphics.density
# ```

df = (x=randn(5000), y=randn(5000), z=rand(["a", "b", "c", "d"], 5000))
data(df) * mapping(:x, layout=:z) * AlgebraOfGraphics.density() |> draw

#

data(df) * mapping(:x, :y, layout=:z) * AlgebraOfGraphics.density(npoints=50) |> draw

#

specs = data(df) * mapping(:x, :y, layout=:z) *
    AlgebraOfGraphics.density(npoints=50) * visual(Surface)

draw(specs, axis=(type=Axis3, zticks=0:0.1:0.2, limits=(nothing, nothing, (0, 0.2))))

# ## Frequency
#
# ```@docs
# frequency
# ```

df = (x=rand(["a", "b", "c"], 100), y=rand(["a", "b", "c"], 100), z=rand(["a", "b", "c"], 100))
specs = data(df) * mapping(:x, layout=:z) * frequency()
draw(specs)

#

specs = data(df) * mapping(:x, layout=:z, color=:y, stack=:y) * frequency()
draw(specs)

#

specs = data(df) * mapping(:x, :y, layout=:z) * frequency()
draw(specs)

# ## Expectation
#
# ```@docs
# expectation
# ```

df = (x=rand(["a", "b", "c"], 100), y=rand(["a", "b", "c"], 100), z=rand(100), c=rand(["a", "b", "c"], 100))
specs = data(df) * mapping(:x, :z, layout=:c) * expectation()
draw(specs)

#

specs = data(df) * mapping(:x, :z, layout=:c, color=:y, dodge=:y) * expectation()
draw(specs)

#

specs = data(df) * mapping(:x, :y, :z, layout=:c) * expectation()
draw(specs)

# ## Linear
#
# ```@docs
# linear
# ```

x = 1:0.05:10
a = rand(1:7, length(x))
y = 1.2 .* x .+ a .+ 0.5 .* randn.()
df = (; x, y, a)
specs = data(df) * mapping(:x, :y, color=:a => nonnumeric) * (linear() + visual(Scatter))
draw(specs)

# ## Smoothing
#
# ```@docs
# smooth
# ```

x = 1:0.05:10
a = rand(1:7, length(x))
y = sin.(x) .+ a .+ 0.1 .* randn.()
df = (; x, y, a)
specs = data(df) * mapping(:x, :y, color=:a => nonnumeric) * (smooth() + visual(Scatter))
draw(specs)


module Convex1d
export minimize

let
    s = read(joinpath(dirname(@__DIR__), "README.md"), String)
    s = replace(s, "```julia" => "```jldoctest")
    @doc s Convex1d
end

function middle(x,y)
    # lets not depend on Statistics for this
    mid = (1//2 * x) + (1//2*y)
    @assert x < mid <y
    mid
end

function find_initial_domain(f, (x_left, x_right)=(-1, 1))

    function grow_once(;x_from,y_from,x_to,y_to)
        if y_to > y_from
            return (x_from=x_from, y_from=y_from, x_to=x_to, y_to=y_to, converged=true)
        end
        step = x_to - x_from
        if step == 0
            msg = """
            Zero step in domain growing:
            x_from = $x_from
            y_from = $y_from
            x_to   = $x_to
            y_to   = $y_to
            step   = $step
            """
            error(msg)
        end
        x_to_new = step + x_to
        y_to_new = f(x_to_new)
        return (x_from=x_from, y_from=y_from, x_to=x_to_new, y_to=y_to_new, converged=false)
    end

    function grow(;x_from, y_from, x_to, y_to)
        item = grow_once(x_from=x_from, y_from=y_from, x_to=x_to, y_to=y_to)
        while !(item.converged)
            item = grow_once(x_from=item.x_from, y_from=item.y_from, x_to=item.x_to, y_to=item.y_to)
        end
        return item
    end

    x_left = float(x_left)
    x_right = float(x_right)
    x_center = middle(x_left, x_right)
    @assert x_left < x_center < x_right
    y_left = f(x_left)
    y_center = f(x_center)
    y_right = f(x_right)
    left = grow(x_from=x_center, y_from=y_center, x_to=x_left, y_to=y_left)
    right = grow(x_from=x_center, y_from=y_center, x_to=x_right, y_to=y_right)
    @assert left.x_to  < x_center < right.x_to
    @assert left.y_to  >= y_center
    @assert right.y_to >= y_center
    (left.x_to, right.x_to)
end

"""
    minimize(f, (x_left, x_right); atol)

Minimize a convex function `f` on the interval `[x_left, x_right]`.

# Arguments
* f: Function that should be minimized, does not have to be differentiable.
* atol: Search tolereance, such that `isapprox(true_minimizer, sol.minimizer, atol=atol)`
will hold.
"""
function minimize(f, (x_left, x_right)=find_initial_domain(f);
        atol=nothing
    )
    if !(x_left < x_right)
        msg = """
        x_left < x_right must hold. Got
        x_left  = $x_left
        x_right = $x_right
        """
        throw(ArgumentError(msg))
    end
    if atol === nothing
        atol = max(sqrt(eps(x_left)), sqrt(eps(x_right)))
    end
    # idea:
    # subdivide interval at five points
    # evaluate f at each point
    # recurse using x_left and x_right the neighbors of the minimizing point
    #
    # also we carefully reuse evals of f

    x1 = x_left
    x5 = x_right
    x3 = middle(x1, x5)
    x2 = middle(x1, x3)
    x4 = middle(x3, x5)
    xs = promote(x1,x2,x3,x4,x5)
    fs = map(f, xs)
    minimize_interval_subdivision_kernel(f, xs, fs, atol)
end

@noinline function minimize_interval_subdivision_kernel(f::F, xs, fs, atol) where {F}
    nevals = 5
    while true
        i = argmin(fs)
        at_left_bdry  = i == 1
        at_right_bdry = i == 5
        at_bdry = at_left_bdry | at_right_bdry
        if at_left_bdry
            ilo = 1; ihi = 2
        elseif at_right_bdry
            ilo = 4; ihi = 5
        else # interiour
            ilo = i-1
            ihi = i+1
        end
        @assert ilo < ihi
        x1 = xs[ilo]
        x5 = xs[ihi]
        f1 = fs[ilo]
        f5 = fs[ihi]
        if at_bdry
            x3 = middle(x1, x5)
            f3 = f(x3) ; nevals += 1
        else
            x3 = xs[i]
            f3 = fs[i]
        end

        converged = (x5 - x1) < atol
        if converged
            return minimize_interval_subdivision_finish((x1,x3,x5), (f1,f3,f5), nevals)
        end

        x2 = middle(x1, x3)
        f2 = f(x2); nevals += 1
        x4 = middle(x3, x5)
        f4 = f(x4); nevals += 1
        xs = (x1,x2,x3,x4,x5)
        fs = (f1,f2,f3,f4,f5)
    end
end

function minimize_interval_subdivision_finish(xs::NTuple{3}, fs::NTuple{3}, nevals)
    x1,x2,x3 = xs
    f1,f2,f3 = fs
    i_opt    = argmin(fs)
    # calculations for ybounds are too unstable
    # m_left   = (f2 - f1) / (x2 - x1)
    # m_right  = (f3 - f2) / (x3 - x2)
    # y_left   = f2 - m_right * (x2 - x1)
    # y_right  = f2 + m_left  * (x3 - x2)
    # @assert y_left <= f1
    # @assert y_right <= f3
    if i_opt == 1
        xbounds = (x1, x2)
        # ybounds = (y_left, f1)
    elseif i_opt == 2
        xbounds = (x1, x3)
        # ybounds = minmax(y_left, y_right)
    elseif i_opt == 3
        xbounds = (x2, x3)
        # ybounds = (y_right, f3)
    else
        error("Unreachable")
    end
    @assert issorted(xbounds)
    # @assert issorted(ybounds)
    return (
        minimizer = xs[i_opt],
        minimum   = fs[i_opt],
        nevals    = nevals,
        xbounds   = xbounds,
        # ybounds   = ybounds,
    )

end

end # module


using LinearAlgebra
import SpecialFunctions: gamma

# Fractional Brownian motion
struct FBM
  t::Vector{Float64}
  n::Int64
  h::Float64 # Hurst index

  function FBM(t::Vector{Float64}, n::Int64, h::Float64)
    t[1] == 0. || error("Starting time point must be equal to 0.0.")
    issorted(t, lt=<=) || error("The time points must be strictly sorted.")
    length(t) == n || error("Number of time points must be equal to the vector holding the time points.")
    0 < h < 1 || error("Hurst index must be between 0 and 1.")
    new(t, n, h)
  end
end

FBM(t::Vector{Float64}, h::Float64) = FBM(t, Int64(length(t)), h)
FBM(t::AbstractRange, h::Float64) = FBM(collect(t), Int64(length(t)), h)
FBM(t::Float64, n::Int64, h::Float64) = FBM(0.0:t/n:t-t/n, h)
FBM(t::Float64, h::Float64) = FBM([t], 1, h)

FBM(t::Matrix{Float64}, h::Float64) = FBM[FBM(t[:, i], h) for i = 1:size(t, 2)]
FBM(t::AbstractRange, np::Int, h::Float64) = FBM[FBM(t, h) for i = 1:np]
FBM(t::Float64, n::Int64, np::Int, h::Float64) = FBM[FBM(t, n, h) for i = 1:np]

# Fractional Gaussian noise
struct FGN
  σ::Float64
  h::Float64 # Hurst index

  function FGN(σ::Float64, h::Float64)
    σ > 0. || error("Standard deviation must be positive.")
    0. < h < 1. || error("Hurst index must be between 0 and 1.")
    new(σ, h)
  end
end

FGN(h::Float64) = FGN(1., h)

convert(::Type{FBM}, p::BrownianMotion) = FBM(p.t, p.n, 0.5)
convert(::Type{FGN}, p::BrownianMotion) = FGN((p.t[end]/(p.n-1))^0.5, 0.5)
convert(::Type{FGN}, p::FBM) = FGN((p.t[end]/(p.n-1))^p.h, p.h)

function autocov(p::FGN, i::Int64, j::Int64)
  twoh::Float64 = 2*p.h
  0.5*abs2(p.σ)*(abs(j-i+1)^twoh+abs(j-i-1)^twoh-2*abs(j-i)^twoh)
end

function autocov!(c::Matrix{Float64}, p::FGN)
  n::Int64 = size(c, 1)

  for i = 1:n
    for j = 1:i
      c[i, j] = autocov(p, i, j)
    end
  end

  for i = 1:n
    for j = (i+1):n
      c[i, j] = c[j, i]
    end
  end

  c
end

autocov(p::FGN, maxlag::Int64) = autocov!(zeros(Float64, maxlag, maxlag), p)

function autocov!(y::Vector{Float64}, p::FGN, lags::IntegerVector)
  nlags = length(lags)
  twoh::Float64 = 2*p.h

  for i = 1:nlags
    y[i] = 0.5*abs2(p.σ)*(abs(lags[i]+1)^twoh+abs(lags[i]-1)^twoh-2*abs(lags[i])^twoh)
  end

  y
end

autocov(p::FGN, lags::IntegerVector) = autocov!(zeros(Float64, length(lags)), p, lags)

function autocov(p::FBM, i::Int64, j::Int64)
  twoh::Float64 = 2*p.h
  0.5*((p.t[i])^twoh+(p.t[j])^twoh-abs(p.t[i]-p.t[j])^twoh)
end

function autocov!(c::Matrix{Float64}, p::FBM)
  n::Int64 = p.n-1

  for i = 1:n
    for j = 1:i
      c[i, j] = autocov(p, i+1, j+1)
    end
  end

  for i = 1:n
    for j = (i+1):n
      c[i, j] = c[j, i]
    end
  end

  c
end

function autocov(p::FBM)
  n::Int64 = p.n-1
  autocov!(zeros(Float64, n, n), p)
end

"""
rand_chol

generates FBM using the method based on Cholesky decomposition.
T. Dieker, Simulation of Fractional Brownian Motion, master thesis, 2004.
The complexity of the algorithm is O(n^3), where n is the number of FBM samples.
"""
function rand_chol(p::FBM, fbm::Bool=true)

  w = (cholesky(autocov(p)).U') * randn(p.n-1)

  # If fbm is true return FBM, otherwise return FGN
  insert!(w, 1, 0.0)
  if !fbm
    w = diff(w)
  end

  w
end

"""
rand_fft

generates FBM using fast Fourier transform (FFT).
The time interval of FBM is [0, 1] with a stepsize of 2^p, where p is a natural number.
The algorithm is known as the Davies-Harte method or the method of circular embedding.
R.B. Davies and D.S. Harte, Tests for Hurst Effect, Biometrika, 74 (1), 1987, pp. 95-102.
For a more recent publication, see P.F. Craigmile, Simulating a Class of Stationary Gaussian Processes Using the
Davies–Harte Algorithm, With Application to Long Memory Processes, Journal of Time Series Analysis, 24 (5), 2003,
pp. 505-511.
The complexity of the algorithm is O(n*log(n)), where n=2^p is the number of FBM samples.
"""
function rand_fft(p::FBM, fbm::Bool=true)
  # Determine number of points of simulated FBM
  pnmone = p.n-1
  n = 1 << Int(ceil(log2(pnmone)))

  # Compute autocovariance sequence of underlying FGN
  c = zeros(Float64, n+1)
  autocov!(c, FGN((1/n)^p.h, p.h), 0:n)

  # Compute square root of eigenvalues of circular autocovariance sequence
  l = real(fft(vcat(1, c, c[end-1:-1:2])))
  all(i->(i>0), l) || error("Non-positive eigenvalues encountered.")
  lsqrt = sqrt.(l)

  # Simulate standard random normal variables
  twon::Int64 = 2n
  z = randn(twon)

  # Generate fractional Gaussian noise (retain only the first p.n-1 values)
  x = sqrt(0.5) .* lsqrt[2:n] .* complex.(z[2*(2:n) .- 2], z[2*(2:n) .- 1])
  w = real(bfft(vcat(1, lsqrt[1]*z[1], x, lsqrt[n+1]*z[twon], conj(reverse(x)))))[1:pnmone]/sqrt(twon)

  # If fbm is true return FBM, otherwise return FGN
  if fbm
    w = insert!(cumsum(w), 1, 0.0)
  end

  w
end

"""
    rand_rl

Implementation of "exact discrete" method for simulating Riemann-Liouville fBm
Based on Muniandy, S. & Lim, S. Modeling of locally self-similar processes using multifractional
Brownian motion of Riemann-Liouville type. Physical Review E 63, 046104. ISSN: 1063-651X (2001).
Specifically, Eqn. 17, 18 and 19.
From Muniandy (2001), Reimann-Liouville fBm is defined as,

```math
B_H(t)=\\frac{1}{\\Gamma(H+0.5)}\\int^{t}_{0}(t-s)^(H-0.5)dB(s), t\\geq0.
```

Here we focus on the discrete time ``t_j=j\\Delta t`` approximation,

```math
B_H(t_j)=\\frac{1}{\\Gamma(H+0.5)}\\sum^{j}_{i=1}\\int^{i\\Delta t}_{(i-1)\\Delta t}(t_j-\\tau)^(H-0.5)dB(\\tau)
```

The exact solution to the interior integral results in a weighting function, implemented here with the :Exact key.
An "improved" weighting function was proposed by
Rambaldi, S. & Pinazza, O. An accurate fractional Brownian motion generator. Physica A 208, 21–30 (1994).
We have implemented this here with the :Improved key.
"""
function rand_rl(p::FBM, fbm::Bool=true; wtype::Symbol=:exact)
  # We are going to require that the time points are evenly spaced to keep things simple
  dt = unique(diff(p.t))
  if length(dt) != 1
	  error("For this simulation technique, the fBm process must be sampled across a uniform temporal grid")
  end

  dt = dt[1]

  w_mat = zeros(p.n-1,p.n-1) # Prealocate a weight matrix. Note that this will be upper triangular

  for j=1:p.n-1
    for i=1:j
      w_mat[i,j] = weight(p.h, (j-i+1)*dt, dt, wtype)
    end
  end

  # Multiply a vector of white noise by the weight matrix and scale appropriately.
	X = dropdims(sqrt(2)*(randn(p.n-1)'*w_mat),dims=1).*sqrt(dt)

	insert!(X, 1, 0.0)  # Snap to zero. Not clear if this causes a discontinuity in the correlation structure.
  #It is worth considering alternative for the above line: generate path X of length p.n, then let X = X-X[1].

  return X
end

weight(H::Float64, t::Float64, dt::Float64, method::Symbol=:exact) = weight(H, t, dt, Val{method})

function weight(H::Float64, t::Float64, dt::Float64, ::Type{Val{:exact}})
	nom = t^(H+.5)-(t-dt)^(H+.5)
	denom = gamma(H+.5)*(H+.5)*dt
	return nom/denom
end

function weight(H::Float64, t::Float64, dt::Float64, ::Type{Val{:improved}})
	nom = t^(2*H)-(t-dt)^(2*H)
	denom = 2*H*dt
	return sqrt(nom/denom)/gamma(H+.5)
end

### API functions

function rand(p::FBM; fbm::Bool=true, method::Symbol=:fft, args...)
  if method == :fft
    rand_fft(p, fbm)
  elseif method == :rl
    rand_rl(p, fbm; args...)
  elseif method == :chol
    rand_chol(p, fbm)
  else
    error("Non-existing method $method")
  end
end

function rand(p::Vector{FBM}; fbm::Bool=true, method::Symbol=:fft, args...)
  n::Int64 = fbm ? p[1].n : p[1].n-1
  np = length(p)
  x = zeros(Float64, n, np)

  for i = 1:np
    x[:, i] = rand(p[i]; fbm=fbm, method=method, args...)
  end

  x
end

function chol_update(r0::Array{Float64,2},A::Array{Float64,2})
# r0 is a lower triangular matrix found via cholesky decomposition of
# A(1:end-1,1:end-1), therefore r0 is a matrix of size (N-1)-by-(N-1) when A
# is of size N-by-N. A is a positive symetric semidefinite matrix by construction.

    m = size(A,1); # Assumes A is square
    r1 = zeros(m,m)
    r1[1:end-1,1:end-1] = r0

  @inbounds  for i=1:m-1
        r1[m,i] =  (1/r1[i,i])*(A[m,i]-sum(r1[m,1:i].*r1[i,1:i]))[1]
    end

    r1[m,m] = sqrt(A[m,m]-sum(r1[m,1:m-1] .* r1[m,1:m-1]))[1]

    return r1
end


"""
Calculate initial vortex condition

κ: vortex strength
μ: decay rate
rc: critical radius

"""
function vortex_ic!(
    prim,
    γ,
    x,
    y,
    κ = 0.25,
    μ = 0.204,
    rc = 0.05;
    x0 = 0.8,
    y0 = 0.5,
)
    T0 = 1 / prim[end]
    s = prim[1]^(1-γ) / (2 * prim[end])

    r = sqrt((x - x0)^2 + (y - y0)^2)
    η = r / rc

    δu = κ * η * exp(μ * (1-η^2)) * (y - y0) / r
    δv = -κ * η * exp(μ * (1-η^2)) * (x - x0) / r
    δT = -(γ-1)*κ^2/(8*μ*γ)*exp(2*μ*(1-η^2))

    ρ = prim[1]^(γ-1) * (T0+δT) / T0^(1/(γ-1))
    prim .= [ρ, prim[2]+δu, prim[3]+δv, 1/(1/prim[4]+δT)]

    return nothing
end


function harmonic(MX, θ0, tgα0, R, N=100)
    tgα0 = -tgα0
    ω=sqrt(1.0-3*MX*G/C/C/R)

    dudθ0 = dudθ(θ0, tgα0, R)
    u0 = 1.0/R

    ϕ = atan(-dudθ0/ω/u0)-ω*θ0
    A = u0/cos(ω*θ0+ϕ)

    ϕm=π/2-ω*ϕ*2.0

    ϕl = linspace(θ0, ϕm, N)
    u = A*cos(ω*ϕl+ϕ)
    du = -A*ω*sin(ω*ϕl+ϕ)

    ixs = find(u.>=u0)
    u=u[ixs]
    du = du[ixs]
    ϕl = ϕl[ixs]

    x = [v for v in -cos(ϕl)./u]
    y = [v for v in sin(ϕl)./u]

    out_angle =  ray_angle( ϕl[end], 1/u[end], du[end])
    in_angle =  atan(tgα0)

    x, y, u, du, -in_angle, out_angle, -out_angle-in_angle, ϕl
end

harmonic200b(z, M200b, Θ0, tgα0, N) =
    harmonic(M200b, Θ0, tgα0, R200b(z, M200b), N)
harmonic200c(z, M200c, Θ0, tgα0, N) =
    harmonic(M200c, Θ0, tgα0, R200c(z, M200c), N)
harmonicVir(z, MVir, Θ0, tgα0, N) =
    harmonic(MVir, Θ0, tgα0, RVIR(z, MVir), N)

function harmonic_e(MX, rmin, tgα, R, N=100)
    tgα =- tgα
    m0 = MX*G/C/C
    ω=sqrt(1.0-3*m0/R)

    ϕ = linspace(-π/2, π/2, N)
    α = ω.*ϕ
    cα = cos(α)
    sα = sin(α)

    u = +cα/rmin + m0/R/rmin/ω/ω * (α.*sα+cα.*log(cα))

    ixs = find(u.>((1./R)))
    u=u[ixs]
    ϕ=ϕ[ixs]

    a1 = atan(tgα)
    dx = -cos(ϕ[2]+π/2)./u[2] + cos(ϕ[1]+π/2)./u[1]
    dy =  sin(ϕ[2]+π/2)./u[2] - sin(ϕ[1]+π/2)./u[1]
    a2 = -atan2(dy,dx)

    @show dα = π/2+a1-a2
    θ = ϕ+dα
    @show dα-π/2

    x = [v for v in -cos(θ)./u]
    y = [v for v in sin(θ)./u]

    # TODO: du, outgoing_angle, deflection_angle
    in_angle = atan(tgα)
    x, y, u, θ, -in_angle
end

harmonic_e200b(z, M200b, Θ0, tgα0, N) =
    harmonic_e(M200b, Θ0, tgα0, R200b(z, M200b), N)
harmonic_e200c(z, M200c, Θ0, tgα0, N) =
    harmonic_e(M200c, Θ0, tgα0, R200c(z, M200c), N)
harmonic_eVir(z, MVir, Θ0, tgα0, N) =
    harmonic_e(MVir, Θ0, tgα0, RVIR(z, MVir), N)


export rnnOrderGraph

function rnnOrderGraph(L,T)
  orderGraph=zeros(Int64,L*T,L*T)
  for i=1:L*T
    if i%L==1
      orderGraph[i,i+1]=1
    elseif i%L!=0
      orderGraph[i,i+1]=1
      if i+L<L*T
        orderGraph[i,i+L]=1
      end
    end
  end
  return orderGraph
end


## resolve the ground truth trajectory from ordinary differential equations

# ddy = g
# dx/dt = Fx + Gu
# [ddy;   = [0  0][dy  + [1][g=-10]
#   dy ]    [1  0]  y]   [0]
function falling(du,u,p,t)
  du[1] = 0    # drag
  du[2] = -10  # gravity
  du[3] = u[1]
  du[4] = u[2]
  nothing
end

u0 = [5,15,-20.0,0.0]
tspan = (0.0,5.0)
prob = ODEProblem(falling,u0,tspan)
sol = solve(prob)
# sol(t)

Plots.plot(sol, vars=(3,4))

## extract the desired measurement information from the physical simulation

tstop = 5.0
nposes = 10
ts = range(0, stop=tstop, length=nposes)
XY = zeros(2,nposes)

for i in 1:nposes
  t = ts[i]
  XY[:,i] = sol(t)[3:4]
end

gx = (t)->sol(t)[3]
gy = (t)->sol(t)[4]

# Calculate the true radar range and angle

ranges = sqrt.(vec(sum(XY.^2, dims=1)))
angles = atan.(XY[2,:], XY[1,:])

XYdense = [gx.(0:0.01:tstop) gy.(0:0.01:tstop)]';
randense = sqrt.(vec(sum(XYdense.^2, dims=1)))
angdense = atan.(XYdense[2,:], XYdense[1,:])


# julia graphs

cd("G:\\My Drive\\18. Github Repo\\DataScience");

# load library
using LightGraphs, Plots, VegaLite, LinearAlgebra, SparseArrays, DataFrames, VegaDatasets, MatrixNetworks

# load data
airports = dataset("airports");
flightsairport = dataset("flights-airport");

# convert to airport
airportsdf = DataFrame(airports);
flightsairportdf = DataFrame(flightsairport);

# show head
first(airportsdf, 10);
first(flightsairportdf, 10);

# only include airport thats in the :origin and :destination
uairports = unique([flightsairportdf[!, :origin]; flightsairportdf[!, :destination]]);
subidx = map(x -> findall(airportsdf[!, :iata] .== uairports[x])[1], 1:length(uairports));
airportsdf_sub = airportsdf[subidx, :];
first(airportsdf_sub, 10)

# build the adjacency matrix
ei_ids = findfirst.(isequal.(flightsairportdf[!,:origin]), [uairports]);
ej_ids = findfirst.(isequal.(flightsairportdf[!,:destination]), [uairports]);
edgeweights = flightsairportdf[!,:count]

A = sparse(ei_ids,ej_ids,1,length(uairports),length(uairports))
A = max.(A,A')

# grapg
G = SimpleGraph(10) #SimpleGraph(nnodes,nedges)
add_edge!(G,7,5) #modifies graph in place.
add_edge!(G,3,5)
add_edge!(G,5,2)

cc = scomponents(A)

degrees = sum(A,dims=2)[:]
p1 = plot(sort(degrees,rev=true),ylabel="log degree",legend=false,yaxis=:log)
p2 = plot(sort(degrees,rev=true),ylabel="degree",legend=false)
plot(p1,p2,size=(600,300))

maxidx = argmax(degrees)
airportsdf_sub[maxidx, :] # ATL

# plot
us10m = dataset("us-10m")
@vlplot(width=500, height=300) +
@vlplot(
    mark={
        :geoshape,
        fill=:lightgray,
        stroke=:white
    },
    data={
        values=us10m,
        format={
            type=:topojson,
            feature=:states
        }
    },
    projection={type=:albersUsa},
) +
@vlplot(
    :circle,
    data=airportsdf_sub,
    projection={type=:albersUsa},
    longitude="longitude:q",
    latitude="latitude:q",
    size={value=10},
    color={value=:steelblue}
)+
@vlplot(
    :rule,
    data=flightsairport,
    transform=[
        {filter={field=:origin,equal=:ATL}},
        {
            lookup=:origin,
            from={
                data=airportsdf_sub,
                key=:iata,
                fields=["latitude", "longitude"]
            },
            as=["origin_latitude", "origin_longitude"]
        },
        {
            lookup=:destination,
            from={
                data=airportsdf_sub,
                key=:iata,
                fields=["latitude", "longitude"]
            },
            as=["dest_latitude", "dest_longitude"]
        }
    ],
    projection={type=:albersUsa},
    longitude="origin_longitude:q",
    latitude="origin_latitude:q",
    longitude2="dest_longitude:q",
    latitude2="dest_latitude:q"
)

# shortest path
ATL_paths = dijkstra(A, maxidx);
ATL_paths[1][maxidx]
maximum(ATL_paths[1])
@show stop1 = argmax(ATL_paths[1])
@show uairports[stop1]

airportsdf_sub[airportsdf_sub[!, :iata] .== "GST", :];

@show stop2 = ATL_paths[2][stop1]
@show uairports[stop2]
airportsdf_sub[airportsdf_sub[!, :iata] .== "JNU", :]

@show stop3 = ATL_paths[2][stop2]
@show uairports[stop3]
airportsdf_sub[airportsdf_sub[!, :iata] .== "SEA", :]

@show stop4 = ATL_paths[2][stop3]
@show uairports[stop4]
airportsdf_sub[airportsdf_sub[!, :iata] .== "ATL", :]

us10m = dataset("us-10m")
airports = dataset("airports")

@vlplot(width=800, height=500) +
@vlplot(
    mark={
        :geoshape,
        fill="#eee",
        stroke=:white
    },
    data={
        values=us10m,
        format={
            type=:topojson,
            feature=:states
        }
    },
    projection={type=:albersUsa},
) +
@vlplot(
    :circle,
    data=airportsdf_sub,
    projection={type=:albersUsa},
    longitude="longitude:q",
    latitude="latitude:q",
    size={value=5},
    color={value=:gray}
) +
@vlplot(
    :line,
    data={
        values=[
            {airport=:ATL,order=1},
            {airport=:SEA,order=2},
            {airport=:JNU,order=3},
            {airport=:GST,order=4}
        ]
    },
    transform=[{
        lookup=:airport,
        from={
            data=airports,
            key=:iata,
            fields=["latitude","longitude"]
        }
    }],
    projection={type=:albersUsa},
    longitude="longitude:q",
    latitude="latitude:q",
    order={field=:order,type=:ordinal}
)

# Minimum Spanning Tree (MST)
@time ti,tj,tv,nverts = mst_prim(A);

df_edges = DataFrame(:ei=>uairports[ti],:ej=>uairports[tj])

@vlplot(width=800, height=500) +
@vlplot(
    mark={
        :geoshape,
        fill="#eee",
        stroke=:white
    },
    data={
        values=us10m,
        format={
            type=:topojson,
            feature=:states
        }
    },
    projection={type=:albersUsa},
) +
@vlplot(
    :circle,
    data=airportsdf_sub,
    projection={type=:albersUsa},
    longitude="longitude:q",
    latitude="latitude:q",
    size={value=20},
    color={value=:gray}
) +
@vlplot(
    :rule,
    data=df_edges, #data=flightsairport,
    transform=[
        {
            lookup=:ei,
            from={
                data=airportsdf_sub,
                key=:iata,
                fields=["latitude", "longitude"]
            },
            as=["originx", "originy"]
        },
        {
            lookup=:ej,
            from={
                data=airportsdf_sub,
                key=:iata,
                fields=["latitude", "longitude"]
            },
            as=["destx", "desty"]
        }
    ],
    projection={type=:albersUsa},
    longitude="originy:q",
    latitude="originx:q",
    longitude2="desty:q",
    latitude2="destx:q"
)

# PageRank
v = MatrixNetworks.pagerank(A,0.85);
insertcols!(airportsdf_sub,7,:pagerank_value => v)

@vlplot(width=500, height=300) +
@vlplot(
    mark={
        :geoshape,
        fill="#eee",
        stroke=:white
    },
    data={
        values=us10m,
        format={
            type=:topojson,
            feature=:states
        }
    },
    projection={type=:albersUsa},
) +
@vlplot(
    :circle,
    data=airportsdf_sub,
    projection={type=:albersUsa},
    longitude="longitude:q",
    latitude="latitude:q",
    size="pagerank_value:q",
    color={value=:steelblue}
)


@testset "soc problems" begin
    @testset "soc1_test" begin
    #=
   primal
       max 0x + 1y + 1z
   s.t.
       x == 1          :w_4
       x >= ||(y,z)||  :w_1, w_2, w_3

   dual
       min -w_4
   s.t.
       w_2 == -1
       w_3 == -1
       w_1 + w_4 == 0
       w_1 >= ||(w_2, w_3)||
    =#
       primal_model = soc1_test()
       dual_model, primal_dual_map = dual_model_and_map(primal_model)

       @test MOI.get(dual_model, MOI.NumberOfVariables()) == 4
       list_of_cons =  MOI.get(dual_model, MOI.ListOfConstraints())
       @test Set(list_of_cons) == Set([
           (SAF{Float64}, MOI.EqualTo{Float64})
           (VVF, MOI.SecondOrderCone)
       ])
       @test MOI.get(dual_model, MOI.NumberOfConstraints{VVF, MOI.SecondOrderCone}()) == 1
       @test MOI.get(dual_model, MOI.NumberOfConstraints{SAF{Float64}, MOI.EqualTo{Float64}}()) == 3
       obj_type = MOI.get(dual_model, MOI.ObjectiveFunctionType())
       @test obj_type == SAF{Float64}
       obj = MOI.get(dual_model, MOI.ObjectiveFunction{obj_type}())
       @test MOI.constant(obj) == 0.0
       @test MOI.coefficient.(obj.terms) == [-1.0]

       eq_con1_fun = MOI.get(dual_model, MOI.ConstraintFunction(), CI{SAF{Float64}, MOI.EqualTo{Float64}}(2))
       eq_con1_set = MOI.get(dual_model, MOI.ConstraintSet(), CI{SAF{Float64}, MOI.EqualTo{Float64}}(2))
       @test MOI.coefficient.(eq_con1_fun.terms) == [1.0; 1.0]
       @test MOI.constant.(eq_con1_fun) == 0.0
       @test MOI.constant(eq_con1_set) == 0.0
       eq_con2_fun = MOI.get(dual_model, MOI.ConstraintFunction(), CI{SAF{Float64}, MOI.EqualTo{Float64}}(3))
       eq_con2_set = MOI.get(dual_model, MOI.ConstraintSet(), CI{SAF{Float64}, MOI.EqualTo{Float64}}(3))
       @test MOI.coefficient.(eq_con2_fun.terms) == [1.0]
       @test MOI.constant.(eq_con2_fun) == 0.0
       @test MOI.constant(eq_con2_set) == -1.0
       eq_con3_fun = MOI.get(dual_model, MOI.ConstraintFunction(), CI{SAF{Float64}, MOI.EqualTo{Float64}}(4))
       eq_con3_set = MOI.get(dual_model, MOI.ConstraintSet(), CI{SAF{Float64}, MOI.EqualTo{Float64}}(4))
       @test MOI.coefficient.(eq_con3_fun.terms) == [1.0]
       @test MOI.constant.(eq_con3_fun) == 0.0
       @test MOI.constant(eq_con3_set) == -1.0

       soc_con = MOI.get(dual_model, MOI.ConstraintFunction(), CI{VVF, MOI.SecondOrderCone}(1))
       @test soc_con.variables == VI.(1:3)

       primal_con_dual_var = primal_dual_map.primal_con_dual_var
       @test primal_con_dual_var[CI{VAF{Float64}, MOI.Zeros}(1)] == [VI(4)]
       @test primal_con_dual_var[CI{VVF, MOI.SecondOrderCone}(2)] == [VI(1); VI(2); VI(3)]

       primal_var_dual_con = primal_dual_map.primal_var_dual_con
       @test primal_var_dual_con[VI(1)] == CI{SAF{Float64}, MOI.EqualTo{Float64}}(2)
       @test primal_var_dual_con[VI(2)] == CI{SAF{Float64}, MOI.EqualTo{Float64}}(3)
       @test primal_var_dual_con[VI(3)] == CI{SAF{Float64}, MOI.EqualTo{Float64}}(4)
   end

   @testset "soc2_test" begin
    #=
   primal
       max 0x + 1y + 1z
   s.t.
       x == 1          :w_4
       x >= ||(y,z)||  :w_1, w_2, w_3

   dual
       min -w_4
   s.t.
       w_2 == -1
       w_3 == -1
       w_1 + w_4 == 0
       w_1 >= ||(w_2, w_3)||
   =#
       primal_model = soc2_test()
       dual_model, primal_dual_map = dual_model_and_map(primal_model)

       @test MOI.get(dual_model, MOI.NumberOfVariables()) == 4
       list_of_cons =  MOI.get(dual_model, MOI.ListOfConstraints())
       @test Set(list_of_cons) == Set([
           (SAF{Float64}, MOI.EqualTo{Float64})
           (VVF, MOI.SecondOrderCone)
       ])
       @test MOI.get(dual_model, MOI.NumberOfConstraints{VVF, MOI.SecondOrderCone}()) == 1
       @test MOI.get(dual_model, MOI.NumberOfConstraints{SAF{Float64}, MOI.EqualTo{Float64}}()) == 3
       obj_type = MOI.get(dual_model, MOI.ObjectiveFunctionType())
       @test obj_type == SAF{Float64}
       obj = MOI.get(dual_model, MOI.ObjectiveFunction{obj_type}())
       @test MOI.constant(obj) == 0.0
       @test MOI.coefficient.(obj.terms) == [-1.0]

       eq_con1_fun = MOI.get(dual_model, MOI.ConstraintFunction(), CI{SAF{Float64}, MOI.EqualTo{Float64}}(2))
       eq_con1_set = MOI.get(dual_model, MOI.ConstraintSet(), CI{SAF{Float64}, MOI.EqualTo{Float64}}(2))
       @test MOI.coefficient.(eq_con1_fun.terms) == [1.0; 1.0]
       @test MOI.constant.(eq_con1_fun) == 0.0
       @test MOI.constant(eq_con1_set) == 0.0
       eq_con2_fun = MOI.get(dual_model, MOI.ConstraintFunction(), CI{SAF{Float64}, MOI.EqualTo{Float64}}(3))
       eq_con2_set = MOI.get(dual_model, MOI.ConstraintSet(), CI{SAF{Float64}, MOI.EqualTo{Float64}}(3))
       @test MOI.coefficient.(eq_con2_fun.terms) == [1.0]
       @test MOI.constant.(eq_con2_fun) == 0.0
       @test MOI.constant(eq_con2_set) == -1.0
       eq_con3_fun = MOI.get(dual_model, MOI.ConstraintFunction(), CI{SAF{Float64}, MOI.EqualTo{Float64}}(4))
       eq_con3_set = MOI.get(dual_model, MOI.ConstraintSet(), CI{SAF{Float64}, MOI.EqualTo{Float64}}(4))
       @test MOI.coefficient.(eq_con3_fun.terms) == [1.0]
       @test MOI.constant.(eq_con3_fun) == 0.0
       @test MOI.constant(eq_con3_set) == -1.0

       soc_con = MOI.get(dual_model, MOI.ConstraintFunction(), CI{VVF, MOI.SecondOrderCone}(1))
       @test soc_con.variables == VI.(2:4)

       primal_con_dual_var = primal_dual_map.primal_con_dual_var
       @test primal_con_dual_var[CI{VAF{Float64}, MOI.Zeros}(1)] == [VI(1)]
       @test primal_con_dual_var[CI{VAF{Float64}, MOI.SecondOrderCone}(2)] == [VI(2); VI(3); VI(4)]

       primal_var_dual_con = primal_dual_map.primal_var_dual_con
       @test primal_var_dual_con[VI(1)] == CI{SAF{Float64}, MOI.EqualTo{Float64}}(2)
       @test primal_var_dual_con[VI(2)] == CI{SAF{Float64}, MOI.EqualTo{Float64}}(3)
       @test primal_var_dual_con[VI(3)] == CI{SAF{Float64}, MOI.EqualTo{Float64}}(4)
   end
end


# Enquire about properties of tensors

"""
    is_iso(C) -> ::Bool

Return `true` if `C` is isotropic.

# Examples
```
julia> is_iso(CIJ.iso(vp=8000, vs=4400))
true

julia> is_iso(CIJ.thom(8000, 4400, 0.01, 0.01, 0.01))
false
```
"""
function is_iso(C; atol=eps(eltype(C)))
    all(x->isapprox(C[1,1], x, atol=atol), (C[2,2], C[3,3])) &&
    all(x->isapprox(C[4,4], x, atol=atol), (C[5,5], C[6,6])) &&
    all(x->isapprox(C[1,1]-2C[4,4], x, atol=atol), (C[1,2], C[1,3], C[2,3])) &&
    all(x->isapprox(0, x, atol=atol), C[i,j] for i in 1:3 for j in 4:6) &&
    all(x->isapprox(0, x, atol=atol), (C[4,5], C[4,6], C[5,6]))
end

"""
    is_stable(C) -> ::Bool

Return `false` if the input 6x6 matrix `C` is not positive definite (symmetric)
and hence not dynamically stable, and `true` otherwise.

# Examples
```
julia> c = CIJ.ol()[1]
6×6 EC{Float64}:
 9.55291e7  2.02981e7  2.13413e7  0.0       0.0        0.0
 2.02981e7  5.85693e7  2.28912e7  0.0       0.0        0.0
 2.13413e7  2.28912e7  6.95976e7  0.0       0.0        0.0
 0.0        0.0        0.0        1.9076e7  0.0        0.0
 0.0        0.0        0.0        0.0       2.29508e7  0.0
 0.0        0.0        0.0        0.0       0.0        2.34575e7

julia> is_stable(c)
true

julia> is_stable(EC(rand(6, 6)))
false
```
"""
is_stable(C) = _isposdef(C)

# It's faster to convert to an Array and then use `isposdef`,
# than it is to catch a PosDefException.
# TODO: https://github.com/JuliaArrays/StaticArrays.jl/pull/886
#       added `check` option to `cholesky` for `StaticArray`s, so when
#       it is safe to require version 1.1.0 (released 28 March 2021), update
#       this method to call `LinearAlgebra.isposdef(c)` instead.
_isposdef(c::StaticArray{Tuple{6,6}}) = LinearAlgebra.isposdef(Array(c))
_isposdef(c::EC) = _isposdef(c.data)

function _isposdef(a::AbstractMatrix)
    is_6x6(a) || throw(ArgumentError("matrix must be 6×6"))
    is_symm(a) || @warn("matrix not symmetrical: using upper half only")
    LinearAlgebra.isposdef(LinearAlgebra.Hermitian(a))
end

"""
    is_symm(C) -> ::Bool

Return `true` if `C` is symmetrical.
"""
function is_symm(C)
    for i = 1:6, j = i+1:6
        if !isapprox(C[i,j], C[j,i]) return false end
    end
    return true
end
is_symm(::EC) = true

"""
    is_6x6(C) -> ::Bool

Return `true` if C is a 6x6 matrix.
"""
is_6x6(C) = size(C) == (6,6)
is_6x6(::EC) = true


using HypothesisTests
using StatsBase
using Base.Test

#Example 1 in R
#Agresti (2007) p. 39

d = [[762,484] [327,239] [468,477]]

m = PowerDivergenceTest(d)

@test_approx_eq m.theta0 [0.25523082406125785,0.19670969099133556,0.11593952361049113,0.08935608756107216,0.1935739395970214,0.1491899341788219]
@test_approx_eq m.thetahat [0.2763873775843308,0.1755531374682626,0.11860718171926006,0.08668842945230323,0.16974972796517954,0.17301414581066377]

c = ci(m)
c0 = [(0.23322451940515054, 0.31882480957222203),
      (0.1323902792890823, 0.21799056945615383),
      (0.0754443235400798, 0.16104461370715129),
      (0.04352557127312297, 0.12912586144019447),
      (0.12658686978599926, 0.21218715995307078),
      (0.12985128763148351, 0.21545157779855498)]

for i = 1:length(c)
    @test c[i][1] ≈ c0[i][1]
    @test c[i][2] ≈ c0[i][2]
end

@test pvalue(m) ≈ 2.9535891832117357e-7
@test m.stat ≈ 30.070149095754687
@test m.df ≈ 2
@test m.n ≈ 2757
@test_approx_eq m.residuals [2.198855766015898,-2.504669492560728,0.4113701700566286,-0.46858294710127296,-2.8432397155451494,3.2386734435365825]
@test_approx_eq m.stdresiduals [4.502053521086705,-4.502053521086705,0.6994517329844298,-0.6994517329844298,-5.315945542704929,5.315945542704929]

m = PowerDivergenceTest(d,lambda=0.0)
testname(m)
pvalue(m)
show(IOBuffer(), m)

m = PowerDivergenceTest(d,lambda=-1.0)
testname(m)
pvalue(m)
show(IOBuffer(), m)

m = PowerDivergenceTest(d,lambda=-2.0)
testname(m)
pvalue(m)
show(IOBuffer(), m)

m = PowerDivergenceTest(d,lambda=-0.5)
testname(m)
pvalue(m)
show(IOBuffer(), m)

m = PowerDivergenceTest(d,lambda=2/3)
testname(m)
pvalue(m)
show(IOBuffer(), m)

m = ChisqTest(d)
m = MultinomialLRT(d)

ci(m, method = :bootstrap)
ci(m, method = :bootstrap, tail=:left)
ci(m, method = :bootstrap, tail=:right)

ci(m, method = :gold)
ci(m, method = :gold, tail=:left)
ci(m, method = :gold, tail=:right)

ci(m, method = :quesenberry_hurst)
ci(m, method = :quesenberry_hurst, tail=:left)
ci(m, method = :quesenberry_hurst, tail=:right)

ci(m, method = :sison_glaz)
ci(m, method = :sison_glaz, correct=false)
ci(m, method = :sison_glaz, tail=:left)
ci(m, method = :sison_glaz, tail=:right)

@test_throws ArgumentError ci(m, method=:FOO)
@test_throws ArgumentError ci(m, tail=:fox)

#Example 3 in R

d = [ 20, 15, 25 ]
m = PowerDivergenceTest(d)

@test_approx_eq m.theta0 [0.3333333333333333,0.3333333333333333,0.3333333333333333]
@test_approx_eq m.thetahat [0.3333333333333333,0.25,0.4166666666666667]

c = ci(m)
c0 = [(0.04999999999999999,0.5833301356192295),(0.0,0.49999680228589616),(0.13333333333333336,0.6666634689525628)]

[ @test_approx_eq c[i][1] c0[i][1] for i in 1:length(c)]
[ @test_approx_eq c[i][2] c0[i][2] for i in 1:length(c)]

@test_approx_eq pvalue(m) 0.2865047968601901
@test_approx_eq m.stat 2.5
@test_approx_eq m.df 2
@test_approx_eq m.n 60
@test_approx_eq m.residuals [0.0,-1.118033988749895,1.118033988749895]
@test_approx_eq m.stdresiduals [0.0,-1.3693063937629153,1.3693063937629153]

m = PowerDivergenceTest(d,lambda=0.0)
testname(m)
pvalue(m)
show(IOBuffer(), m)

m = PowerDivergenceTest(d,lambda=-1.0)
testname(m)
pvalue(m)
show(IOBuffer(), m)

m = PowerDivergenceTest(d,lambda=-2.0)
testname(m)
pvalue(m)
show(IOBuffer(), m)

m = PowerDivergenceTest(d,lambda=-0.5)
testname(m)
pvalue(m)
show(IOBuffer(), m)

m = PowerDivergenceTest(d,lambda=2/3)
testname(m)
pvalue(m)
show(IOBuffer(), m)

m = ChisqTest(d)
m = MultinomialLRT(d)

ci(m, method = :bootstrap)
ci(m, method = :bootstrap, tail=:left)
ci(m, method = :bootstrap, tail=:right)

ci(m, method = :gold)
ci(m, method = :gold, tail=:left)
ci(m, method = :gold, tail=:right)

ci(m, method = :quesenberry_hurst)
ci(m, method = :quesenberry_hurst, tail=:left)
ci(m, method = :quesenberry_hurst, tail=:right)

ci(m, method = :sison_glaz)
ci(m, method = :sison_glaz, correct=false)
ci(m, method = :sison_glaz, tail=:left)
ci(m, method = :sison_glaz, tail=:right)

@test_throws ArgumentError ci(m, method=:FOO)
@test_throws ArgumentError ci(m, tail=:fox)

#
x=[1,2,3,1,2,3]
y=[1,1,1,2,2,3]

d = counts(x,y,3)

ChisqTest(d)
MultinomialLRT(d)

PowerDivergenceTest(x,y,3)
PowerDivergenceTest(x,y,(1:3,1:3))

ChisqTest(x,y,3)
ChisqTest(x,y,(1:3,1:3))

MultinomialLRT(x,y,3)
MultinomialLRT(x,y,(1:3,1:3))


# ------------------------------------------------------------------- #
# Generic methods
# ------------------------------------------------------------------- #

# Base.size(::AbstractExogenousProcess{N}) = N

# AbstractExogenousProcess Interface
# for f in [:step]
#     @eval $(f){T<:AbstractExogenousProcess}(::T, args...) =
#         throw(NotImplementedError("$f not implemented for type $T"))
# end

# simulate a univariate exog process and return a Vector
function simulate{T<:Number}(ep::AbstractExogenousProcess{1}, x0::T,
                             capT::Int=25_000)
    out = Array(T, capT)
    out[1] = x0
    @inbounds for t=2:capT
        out[t] = step(ep, out[t-1])
    end
    return out
end

function simulate{T<:Number,N}(ep::AbstractExogenousProcess{N},
                               x0::Vector{T}=ones(N);
                               capT::Int=25_000)
    out = Array(T, N, capT)
    out[:, 1] = convert(Vector{T}, x0)
    @inbounds for t=2:capT
        out[:, t] = step(ep, out[:, t-1])
    end

    return out
end

# ------------------------------------------------------------------- #
# AR1ConstantVolatility
# ------------------------------------------------------------------- #

# special case the 1d version
immutable ConstantVolatility1 <: AbstractExogenousProcess{1}
    A::Float64
    Bv::Float64
    ϵ::Array{Float64, 1}
    Π::Array{Float64, 1}
    Πcumsum::Array{Float64, 1}
end

function ConstantVolatility1(A::Float64=0., B::Float64=1., vbar::Float64=0.004,
                             nϵ::Int=5)

    Bv = B * sqrt(vbar)
    ϵ, Π = qnwnorm(nϵ, 0., 1.)
    Πcumsum = cumsum(Π)

    ConstantVolatility1(A, Bv, ϵ, Π, Πcumsum)
end

Base.step(ep::ConstantVolatility1, x::Float64, ϵ) = ep.A*x .+ ep.Bv*ϵ
Base.step(ep::ConstantVolatility1, x::Float64) = step(ep, x, randn())

# use matrix form to represent general version
immutable ConstantVolatility{N} <: AbstractExogenousProcess{N}
    A::Matrix{Float64}
    B::Matrix{Float64}
    ϵ::Matrix{Float64}
    Π::Vector{Float64}
    Πcumsum::Vector{Float64}
end

# standard 2d case where we can scale by z1 to reduce dimensionality
function ConstantVolatility(γ::Real, v1::Real, v2::Real, nϵ1::Integer,
                            nϵ2::Integer)

    A = [γ 1-γ; 1-γ γ]
    B = [v1 0; 0 v2]
    ϵ, Π = qnwnorm([nϵ1, nϵ2], zeros(2), eye(2))
    ConstantVolatility{2}(A, B, ϵ, Π, cumsum(Π))
end

Base.step{N}(ex::ConstantVolatility{N}, x, ϵ=randn(N)) = ex.A*x .+ ex.B*ϵ

immutable StochasticVolatility{N} <: AbstractExogenousProcess{N}
    A::Float64
    B::Float64
    τ::Float64
    φv::Float64
    vbar::Float64
    ϵ::Array{Float64, 2}
    Π::Array{Float64, 1}
    Πcumsum::Array{Float64, 1}
end

function StochasticVolatility(A::Float64=0., B::Float64=1., τ::Float64=7.4e-6,
                              φv::Float64=.95, vbar::Float64=.004,
                              nϵ1::Int=5, nϵ2::Int=5)

    ϵ, Π = qnwnorm([nϵ1, nϵ2], zeros(2), eye(2))
    Πcumsum = cumsum(Π)

    StochasticVolatility{2}(A, B, τ, φ, vbar, ϵ, Π, Πcumsum, 2)
end

function Base.step(ep::StochasticVolatility{2}, xt::Float64, vt::Float64, ϵ1, ϵ2)

    xtp1 = ep.A*xt + ep.B*sqrt(vt)*ϵ1
    vtp1 = (1 - ep.φv)*ep.vbar + ep.φv*vt + ep.τ*ϵ2

    return (xtp1, vtp1)
end

Base.step(ep::StochasticVolatility{2}, xt::Float64, vt::Float64) =
    Base.step(ep, xt, vt, randn(2)...)


using StatsBase

xs = sort(rand(1000))
ys = diff(rand(1001));
T = eltype(ys)

g = -0.3:0.025:1.2
nbins = length(g) - 1

ybinned = bin(median, g, xs, ys)
@test length(ybinned) == nbins
@test ybinned isa Vector{T}

ybinned = bin(quantile, g, xs, ys, [0.5])
@test length(ybinned) == nbins
@test ybinned isa Vector{T}

ybinned = bin_mean(g, xs, ys)
@test ybinned isa Vector
@test length(ybinned) == nbins


@testset "MOI interface" begin
    @testset "basic.fzn" begin
        model = optimizer(Int)

        @test MOI.supports_add_constrained_variable(model, MOI.Integer)
        @test MOI.supports_constraint(CP.FlatZinc.Model(), MOI.ScalarAffineFunction{Int}, MOI.LessThan{Int})

        # x ∈ {1, 2, 3}
        x, x_int = MOI.add_constrained_variable(model, MOI.Integer())
        c1 = MOI.add_constraint(model, -1 * x, MOI.LessThan(-1))
        c2 = MOI.add_constraint(model, 1 * x, MOI.LessThan(3))

        @test MOI.is_valid(model, x)
        @test MOI.is_valid(model, x_int)
        @test MOI.is_valid(model, c1)
        @test MOI.is_valid(model, c2)

        MOI.optimize!(model)

        @test MOI.get(model, MOI.TerminationStatus()) === MOI.OPTIMAL
        @test MOI.get(model, MOI.ResultCount()) ≥ 1
        @test MOI.get(model, MOI.VariablePrimal(), x) ∈ Set([1, 2, 3])
        @test MOI.get(model, MOI.VariablePrimal(1), x) ∈ Set([1, 2, 3])
    end

    @testset "Infeasible" begin
        model = optimizer(Int)

        @test MOI.supports_add_constrained_variable(model, MOI.Integer)
        @test MOI.supports_constraint(CP.FlatZinc.Model(), MOI.ScalarAffineFunction{Int}, MOI.LessThan{Int})

        # x ∈ ∅
        x, x_int = MOI.add_constrained_variable(model, MOI.Integer())
        c1 = MOI.add_constraint(model, -1 * x, MOI.LessThan(-5))
        c2 = MOI.add_constraint(model, 1 * x, MOI.LessThan(3))

        @test MOI.is_valid(model, x)
        @test MOI.is_valid(model, x_int)
        @test MOI.is_valid(model, c1)
        @test MOI.is_valid(model, c2)

        MOI.optimize!(model)

        @test MOI.get(model, MOI.TerminationStatus()) === MOI.INFEASIBLE
        @test MOI.get(model, MOI.ResultCount()) == 0
        # TODO: https://github.com/jump-dev/MathOptInterface.jl/issues/1600
        # @test MOI.get(model, MOI.VariablePrimal(), x) ∈ Set([1, 2, 3])
        # @test MOI.get(model, MOI.VariablePrimal(1), x) ∈ Set([1, 2, 3])
    end
end