abstract type priorType end

"""
Represents a uniform prior on range = [a, b]
"""
struct Uniform <:priorType
    range::Array{Real, 2} #admissible ranges for each length scale ([lowerbound_i, upperbound_i]), stacked vertically
    d::Int64 #number of ranges in range object/number length scales
    lengths::Array{Real, 1}
    function Uniform(range) #inner constructor allows one to enforce constraints
        @assert typeof(range)<:Union{Array{T, 2}, Array{T, 1}} where T<:Real
        if length(range) > 1
            @assert prod(range[:, 2]- range[:, 1] .> 0) #ranges must be valid
            lengths = range[:, 2] - range[:, 1]
            if typeof(range)<:Array{Real, 1}
                range = reshape(range, 1, 2)
                return new(range, 1, lengths)
            else
                @assert Base.size(range, 2) == 2
                return new(range, Base.size(range, 1), lengths)
            end
        else
            return new(range, Base.size(range, 1), ones(size(range, 1)))
        end
    end
end

getRange(u::Uniform) = u.range #internal range array
getNum(u::Uniform) = u.d #number of dimensions of interal range array
getLengths(u::Uniform) = u.lengths
logProb(u::Uniform, x) = (@assert length(x) == u.d; l = getLengths(u); -sum(log.(l)))#density function
prob(u::Uniform, x) = (@assert length(x) == u.d; l = getLengths(u); 1/prod(l))#density function
partialx(u::Uniform, x) = 0

"""
Represents a prior on [1/b, 1/a] that induces a uniform distribution on range = [a, b]
"""
struct inverseUniform<:priorType
    range::Array
    d::Int64 #number of ranges in range object/number length scales
    lengths::Array{Real, 1}
    function inverseUniform(range)
        if length(range) > 1
            @assert prod(range[:, 2]- range[:, 1] .> 0) #ranges must be valid
            lengths = range[:, 2] - range[:, 1]
            if typeof(range)<:Array{Real, 1}
                range = reshape(range, 1, 2)
                return new(range, 1, lengths)
            else
                @assert Base.size(range, 2) == 2
                return new(range, Base.size(range, 1), lengths)
            end
        else
            return new(range, Base.size(range, 1), ones(size(range, 1)))
        end
    end
end
getLengths(u::inverseUniform) = u.lengths
prob(u::inverseUniform, x) = (@assert length(x) == u.d; l = getLengths(u); prod([1/x[i]^2 * 1 / (l[i]) for i=1:u.d]))
logProb(u::inverseUniform, x) = (@assert length(x) == u.d; l = getLengths(u); sum([-2*log(x[i]) - log(l[i]) for i=1:u.d])  )#density function

#partialx(u::inverseUniform) = -2/x^3 * 1/(u.range[2]-u.range[1])

#struct univariateExponential <:priorType
#    range::Array
#    stdev::Float64
#    mean::Float64
#    function univariateExponential(range)
#
#    end
#end


@testset "Result" begin
    @testset "Result+PathResult" begin
        R = solve(
            equations(heart()),
            save_all_paths = true,
            seed = 13186,
            show_progress = false,
        )
        @test R isa Result
        @test nat_infinity(R) ≤ 572
        @test nfinite(R) == 4
        @test length(collect(R)) == 576
        @test finite(R) isa Vector{<:PathResult}
        @test seed(R) == 13186

        @test length(finite(R, only_nonsingular = false)) == 4
        @test length(finite(R, only_nonsingular = true)) == 4
        @test length(finite(R, only_singular = true)) == 0
        @test 572 - length(failed(R)) == nat_infinity(R)
        @test length(real(R, tol = 1e-6)) == 2
        @test nreal(R, tol = 1e-6) == 2
        @test length(real_solutions(R)) == 2
        @test_deprecated realsolutions(R)
        @test length(at_infinity(R)) ≤ 572
        @test length(results(R, only_real = true, real_tol = 1e-8)) == 2
        @test length(results(R, only_nonsingular = true, singular_tol = 1e9)) == 4
        @test length(finite(results(R, only_real = true))) == 2
        @test nresults(R, only_nonsingular = true, singular_tol = 1e9) == 4
        @test length(results(R, onlyfinite = false)) == 576
        @test nresults(R, onlyfinite = false) == 576
        @test nnonsingular(R) == 4
        @test length(nonsingular(R)) == 4
        @test length(singular(R)) == 0
        @test length(singular(real(R))) == 0

        @test_nowarn mapresults(solution, R)
        @test_nowarn mapresults(start_solution, R)
        @test_nowarn mapresults(residual, R)
        @test_nowarn mapresults(is_singular, R)
        @test count(mapresults(is_real, R)) == 2
        # test fallback
        @test count(results(is_real, R)) == 2

        @test length(solutions(R)) == 4
        @test solutions(R) isa Vector{Vector{ComplexF64}}
        @test real_solutions(R, real_tol = 1e-8) isa Vector{Vector{Float64}}
        @test length(real_solutions(R, real_tol = 1e-8)) == 2

        @test_nowarn string(R)
        @test_nowarn string(R[end])
        @test_nowarn show(IOContext(devnull, :compact => true), R)
        @test_nowarn show(IOContext(devnull, :compact => true), R[end])

        test_treeviews(R)

        @polyvar x y z
        R = solve([(x - 3)^3, (y - 2)], affine_tracking = true)
        @test R isa Result
        test_treeviews(R)
        @test nnonsingular(R) == 0
        @test nsingular(R) == 1
        @test nsingular(R, counting_multiplicities = true) == 3
        @test_nowarn sprint(show, R)

        R = solve([(x - 3)^3, (y - 2)], affine_tracking = true, system_scaling = nothing)
        @test nnonsingular(R) == 0
        @test nsingular(R) == 1
        @test nsingular(R, counting_multiplicities = true) == 3
        @test_nowarn sprint(show, R)

        @polyvar x y z
        R = solve([(x - 3z), (y - 2z)])
        @test R isa Result{<:ProjectiveVectors.PVector}
        @test_nowarn string(R)
        test_treeviews(R)
        @test nnonsingular(R) == 1

        @polyvar x y z
        R = solve([(x - 3z)^3, (y - 2z)])
        @test R isa Result{<:ProjectiveVectors.PVector}
        test_treeviews(R)
        @test nnonsingular(R) == 0
        @test nsingular(R) == 1
        @test nsingular(R, counting_multiplicities = true) == 3

        @polyvar x y z
        V = x^2 + y^2 - z^2
        L = randn(1, 3) * [x, y, z]
        R = solve([V; L])
        @test_nowarn sprint(show, R)
    end

    @testset "singular result" begin
        @polyvar x
        f = (x - 3)^3 * (x - 2)
        R = solve([f])
        @test length(solutions(R)) == 2
        @test sort(results(multiplicity, R)) == [1, 3]

        @polyvar x y
        f = (x - 3y)^3 * (x - 2y)
        R = solve([f])
        @test nsolutions(R) == 2
        @test sort(results(multiplicity, R)) == [1, 3]
    end

    @testset "multiplicities" begin
        @polyvar x y z
        f = (x - 1 * y) * (x - 1.01 * y)
        g = x - y - z
        S = solve([f, g], threading = false)
        @test length(multiplicities(S, tol = 1e-1)) == 1
        @test length(multiplicities(S, tol = 1e-5)) == 0
        @test length(multiplicities(S.pathresults, tol = 1e-1)) == 1
        @test length(multiplicities(S.pathresults, tol = 1e-5)) == 0

        @polyvar x z
        f = (x - 1) * (x - 1.01)
        g = x - 1 - z
        S = solve([f, g], threading = false)
        @test length(multiplicities(S, tol = 1e-1)) == 1
        @test length(multiplicities(S, tol = 1e-5)) == 0
        @test length(multiplicities(S.pathresults, tol = 1e-1)) == 1
        @test length(multiplicities(S.pathresults, tol = 1e-5)) == 0
        @test norm(solution(S[1]) - solution(S[2])) < 1e-1
    end

    @testset "real projective solutions" begin
        @polyvar x y v w
        res = solve([x * y - 6 * v * w, x^2 - 5 * v^2], variable_groups = [(x, v), (y, w)])
        @test all(is_real, res)
        @test sum(fubini_study.(real_solutions(res), solutions(res))) ≈ 0.0 atol = 1e-6
    end
end


export ItemKNN

"""
    ItemKNN(
        data::DataAccessor,
        k::Int
    )

[Item-based CF](https://dl.acm.org/citation.cfm?id=963776) that provides a way to model item-item concepts by utilizing the similarities of items in the CF paradigm. `k` represents number of neighbors.

Item properties are relatively stable compared to the users' tastes, and the number of items is generally smaller than the number of users. Hence, while user-based CF successfully captures the similarities of users' complex tastes, modeling item-item concepts could be much more promising in terms of both scalability and overall accuracy.

Item-based CF defines a similarity between an item ``i`` and ``j`` as:

```math
s_{i,j} = \\frac{ \\sum_{u \\in \\mathcal{U}_{i \\cap j}}  (r_{u, i} - \\overline{r}_i) (r_{u, j} - \\overline{r}_j)}
{ \\sqrt{\\sum_{u \\in \\mathcal{U}_{i \\cap j}} (r_{u,i} - \\overline{r}_i)^2} \\sqrt{\\sum_{u \\in \\mathcal{U}_{i \\cap j}} (r_{u, j} - \\overline{r}_j)^2} },
```

where ``\\mathcal{U}_{i \\cap j}`` is a set of users that both of ``r_{u,i}`` and ``r_{u, j}`` are not missing, and ``\\overline{r}_i, \\overline{r}_j`` are mean values of ``i``-th and ``j``-th column in ``R``. Similarly to the user-based algorithm, for the ``t``-th nearest-neighborhood item ``\\tau(t)``, prediction can be done by top-``k`` weighted sum of target user's feedbacks:

```math
r_{u,i} = \\frac{\\sum^k_{t=1} s_{i,\\tau(t)} \\cdot r_{u,\\tau(t)} }{ \\sum^k_{t=1} s_{i,\\tau(t)} }.
```

In case that the number of items is smaller than users, item-based CF could be a more reasonable choice than the user-based approach.
"""
struct ItemKNN <: Recommender
    data::DataAccessor
    k::Int
    sim::AbstractMatrix

    function ItemKNN(data::DataAccessor, k::Int)
        n_item = size(data.R, 2)
        new(data, k, matrix(n_item, n_item))
    end
end

ItemKNN(data::DataAccessor) = ItemKNN(data, 5)

isbuilt(recommender::ItemKNN) = isfilled(recommender.sim)

function build!(recommender::ItemKNN; adjusted_cosine::Bool=false)
    # cosine similarity

    R = copy(recommender.data.R)
    n_row, n_col = size(R)

    if adjusted_cosine
        # subtract mean
        for ri in 1:n_row
            indices = broadcast(!isnan, R[ri, :])
            vmean = mean(R[ri, indices])
            R[ri, indices] .-= vmean
        end
    end

    # unlike pearson correlation, matrix can be filled by zeros for cosine similarity
    R[isnan.(R)] .= 0

    # compute L2 nrom of each column
    norms = sqrt.(sum(R.^2, dims=1))

    for ci in 1:n_col
        for cj in ci:n_col
            numer = dot(R[:, ci], R[:, cj])
            denom = norms[ci] * norms[cj]
            s = numer / denom

            recommender.sim[ci, cj] = s
            if (ci != cj); recommender.sim[cj, ci] = s; end
        end
    end

    # NaN similarities are converted into zeros
    recommender.sim[isnan.(recommender.sim)] .= 0
end

function predict(recommender::ItemKNN, u::Int, i::Int)
    check_build_status(recommender)

    numer = denom = 0

    # negative similarities are filtered
    pairs = collect(zip(1:size(recommender.data.R)[2], max.(recommender.sim[i, :], 0)))
    ordered_pairs = sort(pairs, by=tuple->last(tuple), rev=true)[1:recommender.k]

    for (j, s) in ordered_pairs
        r = recommender.data.R[u, j]
        if isnan(r); continue; end

        numer += s * r
        denom += s
    end

    (denom == 0) ? 0 : numer / denom
end


#
# asian-winstonplot.jl
#
# Note:
#
# With OSX I have had some problems using the Pango as part of the Winston package
# to display titles and text etc.
#
# This seems to be a problem with Homebrew and the following worked for me
# It only needs to be run once,
#
# using Homebrew
# Homebrew.update()
# Homebrew.rm("pango")
# Homebrew.add("pango")
#

import Winston;
wn = Winston;

S0  = 100;      # Spot price
K   = 102;      # Strike price
r   = 0.05;     # Risk free rate
q   = 0.0;      # Dividend yield
v   = 0.2;      # Volatility
tma = 0.25;     # Time to maturity
T = 100;        # Number of time steps
dt = tma/T;     # Time increment
S = zeros(Float64,T)
S[1] = S0;

dW = randn(T)*sqrt(dt);
[ S[t] = S[t-1] * (1 + (r - q - 0.5*v*v)*dt + v*dW[t] + 0.5*v*v*dW[t]*dW[t]) for t=2:T ]
x = linspace(1, T, T);
p = wn.FramedPlot(title = "Random Walk, drift 5%, volatility 20%")
wn.add(p, wn.Curve(x,S,color="red"))
wn.display(p)


# # Julia is fast
#
# Very often, benchmarks are used to compare languages.  These benchmarks can lead to long discussions, first as to exactly what is being benchmarked and secondly what explains the differences.  These simple questions can sometimes get more complicated than you at first might imagine.
#
# The purpose of this notebook is for you to see a simple benchmark for yourself.  One can read the notebook and see what happened on the author's Macbook Pro with a 4-core Intel Core I7, or run the notebook yourself.
#
# (This material began life as a wonderful lecture by Steven Johnson at MIT: https://github.com/stevengj/18S096/blob/master/lectures/lecture1/Boxes-and-registers.ipynb.)

#-

# # Outline of this notebook
#
# - Define the sum function
# - Implementations & benchmarking of sum in...
#     - C (hand-written)
#     - C (hand-written with -ffast-math)
#     - python (built-in)
#     - python (numpy)
#     - python (hand-written)
#     - Julia (built-in)
#     - Julia (hand-written)
#     - Julia (hand-written with SIMD)
# - Summary of benchmarks

#-

# # `sum`: An easy enough function to understand

#-

# Consider the  **sum** function `sum(a)`, which computes
# $$
# \mathrm{sum}(a) = \sum_{i=1}^n a_i,
# $$
# where $n$ is the length of `a`.

a = rand(10^7) # 1D vector of random numbers, uniform on [0,1)

#-

sum(a)

# The expected result is 0.5 * 10^7, since the mean of each entry is 0.5

#-

# # Benchmarking a few ways in a few languages

@time sum(a)

#-

@time sum(a)

#-

@time sum(a)

# The `@time` macro can yield noisy results, so it's not our best choice for benchmarking!
#
# Luckily, Julia has a `BenchmarkTools.jl` package to make benchmarking easy and accurate:

## using Pkg
## Pkg.add("BenchmarkTools")

#-

using BenchmarkTools

# #  1. The C language
#
# C is often considered the gold standard: difficult on the human, nice for the machine. Getting within a factor of 2 of C is often satisfying. Nonetheless, even within C, there are many kinds of optimizations possible that a naive C writer may or may not get the advantage of.
#
# The current author does not speak C, so he does not read the cell below, but is happy to know that you can put C code in a Julia session, compile it, and run it. Note that the `"""` wrap a multi-line string.

using Libdl
C_code = """
#include <stddef.h>
double c_sum(size_t n, double *X) {
    double s = 0.0;
    for (size_t i = 0; i < n; ++i) {
        s += X[i];
    }
    return s;
}
"""

const Clib = tempname()   # make a temporary file

## compile to a shared library by piping C_code to gcc
## (works only if you have gcc installed):

open(`gcc -fPIC -O3 -msse3 -xc -shared -o $(Clib * "." * Libdl.dlext) -`, "w") do f
    print(f, C_code)
end

## define a Julia function that calls the C function:
c_sum(X::Array{Float64}) = ccall(("c_sum", Clib), Float64, (Csize_t, Ptr{Float64}), length(X), X)

#-

c_sum(a)

#-

c_sum(a) ≈ sum(a) # type \approx and then <TAB> to get the ≈ symbolb

#-

c_sum(a) - sum(a)

#-

≈  # alias for the `isapprox` function

#-

?isapprox

# We can now benchmark the C code directly from Julia:

c_bench = @benchmark c_sum($a)

#-

println("C: Fastest time was $(minimum(c_bench.times) / 1e6) msec")

#-

d = Dict()  # a "dictionary", i.e. an associative array
d["C"] = minimum(c_bench.times) / 1e6  # in milliseconds
d

#-

using Plots
gr()

#-

using Statistics # bring in statistical support for standard deviations
t = c_bench.times / 1e6 # times in milliseconds
m, σ = minimum(t), std(t)

histogram(t, bins=500,
    xlim=(m - 0.01, m + σ),
    xlabel="milliseconds", ylabel="count", label="")
## # 2. C with -ffast-math
#
## If we allow C to re-arrange the floating point operations, then it'll vectorize with SIMD (single instruction, multiple data) instructions.
const Clib_fastmath = tempname()   # make a temporary file

## The same as above but with a -ffast-math flag added
open(`gcc -fPIC -O3 -msse3 -xc -shared -ffast-math -o $(Clib_fastmath * "." * Libdl.dlext) -`, "w") do f
    print(f, C_code)
end

## define a Julia function that calls the C function:
c_sum_fastmath(X::Array{Float64}) = ccall(("c_sum", Clib_fastmath), Float64, (Csize_t, Ptr{Float64}), length(X), X)

#-

c_fastmath_bench = @benchmark $c_sum_fastmath($a)

#-

d["C -ffast-math"] = minimum(c_fastmath_bench.times) / 1e6  # in milliseconds

# # 3. Python's built in `sum`

#-

# The `PyCall` package provides a Julia interface to Python:

## using Pkg; Pkg.add("PyCall")
using PyCall

#-

## get the Python built-in "sum" function:
pysum = pybuiltin("sum")

#-

pysum(a)

#-

pysum(a) ≈ sum(a)

#-

py_list_bench = @benchmark $pysum($a)

#-

d["Python built-in"] = minimum(py_list_bench.times) / 1e6
d

# # 4. Python: `numpy`
#
# ## Takes advantage of hardware "SIMD", but only works when it works.
#
# `numpy` is an optimized C library, callable from Python.
# It may be installed within Julia as follows:

## using Pkg; Pkg.add("Conda")
using Conda

#-

## Conda.add("numpy")

#-

numpy_sum = pyimport("numpy")["sum"]

py_numpy_bench = @benchmark $numpy_sum($a)

#-

numpy_sum(a)

#-

numpy_sum(a) ≈ sum(a)

#-

d["Python numpy"] = minimum(py_numpy_bench.times) / 1e6
d

# # 5. Python, hand-written

py"""
def py_sum(A):
    s = 0.0
    for a in A:
        s += a
    return s
"""

sum_py = py"py_sum"

#-

py_hand = @benchmark $sum_py($a)

#-

sum_py(a)

#-

sum_py(a) ≈ sum(a)

#-

d["Python hand-written"] = minimum(py_hand.times) / 1e6
d

# # 6. Julia (built-in)
#
# ## Written directly in Julia, not in C!

@which sum(a)

#-

j_bench = @benchmark sum($a)

#-

d["Julia built-in"] = minimum(j_bench.times) / 1e6
d

# # 7. Julia (hand-written)

function mysum(A)
    s = 0.0 # s = zero(eltype(a))
    for a in A
        s += a
    end
    s
end

#-

j_bench_hand = @benchmark mysum($a)

#-

d["Julia hand-written"] = minimum(j_bench_hand.times) / 1e6
d

# # 8. Julia (hand-written w. simd)

function mysum_simd(A)
    s = 0.0 # s = zero(eltype(A))
    @simd for a in A
        s += a
    end
    s
end

#-

j_bench_hand_simd = @benchmark mysum_simd($a)

#-

mysum_simd(a)

#-

d["Julia hand-written simd"] = minimum(j_bench_hand_simd.times) / 1e6
d

# # Summary

for (key, value) in sort(collect(d), by=last)
    println(rpad(key, 25, "."), lpad(round(value; digits=1), 6, "."))
end


export Contingency

"""
Description:

    Contingency factor node

    The contingency distribution is a multivariate generalization of
    the categorical distribution. As a bivariate distribution, the
    contingency distribution defines the joint probability
    over two unit vectors. The parameter p encodes a contingency matrix
    that specifies the probability of co-occurrence.

    out1 ∈ {0, 1}^d1 where Σ_j out1_j = 1
    out2 ∈ {0, 1}^d2 where Σ_k out2_k = 1
    p ∈ [0, 1]^{d1 × d2}, where Σ_jk p_jk = 1

    f(out1, out2, p) = Con(out1, out2 | p)
                     = Π_jk p_jk^{out1_j * out2_k}

    A Contingency distribution over more than two variables requires
    higher-order tensors as parameters; these are not implemented in ForneyLab.

Interfaces:

    1. out1
    2. out2
    3. p

Construction:

    Contingency(id=:some_id)
"""
mutable struct Contingency <: SoftFactor
    id::Symbol
    interfaces::Vector{Interface}
    i::Dict{Symbol,Interface}

    function Contingency(out1, out2, p; id=generateId(Contingency))
        @ensureVariables(out1, out2, p)
        self = new(id, Array{Interface}(undef, 3), Dict{Symbol,Interface}())
        addNode!(currentGraph(), self)
        self.i[:out1] = self.interfaces[1] = associate!(Interface(self), out1)
        self.i[:out2] = self.interfaces[2] = associate!(Interface(self), out2)
        self.i[:p] = self.interfaces[3] = associate!(Interface(self), p)

        return self
    end
end

slug(::Type{Contingency}) = "Con"

format(dist::ProbabilityDistribution{Multivariate, Contingency}) = "$(slug(Contingency))(p=$(format(dist.params[:p])))"

ProbabilityDistribution(::Type{Multivariate}, ::Type{Contingency}; p=1/9*ones(3,3)) = ProbabilityDistribution{Multivariate, Contingency}(Dict(:p=>p))
ProbabilityDistribution(::Type{Contingency}; p=1/9*ones(3,3)) = ProbabilityDistribution{Multivariate, Contingency}(Dict(:p=>p))

dims(dist::ProbabilityDistribution{Multivariate, Contingency}) = (length(size(dist.params[:p])),)

vague(::Type{Contingency}, n_factors::Tuple{Int64, Int64}=(3,3)) = ProbabilityDistribution(Multivariate, Contingency, p=(1/prod(n_factors))*ones(n_factors))

isProper(dist::ProbabilityDistribution{Multivariate, Contingency}) = (abs(sum(dist.params[:p])-1.) < 1e-6)

# Entropy functional
function differentialEntropy(dist::ProbabilityDistribution{Multivariate, Contingency})
    B = dist.params[:p]
    if isa(B, Vector{<:Matrix}) # Contingency tensor
        H = 0.0
        for k=1:length(B)
            H += -sum(B[k].*log.(clamp.(B[k], tiny, Inf)))
        end
    else # Contingency matrix
        H = -sum(B.*log.(clamp.(B, tiny, Inf)))
    end

    return H
end


# ------------------------------------------------------------------
# Licensed under the MIT License. See LICENSE in the project root.
# ------------------------------------------------------------------

"""
    DirectionPartition(direction; tol=1e-6)

A method for partitioning spatial objects along a given `direction`
with bandwidth tolerance `tol`.
"""
struct DirectionPartition{T,N} <: SPredicatePartitionMethod
  direction::SVector{N,T}
  tol::Float64

  function DirectionPartition{T,N}(direction, tol) where {N,T}
    new(normalize(direction), tol)
  end
end

DirectionPartition(direction::SVector{N,T}; tol=1e-6) where {T,N} =
  DirectionPartition{T,N}(direction, tol)

DirectionPartition(direction::NTuple{N,T}; tol=1e-6) where {T,N} =
  DirectionPartition(SVector(direction), tol=tol)

(p::DirectionPartition)(x, y) = begin
  δ = x - y
  d = p.direction
  norm(δ - (δ⋅d)*d) < p.tol
end


#  Copyright 2016, Iain Dunning, Joey Huchette, Miles Lubin, and contributors
#  This Source Code Form is subject to the terms of the Mozilla Public
#  License, v. 2.0. If a copy of the MPL was not distributed with this
#  file, You can obtain one at http://mozilla.org/MPL/2.0/.
#############################################################################
# JuMP
# An algebraic modelling langauge for Julia
# See http://github.com/JuliaOpt/JuMP.jl
#############################################################################
# qcp.jl
#
# A simple quadratically constrained probgram
# Based on http://www.gurobi.com/documentation/5.5/example-tour/node25
#############################################################################

using JuMP

# Will require either Gurobi.jl, CPLEX.jl, or Mosek.jl to run
m = Model()

# Need nonnegativity for (rotated) second-order cone
@variable(m, x)
@variable(m, y >= 0)
@variable(m, z >= 0)

# Maximize x
@objective(m, Max, x)

# Subject to 1 linear and 2 nonlinear constraints
@constraint(m, x + y + z == 1)
@constraint(m, x*x + y*y - z*z <= 0)
@constraint(m, x*x - y*z <= 0)

# Print the model to check correctness
print(m)

# Solve with Gurobi
status = solve(m)

# Solution
println("Objective value: ", getobjectivevalue(m))
println("x = ", getvalue(x))
println("y = ", getvalue(y))


function normalize!{T}(s::Array{T})
# Normalize s so that the squared L2 vector norm of each channel is at most 1

  (m, n, d) = size(s);
  for k=1:d;
    norm_s = vecnorm(s[:, :, k]);
    if norm_s*norm_s>1;
      s[:, :, k] /= norm_s;
    end
  end
end


#=

Authors: Shunsuke Hori

=#
"""
Function to Solve Single Beliefs
"""
function price_single_beliefs(transition::Matrix, dividend_payoff::Vector;
                              β::AbstractFloat=.75)
    # First compute inverse piece
    imbq_inv = inv(eye(size(transition, 1)) - β * transition)

    # Next compute prices
    prices = β * ((imbq_inv * transition) * dividend_payoff)

    return prices
end


#####################################################################################
#Forecast using OLS with lagged aggregate variable, and lagged disaggregate variable.
#The first out-of-sample forecast has vector index of J - (h_lag + h)
#####################################################################################
function OLS_oos_forc(J, h, h_lag, Y, Y_lag, X_lag)
    ## Vector Index for oos forecast goes from J - (h_lag + h) to length(Y)
    ŷ = zeros(Float64, size(J - (h_lag + h):length(Y)))
    for (n, vec_î) in enumerate(J - (h_lag + h):length(Y))
        LHS = Y[1:(vec_î - h)]
        sample_size = length(LHS)
        RHS = [ones(sample_size) Y_lag[1:sample_size] X_lag[1:sample_size, :]]
        β̂ = OLSestimator(LHS, RHS)
        RHS_crnt = [1.0; Y_lag[vec_î]; X_lag[vec_î, :]]
        ŷ[n] = dot(β̂, RHS_crnt)
    end
    RMSE = sqrt(mean((Y[J - (h_lag + h):length(Y)] - ŷ) .^ 2))
    return ŷ, RMSE
end


module QuantilePriors

using ApproxFun
using Statistics

export QuantilePrior,
    mean,
    quantile

"""
    const S = Chebyshev(0..1)

All functions in this package use a Chebyshev basis on the [0, 1] interval.
"""
const S = Chebyshev(0..1)
"""
    const QP_INT = Integral(S)

The integral operator for the Chebyshev basis on the [0, 1] interval. Used for
calculating the mean of a prior.
"""
const QP_INT = Integral(S)

"""
    QuantilePrior{T} where T<:Fun
    QuantilePrior(p, q)

A quantile (inverse CDF) function that can be used as a prior, inspired by [Ben
Goodrich's StanCon 2020 talk](https://youtu.be/_wfZSvasLFk).

Usually specified via a vector of probabilities `p` and corresponding quantiles
`q`. These are interpolated using Chebyshev polynomials from the ApproxFun
package. This must result in a monotone function to represent a valid
distribution (this is tested when the function is constructed).

This would be used in e.g. MCMC settings by sampling from [0, 1] and then using
`quantile` to recover a parameter value that can be used. In this case the
density function of the prior is not needed; it is included implicitly in the
posterior density due to the change of variables.

Note that the resulting prior has a compact support, which may or may not be a
problem.
"""
struct QuantilePrior{T}
    f::T

    function QuantilePrior(f::T) where T <: Fun
        domain(f) == 0..1 || throw(DomainError("Domain must be [0, 1]"))
        ismonotoneinc(f) || throw("Not monotone; specify additional quantiles")
        new{T}(f)
    end
end

function QuantilePrior(p, q)
    all(0 .≤ p .≤ 1) || throw(DomainError("All `p` must be in [0, 1]"))
    V = vandermonde(p)
    f = Fun(S, V \ q)
    QuantilePrior(f)
end

"""
    vandermonde(p)

Construct a Vandermonde matrix at grid locations `p`, as described in ApproxFun
FAQ.
"""
function vandermonde(p)
    T = eltype(p)
    # Following ApproxFun FAQ
    n = length(p)
    V = Array{T}(undef, n, n)
    for k in 1:n
        V[:, k] = Fun(S, [zeros(T, k - 1); 1]).(p)
    end
    V
end

"""
    ismonotoneinc(f::Fun, x)

Test that the derivatives of `f` at `x` are all positive.
"""
function ismonotoneinc(f::Fun, x=range(0, 1, length=1025))
    df = f'.(x)
    all(df .> 0)
end

# Needed for broadcasting
Statistics.quantile(qp::QuantilePrior, p) = qp.f(p)
# Statistics.mean(qp::QuantilePrior) = (QP_INT * qp.f)(1)
Statistics.mean(qp::QuantilePrior) = stop("Not implemented (correctly)")

end


using SparseArrays: spdiagm, sparse, spzeros, SparseMatrixCSC
using LinearAlgebra: diag, I

export differential_operator_sequence
"""
    differential_operator_sequence(m::Metric{N}, mesh::Mesh{N, K}, expr::String,
        k::Int, primal::Bool) where {N, K}

Compute the differential operators defined by the string `expr`. This string must consist
of the characters `d`, `★`, `δ`, and `Δ` indicating the exterior derivative, hodge dual,
codifferential, and Laplace-de Rham operators, respectively. If there are two `★` operators
in a row, a single `★★` operator is computed since this avoids unnecessary calculation.
The dimension of the form on which the operator acts is `k-1` and its primality or duality
is indicated by `primal`.
"""
function differential_operator_sequence(m::Metric{N}, mesh::Mesh{N, K}, expr::String,
    k::Int, primal::Bool) where {N, K}
    ops = SparseMatrixCSC{Float64,Int64}[]
    chars = collect(expr)
    i = length(chars)
    while i > 0
        char = chars[i]
        @assert char in ['d', '★', 'δ', 'Δ']
        if char == 'd'
            comp = primal ? mesh.primal.complex : mesh.dual.complex
            push!(ops, exterior_derivative(comp, k))
            k += 1
        elseif char == '★'
            if (i > 1) && chars[i-1] == '★'
                push!(ops, hodge_square(m, mesh, k, primal))
                i -= 1
            else
                push!(ops, circumcenter_hodge(m, mesh, k, primal))
                primal = !primal
                k = K-k+1
            end
        elseif char == 'δ'
            push!(ops, codifferential(m, mesh, k, primal))
            k -= 1
        elseif char == 'Δ'
            push!(ops, laplace_de_Rham(m, mesh, k, primal))
        end
        i -= 1
    end
    return reverse(ops)
end

export differential_operator
"""
    differential_operator(m::Metric{N}, mesh::Mesh{N}, expr::String, k::Int,
        primal::Bool) where N

Compute the differential operator defined by the string `expr`. This string must consist
of the characters `d`, `★`, `δ`, and `Δ` indicating the exterior derivative, hodge dual,
codifferential, and Laplace-de Rham operators, respectively. The dimension of the form on
which the operator acts is `k-1` and its primality or duality is indicated by `primal`.
"""
function differential_operator(m::Metric{N}, mesh::Mesh{N}, expr::String, k::Int,
    primal::Bool) where N
    ops = differential_operator_sequence(m, mesh, expr, k, primal)
    return reduce(*, ops)
end

"""
    differential_operator(m::Metric{N}, mesh::Mesh{N}, expr::String, k::Int,
        primal::Bool, v::AbstractVector{<:Real}) where N

Apply the differential operator defined by the string `expr` to the vector `v`.
"""
function differential_operator(m::Metric{N}, mesh::Mesh{N}, expr::String, k::Int,
    primal::Bool, v::AbstractVector{<:Real}) where N
    ops = differential_operator_sequence(m, mesh, expr, k, primal)
    return foldr(*, ops, init=v)
end

"""
    circumcenter_hodge(m::Metric{N}, mesh::Mesh{N, K}, k::Int, primal::Bool) where {N, K}

Find the discrete hodge star operator using the circumcentric hodge star construction. If
`primal == true`, this operator takes primal `k-1` forms to dual `K-k+1` forms. Otherwise
it takes dual `k-1` forms to priaml `K-k+1` forms.
"""
function circumcenter_hodge(m::Metric{N}, mesh::Mesh{N, K}, k::Int,
    primal::Bool) where {N, K}
    @assert 1 <= k <= K+1
    if k == K+1
        return spzeros(0,0)
    end
    if primal
        ratios = Float64[]
        for (p_cell, d_cell) in zip(mesh.primal.complex.cells[k],
                mesh.dual.complex.cells[K-k+1])
            p_vol = volume(m, mesh.primal, p_cell)
            d_vol = volume(m, mesh.dual, d_cell)
            @assert p_vol != 0
            push!(ratios, d_vol/p_vol)
        end
    else
        pm = hodge_square_sign(m, K, k)
        v = diag(circumcenter_hodge(m, mesh, K-k+1, true))
        @assert !any(v .== 0)
        ratios = pm ./ v
    end
    return spdiagm(0 => ratios)
end

"""
    hodge_square_sign(m::Metric, K::Int, k::Int)

There is an identity `★★ = sign(det(metric)) * (-1)^((k-1) * (K-k)) * I` for `k-1` forms in
`K-1` dimensions. Compute the coefficient of `I` in this expression.
"""
hodge_square_sign(m::Metric, K::Int, k::Int) = sign(det(collect(m.mat))) *
    (mod((k-1) * (K-k), 2) == 0 ? 1 : -1)

"""
    hodge_square(m::Metric{N}, mesh::Mesh{N, K}, k::Int, primal::Bool) where {N, K}

Compute `★★` without computing the `★` operators by using the identity
`★★ = sign(det(metric)) * (-1)^((k-1) * (K-k)) * I` for `k-1` forms in `K-1` dimensions.
"""
function hodge_square(m::Metric{N}, mesh::Mesh{N, K}, k::Int, primal::Bool) where {N, K}
    comp = primal ? mesh.primal.complex : mesh.dual.complex
    n = k < K+1 ? length(comp.cells[k]) : 0
    return hodge_square_sign(m, K, k) * sparse(I, n, n)
end

"""
    exterior_derivative(comp::CellComplex{N, K}, k::Int) where {N, K}

Find the discrete exterior derivative operator.
"""
function exterior_derivative(comp::CellComplex{N, K}, k::Int) where {N, K}
    @assert 0 <= k <= K
    if k == 0
        return spzeros(length(comp.cells[k+1]), 0)
    end
    row_inds, col_inds, vals = Int[], Int[], Int[]
    for (col_ind, cell) in enumerate(comp.cells[k])
        for p in keys(cell.parents)
            o = cell.parents[p]
            row_ind = findfirst(isequal(p), comp.cells[k+1])
            push!(row_inds, row_ind); push!(col_inds, col_ind); push!(vals, 2 * o - 1)
        end
    end
    num_rows = k+1 <= K ? length(comp.cells[k+1]) : 0
    num_cols = length(comp.cells[k])
    return sparse(row_inds, col_inds, vals, num_rows, num_cols)
end

"""
    codifferential(m::Metric{N}, mesh::Mesh{N, K}, k::Int, primal::Bool) where {N, K}

Compute the codifferential defined by
`δ = sign(det(collect(m.mat))) * (-1)^((K-1) * (k-2) + 1) * ★d★` for `k-1` forms in `K-1`
dimensions.
"""
function codifferential(m::Metric{N}, mesh::Mesh{N, K}, k::Int, primal::Bool) where {N, K}
    ★d★ = differential_operator(m, mesh, "★d★", k, primal)
    s = sign(det(collect(m.mat)))
    return s * (mod((K-1) * (k-2) + 1, 2) == 0 ? 1 : -1) * ★d★
end

"""
    laplace_de_Rham(m::Metric{N}, mesh::Mesh{N, K}, k::Int, primal::Bool) where {N, K}

Compute the Laplace-de Rham operator defined by `Δ = dδ + δd`.
"""
function laplace_de_Rham(m::Metric{N}, mesh::Mesh{N, K}, k::Int, primal::Bool) where {N, K}
    dδ = differential_operator(m, mesh, "dδ", k, primal)
    δd = differential_operator(m, mesh, "δd", k, primal)
    return dδ + δd
end

export sharp
"""
    sharp(m::Metric{N}, comp::CellComplex{N}, form::AbstractVector{<:Real}) where N

Given a 1-form on a cell complex, approximate a vector of length `N` at each vertex using
least squares.
"""
function sharp(m::Metric{N}, comp::CellComplex{N}, form::AbstractVector{<:Real}) where N
    field = Vector{Float64}[]
    for c in comp.cells[1]
        mat = zeros(length(c.parents), N)
        w = zeros(length(c.parents))
        for (row_ind, e) in enumerate(collect(keys(c.parents)))
            mat[row_ind, :] = sum([x.points[1].coords * (2 * x.parents[e] - 1)
                for x in e.children])
            w[row_ind] = form[findfirst(isequal(e), comp.cells[2])]
        end
        push!(field, (mat * m.mat) \ w)
    end
    return field
end


struct Add{T} <: PushBinaryOp{T} end
execute_op(i::Add{T}, a::T, b::T) where {T} = b+a
oldopname(::Type{<:Add}) = "+"

struct Sub{T} <: PushBinaryOp{T} end
execute_op(i::Sub{T}, a::T, b::T) where {T} = b-a # Push order is that element at top of stack (a) is subtracted from the next element (b)
oldopname(::Type{<:Sub}) = "-"

struct Mul{T} <: PushBinaryOp{T} end
execute_op(i::Mul{T}, a::T, b::T) where {T} = b*a
oldopname(::Type{<:Mul}) = "*"

struct Div{T} <: PushBinaryOp{T} end
@inline function execute!(i::Div{T}, interp::AbstractPushInterpreter) where {T}
    stackhas(interp, T, 2) || return(nothing)
    last(stack(interp, T)) == 0 && return(nothing)
    a, b = pop2!(interp, T)
    res = b/a
    push!(interp, res)
end
oldopname(::Type{<:Div}) = "/"

struct Mod{T} <: PushBinaryOp{T} end
@inline function execute!(i::Mod{T}, interp::AbstractPushInterpreter) where {T}
    stackhas(interp, T, 2) || return(nothing)
    last(stack(interp, T)) == 0 && return(nothing)
    a, b = pop2!(interp, T)
    res = div(b, a)
    push!(interp, res)
end
oldopname(::Type{<:Mod}) = "%"

struct Max{T} <: PushBinaryOp{T} end
execute_op(i::Max{T}, a::T, b::T) where {T} = max(a, b)

struct Min{T} <: PushBinaryOp{T} end
execute_op(i::Min{T}, a::T, b::T) where {T} = min(a, b)


# MIT license
# Copyright (c) Microsoft Corporation. All rights reserved.
# See LICENSE in the project root for full license information.

@testset "Lenses" begin
    test_n = 5000

    """Creates a 3D vector uniformly distributed on the sphere by rejection sampling, i.e., discarding all points with norm > 1.0"""
    function randunit()
        let v = rand(3)
            while (norm(v) > 1.0)
                v = rand(3)
            end
            return normalize(v)
        end
    end

    @testset "Refraction" begin
        Random.seed!(SEED)
        ηᵢ = 1.4
        ηₜ = 1.2
        for i in 1:test_n
            nₛ = randunit()
            rᵢ = randunit()
            rₜ = refractedray(ηᵢ, ηₜ, nₛ, rᵢ)
            if !(rₜ === nothing)
                sinθₜ = norm(cross(-nₛ, rₜ))
                sinθᵢ = norm(cross(nₛ, rᵢ))
                a = isapprox(ηₜ * sinθₜ, ηᵢ * sinθᵢ, rtol = RTOLERANCE, atol = ATOLERANCE) #verify snell's law for the the transmitted and incident ray
                b = isapprox(1.0, norm(rₜ), rtol = RTOLERANCE, atol = ATOLERANCE) #verify transmitted ray is unit
                perp = normalize(cross(nₛ, rᵢ))
                c = isapprox(0.0, rₜ ⋅ perp, rtol = RTOLERANCE, atol = ATOLERANCE) #verify incident and transmitted ray are in the same plane
                @test a && b && c
            end
        end
    end # testset refraction

    # snell
    @testset "Snell" begin
        Random.seed!(SEED)
        for i in 1:test_n
            r = randunit()
            nₛ = randunit()
            n1 = 1.0
            n2 = 1.4
            sθ1, sθ2 = snell(nₛ, r, n1, n2)
            sθ3, sθ4 = snell(nₛ, r, n2, n1)
            @test isapprox(sθ1 * n1, sθ2 * n2, rtol = RTOLERANCE, atol = ATOLERANCE) && isapprox(sθ3 * n2, sθ4 * n1, rtol = RTOLERANCE, atol = ATOLERANCE)
        end
    end # testset snell

    @testset "Reflection" begin
        Random.seed!(SEED)
        # reflection
        for i in 1:test_n
            r = randunit()
            nₛ = randunit()
            if abs(r ⋅ nₛ) < 0.9999 # if r and n are exactly aligned then their sum will be zero, not a vector pointing in the direction of the normal
                reflected = reflectedray(nₛ, r)
                #ensure reflected and refracted rays are in the same plane
                a = isapprox(norm(reflected ⋅ cross(r, nₛ)), 0.0, rtol = RTOLERANCE, atol = ATOLERANCE)
                b = isapprox(norm(reflected), 1.0, rtol = RTOLERANCE, atol = ATOLERANCE)
                c = isapprox(0.0, reflected ⋅ nₛ + r ⋅ nₛ, rtol = RTOLERANCE, atol = ATOLERANCE)
                #ensure sum of reflected and origin ray are in the direction of the normal
                d = isapprox(0.0, norm(nₛ - sign(reflected ⋅ nₛ) * (normalize(reflected - r))), rtol = RTOLERANCE, atol = ATOLERANCE)
                @test a & b & c & d
            end
        end
    end # testset reflection

    @testset "Paraxial" begin
        # check that normally incident rays are focussed across the lens
        l = ParaxialLensEllipse(100.0, 10.0, 10.0, [1.0, 1.0, 0.0], [3.0, 3.0, 3.0])
        r = OpticalRay([2.0, 2.0, 3.0], [1.0, 1.0, 0.0], 1.0, 0.55)
        intsct = halfspaceintersection(surfaceintersection(l, r))
        ref, _, _ = OpticSim.processintersection(OpticSim.interface(intsct), OpticSim.point(intsct), OpticSim.normal(intsct), r, 20.0, 1.0, true, true)
        @test isapprox(ref, normalize([1.0, 1.0, 0.0]), rtol = RTOLERANCE, atol = ATOLERANCE)
        l = ParaxialLensRect(100.0, 10.0, 10.0, [1.0, 1.0, 0.0], [3.0, 3.0, 3.0])
        r = OpticalRay([2.3, 2.4, 3.1], [1.0, 1.0, 0.0], 1.0, 0.55)
        intsct = halfspaceintersection(surfaceintersection(l, r))
        fp = [3.0, 3.0, 3.0] + 100 * normalize([1.0, 1.0, 0.0])
        ref, _, _ = OpticSim.processintersection(OpticSim.interface(intsct), OpticSim.point(intsct), OpticSim.normal(intsct), r, 20.0, 1.0, true, true)
        @test isapprox(ref, normalize(fp - OpticSim.point(intsct)), rtol = RTOLERANCE, atol = ATOLERANCE)
    end # testset Paraxial
end # testset lenses


function pulaymix!(vin::Array{Float64,1}, vout::Array{Float64,1},
                   beta::Float64, dvmat, vmat, iter::Int64, mixdim::Int64)
    # output [vnew,df,dv]
    Npoints = size(vin)[1]
    #
    # construct and modify vmat and dvmat
    #
    if iter <= mixdim
        vmat[:,iter] = vin
        dvmat[:,iter] = vout - vin
    else
        # delete the first leading column  ???
        #vmat[:,1] = []; dvmat[:,1] = []
        # append vin and vout-vin at the end of vmat and dvmat;
        vmat[:,mixdim] = vin
        dvmat[:,mixdim] = vout - vin
    end

    thresh = 1e-6
    if iter > 1
        ibeg = 2
        iend = min(iter,mixdim)
        A = dvmat[:,ibeg:iend]
        b = dvmat[:,ibeg-1]
        ncols = size(A)[2]
        A = b*ones(1,ncols) - A
        U,S,V = svd(A,thin=true)
        while S[ncols] <  thresh*S[1]
            ibeg = ibeg + 1
            if ibeg > iend
                @printf("warning: no mixing...\n")
                vnew = vout
                return
            end
            A = dvmat[:,ibeg:iend]
            b = dvmat[:,ibeg-1]
            ncols = size(A)[2]
            A = b*ones(1,ncols) - A
            U,S,V = svd(A,thin=true)
        end
        g2 = V*(Matrix(Diagonal(S))\(U'*b))
        g1 = 1 - sum(g2)
        g = [g1;g2]
        vnew = ( vmat[:,ibeg-1:iend] + beta*dvmat[:,ibeg-1:iend] )*g
    else
        vnew = vout
    end
    return vnew
end


t = -3.1:0.01:3.1
u = 16*sin.(t).^3
v = 13*cos.(t) .- 5*cos.(2t) .- 2*cos.(3t) .- cos.(4t)
plot(u, v, c = :black, leg=false, fill=(0, 0.8,:black),
    xlab = L"u", ylab = L"v", xlim = (-20,20), ylim = (-20,15),
    size = (400, 400))
savefig("figheart.pdf")


## Ball on beam
#TODO make inplace
function ballonbeam_dynamics(x::AbstractVector{T},u::AbstractVector{T}) where T
    g = 9.81
    m1 = .35
    m2 = 2
    l = 0.5

    z = x[1];
    theta = x[2];
    zdot = x[3];
    thetadot = x[4];
    F = u[1];
    zddot = z*thetadot^2-g*sin(theta);
    thetaddot = (F*l*cos(theta) - 2*m1*z*zdot*thetadot - m1*g*z*cos(theta) - (m2*g*l*cos(theta))/2) / (m2*l^2/3 + m1*z^2);

    sys = [zdot; thetadot; zddot; thetaddot];
end
n,m = 4,1
ballonbeam = Model(ballonbeam_dynamics,n,m)


using Pkg
pkg"activate ."
pkg"add JuMP"
pkg"add Cbc"
pkg"add Juniper"
pkg"add Ipopt"

using JuMP, Ipopt, Juniper

# Escolhendo o modelo
optimizer = Juniper.Optimizer
nl_solver = optimizer_with_attributes(Ipopt.Optimizer, "print_level" => 0)
model = Model(optimizer_with_attributes(optimizer, "nl_solver" => nl_solver))

# Definindo os parâmetros que, por sinal, coloquei aleatórios, pois não sei as entradas
m = 4
n = 4
c = rand(1:4, n,m)
a = rand(0:0, m)
b = rand(1000:1000,n)

# Implementando realmente o modelo
@variable(model, x[i = 1:n, g = 1:m], Bin) #Variável
@NLobjective(model, Max, sum(c[i,j] * x[i,g] * x[j,g] for i = 1:n-1, j = i+1:n, g = 1:m)) #Func. objetivo

#Restrições
@constraint(model, [i = 1:n], sum(x[i,g] for g = 1:m) == 1)
@constraint(model, [i = 1:n, g = 1:m], sum(x[i,g] for g = 1:m) ≤ b[g])
@constraint(model, [i = 1:n, g = 1:m], a[g] <= sum(x[i,g] for g = 1:m))

optimize!(model) # Otimizando o modelo

println("Valor da função objetivo = ", objective_value(model))
round.(value.(x))


# Fractional Order Chaotic examples come from
#=
@inproceedings{Petr2011FractionalOrderNS,
title={Fractional-Order Nonlinear Systems: Modeling, Analysis and Simulation},
author={Ivo Petr{\'a}{\vs}},
year={2011}
}
=#
#

#=======Fractional Order Liu System=======#
using FractionalDiffEq

h=0.005
alpha = [0.95, 0.95, 0.95]
x0 = [0.2, 0, 0.5]
tf=100
function Liu!(du, u, p, t)
    a, b, c, e, n, m = 1, 2.5, 5, 1, 4, 4
    du[1] = -a*u[1]-e*u[2]^2
    du[2] = b*u[2]-n*u[1]*u[3]
    du[3] = -c*u[3]+m*u[1]*u[2]
end
prob = FODESystem(Liu!, alpha, x0, tf)
result = solve(prob, h, GLWithMemory())

using Plots
plot3d(result[:, 1], result[:, 2], result[:, 3], title="Fractional Order Liu System")

#=======Fractional Order Duffing System=======#
using FractionalDiffEq

h=0.005
alpha = [0.9, 1]
x0 = [0.21, 0.31]
tf=100
function Duffing!(du, u, p, t)
    α, δ, ω = 0.15, 0.3, 1
    du[1] = u[2]
    du[2] = u[1]-α*u[2]-u[1]^3+δ*cos(ω*t)
end
prob = FODESystem(Duffing!, alpha, x0, tf)
result = solve(prob, h, GLWithMemory())

using Plots
plot(result[:, 1], result[:, 2], title="Fractional Order Duffing System")

#=======Fractional Order Chen System=======#
using FractionalDiffEq

h=0.005
alpha = [0.9, 0.9, 0.9]
x0 = [-9, -5, 14]
tf=100
function Chen!(du, u, p, t)
    a, b, c, d = 35, 3, 28, -7
    du[1] = a*(u[2]-u[1])
    du[2] = d*u[1]-u[1]*u[3]+c*u[2]
    du[3] = u[1]*u[2]-b*u[3]
end
prob = FODESystem(Chen!, alpha, x0, tf)
result = solve(prob, h, GLWithMemory())

using Plots
plot3d(result[:, 1], result[:, 2], result[:, 3], title="Fractional Order Chen System")

#=======Fractional Order Van der Pol Oscillator=======#
using FractionalDiffEq

h=0.005
alpha = [1.2, 0.8]
x0 = [0.2, -0.2]
tf=60
function VanderPol!(du, u, p, t)
    ϵ = 1
    du[1] = u[2]
    du[2] = -u[1]-ϵ*(u[1]^2-1)*u[2]
end
prob = FODESystem(VanderPol!, alpha, x0, tf)
result = solve(prob, h, GLWithMemory())

using Plots
plot(result[:, 1], result[:, 2], title="Fractional Order Van der Pol Oscillator")

#=======Fractional Order Lotka Volterra System=======#
using FractionalDiffEq

h=0.005
alpha = [0.95, 0.95, 0.95]
x0 = [1, 1.4, 1]
tf=50
function LotkaVolterra!(du, u, p, t)
    a, b, c, d, e, p, s = 1, 1, 1, 1, 2, 3, 2.7
    du[1] = a*u[1]+e*u[1]^2-b*u[1]*u[2]-s*u[3]*u[1]^2
    du[2] = -c*u[2]+d*u[1]*u[2]
    du[3] = -p*u[3]+s*u[3]*u[1]^2
end
prob = FODESystem(LotkaVolterra!, alpha, x0, tf)
result = solve(prob, h, GLWithMemory())

using Plots
plot3d(result[:, 1], result[:, 2], result[:, 3], title="Fractional Order Lotka Volterra System")

#=======Fractional Order Lu System=======#
using FractionalDiffEq

h=0.005
alpha = [0.985, 0.99, 0.98]
x0 = [0.2, 0.5, 0.3]
tf=60
function Lu!(du, u, p, t)
    a, b, c = 36, 3, 20
    du[1] = a*(u[2]-u[1])
    du[2] = -u[1]*u[3]+c*u[2]
    du[3] = u[1]*u[2]-b*u[3]
end
prob = FODESystem(Lu!, alpha, x0, tf)
result = solve(prob, h, GLWithMemory())

using Plots
plot3d(result[:, 1], result[:, 2], result[:, 3], title="Fractional Order Lu System")

#=======Fractional Order Rossler System=======#
using FractionalDiffEq

h=0.005
alpha = [0.9, 0.85, 0.95]
x0 = [0.5, 1.5, 0.1]
tf=120
function Rossler!(du, u, p, t)
    a, b, c = 0.5, 0.2, 10
    du[1] = -u[2]-u[3]
    du[2] = u[1]+a*u[2]
    du[3] = b+u[3]*(u[1]-c)
end
prob = FODESystem(Rossler!, alpha, x0, tf)
result = solve(prob, h, GLWithMemory())

using Plots
plot3d(result[:, 1], result[:, 2], result[:, 3], title="Fractional Order Rossler System")

#=======Fractional Order Arneodo System=======#
using FractionalDiffEq

h=0.005
alpha = [0.97, 0.97, 0.96]
x0 = [-0.2, 0.5, 0.2]
tf=100
function Arneodo!(du, u, p, t)
    b1, b2, b3, b4 = -5.5, 3.5, 0.8, -1.0
    du[1] = u[2]
    du[2] = u[3]
    du[3] = -b1*u[1]-b2*u[2]-b3*u[3]+b4*u[1]^3
end
prob = FODESystem(Arneodo!, alpha, x0, tf)
result = solve(prob, h, GLWithMemory())

using Plots
plot3d(result[:, 1], result[:, 2], result[:, 3], title="Fractional Order Arneodo System")

#=======Fractional Order Genesio-Tesi System=======#
using FractionalDiffEq

h=0.005
alpha = [0.95, 0.95, 0.95]
x0 = [-0.1, 0.5, 0.2]
tf=100
function GenesioTesi!(du, u, p, t)
    b1, b2, b3, b4 = 1, 1.1, 0.4, 1.0
    du[1] = u[2]
    du[2] = u[3]
    du[3] -b1*u[1]-b2*u[2]-b3*u[3]+b4*u[1]^2
end
prob = FODESystem(GenesioTesi!, alpha, x0, tf)
result = solve(prob, h, GLWithMemory())

using Plots
plot3d(result[:, 1], result[:, 2], result[:, 3], title="Fractional Order Genesio-Tesi System")


using Sigma
using AbstractDomains
using Base.Test
x = uniform(0,1)
y = uniform(0,1)
@test isequal(prob(x+y > - 1.0), Interval(1.0,1.0))


# This is not optimized, but tries to follow the pseudocode given the Wikipedia entry below.
# Reference: https://en.wikipedia.org/wiki/Stable_marriage_problem#Algorithm

const males = ["abe", "bob", "col", "dan", "ed", "fred", "gav", "hal", "ian", "jon"]
const females = ["abi", "bea", "cath", "dee", "eve", "fay", "gay", "hope", "ivy", "jan"]

const malepreferences = Dict(
  "abe" => ["abi", "eve", "cath", "ivy", "jan", "dee", "fay", "bea", "hope", "gay"],
  "bob" => ["cath", "hope", "abi", "dee", "eve", "fay", "bea", "jan", "ivy", "gay"],
  "col" => ["hope", "eve", "abi", "dee", "bea", "fay", "ivy", "gay", "cath", "jan"],
  "dan" => ["ivy", "fay", "dee", "gay", "hope", "eve", "jan", "bea", "cath", "abi"],
   "ed" => ["jan", "dee", "bea", "cath", "fay", "eve", "abi", "ivy", "hope", "gay"],
 "fred" => ["bea", "abi", "dee", "gay", "eve", "ivy", "cath", "jan", "hope", "fay"],
  "gav" => ["gay", "eve", "ivy", "bea", "cath", "abi", "dee", "hope", "jan", "fay"],
  "hal" => ["abi", "eve", "hope", "fay", "ivy", "cath", "jan", "bea", "gay", "dee"],
  "ian" => ["hope", "cath", "dee", "gay", "bea", "abi", "fay", "ivy", "jan", "eve"],
  "jon" => ["abi", "fay", "jan", "gay", "eve", "bea", "dee", "cath", "ivy", "hope"]
)

const femalepreferences = Dict(
  "abi"=> ["bob", "fred", "jon", "gav", "ian", "abe", "dan", "ed", "col", "hal"],
  "bea"=> ["bob", "abe", "col", "fred", "gav", "dan", "ian", "ed", "jon", "hal"],
 "cath"=> ["fred", "bob", "ed", "gav", "hal", "col", "ian", "abe", "dan", "jon"],
  "dee"=> ["fred", "jon", "col", "abe", "ian", "hal", "gav", "dan", "bob", "ed"],
  "eve"=> ["jon", "hal", "fred", "dan", "abe", "gav", "col", "ed", "ian", "bob"],
  "fay"=> ["bob", "abe", "ed", "ian", "jon", "dan", "fred", "gav", "col", "hal"],
  "gay"=> ["jon", "gav", "hal", "fred", "bob", "abe", "col", "ed", "dan", "ian"],
 "hope"=> ["gav", "jon", "bob", "abe", "ian", "dan", "hal", "ed", "col", "fred"],
  "ivy"=> ["ian", "col", "hal", "gav", "fred", "bob", "abe", "ed", "jon", "dan"],
  "jan"=> ["ed", "hal", "gav", "abe", "bob", "jon", "col", "ian", "fred", "dan"]
)

function pshuf(d)
    ret = Dict()
    for (k,v) in d
        ret[k] = shuffle(v)
    end
    ret
end

# helper functions for the verb: p1 "prefers" p2 over p3
pindexin(a, p) = ([i for i in 1:length(a) if a[i] == p])[1]
prefers(d, p1, p2, p3) = (pindexin(d[p1], p2) < pindexin(d[p1], p3))

function isstable(mmatchup, fmatchup, mpref, fpref)
    for (mmatch, fmatch) in mmatchup
        for f in mpref[mmatch]
            if(f != fmatch && prefers(mpref, mmatch, f, fmatch)
                           && prefers(fpref, f, mmatch, fmatchup[f]))
                println("$mmatch prefers $f and $f prefers $mmatch over their current partners.")
                return false
            end
        end
    end
    true
end

function galeshapley(men, women, malepref, femalepref)
    # Initialize all m ∈ M and w ∈ W to free
    mfree = Dict([(p, true) for p in men])
    wfree = Dict([(p, true) for p in women])
    mpairs = Dict()
    wpairs = Dict()
    while true                    # while ∃ free man m who still has a woman w to propose to
        bachelors = [p for p in keys(mfree) if mfree[p]]
        if(length(bachelors) == 0)
            return mpairs, wpairs
        end
        for m in bachelors
            for w in malepref[m]  # w = first woman on m’s list to whom m has not yet proposed
                if(wfree[w])      # if w is free (else some pair (m', w) already exists)
                    #println("Free match: $m, $w")
                    mpairs[m] = w # (m, w) become engaged
                    wpairs[w] = m # double entry bookeeping
                    mfree[m] = false
                    wfree[w] = false
                    break
                elseif(prefers(femalepref, w, m, wpairs[w])) # if w prefers m to m'
                    #println("Unmatch $(wpairs[w]), match: $m, $w")
                    mfree[wpairs[w]] = true # m' becomes free
                    mpairs[m] = w           # (m, w) become engaged
                    wpairs[w] = m
                    mfree[m] = false
                    break
                end                         # else (m', w) remain engaged, so continue
            end
        end
    end
end

function tableprint(txt, ans, stab)
    println(txt)
    println("   Man     Woman")
    println("   -----   -----")
    show(STDOUT, "text/plain", ans)
    if(stab)
        println("\n  ----STABLE----\n\n")
    else
        println("\n  ---UNSTABLE---\n\n")
    end
end

println("Use the Gale Shapley algorithm to find a stable set of engagements.")
answer = galeshapley(males, females, malepreferences, femalepreferences)
stabl = isstable(answer[1], answer[2], malepreferences, femalepreferences)
tableprint("Original Data Table", answer[1], stabl)

println("To check this is not a one-off solution, run the function on a randomized sample.")
newmpref = pshuf(malepreferences)
newfpref = pshuf(femalepreferences)
answer = galeshapley(males, females, newmpref, newfpref)
stabl = isstable(answer[1], answer[2], newmpref, newfpref)
tableprint("Shuffled Preferences", answer[1], stabl)

# trade abe with bob
println("Perturb this set of engagements to form an unstable set of engagements then check this new set for stability.")
answer = galeshapley(males, females, malepreferences, femalepreferences)
fia1 = (answer[1])["abe"]
fia2 = (answer[1])["bob"]
answer[1]["abe"] = fia2
answer[1]["bob"] = fia1
answer[2][fia1] = "bob"
answer[2][fia2] = "abe"
stabl = isstable(answer[1], answer[2], malepreferences, femalepreferences)
tableprint("Original Data With Bob and Abe Switched", answer[1], stabl)


function onehot(y::Vector)
    n_samples = size(y, 1)
    n_features = length(unique(y))
    Y = zeros(n_features, n_samples)

    # all elements of y must be > 0 because they are treated
    # as indices
    assert(all(0 .< y))
    for i in 1:n_samples
        Y[y[i], i] = 1
    end
    return Y
end

maxindices{T}(X::AA{T, 2}) = [indmax(X[:, i]) for i in 1:size(X, 2)]

function accuracy{T}(y::AA{T, 1}, ŷ::AA{T, 1})
    assert(size(y) == size(ŷ))
    sum(y .== ŷ) / length(y)
end

to_tuple_if_int(x::Real) = (x, x)


#=
# 4. The Kutta condition

For surfaces that contain convex edges, the vortex sheet strength assumes a singular behavior in the vicinity of these edges, with a strength that depends on the interior angle of the edge: sharper edges have more singular behavior. In the discrete representation of the surface, edges are only approximately represented by the sudden disruptions of positions in clusters of adjacent points. The behavior in this discrete form is not quite singular, but the solution for the vortex sheet strength $\mathfrak{f}$ nonetheless exhibits a large and rapid change of amplitude.
=#

# Let's illustrate this with the example of a flat plate. First create the grid.
#md # ```@setup 4.-The-Kutta-condition
#md # using GridPotentialFlow
#md # using Plots
#md # ```
#!md using GridPotentialFlow
#!md using Plots
Δx = 0.01
Lx = 2.0
xlim = (-Lx/2,Lx/2)
ylim = (-Lx/2,Lx/2)
g = PhysicalGrid(xlim,ylim,Δx);

# To create a potential flow model with flat plate, we use the `Plate` constructor from `RigidBodyTools.jl` and transform the body such that its angle of attack is `α`.
c = Lx/2 # chord length
α = -π/6 # angle of attack
plate = Plate(c,4*cellsize(g))
Tr = RigidTransform((0.0,0.0),α)
Tr(plate)
pfb = PotentialFlowBody(plate);
model = VortexModel(g,bodies=[pfb]);

# To simulate the plate in a uniform flow, we have to set the `U∞` field of the vortex model to our desired value.
setU∞(model,(1.0,0.0))
sol = solve(model);

# By plotting a large number streamlines, we see how the flow accelerates as it tries to make its way around the sharp edges of the flat plate. In the continuous case, the velocity field would be singular at the edges.
plot(sol.ψ,g,levels=60);
plot!(plate,linecolor=:black,linewidth=2,xlabel="x",ylabel="y")

# The continous vortex sheet strength shows similar singularities at the edges, and also in our discrete case, vortex sheet strength the indeed assumes an approximate singular behavior near those edges.
plot(sol.f,xlabel="body point index",ylabel="f",legend=false)

#=
If we seek to eliminate this behavior, we must first have some means of exposing it. In fact, for any discretized surface, the essence of this nearly-singular behavior lies in the previously defined vector $\mathfrak{f}_0$, and all other bound vortex sheets associated with the same surface share the same nearly-singular behavior. Thus, we will use a multiplicative decomposition of the vortex sheet strength:

$\mathfrak{f} = \mathfrak{f}_0 \circ \tilde{\mathfrak{f}}$

where $\circ$ is the Hadamard product. This decomposed form isolates the singular behavior into $\mathfrak{f}_0$, and $\tilde{\mathfrak{f}}$ is a relatively smoother vector of surface point data.

=#

# For our flat plate, we create $\mathfrak{f}_0$ in the same way as we did on the previous page.
ones = ScalarData(length(plate))
ones .= 1.0
f₀ = model.system.ibp.Sfact\ones
plot(f₀)

# Then we can plot the smooth $\tilde{\mathfrak{f}}$ as follows.
plot(sol.f./f₀,xlabel="body point index",ylabel="f̃",legend=false)

#=
## Using the Kutta condition in a steady-state problem

The Kutta condition corresponds to annihilating the nearly-singular behavior at a surface point. At such points, we will set the corresponding value of $\tilde{\mathfrak{f}}$ to zero. Suppose we wish to enforce the Kutta condition at an edge corresponding to surface point $k$. The condition is

$\mathfrak{e}_{k}^{T} \tilde{\mathfrak{f}} = 0.$

In the regularization operation on $\mathfrak{f}$, we can absorb $\mathfrak{f}_0$ into $\mathsf{R}$, first noting that the Hadamard product can alternatively be written with the help of a diagonal matrix,

$\mathfrak{f} = \mathfrak{f}_0 \circ \tilde{\mathfrak{f}} = \mathsf{D}{\mathfrak{f}_0} \tilde{\mathfrak{f}}.$

Then, we can define a re-scaled regularization operator,

$\mathsf{R}\mathfrak{f} = \mathsf{R} \mathsf{D}{\mathfrak{f}_0} \tilde{\mathfrak{f}} = \tilde{\mathsf{R}} \tilde{\mathfrak{f}}.$

We will first take the steady-state approach to enforce the Kutta condition: allow the bound circulation to be set appropriately, with the implicit understanding that there is a starting vortex of equal and opposite circulation at infinity that preserves the Kelvin circulation theorem. The Lagrange multiplier for this constraint will not be $\Gamma_b$, but $s_0$.

$\begin{bmatrix}
\mathsf{L} & \tilde{\mathsf{R}} & 0\\
\mathsf{E} &  0 & \mathfrak{1}\\
0 & \mathfrak{e}_{k}^{T} & 0\\
\end{bmatrix} \begin{pmatrix} \mathsf{s} \\ \tilde{\mathfrak{f}} \\ s_0 \end{pmatrix} =
\begin{pmatrix} -\mathsf{w} \\ \mathfrak{s}'_b \\ 0 \end{pmatrix}.$

=#

# In `GridPotentialFlow.jl`, we create a regularized potential flow model by specifying the index of the regularized edges using the `edges` keyword when creating a `PotentialFlowBody`. In this case, we want to regularize the trailing edge, which is the last point of `Plate`, so its index is `length(plate)`.
pfb = PotentialFlowBody(plate,edges=[length(plate)])
model = VortexModel(g,bodies=[pfb],U∞=(1.0,0.0))
sol = solve(model);

# When there are no vortices present in the model, the package solves the above saddle point system and the solution is the steady state solution. Inspection of the streamlines shows that flow now indeed leaves the trailing edge smoothly.
plot(sol.ψ,g);
plot!(plate,linecolor=:black,linewidth=2,xlabel="x",ylabel="y")

# By plotting the vortex sheet strength, we can see that by constraining the trailing edge point of $\tilde{\mathfrak{f}}$, its whole distribution is shifted upward compared to the unregularized case such that the last value equals zero.
plot(plot(sol.f,xlabel="body point index",ylabel="f",legend=false),plot(sol.f./f₀,xlabel="body point index",ylabel="f̃",legend=false),size=[800,300])

#=
## Using the Kutta condition to set a new vortex element

In the previous section, we used the Kutta condition to set the bound circulation but did not explicitly create a new vortex element. This vortex element was assumed to lie at infinity so that its effect was negligible except insofar as it left equal but opposite circulation about the body.

In this section, we will create a new vortex element in the vicinity of the edge at which we are applying the Kutta condition. We will thus seek to establish the strength of this new element and to do so in such a manner that the overall circulation of the flow is conserved. Once the element is created, it will be allowed to advect with the local fluid velocity (minus its own contribution to this velocity).

Let us assume that the new vortex element (which we label with the subscript 1) is introduced at some point in physical space, and that its immersion into the Cartesian grid is described by a grid vector $\mathsf{d}_1$ and that its strength (i.e., its circulation) is $\delta\Gamma_1$. Thus, the fluid vorticity after this new element's introduction can be written as

$\mathsf{w} + \delta\Gamma_1 \mathsf{d}_1.$

The vector $\mathsf{d}_1$ is the discrete Dirac delta function, identical (or similar) to the function that constitutes the regularization $\mathsf{R}$ and interpolation $\mathsf{E}$ matrices.

The Kutta condition is still to be enforced. We also seek to ensure that the total circulation is zero. (We are assuming that the flow has started from rest.) Let us denote the circulation of the existing fluid vorticity $\mathsf{w}$ by

$\Gamma_{\mathsf{w}} = \Delta x \, \mathsf{1}^{T}\mathsf{w},$

where $\mathsf{1}$ is a grid vector of ones. Then, the circulation constraint is

$\mathfrak{1}^{T} \mathfrak{f} + \delta\Gamma_1 + \Gamma_{\mathsf{w}} = 0.$

The circulation of the bound vortex sheet $\mathfrak{f}$ can be re-written in terms of the smooth part of the sheet as $\mathfrak{1}^{T} \mathfrak{f} = \mathfrak{f}_0 \tilde{\mathfrak{f}}$.

With these two constraints, the overall saddle point system of equations is

$\begin{bmatrix}
\mathsf{L} & \tilde{\mathsf{R}} & 0 & \mathsf{d}_1\\
\mathsf{E} &  0 & \mathfrak{1} & 0\\
0 & \mathfrak{e}_{k}^{T} & 0 & 0\\
0 & \mathfrak{f}_{0}^{T} & 0 & 1
\end{bmatrix} \begin{pmatrix} \mathsf{s} \\ \tilde{\mathfrak{f}} \\ s_0 \\ \delta\Gamma_1 \end{pmatrix} =
\begin{pmatrix} -\mathsf{w} \\ \mathfrak{s}'_b \\ 0 \\ -\Gamma_{\mathsf{w}} \end{pmatrix}.$

=#

# To simulate this case for our flat plate, we need to insert a point vortex in the flow, preferably near the trailing edge. This situation corresponds to the flow right after impulsively starting a uniform flow around a flat plate and the point vortex now represents the starting vortex. We position the vortex behind and orthogonal the plate at a distance that is proportional to the convective length from the trailing edge. This position does not matter much here, but is of importance when we step in time and insert more point vortices every time step. Its strength can be set arbitrarily, because any vortices that are used for regularizing edges will be ignored when calculating $\mathsf{w}$ in the right-hand side for the saddle-point system.
Δt = 1e-2
vTE = Vortex(plate.x[end]+3*Δt*cos(α+π/2),plate.y[end]+3*Δt*sin(α+π/2),0.0);

# We include the point vortex when we create our model. Because the package detects that there is a vortex in the model, it solves the unsteady saddle point system and the solution will include the strength for the new point vortex.
pfb = PotentialFlowBody(plate,edges=[length(plate)])
model = VortexModel(g,bodies=[pfb],vortices=[vTE],U∞=(1.0,0.0))
sol = solve(model);
plot(sol.ψ,g);
plot!(plate,linecolor=:black,linewidth=2)
scatter!(model.vortices.x,model.vortices.y,color=:black,markersize=2,xlabel="x",ylabel="y")
# Note that the strength of the new point vortex is not automatically set when calling `solve` on the vortex model

# Because of the proximity of the point vortex to the flat plate, $\tilde{\mathfrak{f}}$ exhibits a quick variation in its value at the surface points that lie closest to the point vortex. The value at the trailing edge point itself is still constrained to zero.
plot(plot(sol.f,xlabel="body point index",ylabel="f",legend=false),plot(sol.f./f₀,xlabel="body point index",ylabel="f̃",legend=false),size=[800,300])

#=
## Applying more than one Kutta condition on a body

Suppose we wish to enforce the Kutta condition at two edges of the body---at points $k_{1}$ and $k_{2}$---instead of one. Each such point has a constraint,

$\mathfrak{e}_{k_j}^{T} \tilde{\mathfrak{f}} = 0,\quad j = 1,2.$

For two such constraints, we need two Lagrange multipliers: the strengths of two new vortices, $\delta\Gamma_1$ and $\delta\Gamma_2$, immersed into the grid with $\mathsf{d}_1$ and $\mathsf{d}_2$, respectively; and we still need the Lagrange multiplier $s_0$ to ensure that Kelvin's circulation theorem is also enforced. The system in the previous section is thus easily generalized to the following:

$\begin{bmatrix}
\mathsf{L} & \tilde{\mathsf{R}} & 0 & \mathsf{d}_1 & \mathsf{d}_2\\
\mathsf{E} &  0 & \mathfrak{1} & 0& 0\\
0 & \mathfrak{e}_{k_1}^{T} & 0 & 0 & 0\\
0 & \mathfrak{e}_{k_2}^{T} & 0 & 0 & 0\\
0 & \mathfrak{f}_{0}^{T} & 0 & 1 & 1
\end{bmatrix} \begin{pmatrix} \mathsf{s} \\ \tilde{\mathfrak{f}} \\ s_0 \\ \delta\Gamma_1 \\ \delta\Gamma_2 \end{pmatrix} =
\begin{pmatrix} -\mathsf{w} \\ \mathfrak{s}'_b \\ 0 \\ 0 \\ -\Gamma_{\mathsf{w}} \end{pmatrix}.$
=#

# We now simulate this case again for our flat plate problem. This solution corresponds to the flow right after impulsively starting a uniform flow around a flat plate, but unlike the previous case, the flow now also separates at the leading edge.

# We create two vortices, one near the leading edge and one near the trailing edge.
vLE = Vortex(plate.x[1]+3*Δt*plate.len*cos(plate.α+π/2),plate.y[1]+3*Δt*plate.len*sin(plate.α+π/2),0.0);
vTE = Vortex(plate.x[end]+3*Δt*cos(α+π/2),plate.y[end]+3*Δt*sin(α+π/2),0.0);

# The model now has two regularized edges, corresponding to the first body point and the last one.
pfb = PotentialFlowBody(plate,edges=[1,length(plate)])
model = VortexModel(g,bodies=[pfb],vortices=[vLE,vTE],U∞=(1.0,0.0))
sol = solve(model);
plot(sol.ψ,g);
plot!(plate,linecolor=:black,linewidth=2)
scatter!(model.vortices.x,model.vortices.y,markersize=2,xlabel="x",ylabel="y")

#

plot(plot(sol.f,xlabel="body point index",ylabel="f",legend=false),plot(sol.f./f₀,xlabel="body point index",ylabel="f̃",legend=false),size=[800,300])


# ---
# title: 650. 2 Keys Keyboard
# id: problem650
# author: Indigo
# date: 2021-06-28
# difficulty: Medium
# categories: Dynamic Programming
# link: <https://leetcode.com/problems/2-keys-keyboard/description/>
# hidden: true
# ---
#
# Initially on a notepad only one character 'A' is present. You can perform two
# operations on this notepad for each step:
#
#   1. `Copy All`: You can copy all the characters present on the notepad (partial copy is not allowed).
#   2. `Paste`: You can paste the characters which are copied **last time**.
#
#
#
# Given a number `n`. You have to get **exactly** `n` 'A' on the notepad by
# performing the minimum number of steps permitted. Output the minimum number of
# steps to get `n` 'A'.
#
# **Example 1:**
#
#
#
#     Input: 3
#     Output: 3
#     Explanation:
#     Intitally, we have one character 'A'.
#     In step 1, we use **Copy All** operation.
#     In step 2, we use **Paste** operation to get 'AA'.
#     In step 3, we use **Paste** operation to get 'AAA'.
#
#
#
#
# **Note:**
#
#   1. The `n` will be in the range [1, 1000].
#
#
#
#
## @lc code=start
using LeetCode

function min_steps650(n::Int)
    res, k = 0, 2
    while n != 1
        while n % k == 0
            res += k
            n ÷= k
        end
        k += 1
    end
    return res
end
## @lc code=end


# used in shortest path calculations
# has_distances{T}(distmx::AbstractArray{T,2}) =
#     issparse(distmx)? (nnz(distmx) > 0) : !isempty(distmx)

type DefaultDistance<:AbstractArray{Int, 2}
    nv::Int
    DefaultDistance(nv::Int=typemax(Int)) = new(nv)
end

getindex(::DefaultDistance, s::Int, d::Int) = 1
getindex(::DefaultDistance, s::UnitRange, d::UnitRange) = DefaultDistance(length(s))
size(d::DefaultDistance) = (d.nv, d.nv)
transpose(d::DefaultDistance) = d
ctranspose(d::DefaultDistance) = d

"""
Calculates the eccentricity[ies] of a vertex `v`, vertex vector `vs`, or the
entire graph. An optional matrix of edge distances may be supplied.

The eccentricity of a vertex is the maximum shortest-path distance between it
and all other vertices in the graph.

Because this function must calculate shortest paths for all vertices supplied
in the argument list, it may take a long time.

The output is either a single float (when a single vertex is provided) or a
vector of floats corresponding to the vertex vector. If no vertex vector is
provided, the vector returned corresponds to each vertex in the graph.

Note: the eccentricity vector returned by `eccentricity()` may be used as input
for the rest of the distance measures below. If an eccentricity vector is
provided, it will be used. Otherwise, an eccentricity vector will be calculated
for each call to the function. It may therefore be more efficient to calculate,
store, and pass the eccentricities if multiple distance measures are desired.
"""
function eccentricity{T}(
    g::SimpleGraph,
    v::Int,
    distmx::AbstractArray{T, 2} = DefaultDistance()
)
    e = maximum(dijkstra_shortest_paths(g,v,distmx).dists)
    e == typemax(T) && error("Infinite path length detected")

    return e
end

eccentricity{T}(
    g::SimpleGraph,
    vs::AbstractArray{Int, 1}=vertices(g),
    distmx::AbstractArray{T, 2} = DefaultDistance()
) =
    [eccentricity(g,v,distmx) for v in vs]

eccentricity{T}(g::SimpleGraph, distmx::AbstractArray{T, 2}) =
    eccentricity(g, vertices(g), distmx)

"""Returns the maximum eccentricity of the graph."""
diameter{T}(all_e::Vector{T}) = maximum(all_e)
diameter{T}(g::SimpleGraph, distmx::AbstractArray{T, 2} = DefaultDistance()) =
    maximum(eccentricity(g, distmx))

"""Returns the set of all vertices whose eccentricity is equal to the graph's
diameter (that is, the set of vertices with the largest eccentricity).
"""
function periphery{T}(all_e::Vector{T})

    diam = maximum(all_e)
    return filter((x)->all_e[x] == diam, 1:length(all_e))
end

periphery{T}(g::SimpleGraph, distmx::AbstractArray{T, 2} = DefaultDistance()) =
    periphery(eccentricity(g, distmx))

"""Returns the minimum eccentricity of the graph."""
radius{T}(all_e::Vector{T}) = minimum(all_e)
radius{T}(g::SimpleGraph, distmx::AbstractArray{T, 2} = DefaultDistance()) =
    minimum(eccentricity(g, distmx))

"""Returns the set of all vertices whose eccentricity is equal to the graph's
radius (that is, the set of vertices with the smallest eccentricity).
"""
function center{T}(all_e::Vector{T})
    rad = radius(all_e)
    return filter((x)->all_e[x] == rad, 1:length(all_e))
end

center{T}(g::SimpleGraph, distmx::AbstractArray{T, 2} = DefaultDistance()) =
    center(eccentricity(g, distmx))


# Visual demonstration of the tracing scheme in different analytical field.
using PyPlot, FieldTracer

include("../../test/utility/dipole.jl")

"""
    test_trace_asymptote(issingle=false)

Test streamline tracing by plotting vectors and associated streamlines through a simple
velocity field where Vx=x, Vy=-y. Support for single and double precision data.
"""
function test_trace_asymptote(issingle=false)

   # Start by creating a velocity vector field.
   if issingle
      xmax, ymax = 200.0f0, 20.0f0
      x = -10.0f0:0.25f0:xmax
      y = -10.0f0:0.25f0:ymax
   else
      xmax, ymax = 200.0, 20.0
      x = -10.0:0.25:xmax
      y = -10.0:0.25:ymax
   end

   xgrid = [i for j in y, i in x]
   ygrid = [j for j in y, i in x]

   if issingle
      vx = xgrid * 1.0f0
      vy = ygrid * -1.0f0

      xstart = 1.0f0
      ystart = 10.0f0
   else
      vx = xgrid * 1.0
      vy = ygrid * -1.0

      xstart = 1.0
      ystart = 10.0
   end

   gridtype = "meshgrid"

   x1, y1 = trace(vx, vy, xstart, ystart, x, y; alg=RK4(), ds=0.1, gridtype)
   x2, y2 = trace(vx, vy, xstart, ystart, x, y; alg=RK4(), ds=0.5, gridtype)
   x3, y3 = trace(vx, vy, xstart, ystart, x, y; alg=RK4(), ds=1.0, gridtype)
   x4, y4 = trace(vx, vy, xstart, ystart, x, y; alg=Euler(), ds=0.1, gridtype)
   x5, y5 = trace(vx, vy, xstart, ystart, x, y; alg=Euler(), ds=0.5, gridtype)
   x6, y6 = trace(vx, vy, xstart, ystart, x, y; alg=Euler(), ds=1.0, gridtype)

   # analytical solution const = x*y
   c = xstart * ystart
   if issingle
      x_anly = 1.0f0:0.001f0:xmax
   else
      x_anly = 1.0:0.001:xmax
   end
   y_anly = c ./ x_anly

   fig = plt.figure(figsize=(8,6))
   ax1 = plt.subplot(111)
   ax1.plot(x_anly, y_anly, "r--", label="Analytic",linewidth=3.0)
   ax1.plot(x1, y1, "b",   label="RK4 ds=0.1", linewidth=1.5)
   ax1.plot(x2, y2, "b--", label="RK4 ds=0.5", linewidth=1.5)
   ax1.plot(x3, y3, "b:",  label="RK4 ds=1.0", linewidth=1.5)
   ax1.plot(x4, y4, "g", label="Euler ds=0.1", linewidth=.75)
   ax1.plot(x5, y5, "g--", label="Euler ds=0.5", linewidth=.75)
   ax1.plot(x6, y6, "g:",  label="Euler ds=1.0", linewidth=.75)
   ax1.legend(loc="upper right")
   if issingle
      ax1.set_title("Runge Kutta 4 vs Euler's: Asymptotic Field, Single Precision")
   else
      ax1.set_title("Runge Kutta 4 vs Euler's: Asymptotic Field, Double Precision")
   end
   ax1.set_xlabel("Normalized X Coordinate")
   ax1.set_ylabel("Normalized Y Coordinate")
   ax1.set_xlim([3, 30])
   ax1.set_ylim([0.25, 2.5 ])

   ax1.annotate("Euler's method diverges strongly\nalong curves except when"*
      " taking \nvery small steps.  RK4 is\nfar more accurate for all dS.",
      xy=(10.5,0.83), xycoords="data", xytext=(9,.3),
      arrowprops=Dict(:fc=>"black",:shrink=>0.05),
      horizontalalignment="center")
   ax1.annotate("Tracing begins at x=1, y=10.",
      xy=(4.8,2.45), xycoords="data", xytext=(6,1.9),
      arrowprops=Dict(:fc=>"black",:shrink=>0.05))
   ax1.annotate("The trace continues until \n"*
      "x=200.  At that point, Euler \n"*
      "dS=0.1 and RK4 dS=1.0 converge\n"*
      " at the same point, despite \n"*
      "the 10X difference in step size.",
      xy=(29,.5), xycoords="data", xytext=(20,1),
      arrowprops=Dict(:fc=>"black",:shrink=>0.005),
      horizontalalignment="center")
end

"Trace field lines through a Earth like magnetic dipole field."
function test_trace_dipole()
   # Start by creating a field of unit vectors.
   x = -100.0:5.0:101.0
   y = -100.0:5.0:101.0

   bx, by = b_hat(x, y)

   fig2 = plt.figure(figsize=(12,6))
   fig2.subplots_adjust(wspace=0.15, left=0.08, right=0.94)
   ax2 = plt.subplot(121)
   ax3 = plt.subplot(322)
   ax4a= plt.subplot(347)
   ax4b= plt.subplot(348)
   ax5 = plt.subplot(326)
   ax2.quiver(x,y, bx, by, units="x", pivot="middle")
   ax3.quiver(x,y, bx, by, units="x", pivot="tip")
   ax4a.quiver(x,y,bx, by, units="x", pivot="tip")
   ax4b.quiver(x,y,bx, by, units="x", pivot="tip")
   ax5.quiver(x,y, bx, by, units="x", pivot="tip")

   # Trace through this field.
   xstart = 10.0 # ystart = 25.0
   ds = 0.1
   gridtype = "meshgrid"
   for ystart in 0.:5.:31.
      x1, y1 = trace(bx, by, xstart, ystart, x, y; alg=RK4(), ds, gridtype)
      l1 = ax2.plot(x1,y1,"b")[1]
      ax3.plot(x1,y1,"b"); ax4b.plot(x1,y1,"b")
      ax5.plot(x1,y1,"b"); ax4a.plot(x1,y1,"b")
      x2, y2 = trace(bx, by, xstart, ystart, x, y; alg=Euler(), ds, gridtype)
      l2 = ax2.plot(x2,y2,"r")[1]
      ax3.plot(x2,y2,"r"); ax4b.plot(x2,y2,"r")
      ax5.plot(x2,y2,"r"); ax4a.plot(x2,y2,"r")
      x3, y3 = b_line(xstart, ystart, npoints=300)
      l3 = ax2.plot(x3,y3,"k--")[1]
      ax3.plot(x3,y3,"k--"); ax4b.plot(x3,y3,"k--")
      ax5.plot(x3,y3,"k--"); ax4a.plot(x3,y3,"k--")

      ax2.set_xlim([-2,  100])
      ax2.set_ylim([-30, 100])
      ax2.set_title("Full View")
      ax2.set_xlabel("Normalized X Coordinate")
      ax2.set_ylabel("Normalized Y Coordinate")
      ax2.legend((l1, l2, l3),("RK4", "Euler", "Analytical"), loc="upper left")

      ax3.set_title("Zoomed Views")
      ax3.set_xlim([8.5, 17.5])
      ax3.set_ylim([3, 33])
      goodpos = ax3.get_position()

      ax4a.set_xlim([20,30])
      ax4a.set_ylim([-12,12])
      pos = ax4a.get_position()
      pos.x0 = goodpos.x0
      pos.x1 = pos.x0 + (goodpos.x1-goodpos.x0)/2.0 -0.01
      ax4a.set_position(pos)

      ax4b.set_xlim([50,60])
      ax4b.set_ylim([-12,12])
      pos = ax4b.get_position()
      pos.x0 = goodpos.x0 + (goodpos.x1-goodpos.x0)/2.0 +0.01
      ax4b.set_position(pos)
      ax4b.yaxis.set_ticklabels([])

      ax5.set_xlim([1,7])
      ax5.set_ylim([-7,-3])
      ax5.set_xlabel("Normalized X Coordinate")

      fig2.suptitle("RK4 vs Euler's Method: Dipole Field for dS="*
         string(round(ds,digits=3)))
   end
end


function lpc2c!(c::AbstractVector, a::AbstractVector, loggain::Bool=false)
    @assert length(a) == length(c)
    m = length(a) - 1
    fill!(c, zero(eltype(c)))
    if loggain
        c[1] = a[1]
    else
        c[1] = log(a[1])
    end
    c[2] = -a[2]
    for k=3:m+1
        d = 0.0
        upperlimit = (k >= m+1) ? m+1 : k
        for i=2:upperlimit
            d += (i-1) *  c[i] * a[k-i+1]
        end
        c[k] = -d / (k-1) - a[k]
    end

    c
end

lpc2c(a::AbstractVector, loggain::Bool=false) = lpc2c!(similar(a), a, loggain)

function lpc2c(state::SpectralParamState{LinearPredictionCoef})
    assert_not_ready_to_filt(state)

    def = paramdef(state)
    newdef = LinearCepstrum(param_order(def))
    data = rawdata(state)
    SpectralParamState(newdef, lpc2c(data, has_loggain(state)),
                       true, gain_normalized(state))
end


using Test
using NeXLCore

@testset "Compton" begin
    cxr = n"U L3-M5"
    @test isapprox(comptonShift(π/4, cxr), 1.0/(1.0+(energy(cxr)/0.511e6)*(1.0-cos(π/4))), rtol=1.0e-5)
    @test isapprox(energy(cxr)*(1.0 - comptonShift(π, cxr)), 689.0, atol=1.0)
    @test isapprox(comptonEnergy(π/3, cxr), energy(cxr)/(1.0+(energy(cxr)/0.511e6)*(1.0-cos(π/3))), rtol=1.0e-5)
    @test isapprox(energy(cxr) - comptonEnergy(π, cxr), 689.0, atol=1.0)
    @test energy(cxr) - comptonEnergy(π/2, cxr) < 689.0
    # These test values are taken from Wikipedia
    @test isapprox(comptonAngular(π/2, 2.75) / comptonAngular(0.0, 2.75), 0.5, atol=0.0001)
    @test isapprox(comptonAngular(π/2, 60.0e3) / comptonAngular(0.0, 60.0e3), 0.4, atol=0.02)
    @test isapprox(comptonAngular(π/2, 511.0e3) / comptonAngular(0.0, 511.0e3), 0.2, atol=0.02)
end


#= single parameter error-free transformations =#

# 'y' must be negated to get the right result
@inline function eftRecip{T<:AbstractFloat}(a::T)
     x = one(T)/a
     y = -(fma(x,a,-1.0)/a)
     x,y
end

#=
   While not strictly an error-free transformation it is quite reliable and recommended for use.
   Augmented precision square roots, 2-D norms and discussion on correctly reounding sqrt(x^2 + y^2)
   by Nicolas Brisebarre, Mioara Joldes, Erik Martin-Dorel, Hean-Michel Muller, Peter Kornerup
=#

@inline function eftSqrt(a::AbstractFloat)
     x = sqrt(a)
     t = fma(x,-x,a)
     y = t / (x*2.0)
     x,y
end

@inline function eftRecipSqrt(a::Float64)
     r = 1.0/a
     x = sqrt(r)
     t = fma(x,-x,r)
     y = t / (x*2.0)
     x,y
end

@inline function eftSquare(a::Float64)
    x = a * a
    y = fma(a, a, -x)
    x,y
end

@inline function eftCube(a::Float64)
    p = a*a; e = fma(a, a, -p)
    x = p*a; p = fma(p, a, -x)
    y = e*a
    x,y
end

#= two parameter error-free transformations =#

@inline function eftSum2{T<:AbstractFloat}(a::T, b::T)
  x = a + b
  t = x - a
  y = (a - (x - t)) + (b - t)
  x,y
end

@inline function eftSum2inOrder{T<:AbstractFloat}(a::T, b::T)
  x = a + b
  y = b - (x - a)
  x,y
end

@inline function eftDiff2{T<:AbstractFloat}(a::T, b::T)
  x = a - b
  t = x - a
  y = (a - (x - t)) - (b + t)
  x,y
end

@inline function eftDiff2inOrder{T<:AbstractFloat}(a::T, b::T)
  x = a - b
  y = (a - x) - b
  x,y
end

@inline function eftProd2{T<:AbstractFloat}(a::T, b::T)
    x = a * b
    y = fma(a, b, -x)
    x,y
end

# 'y' must be negated to get the right result
@inline function eftDiv2{T<:AbstractFloat}(a::T,b::T)
     x = a/b
     y = -(fma(x,b,-a)/b)
     x,y
end

#= three parameter error-free transformations =#

function eftSum3{T<:Float64}(a::T,b::T,c::T)
    s,t = eftSum2(b, c)
    x,u = eftSum2(a, s)
    y,z = eftSum2(u, t)
    x,y = eftSum2inOrder(x, y)
    x,y,z
end

function eftSum3inOrder{T<:Float64}(a::T,b::T,c::T)
    s,t = eftSum2inOrder(b, c)
    x,u = eftSum2inOrder(a, s)
    y,z = eftSum2inOrder(u, t)
    x,y = eftSum2inOrder(x, y)
    x,y,z
end

function eftProd3{T<:Float64}(a::T, b::T, c::T)
    p,e = eftProd2(a,b)
    x,p = eftProd2(p,c)
    y,z = eftProd2(e,c)
    x,y,z
end

function eftFMA{T<:Float64}(a::T, b::T, c::T)
    x = fma(a,b,c)
    u1,u2 = eftProd2(a,b)
    a1,a2 = eftSum2(u2,c)
    b1,b2 = eftSum2(u1,a1)
    g = (b1-x)+b2
    y,z = eftSum2inOrder(g,a2)
    x,y,z
end

function eftFMS{T<:Float64}(a::T, b::T, c::T)
    x = fma(a,b,c)
    u1,u2 = eftProd2(a,b)
    a1,a2 = eftDiff2(u2,c)
    b1,b2 = eftSum2(u1,a1)
    g = (b1-x)+b2
    y,z = eftSum2inOrder(g,a2)
    x,y,z
end


using BenchmarkTools
using FinancialMonteCarlo, CUDA
CUDA.allowscalar(false)

S0 = 100.0;
K = 100.0;
r = 0.02;
T = 1.0;
d = 0.01;
D = 90.0;

Nsim = 10000;
Nstep = 30;
sigma = 0.2;
lam = 5.0;
mu1 = 0.03;
sigma1 = 0.02;
mc = MonteCarloConfiguration(Nsim, Nstep);

mc_2 = MonteCarloConfiguration(Nsim, Nstep, FinancialMonteCarlo.CudaMode());
toll = 0.8;

rfCurve = ZeroRate(r);

FwdData = Forward(T)
EUData = EuropeanOption(T, K)
AMData = AmericanOption(T, K)
BarrierData = BarrierOptionDownOut(T, K, D)
AsianFloatingStrikeData = AsianFloatingStrikeOption(T)
AsianFixedStrikeData = AsianFixedStrikeOption(T, K)
Model = MertonProcess(sigma, lam, mu1, sigma1, Underlying(S0, d));

@show "CUDA_1 fwd"

@show "STD fwd"
@btime FwdPrice = pricer(Model, rfCurve, mc, FwdData);
@show "CUDA_1 fwd"

@show "CUDA_2 fwd"
@btime FwdPrice = pricer(Model, rfCurve, mc_2, FwdData);
@show "std eu"
@btime EuPrice = pricer(Model, rfCurve, mc, EUData);
@show "CUDA_1 eu"

@show "CUDA_2 eu"
@btime EuPrice = pricer(Model, rfCurve, mc_2, EUData);
@show "std am"
@btime AmPrice = pricer(Model, rfCurve, mc, AMData);
@show "CUDA_1 am"

@show "CUDA_2 am"
@btime AmPrice = pricer(Model, rfCurve, mc_2, AMData);


using LinearAlgebra

Cep = Matrix(0.5I, 2, 2);                                                           # quadratic position cost matrix
Cu = Matrix(0.2I, 2, 2);                                                            # quadratic control cost matrix
β_pos = 0.1;                                                                        # relative weight between instant and terminal pos cost
α_col = 100.0;                                                                      # scale parameter for exponential collision cost
β_col = 0.1;                                                                        # relative weight between instant and terminal pos cost
λ_col = 0.2;                                                                        # bandwidth parameter for exponential collision cost
σ_risk = 0.0;                                                                       # risk-sensitiveness parameter


@testset "Finite-differences" begin
    include("basis_functions.jl")
    include("scalar_operators.jl")
    include("inner_products.jl")

    @testset "Function interpolation" begin
        R = FiniteDifferences(20,1.0)
        R⁻¹ = pinv(R)
        r̃ = locs(R)
        χ = R[r̃,:]

        r = axes(R,1)

        y = Inclusion(0.0..5.0)
        @test_throws DimensionMismatch R \ y.^2

        fu = r -> r^2*exp(-r)
        u = R*(R\fu.(r))
        @test norm(χ * (R⁻¹*u) - fu.(r̃)) == 0

        fv = r -> r^6*exp(-r)
        v = R*(R\fv.(r))
        @test norm(χ * (R⁻¹*v) - fv.(r̃)) == 0
    end

    include("derivatives.jl")
end


@defcomp damages begin
    DAMFRAC = Variable(index=[time])    # Damages (fraction of gross output)

    TATM    = Parameter(index=[time])   # Increase temperature of atmosphere (degrees C from 1900)
    YGROSS  = Parameter(index=[time])   # Gross world product GROSS of abatement and damages (trillions 2005 USD per year)
    a1      = Parameter()               # Damage intercept
    a2      = Parameter()               # Damage quadratic term
    a3      = Parameter()               # Damage exponent

    TotSLR  = Parameter(index=[time])   # Path of total SLR
    b1      = Parameter()               # Coefficient on SLR
    b2      = Parameter()               # Coefficient on quadratic SLR term
    b3      = Parameter()               # SLR exponent

    function run_timestep(p, v, d, t)
        # Define function for DAMFRAC
        v.DAMFRAC[t] = p.a1 * p.TATM[t] + p.a2 * p.TATM[t]^p.a3 + p.b1 * p.TotSLR[t] + p.b2 * p.TotSLR[t]^p.b3
    end

end


"""
Variational inference for Bayesian neural network
"""
module BayesNeuralNet
using Distributions

export sample_data_from_prior, sample_data_from_posterior
export VI

function sigmoid(x)
    return 1.0 / (1.0 + exp.(-x[1]))
end

function rho2sig(rho)
    return log.(1 + exp.(rho))
end

function compute_df_dmu(mu, rho, W)
    return (W - mu) ./ rho2sig(rho).^2
end

function compute_df_drho(Y, X, mu, rho, W)
    return -0.5*((W - mu).^2 - rho2sig(rho).^2) .* compute_dprec_drho(rho)
end

function compute_dprec_drho(rho)
    return 2 * rho2sig(rho) .^ (-3) .* (1 ./ (1+exp.(rho))).^2 .* (1 ./ (1+exp.(-rho)))
end

function compute_df_dw(Y, X, sigma2_y, sigma2_w, mu1, rho1, W1, mu2, rho2, W2)
    M, N = size(X)
    Y_err1 = zeros(size(W1)) # MxK
    Y_err2 = zeros(size(W2)) # KxD

    for n in 1 : N
        Z = tanh.(W1'*X[:,n]) # Kx1
        Y_est = W2'*Z
        # 2nd unit, Dx1
        delta2 = Y_est - Y[n]

        # 1st unit, KxD
        delta1 = diagm(1 - Z.^2) * W2 * delta2

        Y_err1 += X[:,n] * delta1'
        Y_err2 += Z * delta2'
    end
    df_dw1 = W1/sigma2_w + (mu1 - W1) ./ rho2sig(rho1).^2 + Y_err1 / sigma2_y
    df_dw2 = W2/sigma2_w + (mu2 - W2) ./ rho2sig(rho2).^2 + Y_err2 / sigma2_y
    return df_dw1, df_dw2
end

"""
Sample data given prior and inputs.
"""
function sample_data_from_prior(X, sigma2_w, sigma2_y, D, K)
    M, N = size(X)

    W1 = sqrt(sigma2_w) * randn(M, K)
    W2 = sqrt(sigma2_w) * randn(K, D)

    # sample function
    Y = [W2'* tanh.(W1'X[:,n]) for n in 1 : N]

    # sample data
    Y_obs = [W2'* tanh.(W1'X[:,n]) + sqrt(sigma2_y)*randn(D) for n in 1 : N]

    return Y_obs, Y, W1, W2
end

"""
Sample data given posterior and inputs.
"""
function sample_data_from_posterior(X, mu1, rho1, mu2, rho2, sigma2_y, D)
    N = size(X, 2)
    ep1 = randn(size(mu1))
    W1_tmp = mu1 + log.(1 + exp.(rho1)) .* ep1
    ep2 = randn(size(mu2))
    W2_tmp = mu2 + log.(1 + exp.(rho2)) .* ep2
    Y_est = [W2_tmp'* tanh.(W1_tmp'X[:,n]) for n in 1 : N]
    Y_obs = [W2_tmp'* tanh.(W1_tmp'X[:,n]) + sqrt(sigma2_y)*randn(D)  for n in 1 : N]
    return Y_est, Y_obs
end

"""
Compute variational parameters.
"""
function VI(Y, X, sigma2_w, sigma2_y, K, alpha, max_iter)
    M, N = size(X)
    D = length(Y[1])

    # initialize
    mu1 = randn(M, K)
    rho1 = randn(M, K)
    mu2 = randn(K, D)
    rho2 = randn(K, D)

    for i in 1 : max_iter
        # sample
        ep1 = randn(size(mu1))
        W1_tmp = mu1 + log.(1 + exp.(rho1)) .* ep1
        ep2 = randn(size(mu2))
        W2_tmp = mu2 + log.(1 + exp.(rho2)) .* ep2

        # calc error
        df_dw1, df_dw2 = compute_df_dw(Y, X, sigma2_y, sigma2_w, mu1, rho1, W1_tmp, mu2, rho2, W2_tmp)

        # 1st unit
        df_dmu1 = compute_df_dmu(mu1, rho1, W1_tmp)
        df_drho1 = compute_df_drho(Y, X, mu1, rho1, W1_tmp)
        d_mu1 = df_dw1 + df_dmu1
        d_rho1 = df_dw1 .* (ep1 ./ (1+exp.(-rho1))) + df_drho1
        mu1 = mu1 - alpha * d_mu1
        rho1 = rho1 - alpha * d_rho1

        # 2nd unit
        df_dmu2 = compute_df_dmu(mu2, rho2, W2_tmp)
            df_drho2 = compute_df_drho(Y, X, mu2, rho2, W2_tmp)
        d_mu2 = df_dw2 + df_dmu2
        d_rho2 = df_dw2 .* (ep2 ./ (1+exp.(-rho2))) + df_drho2
        mu2 = mu2 - alpha * d_mu2
        rho2 = rho2 - alpha * d_rho2
    end
    return mu1, rho1, mu2, rho2
end

end


using LinearAlgebra

import LinearAlgebra: sylvester, lyap

function sylvester(A::StridedMatrix{T},B::StridedMatrix{T},C::StridedMatrix{T}) where T<:BigFloatOrComplex

  _sylvester_lyapunov(A, B, C, sylvester)
end

function _sylvester_lyapunov(A::StridedMatrix{T}, B::Union{StridedMatrix{T},Nothing}, C::StridedMatrix{T}, syly::Function) where T <: BigFloatOrComplex

  # make standard precision copy of all input matrices calculate first approximation
  P = Float64
  S = T <: Real ? P : Complex{P}
  AS = S.(A)
  if B === nothing
    B = A'
    BS = nothing
  else
    BS = S.(B)
  end
  CS = S.(C)
  XS = syly(AS, BS, CS)
  X = T.(XS)
  rtol = eps(real(T))
  ftol = cbrt(eps(P)) # 0.9
  mres = rtol * norm(CS, 1)
  mdx  = rtol * norm(XS, 1)
  pres = nres = pdx = ndx = big"Inf"

  while nres >= mres && ndx >= mdx && nres <= pres && ndx <= pdx
    pres = nres * ftol
    pdx = ndx * ftol
    R = A * X + X * B + C
    nres = norm(R, 1)
    CS = S.(R)
    dX = syly(AS, BS, CS)
    ndx = norm(dX, 1)
    if nres <= pres && ndx <= pdx
      X .+= dX
    end
    println("nres = $(Float64(nres)/Float64(mres)), ndx = $(Float64(ndx)/Float64(mdx))")
  end
  X
end

function lyap(A::StridedMatrix{T}, C::StridedMatrix{T}) where T<:BigFloatOrComplex

  lyapunov(A, B, C) = lyap(A, C)
  _sylvester_lyapunov(A, nothing, C, lyapunov)
end


using TuringModels
#using StatsPlots

# Define a simple Normal model with unknown mean and variance.

@model gdemo(x, y) = begin
  s ~ InverseGamma(2, 3)
  m ~ Normal(0, sqrt(s))
  x ~ Normal(m, sqrt(s))
  y ~ Normal(m, sqrt(s))
end

#  Run sampler, collect results

chn = sample(gdemo(1.5, 2), NUTS(0.65), 1000)

# Summarise results (currently requires the master branch from MCMCChains)

describe(chn) |> display

# Plot and save results

p = plot(chn)
#savefig("basic-example-plot.png")


export Orthogonal
import LinearAlgebra: svd

doc"""
    Orthogonal([gain=1.0])

# References
* Saxe et al., [Exact solutions to the nonlinear dynamics of learning in deep linear neural networks](http://arxiv.org/abs/1312.6120)
"""
struct Orthogonal
    gain::Float64
end
Orthogonal() = Orthogonal(1.0)

function (o::Orthogonal)(::Type{T}, dim1::Int, dim2::Int) where T
    a = randn(T, dim1, dim2)
    F = svd(a)
    # u, _, v = svd(a)
    q = size(F.U) == (dim1,dim2) ? F.U : F.Vt
    q .* T(o.gain)
end


#这种注释方式，不推荐大家使用，除非是需要用到数学公式之类的。
#因为此注释不能被julia插件解析，在vscode编辑器中无法悬停查看函数帮助。
@doc raw"""
    pt(a,b)

勾股定理，英文名为Pythagoras 也称为毕达哥拉斯定理。

在平面上的一个直角三角形中，两个直角边边长的平方加起来等于斜边长的平方。
如果设直角三角形的两条直角边长度分别是`a`和`b`，斜边长度是`c`，那么数学公式为：

``{\rm{c = }}\sqrt {{a^2} + {b^2}}``

返回 斜边长`c`
"""
function pt(a,b)
    c = sqrt(a^2+b^2)
end


using Distributions
using EmpiricalLikelihood
using Ipopt
using JuMP
using Random
using Test
using Statistics

het_moment(X, y, beta) = [y[1] - beta[1] - X[1] * beta[2]]

@testset "Heteroskedasticity" begin
    n = 50
    Random.seed!(213)
    β = [1., 1.]
    β_test = zeros(100, 2)
    for rep in 1:100
        X = rand(LogNormal(), n)
        ϵ = rand(Normal(), n)
        u = ϵ .* @. sqrt(.1 .+ .2 * X .+ .3 * X.^2)
        y = β[1] .+ X * β[2] .+ u
        β_test[rep, :] = optimize_el(1, het_moment, reshape(X, n, 1), reshape(y, n, 1), β, .35)[1]
    end
    @test mean(β_test, dims=1)[1, :] ≈ β atol=1e-1
end

@testset "Test JuMP interface" begin
    n = 50
    Random.seed!(213)
    β = [1., 1.]
    β_test = zeros(100, 2)
    for rep in 1:100
        X = rand(LogNormal(), n)
        ϵ = rand(Normal(), n)
        u = ϵ .* @. sqrt(.1 .+ .2 * X .+ .3 * X.^2)
        y = β[1] .+ X * β[2] .+ u
        model = setup_jump_problem(1, het_moment, reshape(X, n, 1), reshape(y, n, 1), β, .35)[1]
        set_optimizer(model, optimizer_with_attributes(Ipopt.Optimizer, "print_level" => 0))
        optimize!(model)
        β_test[rep, :] = @. value(model[:theta])
    end
    @test mean(β_test, dims=1)[1, :] ≈ β atol=1e-1
end


export ∅, 𝔻, NaturalNumber, @natural

@symbols ϕ
∅ = ExtensionalSet(Set([]))
𝔻 = IntensionalSet(ϕ, ¬(ϕ ∈ ∅))

# von Neumann ordinals
function NaturalNumber(x::Int)
    if x == 0
        return ∅
    end

    ExtensionalSet([NaturalNumber(n) for n ∈ 0:(x-1)])
end

macro natural(assignment)
    @assert assignment.head == :(=) "Operator must be the assignment operator (=)"
    quote
        $(esc(assignment.args[1])) = NaturalNumber($(assignment.args[2]))
    end
end

# special definitions
# natural numbers:
#  ℕ = ...


@testset "NfRel" begin
  @testset "issubfield" begin
    Qx, x = FlintQQ["x"]
    f = x^2 + 12x - 92
    K, a = NumberField(f, "a")
    Ky, y = K["y"]

    L, b = NumberField(y^2 + y + 1, "b")
    M, c = NumberField(y^6 + y^3 + 1, "c")

    d, LtoM = Hecke.issubfield(L, M)

    @test d == true
    @test parent(LtoM(b)) == M
  end

  @testset "isisomorphic" begin
    Qx, x = FlintQQ["x"]
    f = x^2 + 12x - 92
    K, a = NumberField(f, "a")
    Ky, y = K["y"]

    g = y^5 - 5
    L, b = NumberField(g, "b")
    bb = 5*b - 2
    h = minpoly(bb)
    L2, b2 = NumberField(h, "b2")

    c, LtoL2 = Hecke.isisomorphic(L, L2)
    @test c == true
    @test parent(LtoL2(b)) == L2

    #i = g - 1
    #L3, b3 = NumberField(i, "b3")
    #d, LtoL3 = Hecke.isisomorphic(L, L3)
    #@test d == false
  end

  @testset "signature" begin
    Qx, x = FlintQQ["x"]
    f = x^2 + 12x - 92
    K, a = NumberField(f, "a")
    Ky, y = K["y"]

    g = y^5 - 5
    L, b = NumberField(g, "b")
    @test signature(L) == (2, 4)

    Qx, x = FlintQQ["x"]
    f = x^2 + 12x - 92
    K, a = NumberField(f, "a")
    Ky, y = K["y"]

    L, b = NumberField(y^2 + y + 1, "b")
    @test signature(L) == (0, 2)
  end

  @testset "rand" begin
    Qx, x = FlintQQ["x"]
    f = x^2 + 12x - 92
    K, a = NumberField(f, "a")
    Ky, y = K["y"]

    L, b = NumberField(y^2 + y + 1, "b")

    m = make(L, 1:3)
    for x in (rand(L, 1:3), rand(rng, L, 1:3), rand(m), rand(rng, m))
      @test x isa Hecke.NfRelElem{nf_elem}
    end
    @test rand(m, 3) isa Vector{Hecke.NfRelElem{nf_elem}}
    @test reproducible(m)
    @test reproducible(L, 1:3)
  end

  @testset "norm" begin
    K, a = Hecke.rationals_as_number_field()
    Kt, t = K["t"]
    L, b = NumberField(t - 1, "b")
    Lt, t = L["t"]
    M, o = NumberField(t^3 + 2, "o")
    @test -1 == @inferred norm(o + 1)

    K, a = Hecke.rationals_as_number_field()
    Kt, t = K["t"]
    L, b = NumberField(t^3 + 2, "b")
    @test -1 == @inferred norm(b + 1, true)
  end
end