using Plots
using LinearAlgebra
using SparseArrays
using Arpack
using ForwardDiff
using BenchmarkTools
using Formatting
using FileIO

#readfile
println("opening file... \n")
a = open("../programing/efelkismos-simple.unv","r")
include("readfile.jl") #returns array of: nodes, edge, elem, bc

include("elastic_functions.jl") #defines basic functions

#start building global array
n = length(elem[:,1])
dofs = 2*length(nodes[:,1])
global E = Ematrix(220e9,0.5,"stress")
global K = sparse([],[],Float64[],dofs,dofs);
global M = sparse([],[],Float64[],dofs,dofs);

println("calculating stiffness/mass matrix...\n")
h =1
@time begin
for i =1:n
    i1 = elem[i,:]'
    xs = nodes[i1,1]
    ys = nodes[i1,2]
    i1 = [2 .*i1 (i1 .* 2 .-1)]
    i2 = sort(i1,dims=2)
    @show i2
    kre = h .* ke(E,xs,ys)
    mre = h .* 7e3 .* me(xs,ys)
    K[i1,i1] = K[i1,i1][1,:,1,:] + kre
    M[i1,i1] = M[i1,i1][1,:,1,:] + mre
end
end

include("rhs.jl") #calculate F

#gib traction nodes where (u,v) are known
#reminder, ix contains boundary condition nodes INDEXES
println("solving for U....\n")
freedofs = zeros(dofs,1)
fixedofs = edge[bc[ix[1]],:][:]'

freedofs[[2 .*fixedofs (2 .* fixedofs .-1)]] .= 1
freedofs = [i for i=1:dofs if freedofs[i]==0]
fixedofs = [2 .* fixedofs (2 .* fixedofs .-1)]
U = zeros(dofs,1)
rhs = F
rhs = rhs[freedofs] - K[freedofs,fixedofs'][:,:,1]*U[fixedofs']
Kfree = K[freedofs,freedofs]
Mfree = M[freedofs,freedofs]

Ufree = U[freedofs]
U[freedofs] = Kfree\rhs

n = length(elem[:,1])
for i=1:n
    i1 = elem[i,:]'
    xs = nodes[i1,1]
    ys = nodes[i1,2]
    i1 = [2 .*i1 (i1 .* 2 .-1)]
end

println("calculating element average stress....\n")
#calculate element stress
include("stress.jl")
#println(stress')
println("DONE! \n")

i1 = elem[1,:]'
xs = nodes[i1,1]
ys = nodes[i1,2]

println(E*b([1 1 1]./3,xs,ys)*U[[2 .* i1 (2 .* i1 .- 1)]]')

xs = [1 0 0 0.5 0 0.5]
ys = [0 -.5 .5 -.25 0 .25]
l = [1 0 0]


#!/usr/bin/env julia

function fact(n::Int)::Int
    n >= 0 || error("fact argument must be positive")
    n <= 1 && return 1
    return n*fact(n - 1)
end

for n in 5:-1:-1
    print("fact($n) = ")
    println(fact(n))
end


function optimize(f::Function,
                  initial_x::Array;
                  method = NelderMead(),
                  x_tol::Real = 1e-32,
                  f_tol::Real = 1e-32,
                  g_tol::Real = 1e-8,
                  iterations::Integer = 1_000,
                  store_trace::Bool = false,
                  show_trace::Bool = false,
                  extended_trace::Bool = false,
                  show_every::Integer = 1,
                  autodiff::Bool = false,
                  callback = nothing)
    options = OptimizationOptions(;
        x_tol = x_tol, f_tol = f_tol, g_tol = g_tol,
        iterations = iterations, store_trace = store_trace,
        show_trace = show_trace, extended_trace = extended_trace,
        callback = callback, show_every = show_every,
        autodiff = autodiff)
    optimize(f, initial_x, method, options)
end

function optimize(f::Function,
                  g!::Function,
                  initial_x::Array;
                  method = LBFGS(),
                  x_tol::Real = 1e-32,
                  f_tol::Real = 1e-32,
                  g_tol::Real = 1e-8,
                  iterations::Integer = 1_000,
                  store_trace::Bool = false,
                  show_trace::Bool = false,
                  extended_trace::Bool = false,
                  show_every::Integer = 1,
                  callback = nothing)
    options = OptimizationOptions(;
        x_tol = x_tol, f_tol = f_tol, g_tol = g_tol,
        iterations = iterations, store_trace = store_trace,
        show_trace = show_trace, extended_trace = extended_trace,
        callback = callback, show_every = show_every)
    optimize(f, g!, initial_x, method, options)
end

function optimize(f::Function,
                  g!::Function,
                  h!::Function,
                  initial_x::Array;
                  method = Newton(),
                  x_tol::Real = 1e-32,
                  f_tol::Real = 1e-32,
                  g_tol::Real = 1e-8,
                  iterations::Integer = 1_000,
                  store_trace::Bool = false,
                  show_trace::Bool = false,
                  extended_trace::Bool = false,
                  show_every::Integer = 1,
                  callback = nothing)
    options = OptimizationOptions(;
        x_tol = x_tol, f_tol = f_tol, g_tol = g_tol,
        iterations = iterations, store_trace = store_trace,
        show_trace = show_trace, extended_trace = extended_trace,
        callback = callback, show_every = show_every)
    optimize(f, g!, h!, initial_x, method, options)
end

function optimize(d::DifferentiableFunction,
                  initial_x::Array;
                  method = LBFGS(),
                  x_tol::Real = 1e-32,
                  f_tol::Real = 1e-32,
                  g_tol::Real = 1e-8,
                  iterations::Integer = 1_000,
                  store_trace::Bool = false,
                  show_trace::Bool = false,
                  extended_trace::Bool = false,
                  show_every::Integer = 1,
                  callback = nothing)
    options = OptimizationOptions(;
        x_tol = x_tol, f_tol = f_tol, g_tol = g_tol,
        iterations = iterations, store_trace = store_trace,
        show_trace = show_trace, extended_trace = extended_trace,
        callback = callback, show_every = show_every)
    optimize(d, initial_x, method, options)
end

function optimize(d::TwiceDifferentiableFunction,
                  initial_x::Array;
                  method = Newton(),
                  x_tol::Real = 1e-32,
                  f_tol::Real = 1e-32,
                  g_tol::Real = 1e-8,
                  iterations::Integer = 1_000,
                  store_trace::Bool = false,
                  show_trace::Bool = false,
                  extended_trace::Bool = false,
                  show_every::Integer = 1,
                  callback = nothing)
    options = OptimizationOptions(;
        x_tol = x_tol, f_tol = f_tol, g_tol = g_tol,
        iterations = iterations, store_trace = store_trace,
        show_trace = show_trace, extended_trace = extended_trace,
        callback = callback, show_every = show_every)
    optimize(d, initial_x, method, options)
end

function optimize(d,
                  initial_x::Array,
                  method::Optimizer,
                  options::OptimizationOptions = OptimizationOptions())
    optimize(d, initial_x, method, options)
end

function optimize(f::Function,
                  g!::Function,
                  initial_x::Array,
                  method::Optimizer,
                  options::OptimizationOptions = OptimizationOptions())
    d = DifferentiableFunction(f, g!)
    optimize(d, initial_x, method, options)
end

function optimize(f::Function,
                  g!::Function,
                  h!::Function,
                  initial_x::Array,
                  method::Optimizer,
                  options::OptimizationOptions = OptimizationOptions())
    d = TwiceDifferentiableFunction(f, g!, h!)
    optimize(d, initial_x, method, options)
end

function optimize{T}(f::Function,
                  initial_x::Array{T},
                  method::Optimizer,
                  options::OptimizationOptions)
    if !options.autodiff
        d = DifferentiableFunction(f)
    else
        g!(x, out) = ForwardDiff.gradient!(out, f, x)

        function fg!(x, out)
            gr_res = ForwardDiff.GradientResult(zero(T),out)
            ForwardDiff.gradient!(gr_res, f, x)
            ForwardDiff.value(gr_res)
        end
        d = DifferentiableFunction(f, g!, fg!)
    end
    optimize(d, initial_x, method, options)
end

function optimize{T}(f::Function,
                  initial_x::Array{T},
                  method::Union{Newton, NewtonTrustRegion},
                  options::OptimizationOptions)
    if !options.autodiff
        error("No gradient or Hessian was provided. Either provide a gradient and Hessian, set autodiff = true in the OptimizationOptions if applicable, or choose a solver that doesn't require a Hessian.")
    else
        g!(x, out) = ForwardDiff.gradient!(out, f, x)

        function fg!(x, out)
            gr_res = ForwardDiff.GradientResult(zero(T),out)
            ForwardDiff.gradient!(gr_res, f, x)
            ForwardDiff.value(gr_res)
        end

        h! = (x, out) -> ForwardDiff.hessian!(out, f, x)
        d = TwiceDifferentiableFunction(f, g!, fg!, h!)
    end
    optimize(d, initial_x, method, options)
end

function optimize(f::Function,
                  g!::Function,
                  initial_x::Array,
                  method::Union{Newton, NewtonTrustRegion},
                  options::OptimizationOptions)
    if !options.autodiff
        error("No Hessian was provided. Either provide a Hessian, set autodiff = true in the OptimizationOptions if applicable, or choose a solver that doesn't require a Hessian.")
    else
        function fg!(x, out)
            g!(x, out)
            f(x)
        end

        h! = (x, out) -> ForwardDiff.hessian!(out, f, x)
        d = TwiceDifferentiableFunction(f, g!, fg!, h!)
    end
    optimize(d, initial_x, method, options)
end

function optimize(d::DifferentiableFunction,
                  initial_x::Array,
                  method::Union{Newton, NewtonTrustRegion},
                  options::OptimizationOptions)
    if !options.autodiff
        error("No Hessian was provided. Either provide a Hessian, set autodiff = true in the OptimizationOptions if applicable, or choose a solver that doesn't require a Hessian.")
    else
        h! = (x, out) -> ForwardDiff.hessian!(out, d.f, x)
    end
    optimize(TwiceDifferentiableFunction(d.f, d.g!, d.fg!, h!), initial_x, method, options)
end

function optimize(d::DifferentiableFunction,
                  initial_x::Array,
                  method::Optimizer,
                  options::OptimizationOptions)
    optimize(d.f, initial_x, method, options)
end

function optimize(d::TwiceDifferentiableFunction,
                  initial_x::Array,
                  method::Optimizer,
                  options::OptimizationOptions)
    dn = DifferentiableFunction(d.f, d.g!, d.fg!)
    optimize(dn, initial_x, method, options)
end

function optimize{T <: AbstractFloat}(f::Function,
                                      lower::T,
                                      upper::T;
                                      method = Brent(),
                                      rel_tol::Real = sqrt(eps(T)),
                                      abs_tol::Real = eps(T),
                                      iterations::Integer = 1_000,
                                      store_trace::Bool = false,
                                      show_trace::Bool = false,
                                      callback = nothing,
                                      show_every = 1,
                                      extended_trace::Bool = false)
    show_every = show_every > 0 ? show_every: 1
    if extended_trace && callback == nothing
        show_trace = true
    end
    if show_trace
        @printf "Iter     Function value   Gradient norm \n"
    end
    optimize(f, Float64(lower), Float64(upper), method;
             rel_tol = rel_tol,
             abs_tol = abs_tol,
             iterations = iterations,
             store_trace = store_trace,
             show_trace = show_trace,
             show_every = show_every,
             callback = callback,
             extended_trace = extended_trace)
end

function optimize(f::Function,
                  lower::Real,
                  upper::Real;
                  kwargs...)
    optimize(f,
             Float64(lower),
             Float64(upper);
             kwargs...)
end


@testset "symmetries.jl" begin

    function order_test(symm, N)
        indices = 0:2^N-1 #indices
        current = indices
        order = SpinSymmetry._order(symm)
        for i in 1:order-1
            current = symm.(current)
            if current == indices
                println("Order too long! Reported: $order, brute-forced: $i. Symmetry: $symm")
                return false
            end
        end
        ok = symm.(current) == indices
        !ok && println("Order too short! Reported: $order, Symmetry: $symm")
        return ok
    end

    @testset "Shift" begin
        # test default arg
        @test Shift(10) == Shift(10,1)
        @test Shift(10,-1) == Shift(10,9)

        @test hash(Shift(5, 3)) == hash(Shift(5, 3))
        @test hash(Shift(5, 3)) == hash(Shift(5, 8))
        @test hash(Shift(5, 3)) != hash(Shift(5, 4))

        # small correctness test
        @test Shift(2).(0:3) == [0,2,1,3]

        # injectivity test
        for k in 0:9
            @test length(Set(Shift(10, k).(0:2^10-1))) == 2^10
        end

        # order test
        for N in 2:10
            for amount in 1:div(N,2)
                @test order_test(Shift(N, amount), N)
            end
        end
    end

    @testset "Flip" begin
        # test default arg
        @test Flip(10) == Flip(collect(1:10))

        @test Flip(10) == Flip(10)
        @test Flip(9) != Flip(10)
        @test hash(Flip(10)) == hash(Flip(10))
        @test hash(Flip(9)) != hash(Flip(10))

        # small correctness test
        @test Flip(2).(0:3) == [3,2,1,0]
        @test Flip([1,2]).(0:7) == vcat([3,2,1,0], 4 .+ [3,2,1,0])

        # injectivity test
        @test length(Set(Flip(10).(0:2^10-1))) == 2^10
        @test length(Set(Flip(2:2:10).(0:2^10-1))) == 2^10

        # order test
        for N in 2:10
            @test order_test(Flip(N), N)
            @test order_test(Flip(div(N,2)), N)
        end
    end

    @testset "Swap" begin
        # small correctness test
        @test Swap(1,2).(0:3) == [0,2,1,3]
        @test Swap(2,2).(0:31) == 0:31 # identity

        @test Swap(1,2) == Swap(1,2)
        @test Swap(1,2) == Swap(2,1)
        @test Swap(1,2) != Swap(3,2)
        @test hash(Swap(1,2)) == hash(Swap(1,2))
        @test hash(Swap(1,2)) == hash(Swap(2,1))
        @test hash(Swap(1,2)) != hash(Swap(3,2))

        # injectivity test
        for (p1,p2) in [(1,2), (3,4), (1,4), (9,6), (1,9), (5,5), (3,7)]
            @test length(Set(Swap(p1, p2).(0:2^10-1))) == 2^10
        end

        # order test
        for N in 3:2:16
            pos1 = round(Int, N/4)
            pos2 = round(Int, 3*N/4)
            @test order_test(Swap(pos1,pos2), N)
            @test order_test(Swap(1,N), N)
            @test Swap(pos1,pos2).(0:2^N-1) == Swap(pos2,pos1).(0:2^N-1)
        end
    end

    @testset "SpatialReflection" begin
        # small correctness test
        @test SpatialReflection(2).(0:3) == [0,2,1,3]

        @test SpatialReflection(2) == SpatialReflection(2)
        @test SpatialReflection(2) != SpatialReflection(3)
        @test hash(SpatialReflection(2)) == hash(SpatialReflection(2))
        @test hash(SpatialReflection(2)) != hash(SpatialReflection(3))

        # injectivity test
        @test length(Set(SpatialReflection(10).(0:2^10-1))) == 2^10

        # order test
        for N in 4:12
            @test order_test(SpatialReflection(N), N)
            @test order_test(SpatialReflection(div(N,2)), N)
        end
    end

    # rather pointless
    # GenericSymmetry is as correct as the values it contains...
    @testset "GenericSymmetry" begin
        symm = GenericSymmetry(identity, 1)

        @test GenericSymmetry(identity, 1) == GenericSymmetry(identity, 1)
        @test GenericSymmetry(identity, 1) != GenericSymmetry(+, 1)
        @test GenericSymmetry(identity, 1) != GenericSymmetry(identity, 2)
        @test hash(GenericSymmetry(identity, 1)) == hash(GenericSymmetry(identity, 1))
        @test hash(GenericSymmetry(identity, 1)) != hash(GenericSymmetry(+, 1))
        @test hash(GenericSymmetry(identity, 1)) != hash(GenericSymmetry(identity, 2))

        @test symm.(0:255) == 0:255
        @test SpinSymmetry._order(symm) == 1
        @test order_test(symm, 1)
    end

end


#=
    Cheating model.

    The model is a more verbose implementation of the Cheating problem in
    https://github.com/CamDavidsonPilon/Probabilistic-Programming-and-Bayesian-Methods-for-Hackers/blob/master/Chapter2_MorePyMC/Ch2_MorePyMC_PyMC3.ipynb
    """
    What we need is a better algorithm to ask students if they had cheated. Ideally the algorithm
    should encourage individuals to be honest while preserving privacy. The following proposed algorithm
    is a solution I greatly admire for its ingenuity and effectiveness:

        In the interview process for each student, the student flips a coin, hidden from the interviewer.
        The student agrees to answer honestly if the coin comes up heads. Otherwise, if the coin comes
        up tails, the student (secretly) flips the coin again, and answers "Yes, I did cheat" if the
        coin flip lands heads, and "No, I did not cheat", if the coin flip lands tails. This way, the
        interviewer does not know if a "Yes" was the result of a guilty plea, or a Heads on a second
        coin toss. Thus privacy is preserved and the researchers receive honest answers.
    """

    Compared to cheating.jl we add an array (cheated) which collected the students
    that we think cheated. Of course it's just a simulation since we cannot pinpoint
    the specific students that actually cheated.

    But the model works and give the same results as cheating.jl (and cheating2.jl), i.e. that
    it was about 22% of the student that cheated (the p variable).

        Summary Statistics
    parameters      mean       std   naive_se      mcse        ess      rhat   ess_per_sec
        Symbol   Float64   Float64    Float64   Float64    Float64   Float64       Float64

               p    0.2212    0.1192     0.0038    0.0063   307.7500    1.0006       28.8048
        obs_prop    0.3556    0.0435     0.0014    0.0028   377.7931    1.0154       35.3606
         p_prior    0.5006    0.2849     0.0090    0.0085   586.1921    1.0036       54.8664

   Cf cheating.jl and cheating2.jl for ports of the two PyMC3 models from the
      "Probabilistic Programming Hacker" book.

=#

using Turing, StatsPlots, Distributions, StatsBase
using CSV
include("jl_utils.jl")

@model function cheating_model0(N,X)
    p ~ Uniform(0,1) # freq cheating
    tail = 0
    head = 1

    cheated      ~ filldist(Bernoulli(p),N)
    truths       ~ filldist(Bernoulli(p),N)
    first_flips  ~ filldist(Bernoulli(0.5), N)
    second_flips ~ filldist(Bernoulli(0.5), N)

    # val = tzeros(N) # We must defined val to be used as LHS in @.

    # @. val = first_flips*truths + (1 - first_flips)*second_flips

    for i in 1:N
        # """
        # In the interview process for each student, the student flips a coin,
        # hidden from the interviewer.
        # """
        if first_flips[i] == head
            # The student agrees to answer honestly if the coin comes up heads.
            # """
            if cheated[i] == true
                truths[i] ~ Dirac(true)
            else
                truths[i] ~ Dirac(false)
            end
            second_flips[i] ~ Dirac(true)
        else
            # """
            # Otherwise, if the coin comes up tails, the student (secretly) flips the coin
            # again, and answers "Yes, I did cheat" if the coin flip lands heads,
            # and "No, I did not cheat", if the coin flip lands tails.
            # """
            if second_flips[i] == head
                truths[i] ~ Dirac(true)
            else
                truths[i] ~ Dirac(false)
            end
        end
    end

    obs_prop ~ Dirac(sum(truths) / N)
    X ~ Binomial(N, obs_prop )

    p_prior ~ Uniform(0,1)

end

N = 100
yes_response = 35
model = cheating_model0(N, yes_response)

# chns = sample(model, Prior(), 10_000)
# chns = sample(model, MH(), 10_000)
chns = sample(model, PG(5),  1_000)
# chns = sample(model, PG(5),  MCMCThreads(), 10_000, 4)
# chns = sample(model, SMC(), 10_000)
# chns = sample(model, IS(), 10_000)

# chns = sample(model, HMC(0.1,6), 1_000)
# chns = sample(model, NUTS(), 1_000)

display(chns[[:p,:obs_prop,:p_prior]])
display(plot(chns[[:p,:obs_prop,:p_prior]]))

# Another way:
println("mean(truths): $(mean(group(chns, :truths).value.data,dims=1) |> mean)")
println("mean(cheated): $(mean(group(chns, :cheated).value.data,dims=1) |> mean)")


using Sundials, Test, SparseArrays, DiffEqOperators

# Test for Jacobian usage
function Lotka(du, u, p, t)
    du[1] = u[1] - u[1] * u[2] # REPL[7], line 3:
    du[2] = -3 * u[2] + 1 * u[1] * u[2]
    nothing
end

jac_called = false
function Lotka_jac(J, u, p, t)
    global jac_called
    jac_called = true
    J[1, 1] = 1.0 - u[2]
    J[1, 2] = -u[1]
    J[2, 1] = 1 * u[2]
    J[2, 2] = -3 + u[1]
    nothing
end

Lotka_f = ODEFunction(Lotka, jac = Lotka_jac)
prob = ODEProblem(Lotka_f, ones(2), (0.0, 10.0))
good_sol = solve(prob, CVODE_BDF())
@test jac_called == true

Lotka_f = ODEFunction(
    Lotka,
    jac = Lotka_jac,
    jac_prototype = sparse([1, 2, 1, 2], [1, 1, 2, 2], zeros(4)),
)

prob = ODEProblem(Lotka_f, ones(2), (0.0, 10.0))
jac_called = false
sol9 = solve(prob, CVODE_BDF(linear_solver = :KLU))
@test jac_called == true
@test Array(sol9) ≈ Array(good_sol)

Lotka_fj = ODEFunction(Lotka, jac_prototype = JacVecOperator{Float64}(Lotka, ones(2)))

prob = ODEProblem(Lotka_fj, ones(2), (0.0, 10.0))
sol9 = solve(prob, CVODE_BDF(linear_solver = :GMRES))

function f2!(res, du, u, p, t)
    res[1] = 1.01du[1]
    return
end

jac_called = false
function f2_jac!(out, du, u, p, gamma, t)
    global jac_called
    jac_called = true
    out[1] = 1.01
end

f2_f = DAEFunction(f2!, jac = f2_jac!)

u0 = [0.0]
tspan = (0.0, 10.0)
du0 = [0.0]
dae_prob = DAEProblem(f2_f, du0, u0, tspan, differential_vars = [true])
good_sol = solve(dae_prob, IDA())
@test jac_called == true

function testjac(res, du, u, p, t)
    res[1] = du[1] - 1.5 * u[1] + 1.0 * u[1] * u[2]
    res[2] = du[2] + 3 * u[2] - u[1] * u[2]
end

jac_called = false
function testjac_jac(J, du, u, p, gamma, t)
    global jac_called
    jac_called = true
    J[1, 1] = gamma - 1.5 + 1.0 * u[2]
    J[1, 2] = 1.0 * u[1]
    J[2, 1] = -1 * u[2]
    J[2, 2] = gamma + 3 - u[1]
    nothing
end

testjac_f = DAEFunction(testjac, jac = testjac_jac)

prob3 = DAEProblem(
    testjac_f,
    [0.5, -2.0],
    ones(2),
    (0.0, 10.0),
    differential_vars = [true, true],
)
sol3 = solve(prob3, IDA())

@test jac_called == true
jac_called = false
prob4 = DAEProblem(testjac, [0.5, -2.0], ones(2), (0.0, 10.0))
sol4 = solve(prob4, IDA())

@test jac_called == false

println("Jacobian vs no Jacobian difference:")
println(maximum(sol3 - sol4))
@test maximum(sol3 - sol4) < 1e-6


# integration.jl

"""
    integrate(a, c0)

Integrates the one-variable Taylor Model (`TaylorModel1`
or `RTaylorModel1`) with respect to the independent variable; `c0` is
the interval representing the integration constant; if omitted
it is considered as the zero interval.
"""
function integrate(a::TaylorModel1{T,S}, c0::T) where {T,S}
    integ_pol = integrate(a.pol, c0)
    δ = a.dom-a.x0

    # Remainder bound after integrating.
    Δ = bound_integration(a, δ)

    return TaylorModel1( integ_pol, Δ, a.x0, a.dom )
end
integrate(a::TaylorModel1{T,S}) where {T,S} = integrate(a, zero(T))

function integrate(a::RTaylorModel1{T,S}, c0::T) where {T,S}
    order = get_order(a)
    integ_pol = integrate(a.pol, c0)

    # Remainder bound after integrating...
    Δ = (a.dom-a.x0) * remainder(a)
    Δ = Δ/(order+2) + a.pol[order]/(order+1)

    return RTaylorModel1( integ_pol, Δ, a.x0, a.dom )
end
integrate(a::RTaylorModel1{T,S}) where {T,S} = integrate(a, zero(T))

"""
    bound_integration(xTM1::TaylorModel1{Interval{S},S}, δt::Interval{S})
    bound_integration(xTM1::Vector{TaylorModel1{Interval{S},S}}, δt::Interval{S})

Remainder bound for the integration of a series, given by
``δ * remainder(a) +  a.pol[order] * δ^(order+1) / (order+1)``.
This is tighter that the one used by Berz+Makino, which corresponds to
``Δ = aux * remainder(a) +  a.pol[order] * aux^(order+1)``.

"""
function bound_integration(a::TaylorModel1{T,S}, δ) where {T,S}
    order = get_order(a)
    aux = δ^order / (order+1)
    Δ = δ * (remainder(a) + getcoeff(polynomial(a),order) * aux)
    return Δ
end
function bound_integration(a::Vector{TaylorModel1{T,S}}, δ) where {T,S}
    order = get_order(a[1])
    aux = δ^order / (order+1)
    Δ = δ .* (remainder.(a) .+ getcoeff.(polynomial.(a), order) .* aux)
    return IntervalBox(Δ)
end

"""
    picard_lindelöf(f, tm::T, xm::T, x0::Interval)
    𝒫(f, tm::T, xm::T, x0::Interval)

Returns the application of the Picard-Lindelöf operator
associated to the ODE ``\\dot{x} = f(t,x)``,
with initial condition `x0`. Here, `tm` and `xm` are
(one-variable) Taylor Models (`TaylorModel1` or `RTaylorModel1`).

𝒫 is an abbreviation of this operator, which is obtained
as `\\mscrP<TAB>`.)
"""
picard_lindelöf(f, tm::T, xm::T, x0::Interval) where
    {T<:Union{TaylorModel1, RTaylorModel1}} = integrate(f(tm, xm), x0)

const 𝒫 = picard_lindelöf

"""
    check_existence(f, tm::T, xm::T, x0::Interval, x_test::Interval)

Checks that the range of one iterate of the Picard-Lindelöf
operator is contained in the a-priori interval `x_test` (of
the dependent variable) that bounds the solution of
the ODE defined by `f`. This function returns an interval
of the independent variable where the a-priori solution
is warranted to exist; see [`shrink_for_existance`](@ref).
Here, `tm` and `xm` are Taylor Models (`TaylorModel1` or `RTaylorModel1`)
defined for the independent and dependent variables, and `x0`
is the initial condition.
"""
function check_existence(f, tm::T, xm::T, x0::Interval, x_test::Interval,
        max_steps::Integer=20) where {T<:Union{TaylorModel1, RTaylorModel1}}

    pl = 𝒫(f, tm, xm, x0)
    tt = shrink_for_existance(pl, tm.dom, x_test, max_steps)
    if pl(tt-tm.x0) ⊆ x_test
        return tt
    else
        return emptyinterval(tt)
    end
end

"""
    shrink_for_existance(xm::T, t_interval, x_test, max_steps::Integer=20)

Shrinks the a-priori independent-variable interval `t_interval`
so the range of `xm`, the Taylor Model (`TaylorModel1` or `RTaylorModel1`)
associated with the dependent variable, is contained in the
a-priori interval `x_test`. The method used is some sort
of bisection. If no independent-variable interval is found within
`max_steps`, an empty interval is returned.
"""
function shrink_for_existance(xm::T, t_interval, x_test, max_steps::Integer=20) where
        {T<:Union{TaylorModel1, RTaylorModel1}}

    tt = t_interval
    t1 = t_interval

    nsteps = 0
    xm_pol = xm.pol

    while nsteps < max_steps
        if xm(t1) ⊆ x_test
            (t_interval.hi-t1.hi < 1.0e-3) && return t1
            tt = t1
            mm = (tt.hi+t_interval.hi)*0.5
            t1 = interval(t1.lo, mm)
        else
            t_interval = t1
            if t1 == tt
                mm = (tt.lo+t1.hi)*0.5
                t1 = interval(t1.lo, mm)
                tt = t1
            else
                mm = (tt.hi+t1.hi)*0.5
                t1 = interval(t1.lo, mm)
            end
        end
        nsteps += 1
    end

    xm(t1) ⊆ x_test && return t1
    xm(tt) ⊆ x_test && return tt

    return emptyinterval(tt)
end

"""
    tight_remainder(f, tm::T, xm::T, x0::Interval, max_steps::Integer=20)

Returns a Taylor Model for the dependent variable, with a tighter
remainder, which is obtained after successive iteration
of the Picard-Lindelöf. If the remainder is not tighter
(and identity with the former iterate is not obtained) a
Taylor Model with an empty interval is returned.

"""
function tight_remainder(f, tm::T, xm::T, x0::Interval, max_steps::Integer=20) where
        {T<:Union{TaylorModel1, RTaylorModel1}}

    xOld = deepcopy(xm)
    for ind = 1:max_steps
        xNew = 𝒫(f, tm, xOld, x0)
        if diam(remainder(xNew)) ≥ diam(remainder(xOld))
            xOld == xNew && return xOld
            return T(xOld.pol, emptyinterval(xOld.rem), xOld.x0, xOld.dom)
        end
        xOld = xNew
    end
    return xOld
end


# Test get/set objective coefficients in LP

using MathProgBase
using CPLEX
using Base.Test

env = CPLEX.Env()

# original model
#
#   maximize  2x + 2y
#
# s.t. 0.2 <= x, y <= 1
#

model = CPLEX.cplex_model(env;
  name="lp_03",
  sense=:Max,
  f=[2.0, 2.0],
  lb=[0.2, 0.2],
  ub=[1.0, 1.0])

lb_ = CPLEX.get_varLB(model)
ub_ = CPLEX.get_varUB(model)
c_ = CPLEX.get_obj(model)

@test lb_ == [0.2, 0.2]
@test ub_ == [1.0, 1.0]
@test c_ == [2.0, 2.0]

CPLEX.optimize!(model)

println()
println("soln = $(CPLEX.get_solution(model))")
println("objv = $(CPLEX.get_objval(model))")

# change objective (warm start)
#
# maximize x - y
#
# s.t. 0.2 <= x, y <= 1
#

CPLEX.set_obj!(model, [1, -1])

c_ = CPLEX.get_obj(model)
@test c_ == [1.0, -1.0]

CPLEX.optimize!(model)

println()
println("soln = $(CPLEX.get_solution(model))")
println("objv = $(CPLEX.get_objval(model))")


function _validitycheck(a)
    if a == 0
        throw(DomainError(a, "'a' is zero - the quadratic function is degenerated"))
    end
end

function calcdelta(a, b, c)
    delta = b^2 - (4 * a * c)
    return delta
end


struct ConsistencyBars{T<:Real}
    samples::Int
    quantiles::Tuple{T,T}

    @doc """
        ConsistencyBars(; samples::Int=1_000, coverage::Real=0.95)

    Create consistency bars that cover a proportion of `coverage` samples obtained
    by consistency resampling with `samples` resampling steps.

    # References

    Bröcker, J. and Smith, L.A. (2007). *Increasing the reliability of reliability diagrams*.
    Weather and forecasting, 22(3), pp. 651-661.
    """
    function ConsistencyBars(; samples::Int=1_000, coverage::Real=0.95)
        samples > 0 || error("number of samples must be positive")
        zero(coverage) < coverage < one(coverage) ||
            error("coverage must be a number in (0, 1)")
        αo2 = (1 - coverage) / 2
        quantiles = promote(αo2, 1 - αo2)
        return new{eltype(quantiles)}(samples, quantiles)
    end
end


"""
    KeplerianElements

Struct representing the standard classical orbital elements.
"""
struct KeplerianElements <: AbstractElements
    "Semi-Major Axis"
    a::Float64
    "Eccentricity"
    e::Float64
    "Inclination"
    i::Float64
    "Argument of Periapsis"
    ω::Float64
    "Right Ascension of the Ascending Node"
    Ω::Float64
    "True Anomaly"
    ν::Float64
end

# Helper validation functions for keplerian elements
_valid_sma(sma) = sma != 0
_valid_ecc(ecc) = ecc >= 0
_valid_inc(inc) = 0 ≤ inc ≤ π
_valid_aop(aop) = true
_valid_raan(raan) = true
_valid_ta(ta) = true

function _validate(ke::KeplerianElements)
    a = semi_major_axis(ke)
    e = eccentricity(ke)
    i = inclination(ke)
    ω = argument_of_periapsis(ke)
    Ω = right_ascension(ke)
    ν = true_anomaly(ke)
    _valid_sma(a) || throw(DomainError(a, "Invalid semi-major axis"))
    _valid_ecc(e) || throw(DomainError(e, "Invalid eccentricity"))
    _valid_inc(i) || throw(DomainError(i, "Invalid inclination"))
    _valid_aop(ω) || throw(DomainError(ω, "Invalid argument of periapsis"))
    _valid_raan(Ω) || throw(DomainError(raan, "Invalid right ascension"))
    _valid_ta(ν) || throw(DomainError(ta, "Invalid true anomaly"))

    asign = a > 0
    ic = isclosed(ke)

    if (ishyperbolic(ke) && asign) || isparabolic(ke) && !isinf(a) || (ic && !asign)
        throw(DomainError((a=a, e=e), "Invalid eccentricity and semi-major axis pair"))
    end

    return ke
end

"""
    KeplerianElements(sma; ecc=0, inc=0, aop=0, raan=0, ta=0)

Contruct a new [`KeplerianElements`](@ref) struct specifying optional orbit elements.

See Also:
[`semi_major_axis`](@ref),
[`eccentricity`](@ref)
[`inclination`](@ref),
[`argument_of_periapsis`](@ref),
[`right_ascension`](@ref),
[`true_anomaly`](@ref)
"""
function KeplerianElements(sma; ecc=0.0, inc=0.0, aop=0.0, raan=0.0, ta=0.0)
    return KeplerianElements(
        sma,
        ecc,
        inc,
        aop,
        raan,
        ta
    ) |> _validate
end

semi_major_axis(k::KeplerianElements) = k.a
eccentricity(k::KeplerianElements) = k.e
inclination(k::KeplerianElements) = k.i
argument_of_periapsis(k::KeplerianElements) = k.ω
right_ascension(k::KeplerianElements) = k.Ω
true_anomaly(k::KeplerianElements) = k.ν


using Tables
import Petri

sir_petri = PetriNet(3, ((1, 2), (2, 2)), (2, 3))
sir_lpetri = LabelledPetriNet([:S, :I, :R], :inf=>((:S, :I), (:I, :I)), :rec=>(:I, :R))
sir_rxn = ReactionNet{Number, Int}([990, 10, 0], (.001, ((1, 2)=>(2,2))), (.25, (2=>3)))
sir_lrxn = LabelledReactionNet{Number, Int}((:S=>990, :I=>10, :R=>0), (:inf, .001)=>((:S, :I)=>(:I,:I)), (:rec, .25)=>(:I=>:R))

sir_tpetri= PetriNet(TransitionMatrices(sir_petri))

@test snames(sir_petri) == 1:3
@test tnames(sir_petri) == 1:2
@test snames(sir_lpetri) == [:S, :I, :R]
@test tnames(sir_lpetri) == [:inf, :rec]
@test snames(sir_rxn) == 1:3
@test tnames(sir_rxn) == 1:2
@test snames(sir_lrxn) == [:S, :I, :R]
@test tnames(sir_lrxn) == [:inf, :rec]

for pn ∈ [sir_petri, sir_lpetri, sir_rxn, sir_lrxn]
  @test PetriNet(pn) == sir_petri
end

for pn ∈ [sir_lpetri, sir_lrxn]
  @test LabelledPetriNet(pn) == sir_lpetri
end
for pn ∈ [sir_petri, sir_lpetri, sir_rxn, sir_lrxn]
  @test LabelledPetriNet(pn, [:S, :I, :R], [:inf, :rec]) == sir_lpetri
end

for pn ∈ [sir_rxn, sir_lrxn]
  @test ReactionNet{Number, Int}(pn) == sir_rxn
end
for pn ∈ [sir_petri, sir_lpetri, sir_rxn, sir_lrxn]
  @test ReactionNet{Number, Int}(pn, [990, 10, 0], [.001, .25]) == sir_rxn
end

for pn ∈ [sir_lrxn]
  @test LabelledReactionNet{Number, Int}(pn) == sir_lrxn
end
for pn ∈ [sir_petri, sir_rxn]
  @test LabelledReactionNet{Number, Int}(pn, [:S=>990, :I=>10, :R=>0], [:inf=>.001, :rec=>.25]) == sir_lrxn
end
for pn ∈ [sir_petri, sir_lpetri, sir_rxn, sir_lrxn]
  @test LabelledReactionNet{Number, Int}(pn, [:S, :I, :R], [:inf, :rec], [990, 10, 0], [.001, .25]) == sir_lrxn
end

β(u,t) = 1 / sum(u)
γ = .25
sir_rxn = ReactionNet{Function, Int}([990, 10, 0], (β)=>((1, 2)=>(2,2)), (t->γ)=>(2=>3))
open_sir_rxn = Open([1,2], sir_rxn, [3])
open_sir_lrxn = Open([:S,:I], sir_lrxn, [:R])

@test sir_tpetri == sir_petri
@test Petri.Model(sir_petri) == Petri.Model(sir_rxn)
@test Petri.Model(sir_lpetri) == Petri.Model(sir_lrxn)

@test typeof(Graph(sir_petri)) == Graph
@test typeof(Graph(sir_lpetri)) == Graph
@test typeof(Graph(sir_rxn)) == Graph
@test typeof(Graph(open_sir_rxn)) == Graph
@test typeof(Graph(sir_lrxn)) == Graph
@test typeof(Graph(open_sir_lrxn)) == Graph

@test inputs(sir_petri, 1) == [1,2]
@test outputs(sir_petri, 1) == [2,2]
@test concentration(sir_rxn, 1) == 990
@test rate(sir_rxn, 1) == β

@test concentrations(sir_rxn) == [990, 10, 0]
@test typeof(rates(sir_rxn)) <: Array{Function}

@test concentrations(sir_lrxn) == LVector(S=990, I=10, R=0)
@test rates(sir_lrxn) == LVector(inf=.001, rec=.25)

@test length(Tables.rows(tables(dom(open_sir_rxn).ob).S)) == length(Tables.rows(tables(dom(open_sir_lrxn).ob).S))

du = [0.0, 0.0, 0.0]
out = vectorfield(sir_rxn)(du, concentrations(sir_rxn), rates(sir_rxn), 0.01)
@test out[1] ≈ -9.9
@test out[2] ≈ 7.4
@test out[3] ≈ 2.5

du = LVector(S=0.0, I=0.0, R=0.0)
out = vectorfield(sir_lrxn)(du, concentrations(sir_lrxn), rates(sir_lrxn), 0.01)
@test out.S ≈ -9.9
@test out.I ≈ 7.4
@test out.R ≈ 2.5

@test ns(sir_petri) == 3
add_species!(sir_petri)
@test ns(sir_petri) == 4

@test nt(sir_petri) == 2
add_transitions!(sir_petri, 2)
@test nt(sir_petri) == 4

@test ni(sir_petri) == 3
@test no(sir_petri) == 3
add_input!(sir_petri, 3, 1)
add_output!(sir_petri, 3, 4)
add_input!(sir_petri, 4, 4)
add_output!(sir_petri, 4, 3)
@test ni(sir_petri) == 5
@test no(sir_petri) == 5
@test sir_petri == PetriNet(4, ((1, 2), (2, 2)), (2, 3), (1, 4), (4, 3))


"
  Name     : c4_19_scatter_plot_PyPlot.jl
  Book     : Hands-on Data Science with Anaconda )
  Publisher: Packt Publishing Ltd.
  Author   : Yuxing Yan and James Yan
  Date     : 1/25/2018
  email    : yany@canisius.edu
             paulyxy@hotmail.com
"
using PyPlot
n=50
x = 100*rand(n)
y = 100*rand(n)
areas = 800*rand(n)
fig = figure("pyplot_scatterplot",figsize=(10,10))
ax = axes()
scatter(x,y,s=areas,alpha=0.5)
title("Scatter Plot")
xlabel("X")
ylabel("Y")
grid("on")


export triangulation, triangulation_indices

# Implementation of Cohen & Hickey algorithm for triangulation; see [CH79] and [BEK00].
#
# [CH79] Cohen, J., & Hickey, T. (1979). Two algorithms for determining volumes of convex polyhedra. Journal of the ACM (JACM), 26(3), 401-414.
# [BEK00] Büeler, B., Enge, A., & Fukuda, K. (2000). Exact volume computation for polytopes: a practical study. In Polytopes—combinatorics and computation (pp. 131-154). Birkhäuser, Basel.

# TODO parallelize: this is inherently parallelizable
function _triangulation(Δs, Δ, v_idx, h_idx, incident_idx, is_weak_adjacent)
    isempty(v_idx) && return
    v = first(v_idx)
    Δ = push!(copy(Δ), v)
    if isone(length(v_idx))
        push!(Δs, Δ)
        return
    end
    tail = true
    for h in h_idx
        if !(v in incident_idx[h])
            tail = false
            # The adjacency may be outside the current face under scrutiny but that's ok,
            # it will simply end up calling `_triangulation` with an empty `v_idx`.
            weak_adjacent = [hj for hj in h_idx if hj != h && is_weak_adjacent[(h, hj)]]
            active = [point for point in v_idx if point in incident_idx[h]]
            _triangulation(Δs, Δ, active, weak_adjacent, incident_idx, is_weak_adjacent)
        end
    end
end
function triangulation_indices(p::Polyhedron)
    hasrays(p) && error("Triangulation only supported for polytope.")
    v_idx = eachindex(points(p))
    h_idx = eachindex(halfspaces(p))
    Δ = eltype(v_idx)[]
    Δs = typeof(Δ)[]
    incident_idx = Dict(h => Set(incidentpointindices(p, h)) for h in h_idx)
    is_weak_adjacent = Dict((hi, hj) => !isempty(incident_idx[hi] ∩ incident_idx[hj]) for hi in h_idx for hj in h_idx)
    _triangulation(Δs, Δ, v_idx, h_idx, incident_idx, is_weak_adjacent)
    return Δs
end
function triangulation(p::Polyhedron)
    return map(Δ -> vrep(get.(p, Δ)), triangulation_indices(p))
end


using ArgParse, MPI, Knet
include("Particle.jl")

function single_iter(problem::Problem, ps::Array{AbsParticle,1}, gbest::AbsParticle)
    for p in ps
        myfitness = fitness(p, problem)
        if myfitness < p.pbest_fitness
            p.pbest_fitness = myfitness
            copypbest!(p)

            if myfitness < gbest.pbest_fitness
                #(MPI.Comm_rank(MPI.COMM_WORLD) == 0) && println("Before: $(forw(gbest.x, problem.x))")
                gbest.pbest_fitness = myfitness
                copygbest!(gbest, p)

                #(MPI.Comm_rank(MPI.COMM_WORLD) == 0) && println("After: $(forw(gbest.x, problem.x))")
            end
        end
    end
end

function search!(problem::Problem, ps::Array{AbsParticle, 1}, gbest::AbsParticle, epoch, phi1, phi2, w, vmin, vmax, parallel)
    if parallel
        comm = MPI.COMM_WORLD
        world = MPI.Comm_size(comm)
        rank = MPI.Comm_rank(comm)
    end

    for e=1:epoch
        single_iter(problem, ps, gbest)# calculate local best
        if parallel
            curr_bests = MPI.Allgather([gbest.pbest_fitness], comm)# send each other best fitness value
            #check whether there is a new gbest
            best_p = -1
            for i=1:world
                if curr_bests[i] <= gbest.pbest_fitness
                    gbest.pbest_fitness = curr_bests[i]
                    best_p = i - 1
                end
            end

            #if there is a new gbest broadcast it
            if best_p != -1
                #(MPI.Comm_rank(MPI.COMM_WORLD) == 0) && println("$(best_p) Before: $(forw(gbest.x, problem.x))")
                Bcast!(gbest, best_p, comm)
                #(MPI.Comm_rank(MPI.COMM_WORLD) == 0) && println("$(best_p) After : $(forw(gbest.x, problem.x))")
            end
        end

        update!(problem, ps, gbest, phi1, phi2, w, vmin, vmax)
    end
end

function pso(problem::Problem, num = 100, epoch = 100, phi1=2.0, phi2=2.0, w=0.8, vmin=-1.0, vmax=1.0, parallel=false)

    if parallel
        comm = MPI.COMM_WORLD
        rank = MPI.Comm_rank(comm)
        world = MPI.Comm_size(comm)

        num = Int(floor(num / world))
    end

    ps, gbest = init_particles(num, problem)

    if parallel
        (rank == 0) && tic()
    else
        tic()
    end

    search!(problem, ps, gbest, epoch, phi1, phi2, w, vmin, vmax, parallel)

    if parallel
        (rank == 0) && toc()
    else
        toc()
    end

    return gbest
end


# ---
# title: 165. Compare Version Numbers
# id: problem165
# author: Indigo
# date: 2021-01-19
# difficulty: Medium
# categories: String
# link: <https://leetcode.com/problems/compare-version-numbers/description/>
# hidden: true
# ---
#
# Given two version numbers, `version1` and `version2`, compare them.
#
# Version numbers consist of **one or more revisions** joined by a dot `'.'`.
# Each revision consists of **digits**  and may contain leading **zeros**. Every
# revision contains **at least one character**. Revisions are **0-indexed from
# left to right** , with the leftmost revision being revision 0, the next
# revision being revision 1, and so on. For example `2.5.33` and `0.1` are valid
# version numbers.
#
# To compare version numbers, compare their revisions in **left-to-right
# order**. Revisions are compared using their  **integer value ignoring any
# leading zeros**. This means that revisions `1` and `001` are considered
# **equal**. If a version number does not specify a revision at an index, then
# **treat the revision as  `0`**. For example, version `1.0` is less than
# version `1.1` because their revision 0s are the same, but their revision 1s
# are `0` and `1` respectively, and `0 < 1`.
#
# _Return the following:_
#
#   * If `version1 < version2`, return `-1`.
#   * If `version1 > version2`, return `1`.
#   * Otherwise, return `0`.
#
#
#
# **Example 1:**
#
#
#
#     Input: version1 = "1.01", version2 = "1.001"
#     Output: 0
#     Explanation: Ignoring leading zeroes, both "01" and "001" represent the same integer "1".
#
#
# **Example 2:**
#
#
#
#     Input: version1 = "1.0", version2 = "1.0.0"
#     Output: 0
#     Explanation: version1 does not specify revision 2, which means it is treated as "0".
#
#
# **Example 3:**
#
#
#
#     Input: version1 = "0.1", version2 = "1.1"
#     Output: -1
#     Explanation:  version1's revision 0 is "0", while version2's revision 0 is "1". 0 < 1, so version1 < version2.
#
#
# **Example 4:**
#
#
#
#     Input: version1 = "1.0.1", version2 = "1"
#     Output: 1
#
#
# **Example 5:**
#
#
#
#     Input: version1 = "7.5.2.4", version2 = "7.5.3"
#     Output: -1
#
#
#
#
# **Constraints:**
#
#   * `1 <= version1.length, version2.length <= 500`
#   * `version1` and `version2` only contain digits and `'.'`.
#   * `version1` and `version2`  **are valid version numbers**.
#   * All the given revisions in `version1` and `version2` can be stored in a  **32-bit integer**.
#
#
## @lc code=start
using LeetCode

function compare_version(version1::String, version2::String)
    s_v1 = parse.(Int, split(version1, "."))
    s_v2 = parse.(Int, split(version2, "."))
    if length(s_v1) > length(s_v2)
        append!(s_v2, fill(0, length(s_v1) - length(s_v2)))
    else
        append!(s_v1, fill(0, length(s_v2) - length(s_v1)))
    end
    for i in 1:length(s_v1)
        if s_v1 > s_v2
            return 1
        elseif s_v1 < s_v2
            return -1
        end
    end
    return 0
end
## @lc code=end


using Multisets, IterTools

function print_map_est(output::SisPosteriorMcmcOutput; top_num::Int=5)
    d = proportionmap(output.S_sample)
    # for key in keys(d)
    #     d[key] /= length(output.S_sample)
    # end
    props = sort(collect(d), by=x->x[2], rev=true)
    title = "\nPosterior probability of mode"
    println(title)
    println("-"^length(title), "\n")
    for i in 1:min(top_num, length(d))
        println(props[i][2],"  ", props[i][1])
    end
    println("\n...showing top $(min(top_num, length(d))) interaction sequences.")
end

function print_map_est(sample::InteractionSequenceSample; top_num::Int=5)
    d = proportionmap(sample)
    # for key in keys(d)
    #     d[key] /= length(output.S_sample)
    # end
    props = sort(collect(d), by=x->x[2], rev=true)
    title = "\nPosterior probability of mode"
    println(title)
    println("-"^length(title), "\n")
    for i in 1:min(top_num, length(d))
        println(props[i][2],"  ", props[i][1])
    end
    println("\n...showing top $(min(top_num, length(d))) interaction sequences.")
end

function mean_dists_summary(
    samples::Vector{InteractionSequenceSample{T}},
    d::Union{Metric,Metric}
    ) where {T<:Union{Int,String}}

    tot_var = sample_frechet_var(
            vcat(samples...),
            d,
            with_memory=true, show_progress=true
        )
    mean_of_var = mean(
        x->sample_frechet_var(
            x,
            d,
            with_memory=true, show_progress=true), samples
    )
    means = map(x->sample_frechet_mean(x, d, with_memory=true)[1], samples)
    var_of_mean = sample_frechet_var(means, d)

    out = Dict{String,Float64}(
        "tot_var" => tot_var,
        "mean_of_vars" => mean_of_var,
        "var_of_means" => var_of_mean
    )
    return out
end

function mean_dists_summary(
    chains::Vector{SisPosteriorMcmcOutput}
    )
    samples = [x.S_sample for x in chains]
    d = chains[1].posterior.dist
    return mean_dists_summary(samples, d)
end

function print_map_est(output::Union{SimPosteriorMcmcOutput,SimPosteriorModeConditionalMcmcOutput}; top_num::Int=5)
    multiset_sample = Multiset.(output.S_sample)
    d = proportionmap(multiset_sample)
    # for key in keys(d)
    #     d[key] /= length(output.S_sample)
    # end
    props = sort(collect(d), by=x->x[2], rev=true)
    title = "\nPosterior probability of mode"
    println(title)
    println("-"^length(title), "\n")
    for i in 1:min(top_num, length(d))
        println(props[i][2],"  ", props[i][1])
    end
    println("\n...showing top $(min(top_num, length(d))) interaction multisets.")
end


using PrettyTables
using ProgressMeter

using RollingFunctions
using Clustering

if VERSION < v"1.1-DEV"
    isnothing(::Nothing) = true
    isnothing(::Any) = false
end

function estimate_convergence_rate(ρ, ϵ)
    i = findall(!iszero, ϵ)
    x = log10.(ρ[i])
    y = log10.(abs.(ϵ[i]))

    f = (x,y) -> ([x ones(length(x))] \ y)[1]

    # Find all slopes for consecutive groups of three errors
    slopes = rolling(f, x, y, 3)
    e = findfirst(s -> s < 0, slopes)
    if e !== nothing && e > 5
        slopes = slopes[1:max(1,e-1)]
    end

    k = 3 # Number of clusters requested
    fit = kmeans(reshape(slopes, 1, :), k)
    maximum(fit.centers)
end

# https://stackoverflow.com/a/52507859
function squeeze(A::AbstractArray)
    singleton_dims = tuple((d for d in 1:ndims(A) if size(A, d) == 1)...)
    dropdims(A, dims=singleton_dims)
end

function test_convergence_rates(fun::Function,
                                Rfun::Function, rₘₐₓ::Number,
                                Ns::AbstractVector{<:Integer},
                                ρcol::Int, # Which column contains the grid spacing?
                                errors::Vector{<:Tuple{Int,String,<:Real,<:Real}},
                                headers::Vector{String},
                                other_vectors::AbstractVector...;
                                convergence_rate_atol=1e-1,
                                plot_fun::Function = (i,data) -> nothing,
                                verbosity=1)
    ovs = Iterators.product(other_vectors...)

    progress = Progress(length(Ns))
    data = map(Ns) do N
        R,ρ = Rfun(rₘₐₓ, N)
        r = axes(R,1)
        cache = (R = R,
                 ρ = ρ,
                 V = vandermonde(R),
                 S = R'R,
                 r⁻¹ = R'*QuasiDiagonal(1 ./ r)*R,
                 poissons = Dict{Int,PoissonProblem}())
        d = map(o -> fun(cache, o...), ovs)
        verbosity > 1 || ProgressMeter.next!(progress)
        d
    end
    data = permutedims(squeeze(hcat(data...)))
    convergence_rates = map(enumerate(ovs)) do (i,o)
        case_data = vcat(filter(!isnothing,data[i,:])...)
        if isempty(case_data)
            ProgressMeter.next!(progress)
            return nothing
        end
        verbosity > 1 && pretty_table(case_data, headers)

        ρ = case_data[:,ρcol]
        label = join(string.(filter(c -> !(c isa Function), [o...])), ", ")

        plot_fun(i, case_data)

        e = map(errors) do (ecol,elabel,eexpected,rateexpected)
            curerr = case_data[:,ecol]
            minerr = minimum(abs, curerr)
            γ = estimate_convergence_rate(ρ, curerr)
            # We deem that the integration has passed if the minimal
            # error is below the requested threshold and the
            # convergence rate is approximately equal to or larger
            # than the requested order.
            pass = minerr < eexpected &&
                (isapprox(γ, rateexpected, atol=convergence_rate_atol) || γ > rateexpected)
            Any[minerr γ pass] # Without Any, pass will be coerced to Float64
        end |> e -> hcat(label, e...)

        verbosity > 1 || ProgressMeter.next!(progress)

        e
    end |> e -> vcat(filter(!isnothing, e)...)
    println()

    passjs = 1 .+ 3*(1:length(errors))

    if verbosity > 0
        pass = Highlighter((data,i,j) -> j>1 && mod(j-1,3)==0 && data[i,j],
                           bold = true, foreground = :green)
        fail = Highlighter((data,i,j) -> j>1 && mod(j-1,3)==0 && !data[i,j],
                           bold = true, foreground = :red)
        # formatter = merge(ft_printf("%2.3e", passjs .- 2),
        #                   ft_printf("%2.3f", passjs .- 1),
        #                   Dict(j => (v,i) -> v ? "✓" : "⨯" for j in passjs))

        fails = vec(.!reduce(&, convergence_rates[:,passjs], dims=2))
        sel = verbosity == 1 ? findall(fails) : (1:size(convergence_rates,1))
        if !isempty(sel)
            pretty_table(convergence_rates[sel, :],
                         vcat("Case", [["$elabel, minδ [$eexpected]", "$elabel, rate [$rateexpected]", "Pass"]
                                       for (_,elabel,eexpected,rateexpected) in errors]...),
                         highlighters=(pass,fail),
                         # formatter=formatter,
                         crop=:none)
        end
        fails = sum([count(.!convergence_rates[:,j]) for j in passjs])
        num_cases = length(errors)*size(convergence_rates,1)
        println("Number of fails: $fails ≡ $(100.0fails/num_cases)% fail rate")
    end

    all([all(convergence_rates[:,j]) for j in passjs])
end


export checkDerivativeMax

function checkDerivativeMax(f::Function,df::Function,x0;kwargs...)
	function testFun(x,v=[])
		if !(isempty(v))
			return f(x), df(v,x)
		else
			return f(x)
		end
	end
	return checkDerivativeMax(testFun,x0;kwargs...)
end

function getRandomTestDirection(x0::Float64)
	return randn()
end

function getRandomTestDirection(x0::Array{Float64})
	return randn(size(x0))
end

function checkDerivativeMax(f::Function,x0;out::Bool=true,tol::Float64=1.9,nSuccess::Int=3,v=getRandomTestDirection(x0))
# checkDerivative(f::Function,x0;out::Bool=true,tol::Float64=1.9,nSuccess::Int=3)
	if out
		println(@sprintf("%9s\t%9s\t%9s\t%9s\t%9s\t%5s","h","E0","E1","O1","O2","OK?"))
	end

	f0,dvf  = f(x0,v)
	f0      = vec(f0)
	nf0     = norm(f0)
	#dvf    = real(dvf)
	Error   = zeros(10,2)
	Order   = zeros(10,2)
	Success = zeros(10)
	for j=1:10
		ft = f(x0+10.0^(-j)*v)                # function value
		ft = vec(ft)
		Error[j,1] = norm(f0-ft)/nf0          # Error TaylorPoly 0
		Error[j,2] = norm(f0 .+10.0^(-j)*dvf .- ft)/nf0 # Error TaylorPoly 1
		if j>1
			Order[j,:] = log10(Error[j-1,:]./Error[j,:]);
		end
		if (Order[j,2]>tol) || (Error[j,1]/Error[j,2] > 100); Success[j]=1; end
		if out
			println(@sprintf("%1.3e\t%1.3e\t%1.3e\t%1.3e\t%1.3e\t%5d",
							10.0^(-j), Error[j,1],Error[j,2], Order[j,1],Order[j,2],Success[j]))
		end
	end
	pass = sum(Success) > nSuccess
	return  pass,Error,Order
end


println([1, 2, 3] .^ 3)
# `.` is a broadcast operator
# a .^ b equal to (^).(a, b)


using LazySets, MathematicalPredicates, SparseArrays

function helicopter_specification()
    # initial set:
    # xᵢ ∈ [-0.1, 0.1] for i ∈ [1, 8]
    # xᵢ = 0 otherwise
    X0 = Hyperrectangle(spzeros(28), sparsevec(1:8, fill(0.1, 8), 28))

    # safety property: x1 ≤ 0.12
    property = is_contained_in(HalfSpace(sparsevec([1], [1.0], 28), 0.12))

    # time horizon: 20 time units
    time_horizon = 20.0

    # specification
    O = Dict(:T => time_horizon, :property => property)

    return X0, O
end


using Distributions
using Distances
include("./Spelliptic.jl")
using Spelliptic
include("./CovFuncs.jl")

#setting the random seed
srand(123)

#Setting problem parameters
d = 2
N = 50000
l = 1.0
nu = 0.5
rho = 4.

#defining covariance function
cov = r -> matern( r, l, nu )

#generating point cloud
dUniform = Uniform(-3,3)
x = rand( dUniform, d, N )

println( "Test without nugget, using SpelMat:" )
#Generating a SpelMat structure
@time K = SpelMat( x , rho )
#Setting the content of the spelliptic struct according to the covariance
#matrix and point cloud:
@time setCovStat!( K, x, cov )

#Measuring elementwise approximation error
trueSol, appSol, error, distance = testAccuracy( K, x, cov, 500 )
println( "mean error = ", mean( error ), " median error = ", median( error ) )

vec = randn( N )
#Matrix vector product:
K * vec;
#Solution of linear system
K \ vec
#computing the logdeterminant
println( "logdet(K) = ", logdet( K ))

println( "Test with large nugget, using SpelMatIter:")
sigma = 10.
@time KIter = SpelMatIter( x , rho )
@time setCovStat!( KIter, x, cov, sigma )

#Matrix vector product:
KIter * vec;
#Solution of linear system
KIter \ vec


using JuMP
using CPLEX

include("parser.jl")

start = time()

function P1(MyFileName::String)
  P, h, n, w1, w2, l, g, t = read_instance(MyFileName)
  w1 = 1
  w2 = 5

  # Create the model
  m = Model(CPLEX.Optimizer)

  ## Variables
  @variable(m, x[1:n+2,1:h+2], Bin)
  @variable(m, d[1:n+2,1:h+2] >= 0)

  ## Constraints
  @constraint(m, [i in 2:n+1, j in 2:h+1], d[i,j] <= x[i,j+1] + x[i+1,j] + x[i-1,j] + x[i,j-1] - 4 * (1 - x[i,j]))
  @constraint(m, sum(x[i,j] for i in 2:n+1, j in 2:h+1) >= 60)

  ## Objective
  @objective(m, Max, w1 * sum(t[i-1,j-1]*(1-x[i,j]) for i in 2:n+1, j in 2:h+1) + w2*g*l*sum(4*x[i,j] - d[i,j] for i in 2:n+1, j in 2:h+1))

  #resolution
  optimize!(m)

  # solution_summary(m, verbose=true)
  # z_vals = JuMP.getvalue.( m[:z] )
  # println(z_vals)

end

function P2(MyFileName::String)
  P, h, n, w1, w2, l, g, t = read_instance(MyFileName)
  w1 = 1
  w2 = 5

  # Create the model
  m = Model(CPLEX.Optimizer)

  ## Variables
  @variable(m, x[1:n+2,1:h+2] >= 0)
  @variable(m, y[1:n+2,1:h+2,1:n+2,1:h+2] >= 0)

  ## Constraints
  @constraint(m, [i in 1:n+2, j in 1:h+2], x[i,j] <= 1)
  @constraint(m, [i in 2:n+1, j in 2:h+1], y[i,j,i-1,j] <= x[i,j])
  @constraint(m, [i in 2:n+1, j in 2:h+1], y[i,j,i+1,j] <= x[i,j])
  @constraint(m, [i in 2:n+1, j in 2:h+1], y[i,j,i,j-1] <= x[i,j])
  @constraint(m, [i in 2:n+1, j in 2:h+1], y[i,j,i,j+1] <= x[i,j])

  @constraint(m, [i in 2:n+1, j in 2:h+1], y[i,j,i-1,j] <= 1 - x[i-1,j])
  @constraint(m, [i in 2:n+1, j in 2:h+1], y[i,j,i+1,j] <= 1 - x[i+1,j])
  @constraint(m, [i in 2:n+1, j in 2:h+1], y[i,j,i,j-1] <= 1 - x[i,j-1])
  @constraint(m, [i in 2:n+1, j in 2:h+1], y[i,j,i,j+1] <= 1 - x[i,j+1])

  @constraint(m, sum(x[i,j] for i in 2:n+1, j in 2:h+1) >= 60)

  ## Objective
  @objective(m, Max, w1 * sum(t[i-1,j-1]*(1-x[i,j]) for i in 2:n+1, j in 2:h+1) + w2*g*l*sum(y[i,j,i-1,j] +  y[i,j,i+1,j] + y[i,j,i,j-1] + y[i,j,i,j+1] for i in 2:n+1, j in 2:h+1))

  #resolution
  optimize!(m)

  println(JuMP.value.(x))
  println(time() - start)

end

P2("output30.txt")


include("incompressible_navier_stokes.jl");
include("tocsv.jl");

timesteps=1000;
quadlen=20;
dt=0.001;
order=5;
p=filter_dependent_modes(order);
len=length(p);
u=rand(Complex128,(len,));
M=curl_basis_mass_matrix(p::Array{Int64},order::Int64);
RTR=cholfact(M);

(x,y,z,w)=trapezoid_tensor_product(-pi,pi,quadlen);

vx=linspace(-pi,pi,30);
vy=linspace(-pi,pi,30);
vz=linspace(-pi,pi,30);

V=curl_basis_vandermondes(order,p,x,y,z,w);
GV=curl_basis_gradient_vandermondes(order,p,x,y,z,w);
W=Diagonal(w[:]);

fn=tu->ns_eval(RTR,tu,V,GV,W);
#RK4
for n=1:timesteps
    t=time();
    print("writing...\n");
    tocsv(vx,vy,vz,(tx,ty,tz)->eval_curl_soln(u,p,order,tx,ty,tz),"data/ns.csv.$(n)");
    print("done writing...\n");
    print("time taken to write = $(time()-t)\n");
    t=time();
    k1=fn(u);
    k2=fn(u+0.5*dt*k1);
    k3=fn(u+0.5*dt*k2);
    k4=fn(u+dt*k3);
    u=u+(dt/6.0)*(k1+2.0*k2+2.0*k3+k4);
    print("iteration $(n) done\n");
    print("iteration time = $(time()-t)\n");
end


using CSV
using DataFrames
using DiscreteChoiceModels

data = CSV.read("../data/biogeme_swissmetro.dat", DataFrame, delim='\t')
data = data[in.(data.PURPOSE, [Set([1, 3])]) .& (data.CHOICE .!= 0), :]

@assert nrow(data) == 6768

model = multinomial_logit(
    @utility(begin
        1 ~ :αtrain + :βtravel_time * TRAIN_TT / 100 + :βcost * (TRAIN_CO * (GA == 0)) / 100
        2 ~ :αswissmetro + :βtravel_time * SM_TT / 100 + :βcost * SM_CO * (GA == 0) / 100
        3 ~ :αcar + :βtravel_time * CAR_TT / 100 + :βcost * CAR_CO / 100

        :αswissmetro = 0f  # fix swissmetro ASC to zero
    end),
    data.CHOICE,
    data,
    availability=[
        1 => (data.TRAIN_AV .== 1) .& (data.SP .!= 0),
        2 => data.SM_AV .== 1,
        3 => (data.CAR_AV .== 1) .& (data.SP .!= 0),
    ]
)

println(DiscreteChoiceModels.summary(model))


using Plots
using Convex
using Mosek, MosekTools
using SCS
using DelimitedFiles
using SparseArrays

data = readdlm(joinpath(dirname(@__FILE__),"data","snp500.txt"))
n = size(data)[1]

#p = plot(1:length(data), data)

N = length(data)
H =  spdiagm(0=>ones(N-2), 1=>-2*ones(N-2), 2=>ones(N-2))

x = Variable(N)

λ = 50
problem = minimize(0.5*sumsquares(x - data) + λ*norm(H*x, 1))

# Solve the problem by calling solve!
solve!(problem, Mosek.Optimizer)
solve!(problem, SCS.Optimizer)

# Check the status of the problem
problem.status # :Optimal, :Infeasible, :Unbounded etc.

# Get the optimal value
problem.optval

plot(data)
plot!(evaluate(x))


function test_user_mean()
    println("-- Testing UserMean recommender")

    data = DataAccessor(sparse([1 2 3; 4 5 6]))
    recommender = UserMean(data)
    build!(recommender)
    actual = predict(recommender, 1, 1)

    @test actual == 2.0
end

test_user_mean()


### A Pluto.jl notebook ###
# v0.14.0

using Markdown
using InteractiveUtils

# This Pluto notebook uses @bind for interactivity. When running this notebook outside of Pluto, the following 'mock version' of @bind gives bound variables a default value (instead of an error).
macro bind(def, element)
    quote
        local el = $(esc(element))
        global $(esc(def)) = Core.applicable(Base.get, el) ? Base.get(el) : missing
        el
    end
end

# ╔═╡ 7dea2baa-8f2c-11eb-184b-cb353ca6ca79
using PlutoUI

# ╔═╡ 9fd099fc-8f2c-11eb-286e-a548737f4572
using Plots

# ╔═╡ 9633d3f8-9000-11eb-3d5a-b3dd2a013890
using LinearAlgebra

# ╔═╡ 2c140db8-9013-11eb-21d9-f7718dc9cf57
using Optim

# ╔═╡ 56018eb0-90e1-11eb-18c0-e1f1aecf307c
using Vec

# ╔═╡ 5fead5d4-8f2d-11eb-2d3c-a34658637923
∇fline(x) = 0.75 * exp(-0.5 * (x - 7)^2) * (x - 7) + exp(-0.25 * (x - 3)^2) * 0.5 *(x - 3);

# ╔═╡ bf810ea8-8f2c-11eb-2d6e-3dcb7961b1fa
fline(x) = 1.5 - exp(-0.25 * (x - 3)^2)  - 0.75 * exp(-0.5 * (x - 7)^2);

# ╔═╡ 99e23b7a-8ffd-11eb-1b86-9940c74de5fc
rosenbrock(x) = (1 - x[1])^2 + 5 * (x[2] - x[1]^2)^2;

# ╔═╡ 9d46db42-8ffe-11eb-1f0f-1d6553bcd608
rosenbrock(x, y) = (1 - x)^2 + 5 * (y - x^2)^2;

# ╔═╡ 9a62773e-8ffd-11eb-315a-611790d3d945
∇rosenbrock(x) = [2 * (10 * x[1]^3 - 10 * x[1] * x[2] + x[1] - 1), 10 * (x[2] - x[1]^2)];

# ╔═╡ fb5587cc-9012-11eb-313a-512bd0f9651b
Hrosenbrock(x) = [-20 * (x[2] - x[1]^2) + 40 * x[1]^2 + 2 -20*x[1];
	              -20*x[1] 10];

# ╔═╡ fb7ea5ef-a2be-43a5-aae6-1bacc3a65f1e
rosenbrock2(x) = (1 - x[1])^2 + 100 * (4x[2] - x[1]^2)^2;

# ╔═╡ db569392-347f-45b1-b7c6-e2a0c235cff9
rosenbrock2(x, y) = (1-x)^2 + 100 * (4y - x^2)^2;

# ╔═╡ 1d09ca7a-8b25-4f3a-9f7d-ef586dd80be8
rosenbrock3(x, y) = (1-x)^2 + 100 * (y - x^2)^2;

# ╔═╡ cb524eeb-d78d-4125-8e5a-5da5e14644bb
∇rosenbrock2(x) = [2 * (200x[1]^3 - 800x[1] * x[2] + x[1] - 1), -800 * (x[1]^2 - 4x[2])];

# ╔═╡ 5e9fcab4-d2fe-445d-9b8e-2fe9fe50444d
md"""
# Descent Direction Iteration

1. Determine if out current x satisfies some termination condition. If so, terminate. If not, proceed.
2. Determine the **descent direction**.
3. Determine the **step size**.
4. Compute the next design point according to $\mathbf{x}^{(k+1)} \gets \mathbf{x}^{(k)} + \alpha^{(k)}\mathbf{d}^{(k)}$.
"""

# ╔═╡ 70855f33-c00d-49c3-82ec-f891ae4b40c5
md"""
nsteps = $(@bind nsteps1d NumberField(0:1:10, default = 0))

α = $(@bind α1d NumberField(0.1:0.1:3, default = 0.1))
"""

# ╔═╡ a3ae828c-3354-43e4-bf49-2284e465660d
begin
	p10 = plot(x -> x^2, legend = false, xlabel = "x", ylabel = "f(x)", color = :black, lw = 3, xlims = (-4,4), ylims = (0, 15))

	function run_gd(α, nsteps)
		gd_xs = [-3.5]
		gd_ys = [gd_xs[end]^2]

		for i = 1:nsteps
			next_x = gd_xs[end] - α * 2 * gd_xs[end]
			push!(gd_xs, next_x)
			push!(gd_ys, next_x^2)
		end

		return gd_xs, gd_ys
	end

	gd_xs, gd_ys = run_gd(α1d, nsteps1d)

	scatter!(gd_xs, gd_ys, color = :aqua, markersize = 5)
	scatter!(gd_xs, zeros(length(gd_xs)), color = :black, markersize = 5)
	for i = 1:length(gd_xs)
		plot!([gd_xs[i], gd_xs[i]], [0.0, gd_ys[i]], color = :black)
	end
	p10
end

# ╔═╡ 0259e325-9558-4dfc-9ca8-06daade248b9
md"""
# Multiple Dimensions
"""

# ╔═╡ e54c3874-667d-42a8-ba9f-5376f3ae3bd6
md"""
nsteps = $(@bind nsteps2d NumberField(0:1:10, default = 0))

α = $(@bind α2d NumberField(0.1:0.1:3, default = 0.1))
"""

# ╔═╡ 33f938e6-8229-4baa-9e1c-3c0215c53197
begin
	xvec = -4:0.1:4
	yvec = -4:0.1:4
	fbowl(x, y) = (x^2 + y^2) / 2

	function run_gd_2d(α, nsteps)
		gd_xs = [-3.5]
		gd_ys = [3.5]
		gd_zs = [fbowl(gd_xs[end], gd_ys[end])]

		for i = 1:nsteps
			next_x = gd_xs[end] - α * 2 * gd_xs[end]
			next_y = gd_ys[end] - α * 2 * gd_ys[end]
			push!(gd_xs, next_x)
			push!(gd_ys, next_y)
			push!(gd_zs, fbowl(gd_xs[end], gd_ys[end]))
		end

		return gd_xs, gd_ys, gd_zs
	end

	gd_xs2, gd_ys2, gd_zs2 = run_gd_2d(α2d, nsteps2d)

	sp1 = plot(xvec, yvec, fbowl, st = :surface, c = cgrad(:viridis, rev = true), colorbar = false, size = (600, 300), alpha = 0.5, legend = false, xlabel = "x₁", ylabel = "x₂", zlabel = "f(x)")
	#for i = 1:length(gd_xs)
	scatter!(sp1, gd_xs2, gd_ys2, gd_zs2, color = :red)
	plot!(sp1, gd_xs2, gd_ys2, gd_zs2, color = :black)
		#plot!(sp1, [-3.5, -3.5], [3.5, 3.5], [0.0, 9.0])

	sp2 = contour(xvec, yvec, fbowl, c = cgrad(:viridis, rev = true), colorbar = false, aspectratio = :equal, size = (600, 300), xlabel = "x₁", ylabel = "x₂", legend = false)
	scatter!(sp2, gd_xs2, gd_ys2, color = :red)
	plot!(sp2, gd_xs2, gd_ys2, color = :black)

	plot(sp1, sp2)
end

# ╔═╡ 9402e2b6-757e-4632-aac2-7251b28a768a
# begin
# # 	# plot(x -> x^2, legend = false, xlabel = "x", ylabel = "f(x)", color = :black, lw = 3, xlims = (-4,4), ylims = (0, 15))
# 	sp10 = plot(xvec, yvec, fbowl, st = :surface, c = cgrad(:viridis, rev = true), colorbar = false, size = (600, 400), alpha = 0.5, legend = false, xlabel = "x₁", ylabel = "x₂", zlabel = "f(x)")
# 	#scatter!(sp10, [-3.5], [3.5], [fbowl(-3.5, 3.5)], color = :red, markersize = 6)
# 	xsline = collect(-4:0.1:4)
# 	ysline = -xsline
# 	fsline = [fbowl(xsline[i], ysline[i]) for i = 1:length(xsline)]

# 	plot!(xsline, ysline, fsline, color = :white, lw = 6)
# 	plot!(xsline, ysline, zeros(length(xsline)), color = :black, lw = 6)
# end

# ╔═╡ a364ef52-8f2c-11eb-26a6-5d47e19a7230
md"""
# Line Search
"""

# ╔═╡ 24b2fe6c-9004-11eb-1e5f-0317dd2e85aa
md"""
### First Wolfe Condition
"""

# ╔═╡ 4794ee64-8f34-11eb-1ef2-5981f0e059df
β = 0.2

# ╔═╡ a00a4ec4-8f34-11eb-3db7-49be2cca0e5f
x₀ = 1.5

# ╔═╡ 80e1c23c-8f2e-11eb-3beb-519be0766468
md"""
α = $(@bind αlinedemo Slider(0.05:0.05:5.0, default = 5.0))
"""

# ╔═╡ ed8b99da-8f2c-11eb-14c5-0393c6d531c4
begin
	p1 = plot(fline, xlims = (0, 7.0), ylims = (0.45, 1.65), color = :black, xlabel = "x", ylabel = "f(x)", label = "")
	first_wolf_bound = fline(x₀) + β * αlinedemo * ∇fline(x₀)
	color = fline(x₀ + αlinedemo) < first_wolf_bound ? :green : :red
	plot!(Shape([0.0, 7.0, 7.0, 0.0, 0.0], [0.0, 0.0, first_wolf_bound, first_wolf_bound, 0.0]), color = color, alpha = 0.3, label = "")
	scatter!([x₀], [fline(x₀)], color = :black, label = "x₀")
	scatter!([x₀ + αlinedemo], [fline(x₀ + αlinedemo)], color = :gray, label = "f(x₀ + αd)")
	if fline(x₀ + αlinedemo) < first_wolf_bound
		xs = collect(x₀ + αlinedemo:0.05:4.05)
		ys = fline.(xs)
		plot!(xs, ys, color = :green, linewidth = 6, label = "")
	end
	p1
end

# ╔═╡ 4c0b5eac-8f35-11eb-06b4-a75f220c7f78
md"""
**f(x₀ + α):** $(fline(x₀ + αlinedemo))

**f(x₀) + βα∇f(x₀):** $first_wolf_bound
"""

# ╔═╡ 3954970e-9004-11eb-3be9-5f7adea26bc2
md"""
### Backtracking Line Search
"""

# ╔═╡ 5479f7d8-8fee-11eb-134b-a781d0affff4
function backtracking_line_search(f, ∇f, x, d, α; p = 0.5, β = 1e-4)
	y, g = f(x), ∇f(x)
	while f(x + α*d) > y + β*α*(g'd)
		α *= p
	end
	α
end

# ╔═╡ f87f64da-8fee-11eb-15fe-8f8090d4d226
md"""
p = $(@bind p NumberField(0.3:0.1:0.9, default = 0.8))
"""

# ╔═╡ ecaf9fe4-8fee-11eb-1c05-d76ef0f7ee1e
begin
	plot(fline, xlims = (0, 7.0), ylims = (0.45, 1.65), color = :black, xlabel = "x", ylabel = "f(x)", label = "", legend = false)

	xs1 = collect(x₀:0.05:4.05)
	ys1 = fline.(xs1)

	plot!(xs1, ys1, color = :green, linewidth = 6)

	function get_trace()
		αbt = 5.0
		attempted_xs = [x₀ + αbt]
		attempted_fs = [fline(x₀ + αbt)]

		while fline(x₀ + αbt) > fline(x₀) + β * αbt * ∇fline(x₀)
			αbt *= p
			push!(attempted_xs, x₀ + αbt)
			push!(attempted_fs, fline(x₀ + αbt))
		end
		return attempted_xs, attempted_fs
	end

	attempted_xs, attempted_fs = get_trace()

	scatter!(attempted_xs[1:end-1], attempted_fs[1:end-1], color = :red, markersize = 6)
	scatter!([attempted_xs[end]], [attempted_fs[end]], color = :green, markersize = 6)
	scatter!([x₀], [fline(x₀)], color = :black, label = "x₀", markersize = 6)
end

# ╔═╡ 827f2e30-8ffd-11eb-2cd6-9b1edfa701f9
md"""
### Backtracking line search on the Rosenbrock function
"""

# ╔═╡ 3f536d2c-9003-11eb-23c7-7b0b0a92d51a
md"""
nsteps = $(@bind nstepsbls NumberField(0:1:7, default = 0))
"""

# ╔═╡ e0fea120-8ffd-11eb-2cac-4f6e4547cc26
begin
	xr = -2:0.1:2
	yr = -2:0.1:2

	# Contour of rosenbrock
	c = contour(xr, yr, rosenbrock, levels=[0.5,1,2,3,5,10,20,50,100], colorbar = false, c = cgrad(:viridis, rev = true), legend = false, xlims = (-2, 2), ylims = (-2, 2), xlabel = "x₁", ylabel = "x₂", aspectratio = :equal)
	xbls = [-1.75, -1.75]
	scatter!([xbls[1]], [xbls[2]], color = :black)

	pbls = 0.5
	βbls = 1e-4
	αmaxbls = 6.0
	#nstepsbl = 7
	for k in 1:nstepsbls
		d = -∇rosenbrock(xbls)
		α = αmaxbls
		y, g = rosenbrock(xbls), ∇rosenbrock(xbls)

		dir = normalize(d, 2)
		plot!([xbls[1], xbls[1] + 6dir[1]], [xbls[2], xbls[2] + 6dir[2]], color = :red)

		while rosenbrock(xbls + α*d) ≥ y + βbls*α*(g⋅d)
			x_push = xbls + d*α
			if -2 ≤ x_push[1] ≤ 2 && -2 ≤ x_push[2] ≤ 2
				scatter!([x_push[1]], [x_push[2]], color = :red)
			end
			α *= pbls
		end
		scatter!([xbls[1] + d[1]*α], [xbls[2] + d[2]*α], color = :black)
		plot!([xbls[1], xbls[1] + d[1]*α], [xbls[2], xbls[2] + d[2]*α], color = :black)
		xbls += d*α
	end
	c
end

# ╔═╡ 4848aa34-9004-11eb-1408-5598d9e145f3
md"""
### Curvature Condition
"""

# ╔═╡ 3ae1a64c-9005-11eb-32cd-adbd7cb6651a
md"""
σ = $(@bind σcurve NumberField(0.1:0.1:1, default = 0.4))
"""

# ╔═╡ 55f1ca30-9004-11eb-2658-972069811aa5
begin
	p2 = plot(fline, xlims = (0, 7.0), ylims = (0.45, 1.65), color = :black, xlabel = "x", ylabel = "f(x)", label = "")
	scatter!([x₀], [fline(x₀)], color = :black, label = "x₀")

	gradx₀ = ∇fline(x₀)
	points_curv = collect(x₀:0.05:7)
	gradpoints_curv = ∇fline.(points_curv)
	inds_curv = findall(gradpoints_curv .≥ σcurve * gradx₀)

	diffs_inds_curv = [inds_curv[i+1] - inds_curv[i] for i = 1:length(inds_curv)-1]
	breaks_curv = findall(diffs_inds_curv .> 1)

	if breaks_curv != nothing
		for i = 1:length(breaks_curv)
			if i == 1
				start_ind = 1
			else
				start_ind = breaks_curv[i-1] + 1
			end
			plot_inds = inds_curv[start_ind:breaks_curv[i]]
			plot!(points_curv[plot_inds], fline.(points_curv[plot_inds]), color = :green, linewidth = 6, label = "")
		end
		plot!(points_curv[inds_curv[breaks_curv[end]+1:end]], fline.(points_curv[inds_curv[breaks_curv[end]+1:end]]), color = :green, linewidth = 6, label = "")
	else
		plot!(points_curv[inds_curv], fline.(points_curv[inds_curv]), color = :green, linewidth = 6, label = "")
	end
	p2
end

# ╔═╡ 529ef5e6-9007-11eb-05b2-a70d8ad51908
md"""
### Strong Curvature Condition (Second Wolfe Condition)
"""

# ╔═╡ 9afa6fa8-9007-11eb-0d4e-01881e229cc7
md"""
σ = $(@bind σscurve NumberField(0.1:0.1:1, default = 0.4))
"""

# ╔═╡ 63249a1a-9007-11eb-2630-0523c391ee59
begin
	p3 = plot(fline, xlims = (0, 7.0), ylims = (0.45, 1.65), color = :black, xlabel = "x", ylabel = "f(x)", label = "")
	scatter!([x₀], [fline(x₀)], color = :black, label = "x₀")

	points_scurv = collect(x₀:0.05:7)
	gradpoints_scurv = ∇fline.(points_scurv)
	inds_scurv = findall(abs.(gradpoints_scurv) .≤ -σscurve * gradx₀)

	diffs_inds_scurv = [inds_scurv[i+1] - inds_scurv[i] for i = 1:length(inds_scurv)-1]
	breaks_scurv = findall(diffs_inds_scurv .> 1)

	if breaks_scurv != nothing
		for i = 1:length(breaks_scurv)
			if i == 1
				start_ind = 1
			else
				start_ind = breaks_scurv[i-1] + 1
			end
			plot_inds = inds_scurv[start_ind:breaks_scurv[i]]
			plot!(points_scurv[plot_inds], fline.(points_scurv[plot_inds]), color = :green, linewidth = 6, label = "")
		end
		plot!(points_scurv[inds_scurv[breaks_scurv[end]+1:end]], fline.(points_scurv[inds_scurv[breaks_scurv[end]+1:end]]), color = :green, linewidth = 6, label = "")
	else
		plot!(points_scurv[inds_scurv], fline.(points_scurv[inds_scurv]), color = :green, linewidth = 6, label = "")
	end
	p3
end

# ╔═╡ e7ca1b5c-9008-11eb-2e5b-972ffbcdad4b
md"""
### Wolfe Conditions
"""

# ╔═╡ 0201e428-9009-11eb-3b2f-d7f3bb5b1831
begin
	p4 = plot(fline, xlims = (0, 7.0), ylims = (0.45, 1.65), color = :black, xlabel = "x", ylabel = "f(x)", label = "")
	scatter!([x₀], [fline(x₀)], color = :black, label = "x₀")

	points_wolfe = collect(x₀:0.05:7)
	gradpoints_wolfe = ∇fline.(points_wolfe)
	inds_wolfe1 = findall(fline.(points_wolfe) .≤ fline(x₀) .+ (β .* (points_wolfe .- x₀) .* ∇fline(x₀)))
	inds_wolfe2 = findall(abs.(gradpoints_wolfe) .≤ -σscurve * gradx₀)

	diffs_inds_wolfe2 = [inds_wolfe2[i+1] - inds_wolfe2[i] for i = 1:length(inds_wolfe2)-1]
	breaks_wolfe2 = findall(diffs_inds_wolfe2 .> 1)

	if breaks_wolfe2 != nothing
		for i = 1:length(breaks_wolfe2)
			if i == 1
				start_ind = 1
			else
				start_ind = breaks_wolfe2[i-1] + 1
			end
			plot_inds = inds_wolfe2[start_ind:breaks_wolfe2[i]]
			plot!(points_wolfe[plot_inds], fline.(points_wolfe[plot_inds]), color = :red, linewidth = 6, label = "")
		end
		plot!(points_wolfe[inds_wolfe2[breaks_wolfe2[end]+1:end]], fline.(points_wolfe[inds_wolfe2[breaks_wolfe2[end]+1:end]]), color = :red, linewidth = 6, label = "2nd Wolfe")
	else
		plot!(points_wolfe[inds_wolfe2], fline.(points_wolfe[inds_wolfe2]), color = :red, linewidth = 6, label = "2nd Wolfe")
	end

	plot!(points_wolfe[inds_wolfe1], fline.(points_wolfe[inds_wolfe1]), color = :blue, linewidth = 6, label = "1st Wolfe")

	inds_both_wolfe = intersect(inds_wolfe1, inds_wolfe2)

	plot!(points_wolfe[inds_both_wolfe], fline.(points_wolfe[inds_both_wolfe]), color = :green, linewidth = 6, label = "Both")

	p4
end

# ╔═╡ 631fc672-900b-11eb-1fe2-a531dac443b1
md"""
### Strong Backtracking Line Search
"""

# ╔═╡ a738a7fb-0ebb-42c5-bc2d-af1a4ea72bfc
md"""
If one of the following conditions hold, an interval is guaranteed to contain step lengths satisfying the Wolfe conditions:

$f(\mathbf{x} + \alpha^{(k)}\mathbf{d}) \geq f(\mathbf{x})$

$f(\mathbf{x^{(k)}} + \alpha^{(k)}\mathbf{d^{(k)}}) \geq f(\mathbf{x^{(k)}}) + \beta\alpha\nabla_{\mathbf{d}^{(k)}}f(\mathbf{x}^{(k)})$

$\nabla f(\mathbf{x} + \alpha^{(k)}\mathbf{d}) \geq \mathbf{0}$
"""

# ╔═╡ 272c43fc-9012-11eb-07f6-3d0625cafbab
md"""
nsteps = $(@bind nstepssbls NumberField(0:1:7, default = 0))
"""

# ╔═╡ 75b076d0-900b-11eb-2875-b9b74ab9a943
begin
	x_init = 0.0
	cond1(α_hi) = ∇fline(x_init + α_hi) >= 0
	cond2(α_hi) = fline(x_init + α_hi) > fline(x_init) + β*α_hi*∇fline(x_init)
	cond3(α_hi) = fline(x_init + α_hi) >= fline(x_init)

	p5 = plot(fline, xlims = (0, 7.0), ylims = (0.45, 1.65), color = :black, xlabel = "x", ylabel = "f(x)", label = "")
	scatter!([x_init], [fline(x_init)], color = :black, label = "x₀", markersize = 6)

	points_cd = collect(0.5:0.05:7)
	cd1 = cond1.(points_cd)
	cd2 = cond2.(points_cd)
	cd3 = cond3.(points_cd)
	inds_cd = findall(cd1 .| cd2 .| cd3)

	diffs_inds_cd = [inds_cd[i+1] - inds_cd[i] for i = 1:length(inds_cd)-1]
	breaks_cd = findall(diffs_inds_cd .> 1)

	if breaks_cd != nothing
		for i = 1:length(breaks_cd)
			if i == 1
				start_ind = 1
			else
				start_ind = breaks_cd[i-1] + 1
			end
			plot_inds = inds_cd[start_ind:breaks_cd[i]]
			plot!(points_cd[plot_inds], fline.(points_cd[plot_inds]), color = :blue, linewidth = 6, label = "")
		end
		plot!(points_cd[inds_cd[breaks_cd[end]+1:end]], fline.(points_cd[inds_cd[breaks_cd[end]+1:end]]), color = :blue, linewidth = 6, label = "Interval Conditions")
	else
		plot!(points_cd[inds_cd], fline.(points_cd[inds_cd]), color = :blue, linewidth = 6, label = "Interval Conditions")
	end

	function sbls()
		α = 0.6
		y0, g0, y_prev, α_prev = fline(x_init), ∇fline(x_init), NaN, 0

		for k = 1:nstepssbls
			y = fline(x_init + α)

			scatter!([α], [y], color = :black, label = "", markersize = 6)

			if y > y0 + β*α*g0 || (!isnan(y_prev) && y >= y_prev)

				α = 0.75α # bisection
				y = fline(α)
				scatter!([α], [y], color = :red, label = "", markersize = 6)
			end
			g = ∇fline(x_init + α)
			if abs(g) <= -σscurve * g0
				break
			elseif g >= 0
				break
			end
			α_prev, α = α, 2α
			y_prev = y
		end
	end

	sbls()
	p5
end

# ╔═╡ 81f2c770-9012-11eb-099f-dda2f9c4246f
md"""
# Trust Region Methods
"""

# ╔═╡ e9e89ffc-9018-11eb-1028-856604a9f26b
md"""
nsteps = $(@bind nstepstrust NumberField(0:1:10, default = 0))
"""

# ╔═╡ 447b2884-9013-11eb-1457-0b057903165f
begin
	function solve_trust_region_subproblem(∇f, H, x0, δ)

		Hx0 = H(x0)
		∇fx0 = ∇f(x0)

		# Unconstrained
		f = x -> ∇fx0⋅(x-x0) + ((x-x0)'*Hx0)⋅(x-x0)/2

		# Constrained with huge penalty
		g = x -> norm(x-x0) <= δ ? f(x) : 9999999.0

		result = optimize(g, x0, NelderMead())

		return (result.minimizer, result.minimum)
	end

	# Contour of rosenbrock
	cr = contour(xr, yr, rosenbrock, levels=[0.5,1,2,3,5,10,20,50,100], colorbar = false, c = cgrad(:viridis, rev = true), legend = false, xlims = (-2, 2), ylims = (-2, 2), xlabel = "x₁", ylabel = "x₂", aspectratio = :equal)

	#scatter!([xbls[1]], [xbls[2]], color = :blue)

	function trust_region_rosenbrock(k_max)
		x₀ = [-1.75,-1.75]
		x = deepcopy(x₀)
		y = rosenbrock(x)
		δ = 1.0
		θs = range(0,stop=2π,length=101)
		η1=0.25; η2=0.5; γ1=0.5; γ2=2.0; δ=1.0

		for k = 1:k_max
			scatter!([x[1]], [x[2]], color = :black, alpha = k/k_max)
			plot!([x[1] + δ*cos(θ) for θ in θs], [x[2] + δ*sin(θ) for θ in θs], color = :black, alpha = k/k_max)

			x′, y′ = solve_trust_region_subproblem(∇rosenbrock, Hrosenbrock, x, δ)
			r = (y - rosenbrock(x′)) / (y - y′)
			if r < η1
				δ *= γ1
			else
				x, y = x′, rosenbrock(x′)
				if r > η2
					δ *= γ2
				end
			end
		end
	end

	trust_region_rosenbrock(nstepstrust)

	cr
end

# ╔═╡ 82a6b55a-90df-11eb-119b-9d94c9420aae
md"""
# First-Order Methods
"""

# ╔═╡ 9c3268ac-90df-11eb-3a09-cd5b1e01213d
md"""
### Gradient Descent
"""

# ╔═╡ 9397f2be-90e1-11eb-2a0c-559077244bd9
md"""
nsteps = $(@bind nstepsgd NumberField(0:1:10, default = 0))
"""

# ╔═╡ a9d3e5b2-90df-11eb-388f-eb7a9433a6dd
begin
	# Contour of rosenbrock
	cr1 = contour(xr, yr, rosenbrock, levels=[0.5,1,2,3,5,10,20,50,100], colorbar = false, c = cgrad(:viridis, rev = true), legend = false, xlims = (-2, 2), ylims = (-2, 2), xlabel = "x₁", ylabel = "x₂", aspectratio = :equal)
	scatter!([-1.0], [-1.0], color = :black)

	# For solving for α
	function secant_method(df, x1, x2, ϵ)
	    df1 = df(x1)

	    delta = Inf
	    while abs(delta) > ϵ
	    	df2 = df(x2)
	        delta = (x2 - x1)/(df2 - df1)*df2
	        x1, x2, df1 = x2, x2 - delta, df2
	    end
	    x2
	end

	# Run
	df(x,y) = [2*(10*x^3-10*x*y+x-1), 10*(y-x^2)]

	function grad_descent(nsteps)
		p0 = (-1, -1)
		pts_grad = Tuple{Float64,Float64}[p0]
		for i in 1 : nsteps
			x, y = pts_grad[end]
			dp = normalize(-VecE2{Float64}(df(x, y)...))

			f1d = a -> begin
				x2 = x + a*dp.x
				y2 = y + a*dp.y

				da = df(x2, y2)
				pa = VecE2{Float64}(da[1], da[2])
				proj(pa, dp, Float64)
			end
			alpha = secant_method(f1d, 0.0, 1.0, 0.0001)
			push!(pts_grad, (x + alpha*dp.x, y + alpha*dp.y))
		end
		return pts_grad
	end

	pts_grad = grad_descent(10)

	plot!([pts_grad[i][1] for i = 1:nstepsgd+1], [pts_grad[i][2] for i = 1:nstepsgd+1], color = :black, linewidth = 2)
end

# ╔═╡ d59fd8f2-90e1-11eb-102a-f36334c851b4
md"""
### Conjugate Gradient Descent
"""

# ╔═╡ 0263d884-90e7-11eb-2d19-37a0ebcf527b
md"""
nsteps = $(@bind nstepsgd_conj NumberField(0:1:10, default = 0))
"""

# ╔═╡ c589570a-90e5-11eb-2100-45f865a96a02
begin
	# Contour of rosenbrock
	cr2 = contour(xr, yr, rosenbrock, levels=[0.5,1,2,3,5,10,20,50,100], colorbar = false, c = cgrad(:viridis, rev = true), legend = false, xlims = (-2, 2), ylims = (-2, 2), xlabel = "x₁", ylabel = "x₂", aspectratio = :equal)
	scatter!([-1.0], [-1.0], color = :black)

	function conj_grad(nsteps)
		x0 = Float64[-1, -1]
		pts_conj = Vector{Float64}[x0]
		g0 = ∇rosenbrock(x0)
		d0 = -g0
		f1d = a -> begin
			da = ∇rosenbrock(x0 + a*d0)
			proj(VecE2{Float64}(da[1], da[2]), VecE2{Float64}(d0[1], d0[2]), Float64)
		end
		alpha = secant_method(f1d, 0.0, 0.1, 0.0001) # NOTE: is sensitive to the value of 0.1, does not work with 1.0!
		push!(pts_conj, x0 + alpha*d0)

		for i in 1 : nsteps
			x = pts_conj[end]
			g1 = ∇rosenbrock(x)
			β = max(0.0, dot(g1, g1-g0)/dot(g0,g0))
			d1 = -g1 + β*d0

			f1d = a -> begin
				da = ∇rosenbrock(x + a*d1)
				proj(VecE2{Float64}(da[1], da[2]), VecE2{Float64}(d1[1], d1[2]), Float64)
			end
			alpha = secant_method(f1d, 0.0, 0.1, 0.0001)
			push!(pts_conj, x + alpha*d1)

			d0 = d1
			g0 = g1
		end
		return pts_conj
	end

	pts_conj = conj_grad(10)

	plot!([pts_grad[i][1] for i = 1:nstepsgd_conj+1], [pts_grad[i][2] for i = 1:nstepsgd_conj+1], color = :black, linewidth = 2, alpha = 0.5)
	plot!([pts_conj[i][1] for i = 1:nstepsgd_conj+1], [pts_conj[i][2] for i = 1:nstepsgd_conj+1], color = :black, linewidth = 2)
end

# ╔═╡ 04834139-2837-4891-9ba1-4f16c32df04f
md"""
### Momentum
"""

# ╔═╡ 50688a3b-c1bf-4cde-a820-874158384fae
md"""
nsteps = $(@bind nstepsgd_mom NumberField(0:1:40, default = 0))
"""

# ╔═╡ 7967346e-bf2f-4791-8e5a-11fee51ba9de
begin
	xr2 = -3:0.1:2
	yr2 = -0.5:0.1:2
	# Contour of rosenbrock 2
	cr3 = contour(xr2, yr2, rosenbrock2, levels=[25,50,100,200,250,300], colorbar = false, c = cgrad(:viridis, rev = true), legend = false, xlims = (-3, 2), ylims = (-0.5, 2), xlabel = "x₁", ylabel = "x₂", aspectratio = :equal, clim = (2, 500))
	scatter!([-2.0], [1.5], color = :black)

	abstract type DescentMethod end

	function this_step!(M::DescentMethod, v::VecE2{Float64})
	    x = Float64[v.x, v.y]
	    return VecE2{Float64}(step!(M, rosenbrock2, ∇rosenbrock2, x)...)
	end

	function run_descent_method(M::DescentMethod, x₀::VecE2{Float64}, N::Int)
	    pts = [x₀]
	    init!(M, rosenbrock2, ∇rosenbrock2, Float64[x₀.x, x₀.y])
	    for i in 1 : N
	        push!(pts, this_step!(M, pts[end]))
	    end
	    return pts
	end

	struct GradientDescent <: DescentMethod
		α
	end

	function init!(M::GradientDescent, f, ∇f, x)
			return M
	end

	function step!(M::GradientDescent, f, ∇f, x)
		α, g = M.α, ∇f(x)
		return x - α*g
	end

	mutable struct Momentum <: DescentMethod
		α # learning rate
		β # momentum decay
		v # momentum
	end

	function init!(M::Momentum, f, ∇f, x)
		M.v = zeros(length(x))
		return M
	end

	function step!(M::Momentum, f, ∇f, x)
		α, β, v, g = M.α, M.β, M.v, ∇f(x)
		v[:] = β*v - α*g
		return x + v
	end

	mutable struct NesterovMomentum <: DescentMethod
		α # learning rate
		β # momentum decay
		v # momentum
	end

	function init!(M::NesterovMomentum, f, ∇f, x)
		M.v = zeros(length(x))
		return M
	end

	function step!(M::NesterovMomentum, f, ∇f, x)
		α, β, v = M.α, M.β, M.v
		v[:] = β*v - α*∇f(x + β*v)
		return x + v
	end

	mutable struct HyperGradientDescent <: DescentMethod
		α0 # initial learning rate
		μ # learning rate of the learning rate
		α # current learning rate
		g_prev # previous gradient
	end

	function init!(M::HyperGradientDescent, f, ∇f, x)
		M.α = M.α0
		M.g_prev = zeros(length(x))
		return M
	end

	function step!(M::HyperGradientDescent, f, ∇f, x)
		α, μ, g, g_prev = M.α, M.μ, ∇f(x), M.g_prev
		α = α + μ*(g⋅g_prev)
		M.g_prev, M.α = g, α
		return x - α*g
	end

	mutable struct HyperNesterovMomentum <: DescentMethod
		α0 # initial learning rate
		μ # learning rate of the learning rate
		β # momentum decay
		v # momentum
		α # current learning rate
		g_prev # previous gradient
	end

	function init!(M::HyperNesterovMomentum, f, ∇f, x)
		M.α = M.α0
		M.v = zeros(length(x))
		M.g_prev = zeros(length(x))
		return M
	end

	function step!(M::HyperNesterovMomentum, f, ∇f, x)
		α, β, μ = M.α, M.β, M.μ
		v, g, g_prev = M.v, ∇f(x), M.g_prev
		α = α - μ*(g⋅(-g_prev - β*v))
		v[:] = β*v + g
		M.g_prev, M.α = g, α
		return x - α*(g + β*v)
	end

	N = 40

	pts_gd = run_descent_method(GradientDescent(0.0003), VecE2{Float64}(-2,1.5), N)
	pts_mom = run_descent_method(Momentum(0.0003, 0.9, zeros(2)), VecE2{Float64}(-2,1.5), N)

	plot!([pts_gd[i][1] for i = 1:nstepsgd_mom + 1], [pts_gd[i][2] for i = 1:nstepsgd_mom + 1], color = :black, linewidth = 2, alpha = 0.5)
	plot!([pts_mom[i][1] for i = 1:nstepsgd_mom + 1], [pts_mom[i][2] for i = 1:nstepsgd_mom + 1], color = :black, linewidth = 2)
end

# ╔═╡ bf301087-6624-4c56-a8d2-dc7a9856d33e
md"""
### Nesterov Momentum
"""

# ╔═╡ 5c4506ab-c281-4f94-a60a-ef5d727fe125
md"""
nsteps = $(@bind nstepsnest_mom NumberField(0:1:40, default = 0))
"""

# ╔═╡ b638d908-35a9-4203-945a-1086f4dba559
begin
	# Contour of rosenbrock 2
	cr4 = contour(xr2, yr2, rosenbrock2, levels=[25,50,100,200,250,300], colorbar = false, c = cgrad(:viridis, rev = true), legend = false, xlims = (-3, 2), ylims = (-0.5, 2), xlabel = "x₁", ylabel = "x₂", aspectratio = :equal, clim = (2, 500))
	scatter!([-2.0], [1.5], color = :black)

	pts_nest_mom = run_descent_method(NesterovMomentum(0.0002, 0.92, zeros(2)), VecE2{Float64}(-2,1.5), N)

	plot!([pts_mom[i][1] for i = 1:nstepsnest_mom + 1], [pts_mom[i][2] for i = 1:nstepsnest_mom + 1], color = :black, linewidth = 2, alpha = 0.5)
	plot!([pts_nest_mom[i][1] for i = 1:nstepsnest_mom + 1], [pts_nest_mom[i][2] for i = 1:nstepsnest_mom + 1], color = :black, linewidth = 2)
end

# ╔═╡ d0a4ca13-8ea4-4451-ba88-b46d28a1ba89
md"""
### Hypergradient Descent and Hyper-Nesterov
"""

# ╔═╡ b3759a50-11d4-4d96-a80b-bcb6a76ba944
md"""
nsteps = $(@bind nstepshyper NumberField(0:1:40, default = 0))
"""

# ╔═╡ 36099212-4d39-4e22-a142-0b9b88263de2
begin
	# Contour of rosenbrock 2
	cr5 = contour(xr2, yr2, rosenbrock2, levels=[25,50,100,200,250,300], colorbar = false, c = cgrad(:viridis, rev = true), legend = false, xlims = (-3, 2), ylims = (-0.5, 2), xlabel = "x₁", ylabel = "x₂", aspectratio = :equal, clim = (2, 500))
	scatter!([-2.0], [1.5], color = :black)

	pts_hyp_gd = run_descent_method(HyperGradientDescent(0.0004, 8e-13, NaN, zeros(2)), VecE2{Float64}(-2,1.5), N)
	pts_hyp_nest = run_descent_method(HyperNesterovMomentum(0.00023, 1e-12, 0.93, zeros(2), NaN, zeros(2)), VecE2{Float64}(-2,1.5), N)

	plot!([pts_hyp_gd[i][1] for i = 1:nstepshyper + 1], [pts_hyp_gd[i][2] for i = 1:nstepshyper + 1], color = :black, linewidth = 2, alpha = 0.5)
	plot!([pts_hyp_nest[i][1] for i = 1:nstepshyper + 1], [pts_hyp_nest[i][2] for i = 1:nstepshyper+ 1], color = :black, linewidth = 2)
end

# ╔═╡ Cell order:
# ╠═7dea2baa-8f2c-11eb-184b-cb353ca6ca79
# ╠═9fd099fc-8f2c-11eb-286e-a548737f4572
# ╠═9633d3f8-9000-11eb-3d5a-b3dd2a013890
# ╠═2c140db8-9013-11eb-21d9-f7718dc9cf57
# ╠═56018eb0-90e1-11eb-18c0-e1f1aecf307c
# ╠═5fead5d4-8f2d-11eb-2d3c-a34658637923
# ╠═bf810ea8-8f2c-11eb-2d6e-3dcb7961b1fa
# ╠═99e23b7a-8ffd-11eb-1b86-9940c74de5fc
# ╠═9d46db42-8ffe-11eb-1f0f-1d6553bcd608
# ╠═9a62773e-8ffd-11eb-315a-611790d3d945
# ╠═fb5587cc-9012-11eb-313a-512bd0f9651b
# ╠═fb7ea5ef-a2be-43a5-aae6-1bacc3a65f1e
# ╠═db569392-347f-45b1-b7c6-e2a0c235cff9
# ╠═1d09ca7a-8b25-4f3a-9f7d-ef586dd80be8
# ╠═cb524eeb-d78d-4125-8e5a-5da5e14644bb
# ╟─5e9fcab4-d2fe-445d-9b8e-2fe9fe50444d
# ╟─70855f33-c00d-49c3-82ec-f891ae4b40c5
# ╟─a3ae828c-3354-43e4-bf49-2284e465660d
# ╟─0259e325-9558-4dfc-9ca8-06daade248b9
# ╟─e54c3874-667d-42a8-ba9f-5376f3ae3bd6
# ╟─33f938e6-8229-4baa-9e1c-3c0215c53197
# ╟─9402e2b6-757e-4632-aac2-7251b28a768a
# ╟─a364ef52-8f2c-11eb-26a6-5d47e19a7230
# ╟─24b2fe6c-9004-11eb-1e5f-0317dd2e85aa
# ╟─4794ee64-8f34-11eb-1ef2-5981f0e059df
# ╟─a00a4ec4-8f34-11eb-3db7-49be2cca0e5f
# ╟─80e1c23c-8f2e-11eb-3beb-519be0766468
# ╟─4c0b5eac-8f35-11eb-06b4-a75f220c7f78
# ╟─ed8b99da-8f2c-11eb-14c5-0393c6d531c4
# ╟─3954970e-9004-11eb-3be9-5f7adea26bc2
# ╠═5479f7d8-8fee-11eb-134b-a781d0affff4
# ╟─f87f64da-8fee-11eb-15fe-8f8090d4d226
# ╟─ecaf9fe4-8fee-11eb-1c05-d76ef0f7ee1e
# ╟─827f2e30-8ffd-11eb-2cd6-9b1edfa701f9
# ╟─3f536d2c-9003-11eb-23c7-7b0b0a92d51a
# ╟─e0fea120-8ffd-11eb-2cac-4f6e4547cc26
# ╟─4848aa34-9004-11eb-1408-5598d9e145f3
# ╟─3ae1a64c-9005-11eb-32cd-adbd7cb6651a
# ╟─55f1ca30-9004-11eb-2658-972069811aa5
# ╟─529ef5e6-9007-11eb-05b2-a70d8ad51908
# ╟─9afa6fa8-9007-11eb-0d4e-01881e229cc7
# ╟─63249a1a-9007-11eb-2630-0523c391ee59
# ╟─e7ca1b5c-9008-11eb-2e5b-972ffbcdad4b
# ╟─0201e428-9009-11eb-3b2f-d7f3bb5b1831
# ╟─631fc672-900b-11eb-1fe2-a531dac443b1
# ╟─a738a7fb-0ebb-42c5-bc2d-af1a4ea72bfc
# ╟─272c43fc-9012-11eb-07f6-3d0625cafbab
# ╟─75b076d0-900b-11eb-2875-b9b74ab9a943
# ╟─81f2c770-9012-11eb-099f-dda2f9c4246f
# ╟─e9e89ffc-9018-11eb-1028-856604a9f26b
# ╟─447b2884-9013-11eb-1457-0b057903165f
# ╟─82a6b55a-90df-11eb-119b-9d94c9420aae
# ╟─9c3268ac-90df-11eb-3a09-cd5b1e01213d
# ╟─9397f2be-90e1-11eb-2a0c-559077244bd9
# ╟─a9d3e5b2-90df-11eb-388f-eb7a9433a6dd
# ╟─d59fd8f2-90e1-11eb-102a-f36334c851b4
# ╟─0263d884-90e7-11eb-2d19-37a0ebcf527b
# ╟─c589570a-90e5-11eb-2100-45f865a96a02
# ╟─04834139-2837-4891-9ba1-4f16c32df04f
# ╟─50688a3b-c1bf-4cde-a820-874158384fae
# ╟─7967346e-bf2f-4791-8e5a-11fee51ba9de
# ╟─bf301087-6624-4c56-a8d2-dc7a9856d33e
# ╟─5c4506ab-c281-4f94-a60a-ef5d727fe125
# ╟─b638d908-35a9-4203-945a-1086f4dba559
# ╟─d0a4ca13-8ea4-4451-ba88-b46d28a1ba89
# ╟─b3759a50-11d4-4d96-a80b-bcb6a76ba944
# ╟─36099212-4d39-4e22-a142-0b9b88263de2


## Delta rule
function delta_rule!(Q, s::Int, a::Int, r, α)
    Q[a,s] += α*(r - Q[a,s])
end

function delta_rule!(Q, a::Int, r::T, α) where T <: Real
    Q[a] += α*(r - Q[a])
end

function delta_rule!(Q, s::Int, r::T, α) where T <: AbstractArray
    q = @view(Q[:,s])
    @. q += α * (r - q)
end


function outer_vals(v::Vector{<:Real}, bounds)
    length(bounds) == 2 || error("bounds must contain 2 values")
    (bounds[1] < v[1] || bounds[2] > v[end]) && error("$bounds is outside of vector range")
    lower = v[findfirst(x -> x > bounds[1], v)-1]
    upper = v[findfirst(x -> x >= bounds[2], v)]
    lower, upper
end

function inner_vals(v::Vector{<:Real}, bounds)
    length(bounds) == 2 || error("bounds must contain 2 values")
    (bounds[1] < v[1] || bounds[2] > v[end]) && error("$bounds is outside of vector range")
    lower = v[findfirst(x -> x >= bounds[1], v)]
    upper = v[findfirst(x -> x > bounds[2], v)-1]
    lower, upper
end

function inner_area(area::Vector{<:Real}, gridres::Real)
    alons = -180.0:gridres:180.0 |> collect
    alats = -90.0:gridres:90.0 |> collect

    inlons = inner_vals(alons, (area[2], area[4]))
    inlats = inner_vals(alats, (area[3], area[1]))
    round.([inlats[2], inlons[1], inlats[1], inlons[2]], digits=6)
end

function outer_area(area::Vector{<:Real}, gridres::Real)
    alons = -180.0:gridres:180.0 |> collect
    alats = -90.0:gridres:90.0 |> collect

    inlons = outer_vals(alons, (area[2], area[4]))
    inlats = outer_vals(alats, (area[3], area[1]))
    round.([inlats[2], inlons[1], inlats[1], inlons[2]], digits=6)
end

round_area(area::Vector{<:Real}, mult=1) = return [ceil(area[1]*mult)/mult, floor(area[2]*mult)/mult, floor(area[3]*mult)/mult, ceil(area[4]*mult)/mult]


struct DiGraph
    n :: Int32 # |V|
    m :: Int32 # |E|
    V :: Array{String, 1} # V[i] = Real Label of node i
    E :: Array{Tuple{Int32, Int32, Float64}, 1} # <u, v, w> in Edge Set
end

function readDiGraph(fileName) # Read a directed graph from a file
    n = 0
    V = Array{String, 1}()
    label = Dict{String, Int32}()
    W = Dict{Tuple{Int32, Int32}, Float64}()

    getid(x) = begin
        if !haskey(label, x)
            n += 1
            label[x] = n
            push!(V, x)
            return n
        end
        return label[x]
    end

    open(fileName) do f1
        for line in eachline(f1)
            buf = split(line)
            u = getid(buf[1])
            v = getid(buf[2])
            if u == v
                continue
            end
            if haskey(W, (u, v))
                W[(u, v)] += 1.0
            else
                W[(u, v)] = 1.0
            end
        end
    end

    m = length(W)
    E = Array{Tuple{Int32, Int32, Float64}, 1}()
    for (u, v) in keys(W)
        push!(E, (u, v, W[(u, v)]))
    end

    return DiGraph(n, m, V, E)
end

function getAdjacentList(G :: DiGraph)
    g = Vector{Vector{Int32}}(undef, G.n)
    foreach(i -> g[i] = Vector{Int32}(), 1 : G.n)
    for (u, v, w) in G.E
        push!(g[u], v)
    end
    return g
end

function SubDiGraph(G :: DiGraph, C, xx)
    n = 0
    h = zeros(Int32, G.n)
    V = Vector{String}()
    for i = 1 : G.n
        if C[i] == xx
            n += 1
            h[i] = n
            push!(V, G.V[i])
        end
    end

    E = Vector{Tuple{Int32, Int32, Float64}}()
    for (u, v, w) in G.E
        if h[u] != 0 && h[v] != 0
            push!(E, (h[u], h[v], w))
        end
    end

    return DiGraph(n, size(E, 1), V, E)
end

function LSCC(G :: DiGraph)
    g = getAdjacentList(G)
    dfn = zeros(Int32, G.n)
    low = zeros(Int32, G.n)
    index = 0
    S = Vector{Int32}()
    C = zeros(Int32, G.n)
    inStack = zeros(Bool, G.n)

    dfs(u) = begin
        index += 1
        dfn[u] = index
        low[u] = index
        push!(S, u)
        inStack[u] = true
        for v in g[u]
            if dfn[v] == 0
                dfs(v)
                low[u] = min(low[u], low[v])
            elseif inStack[v]
                low[u] = min(low[u], dfn[v])
            end
        end
        if dfn[u] == low[u]
            while true
                v = pop!(S)
                inStack[v] = false
                C[v] = u
                if u == v
                    break
                end
            end
        end
    end

    for i = 1 : G.n
        if dfn[i] == 0
            dfs(i)
        end
    end

    sz = zeros(Int32, G.n)
    foreach(x -> sz[x] += 1, C)
    xx = argmax(sz)

    return SubDiGraph(G, C, xx)
end


ENV["JULIA_CUDA_SILENT"] = true
# Change the working directory to the file location
cd(@__DIR__)

using DrMZ
using Parameters: @with_kw
using Flux.NNlib
using LaTeXStrings
using ColorSchemes
using Flux
using Plots; pyplot()
using Plots.PlotMeasures
using Random
using Printf

default(fontfamily="serif",frame=:box,grid=:hide,palette=:viridis,markeralpha=0.4,dpi=200,legendfontsize=6);
PyPlot.matplotlib.rc("mathtext",fontset="cm")

@with_kw mutable struct Args
    num_sensors::Int = 128;
    num_train_functions::Int = 500;
    num_test_functions::Int = Int(2*num_train_functions);
    num_sol_points::Int = 100;
    L1::Float64 = 0.0;
    L2::Float64 = 2*pi;
    tspan::Tuple = (0.0,1.0);
    n_epoch::Int = 25000;
    N::Int = num_sensors;
    branch_layers::Int = 2; # Branch depth
    branch_neurons::Array{Tuple} = [(Int(num_sensors),num_sensors),(num_sensors,N)]; # Branch layers input and output dimensions - quantity of tuples must match branch depth
    branch_activations::Array = [tanh,identity]; # Activation functions for each branch layer
    trunk_layers::Int = 3; # Trunk depth
    trunk_neurons::Array{Tuple} = [(2,num_sensors),(num_sensors,num_sensors),(num_sensors,N)]; # Trunk layers input and output dimensions - quantity of tuples must match trunk depth
    trunk_activations::Array = [tanh,tanh,tanh]; # Activation functions for each trunk layer
end

function generate_train(pde_function,pde_function_handle;kws...)

    args = Args(;);

    # Generate training and testing data
    train_data, test_data = generate_periodic_train_test(args.tspan,args.num_sensors,args.num_train_functions,args.num_test_functions,args.num_sol_points,pde_function_handle);
    save_data(train_data,test_data,args.num_train_functions,args.num_test_functions,args.num_sol_points,pde_function)

    branch = build_dense_model(args.branch_layers,args.branch_neurons,args.branch_activations);
    trunk = build_dense_model(args.trunk_layers,args.trunk_neurons,args.trunk_activations)

    # Train the operator neural network
    branch, trunk = train_model(branch,trunk,args.n_epoch,train_data,test_data,pde_function);
    save_model(branch,trunk,args.n_epoch,pde_function)

end

function generate_opnn_results(pde_function,pde_function_handle;random_integer = "none",kws...)

    args = Args(;);

    # Load all the data
    branch, trunk, train_ic, train_loc, train_sol, test_ic, test_loc, test_sol = load_data(args.n_epoch,args.num_train_functions,args.num_test_functions,pde_function)

    train_MSE = loss_all(branch,trunk,train_ic,train_loc,train_sol);
    test_MSE = loss_all(branch,trunk,test_ic,test_loc,test_sol);
    println("Training set MSE: $train_MSE, test set MSE: $test_MSE")

    dL = abs(args.L2 - args.L1);
    j = reduce(vcat,[0:1:args.num_sensors-1]);
    x = (dL.*j)./args.num_sensors;
    dt = 1e-3;
    t = reduce(vcat,[0:dt:args.tspan[2]]);

    if random_integer == "none"
        rand_int = rand(1:args.num_test_functions);
        ic = test_ic[:,rand_int*args.num_sol_points];
    elseif random_integer == "exact"
        ic = sin.(x/2).^2;
        rand_int = 0;
    else
        rand_int = random_integer;
        ic = test_ic[:,rand_int*args.num_sol_points];
    end

    # Comparisons vs random test IC
    pltrand_test = plot(x,ic,label = "IC",linewidth=:2,xlims=(x[1],x[end]),xlabel=L"x",ylabel=L"u(x)")
    plot!(pltrand_test,x,ic,label = "Sensor locations",seriestype=:scatter)
    display(pltrand_test)
    savefig(pltrand_test, @sprintf("random_ic_%s_%i.png",pde_function,rand_int))

    # M size Fourier solution for comparisons
    u_fourier_M = generate_fourier_solution(args.L1,args.L2,args.tspan,args.num_sensors,ic,pde_function_handle)[1];

    pltexact = plot(heatmap(periodic_fill_domain(x),t,periodic_fill_solution(u_fourier_M),aspect_ratio=dL/args.tspan[2]),fillcolor=cgrad(ColorSchemes.viridis.colors),label=false,xlabel = L"x", ylabel = L"t",xlims=(x[1],x[end]),ylims=(0,args.tspan[2]))
    display(pltexact)
    savefig(pltexact, @sprintf("exact_fourier_solution_random_ic_%s_%i.png",pde_function,rand_int))

    # Operator neural network prediction
    u_predict = predict(branch,trunk,ic,x,t);

    pltpredict = plot(heatmap(periodic_fill_domain(x),t,periodic_fill_solution(u_predict),aspect_ratio=dL/args.tspan[2]),fillcolor=cgrad(ColorSchemes.viridis.colors),label=false,xlabel = L"x", ylabel = L"t",xlims=(x[1],x[end]),ylims=(0,args.tspan[2]))
    display(pltpredict)
    savefig(pltpredict, @sprintf("opnn_solution_random_ic_epochs_%i_%s_%i.png",args.n_epoch,pde_function,rand_int))

    opnn_error = norm_rel_error(u_fourier_M,u_predict);
    opnn_error = plot(t,opnn_error,legend=:outerbottom,foreground_color_legend = nothing,label="Operator network average error $(@sprintf("%.4e",average_error(t,opnn_error)))",xlabel = L"t", ylabel = "Rel. Error",xlims=(t[1],t[end]))
    display(opnn_error)
    savefig(opnn_error, @sprintf("opnn_vs_exact_solution_random_ic_epochs_%i_%s_%i.png",args.n_epoch,pde_function,rand_int))

end

# Generate data and train the model
generate_train("advection_equation",advection_pde!)

# Generate results - enter "none", an integer value (e.g. 975), or "exact" to specify an initial condition
# If testing ability of network to extrapolate beyond the training interval, comment out the generation functions above and adjust the Arg tspan
generate_opnn_results("advection_equation",advection_pde!;random_integer = "exact")


### A Pluto.jl notebook ###
# v0.12.7

using Markdown
using InteractiveUtils

# ╔═╡ 8e1bdc16-2342-11eb-22a6-cb3487fd6818
using LinearAlgebra

# ╔═╡ c810852c-2342-11eb-3c4c-b35d8ca9bb77
using Plots

# ╔═╡ e2bdc99a-2340-11eb-36d1-5b26c095e69d
md"""
# Ma première cellule `Markdown`

Une première équation ``\LaTeX``
```math
\overline{\rho}_i \equiv \frac{n_i}{V}.
```

Pour exécuter une cellule, deux options :

1. Cliquer sur le bouton *play* en bas à droite de la cellule,
1. Presser *Shift* puis *Enter* séquentiellement en maintenant *Shift* appuyé.

"""

# ╔═╡ 7e43d9a4-2341-11eb-2a9b-6f2fab9f6803
function hello(name = "Louis")
	println("Hello, $name !")
end

# ╔═╡ aab3a9b0-2341-11eb-37a1-476a3acf758b
hello("Martin")

# ╔═╡ bccedea6-2341-11eb-3a3c-65363b8441ec
begin
	a = 3
	b = 2
end

# ╔═╡ d52c2852-2341-11eb-1ff2-393759544efe
a, b

# ╔═╡ df4ac668-2341-11eb-0203-2f2059b0c3e9
md"""
Voici grossièrement comment un système de notebook fonctionne.

* Jupyter,
* Pluto.

Le système Pluto.jl est ce qu'on appelle un environnement réactif.

"""

# ╔═╡ 3512729e-2342-11eb-161c-f31631663ff0
c = [1., 2., 4., 10.]

# ╔═╡ 646d1382-2342-11eb-1e6a-c9f8f95c9d0d
typeof(c)

# ╔═╡ 68594ec0-2342-11eb-1b54-8133d0d00cc1
A = [1. 0. 3. 4.
	5. 6. 7. 8.
	1. 0. 3. 4.
	5. 6. 7. 8.]

# ╔═╡ 85dcfd16-2342-11eb-1759-bf7897e56472
typeof(A)

# ╔═╡ 8aed5238-2342-11eb-0bc9-8fbe5e1929fb
A * c

# ╔═╡ 962ebdbc-2342-11eb-1a08-85f6c5fb0acd
eigvals(A)

# ╔═╡ d12ab39e-2342-11eb-181b-895534c67768
plot(exp, lw = 2, xlim = (-2, 3), label = "exp")

# ╔═╡ a2a81b28-2343-11eb-2190-63139c0ce9af
md"""
Le package `DifferentialEquations.jl` permet de résoudre : EDO, DAE, DDE, stochastique

* Problèmes de Cauchy,
* Problèmes aux limites,
* ...

"""

# ╔═╡ Cell order:
# ╟─e2bdc99a-2340-11eb-36d1-5b26c095e69d
# ╠═7e43d9a4-2341-11eb-2a9b-6f2fab9f6803
# ╠═aab3a9b0-2341-11eb-37a1-476a3acf758b
# ╠═bccedea6-2341-11eb-3a3c-65363b8441ec
# ╠═d52c2852-2341-11eb-1ff2-393759544efe
# ╟─df4ac668-2341-11eb-0203-2f2059b0c3e9
# ╠═3512729e-2342-11eb-161c-f31631663ff0
# ╠═646d1382-2342-11eb-1e6a-c9f8f95c9d0d
# ╠═68594ec0-2342-11eb-1b54-8133d0d00cc1
# ╠═85dcfd16-2342-11eb-1759-bf7897e56472
# ╠═8aed5238-2342-11eb-0bc9-8fbe5e1929fb
# ╠═8e1bdc16-2342-11eb-22a6-cb3487fd6818
# ╠═962ebdbc-2342-11eb-1a08-85f6c5fb0acd
# ╠═c810852c-2342-11eb-3c4c-b35d8ca9bb77
# ╠═d12ab39e-2342-11eb-181b-895534c67768
# ╟─a2a81b28-2343-11eb-2190-63139c0ce9af