 using JuPOT

             # Collect data from ARGS array
             data = float(split(ARGS[1], ","))
             returns = round(float(split(ARGS[2], ",")),3)
             tickers = split(ARGS[3], ",")
             target_return = float(ARGS[4])

             # Calculate number of stocks and days from data
             stocks = length(tickers)
             days = Int(length(data)/stocks)

             # Form data into matrix
             matrix = reshape(data,days,stocks)

             # Produce covariance matrix
             covariance = cov(matrix)

             # Assets data structure containing, tickers, expected returns and covarariance
             assets = AssetsCollection(tickers, returns, covariance)

             # Create simple MVO object using assets and target_return
             mvo = SimpleMVO(assets, target_return; short_sale=false)

             # Optimise MVO object and print result to file
             outfile = "output.txt"
                          f = open(outfile, "w")
                          println(f, optimize(mvo))
                          close(f)

             # No. Of Assets
             # n = 3
             # No. of days data
             # m = 6
             # d = [0.5,0.7,0.3,0.2,0.56,0.26,0.2,0.4,0.6,0.2,0.15,0.58,0.8,0.3,0.9,0.2,0.51,0.15]
             # Returns a matrix of size(n) with entries between 0-1s
             # returns =  [0.5,0.4,0.6]
             # matrix = reshape(d,m,n)
             #covariance matrix of returns
             # covariance = cov(matrix)
             # List of asset names
             # tickers = [randstring(3) for i in 1:n]

             # Assets data structure containing, names, expected returns, covarariance
             # assets = AssetsCollection(tickers, returns, covariance)

             # target_return = 0.2
             # mvo = SimpleMVO(assets, target_return; short_sale=false)

             # println(optimize(mvo))


function solve()
    n = parse(Int, readline())
    as = [parse(Int, x) for x in split(readline())]

    bs = zeros(Int, n)
    bs[1] = as[1]
    for i in 2:n
        bs[i] = as[i] + bs[i-1]
    end
    ret = (n-1) * sum(x->x^2, as)
    for i in 2:n
        ret -= 2*as[i]*bs[i-1]
    end
    ret
end

println(solve())


cd(@__DIR__)

function normalMean(x)
    N    = length(x)
    sumx = 0.0
    for i in 1:N
       sumx += x[i]
    end
    return sumx/N
end

function onlineMean(oldMean,oldN,newX)
     oldSum  = oldMean * oldN
     newN    = oldN+1
     newMean = (oldSum + newX) / newN
     return (newMean, newN)
end

X = [1,2,3,4,5]

nMean = normalMean(X)

function computeMeanInLargeFile(filename)
    currentMean = 0.0
    currentN    = 0.0
    open(filename,"r") do f
        for ln in eachline(f)
            if ln != ""
                (currentMean, currentN) =  onlineMean(currentMean,currentN,parse(Float64,ln))
                println("Current mean: $currentMean")
            end
        end
    end

    return currentMean
end

olMean = computeMeanInLargeFile("x.txt")


using Images, JLD

function main()
    w = 2000
    h = 2000
    points = load("./sets/15k.jld", "points")
    img = zeros(RGB,w,h)
    for c::ComplexF64 in points
        i::Int = 0
        z::ComplexF64 = c
        while abs(z) < 3 && i < 5000
            x::Int = trunc(Int,(real(z) + 2)*w/3); y::Int = trunc(Int,(imag(z) + 1.5)*h/3)
            0<x<w && 0<y<h && i > 0 ? img[x,y] += RGB(0.0005,0,0) : nothing #log10(i)/3
            z = z*z + c
            i += 1
        end
    end
    return map(clamp01nan, img)
end

@time save("./out/buddhared.png", main())


"""
Transient heating of a rectangular steel block suddenly subject to surface temperature of 100° C.

The initial temperature of the block is 20° C. The temperature at the center
climbs from 20° C to approximately 90° C in 1200 seconds.
"""

module steel_block
using FinEtools
using FinEtoolsHeatDiff
using FinEtoolsHeatDiff.AlgoHeatDiffModule
import LinearAlgebra: cholesky, mul!
using Gnuplot

function steel_block_m()
	# Thermal conductivity
	thermal_conductivity =  [i==j ? 44.0*phun("W/K/m") : zero(FFlt) for i=1:3, j=1:3]; # conductivity matrix
	# Specific heat: the specific heat is per unit volume
	rho = 8000*phun("kg/m^3")
	specific_heat = 470.0*phun("J/kg/K")*rho
	theta = 1.0; # generalized trapezoidal method parameter
	dt = 1.0; # time step
	tend = 1200; # length of the time interval
	T0(x, y, z) = 20.0;# Initial distribution of temperature
	Ts = 100.0 # Prescribed surface temperature
	dimensions = (0.2, 0.15, 0.3).*phun("m")
	nels = (10, 10, 10)

	tolerance =min(dimensions...)/max(nels...)/100;

	fens,fes = H8block(dimensions..., nels...)
	@show lcenter  = selectnode(fens; nearestto=[0.0 0.0 0.0],  inflate = tolerance)

	geom = NodalField(fens.xyz)
	Temp = NodalField(zeros(size(fens.xyz,1),1))

	# Initial temperature
	for i in 1:count(fens)
		Temp.values[i] = T0(fens.xyz[i, :]...)
	end

	lx  = selectnode(fens; box=[dimensions[1] dimensions[1] -Inf Inf -Inf Inf],  inflate = tolerance)
	ly  = selectnode(fens; box=[-Inf Inf dimensions[2] dimensions[2] -Inf Inf],  inflate = tolerance)
	lz  = selectnode(fens; box=[-Inf Inf -Inf Inf dimensions[3] dimensions[3]],  inflate = tolerance)
	for (i, l) in enumerate([lx, ly, lz])
		setebc!(Temp, l, true, 1, Ts)
	end
	applyebc!(Temp)
    numberdofs!(Temp)

    centerdof = Temp.dofnums[lcenter]

    material = MatHeatDiff(thermal_conductivity, specific_heat)
    femm = FEMMHeatDiff(IntegDomain(fes, GaussRule(3, 3)), material)

    K = conductivity(femm, geom, Temp)
    C = capacity(femm, geom, Temp)
    L = nzebcloadsconductivity(femm, geom, Temp)

    A = cholesky((1/dt*C+theta*K))
    B = (1/dt*C-(1-theta)*K)

    Tn = gathersysvec(Temp)
    Tn1 = deepcopy(Tn)

    tstart = time()
    t = 0.0
    ts = Float64[t]
    @show Tn1[centerdof, 1][1]
    center_T = Float64[Tn1[centerdof, 1][1]];
    while true # Time stepping
    	t = t+dt; # Advance the current time
    	Tn .= Tn1 # current temperature system vector

    	# Solve for the new temperature  vector
    	rhs = B*Tn + L; # complete right-hand side (loads)
    	Tn1 .= A\rhs; # compute next temperature
    	push!(ts, t)
    	push!(center_T, Tn1[centerdof, 1][1])
    	if (t == tend) # Have we reached the end?  If so jump out.
	    	break;
	    end
	    if (t+dt > tend) # Adjust the last time step so that we exactly reach tend
	    	dt = tend-t;
	    end
	end
	println("Time = $(time() - tstart)")

	@show minimum(center_T), maximum(center_T)
	@gp "set terminal windows 0 " :-
	@gp  :- vec(ts) vec(center_T) "lw 2 lc rgb 'red' with lines title 'Temperature at the corner' "
	@gp  :- "set xlabel 'Time [s]'"
	@gp  :- "set ylabel 'Temperature [degrees C]'"

    true
end

end # module steel_block

using .steel_block
steel_block.steel_block_m()


# This file contains code that was formerly part of Julia. License is MIT: http://julialang.org/license

using QuadGK, Test

@testset "quadgk" begin
    @test quadgk(cos, 0,0.7,1)[1] ≈ sin(1)
    @test quadgk(x -> exp(im*x), 0,0.7,1)[1] ≈ (exp(1im)-1)/im
    @test quadgk(x -> exp(im*x), 0,1im)[1] ≈ -1im*expm1(-1)
    @test isapprox(quadgk(cos, 0,BigFloat(1),order=40)[1], sin(BigFloat(1)),
                   atol=1000*eps(BigFloat))
    @test quadgk(x -> exp(-x), 0,0.7,Inf)[1] ≈ 1.0
    @test quadgk(x -> exp(x), -Inf,0)[1] ≈ 1.0
    @test quadgk(x -> exp(-x^2), -Inf,Inf)[1] ≈ sqrt(pi)
    @test quadgk(x -> [exp(-x), exp(-2x)], 0, Inf)[1] ≈ [1,0.5]
    @test quadgk(cos, 0,0.7,1, norm=abs)[1] ≈ sin(1)
end

module Test19626
    using QuadGK
    using Test

    # Define a mock physical quantity type
    struct MockQuantity <: Number
        val::Float64
    end

    # Following definitions needed for quadgk to work with MockQuantity
    import Base: +, -, *, abs, isnan, isinf, isless
    +(a::MockQuantity, b::MockQuantity) = MockQuantity(a.val+b.val)
    -(a::MockQuantity, b::MockQuantity) = MockQuantity(a.val-b.val)
    *(a::MockQuantity, b::Number) = MockQuantity(a.val*b)
    abs(a::MockQuantity) = MockQuantity(abs(a.val))
    isnan(a::MockQuantity) = isnan(a.val)
    isinf(a::MockQuantity) = isinf(a.val)
    isless(a::MockQuantity, b::MockQuantity) = isless(a.val, b.val)

    # isapprox only needed for test purposes
    Base.isapprox(a::MockQuantity, b::MockQuantity) = isapprox(a.val, b.val)

    # Test physical quantity-valued functions
    @test QuadGK.quadgk(x->MockQuantity(x), 0.0, 1.0, atol=MockQuantity(0.0))[1] ≈
        MockQuantity(0.5)
end


using Random
using Printf

include("Algencan/AlgencanWrapper.jl")
include("sh.jl")

import Voronoi
import Covering

drawing = (haskey(ENV, "DRAW") && ENV["DRAW"] != "" && ENV["DRAW"] != "0")
drawingAll = (haskey(ENV, "DRAWALL") && ENV["DRAWALL"] != "" && ENV["DRAWALL"] != "0")

WIDTH = 1.0
HEIGHT = 1.0

A = Array{Covering.Section, 1}([
        Covering.Section(Array{Tuple{Real, Real}, 1}([
            (0.2, 0.5),
            (0.8, 0.5),
            (0.8, 0.75),
            (0.2, 0.75),
        ])),
        Covering.Section(Array{Tuple{Real, Real}, 1}([
            (0.4, 0.5),
            (0.4, 0.25),
            (0.6, 0.25),
            (0.6, 0.5),
        ]))
    ])

#A = Array{Covering.Section, 1}([
#        Covering.Section(Array{Tuple{Real, Real}, 1}([
#            (0, 0.125),
#            (0.75, 0.5),
#            (1 ,0.875),
#            (0.25, 0.5),
#        ])),
#        Covering.Section(Array{Tuple{Real, Real}, 1}([
#            (0, 1),
#            (0, 0.5),
#            (0.5, 1),
#        ]))
#    ])

function f()
    total_area = 0
    for Aⱼ in A
        t, _, _ = Covering.areaAndGradient(Aⱼ, 0)
        total_area += t
    end
    return total_area
end
total_area = f()

#function avoidRepeats(points)
#    println("LÁ VAMOS NÓS")
#    println("points antes: ", points)
#    repeat = []
#    for i in 1:length(points)
#        for j in 1:length(points)
#            if j == i
#                continue
#            end
#
#            if points[i][1] == points[j][1] && points[i][2] == points[j][2]
#                push!(repeat, points[j])
#                deleteat!(points, j)
#            end
#        end
#    end
#
#    println("points depois: ", points)
#    println("repeats depois: ", repeat)
#    return points, repeat
#end

function coverWithCircles(n, WIDTH, HEIGHT, r, points)
   function pack(r, points)
       x = zeros(2n + 1)
       x[1] = r
       for (i, p) in enumerate(points)
           x[2i] = p[1]
           x[2i + 1] = p[2]
       end

       return x
   end

   function unpack(x)
       r = x[1]

       points = Array{Tuple{Real, Real}}([])

       for i in 1:n
           append!(points, [(x[2i], x[2i + 1])])
       end

       return r, points
   end

   function draw(r, points)
       if !drawing
           return
       end

       V = Voronoi.Fortune.compute(points, WIDTH, HEIGHT)
       Voronoi.Intersect.intersect(V, Voronoi.Intersect.Rectangle(WIDTH, HEIGHT))

       Draw.init(WIDTH, HEIGHT)
       Draw.circles(r, points)

       for Aⱼ in A
           W = SH.intersect(V, Aⱼ, r)
           Draw.coveringPartition(W, "xkcd:pink")
           Draw.filledPolygon(Aⱼ, "xkcd:black")
       end

       Draw.commit()
       print("")
   end

   function f(x)
       r, points = unpack(x)

       if drawing && drawingAll
           draw(r, points)
       end

       #println("R: ", r)
       return r
   end

   function ∇f(x)
       grad = zeros(2n + 1)
       grad[1] = 1
       #println("GRAD: ", grad)
       return grad
   end

   function ∇∇f(x)
       return [], [], []
   end

   function c(ind, x)
       r, points = unpack(x)

       #points, repeats = avoidRepeats(points)

       V = Voronoi.Fortune.compute(points, WIDTH, HEIGHT)
       Voronoi.Intersect.intersect(V, Voronoi.Intersect.Rectangle(WIDTH, HEIGHT))

       uncovered_area = total_area
       for Aⱼ in A
           W = SH.intersect(V, Aⱼ, r)
           area, _, _ = Covering.areaAndGradient(W)
           uncovered_area -= area
       end

       #println("C: ", uncovered_area)
       return uncovered_area
   end

   function ∇c(ind, x)
       r, points = unpack(x)

       #points, repeats = avoidRepeats(points)

       V = Voronoi.Fortune.compute(points, WIDTH, HEIGHT)
       Voronoi.Intersect.intersect(V, Voronoi.Intersect.Rectangle(WIDTH, HEIGHT))

       gᵣ = 0
       gₛ = fill((0.0, 0.0), n)
       for Aⱼ in A
           W = SH.intersect(V, Aⱼ, r)
           _, gᵣⱼ, gₛⱼ = Covering.areaAndGradient(W)
           gᵣ += gᵣⱼ
           for i in 1:n
               gₛ[i] = (gₛ[i][1] + gₛⱼ[i][1], gₛ[i][2] + gₛⱼ[i][2])
           end
       end

       #for i in repeats
       #    push!(gₛ, (0, 0))
       #end
       #println("GRAD C.: ", pack(gᵣ, gₛ))
       #println("Aperte Enter pra continuar")
       #readline(stdin)

       return collect(1:2n+1), pack(gᵣ, gₛ)
   end

   l = zeros(2n + 1)
   l[1] = 1e-14

   u = zeros(2n + 1)
   u[1] = 10e20
   for i in 2:2n+1
       u[i] = 1.0
   end

   draw(r, points)
   #println("Aperte Enter pra continuar")
   #readline(stdin)

   try
       x, fx = AlgencanWrapper.optimize(n = 2n + 1, m = 1,

                                        f = f, g = ∇f,# h = ∇∇f,
                                        equatn = [1],
                                        c = c, jac = ∇c,

                                        x = pack(r, points),
                                        l = l,
                                        u = u,

                                        nvparam = 2,
                                        vparam = [
                                                  "ITERATIONS-OUTPUT-DETAIL 10",
                                                  "PENALTY-PARAMETER-INITIAL-VALUE 1000"
                                                 ],
                                        #checkder = 1,
                                        epsopt = 1.0e-6,
                                        epsfeas = 1.0e-6
                                        )

       r, points = unpack(x)
       draw(r, points)

       println("\n\n=================\nRESULTADO\n=================\n")
       println("r = ", r)
       println("s = ", 1/r)
       println("pontos: ")
       for p in points
           println("(", p[1], ", ", p[2], ")")
       end
       if drawing
           println("Digite ENTER para sair.")
           readline(stdin)
       end
   catch exc
       println("Exception: ", exc)
       return 1e10, []
   end
end

if length(ARGS) == 0
    println("USO: cover.sh N_DE_PONTOS [DESENHAR] [DESENHAR_TODA_AVALIAÇÃO]")
else
    if drawing
        import Draw
    end

    n = tryparse(Int64, ARGS[1])

    function randf(start, finish, n)
        v = rand(n)
        return map(x -> start + x*(finish-start), v)
    end

    Random.seed!(1)
    points = convert(Array{Tuple{Real, Real}}, collect(zip(randf(1, WIDTH-1, n), randf(1, HEIGHT-1, n))))

    r = (0.5 + (rand(1)[1]))/n

    function insideHalfSpace(point, p, q)
        pq = Voronoi.Geometry.subVector(q, p)
        v = Voronoi.Geometry.rotateVectorCCW(pq)

        function dot(a, b)
            return a[1]*b[1] + a[2]*b[2]
        end

        u = Voronoi.Geometry.subVector(point, p)

        return (dot(u, v) >= 0)
    end

    function pointIsInsideConvexPolygon(point, polygon)
        p = polygon.borderHead

        first = true
        while p != polygon.borderHead || first
            first = false
            q = p.next
            if !insideHalfSpace(point, p.origin, q.origin)
                return false
            end

            p = p.next
        end

        return true
    end

    function pointIsInsideSomeConvexPolygon(point, polygons)
        for polygon in polygons
            if pointIsInsideConvexPolygon(point, polygon)
                return true
            end
        end

        return false
    end

    for (i, p) in enumerate(points)
        while !pointIsInsideSomeConvexPolygon(points[i], A)
            #println("Ponto $i ainda está fora. Re-sorteando.")
            points[i] = (randf(1, WIDTH-1, 1)[1], randf(1, HEIGHT-1, 1)[1])
        end
    end

    try
        coverWithCircles(n, WIDTH, HEIGHT, r, points)
    catch e
        @error "Deu pau:" exception=(e, catch_backtrace())
    end
end


module day12

using StaticArrays, Combinatorics, ProgressMeter

readInput() = [parse.(Int, split(x, r"[^\d|-]", keepempty=false)) for x in readlines(joinpath(@__DIR__, "input.txt"))]

struct Moon
    pos::MVector{3, Int}
    vel::MVector{3, Int}
end
Moon(pos) = Moon(pos, [0,0,0])

compare(a, b) = a == b ? 0 : a < b ? +1 : -1
function applyGravity!(a::Moon, b::Moon)
    Δ = compare.(a.pos, b.pos)
    a.vel .+= Δ
    b.vel .-= Δ
end
applyVelocity!(m::Moon) = m.pos .+= m.vel
function stepUniverse!(moons::Vector{Moon})
    for (a, b) in combinations(moons, 2)
        applyGravity!(a, b)
    end
    applyVelocity!.(moons)
end

potentialEnergy(m::Moon) = sum(abs, m.pos)
kineticEnergy(m::Moon) = sum(abs, m.vel)
energy(m::Moon) = potentialEnergy(m) * kineticEnergy(m)

function part1(input, steps = 1000)
    moons = Moon.(input)
    for _ = 1:steps
        stepUniverse!(moons)
    end
    sum(energy, moons)
end

function part2(input)
    moons = Moon.(input)
    x0, y0, z0 = (Tuple((m.pos[i], m.vel[i]) for m in moons) for i = 1:3)
    cycles = [0, 0, 0]
    iter = 1
    while any(==(0), cycles)
        stepUniverse!(moons)
        x, y, z = (Tuple((m.pos[i], m.vel[i]) for m in moons) for i = 1:3)
        x == x0 && cycles[1] == 0 && (cycles[1] = iter)
        y == y0 && cycles[2] == 0 && (cycles[2] = iter)
        z == z0 && cycles[3] == 0 && (cycles[3] = iter)
        iter += 1
    end
    lcm(cycles)
end

function part2Plot(input, nbIter=100, smooth=10)
    moons = Moon.(input)
    anim = Animation()
    @showprogress for i = 1:nbIter

        x0, y0, z0 = ([m.pos[i] for m in moons] for i = 1:3)
        stepUniverse!(moons)
        x, y, z = ([m.pos[i] for m in moons] for i = 1:3)

        rx = range.(x0, x, length=smooth)
        ry = range.(y0, y, length=smooth)
        rz = range.(z0, z, length=smooth)
        for j = 1:smooth
            xx, yy, zz = (r -> r[j]).(rx), (r -> r[j]).(ry), (r -> r[j]).(rz)
            p = scatter3d(reshape(xx, 1, :), reshape(yy, 1, :), reshape(zz, 1, :), xlims=(-20,20), ylims=(-20,20), zlims=(-20,20),
                label=["Io" "Europa" "Ganymede" "Callisto"], color= [:green :orange :black :purple])
            frame(anim, p)
        end
    end
    anim
end

end


using Zygote, NNlib, Test, Random, LinearAlgebra
using Zygote: gradient
using NNlib: conv
import Random

function ngradient(f, xs::AbstractArray...)
  grads = zero.(xs)
  for (x, Δ) in zip(xs, grads), i in 1:length(x)
    δ = sqrt(eps())
    tmp = x[i]
    x[i] = tmp - δ/2
    y1 = f(xs...)
    x[i] = tmp + δ/2
    y2 = f(xs...)
    x[i] = tmp
    Δ[i] = (y2-y1)/δ
  end
  return grads
end

gradcheck(f, xs...) =
  all(isapprox.(ngradient(f, xs...),
                gradient(f, xs...), rtol = 1e-5, atol = 1e-5))

gradtest(f, xs::AbstractArray...) = gradcheck((xs...) -> sum(sin.(f(xs...))), xs...)
gradtest(f, dims...) = gradtest(f, rand.(Float64, dims)...)

Random.seed!(0)

@test gradient(//, 2, 3) === (1//3, -2//9)

@test gradtest((x, W, b) -> σ.(W*x .+ b), 5, (2,5), 2)
@test gradtest((x, W, b) -> σ.(W*x .+ b), (5,3), (2,5), 2)
@test gradtest((x, W, b) -> logσ.(W*x .+ b), 5, (2,5), 2)
@test gradtest((x, W, b) -> logσ.(W*x .+ b), (5,3), (2,5), 2)

@test gradtest((w, x) -> w'*x, randn(10, 2), randn(10))
@test gradtest((w, x) -> transpose(w)*x, randn(5,5), randn(5,5))

@test gradtest(x -> sum(x, dims = (2, 3)), (3,4,5))
@test gradtest(x -> prod(x, dims = (2, 3)), (3,4,5))
@test gradtest(x -> prod(x), (3,4,5))

@test gradtest(x -> softmax(x).*(1:3), 3)
@test gradtest(x -> softmax(x).*(1:3), (3,5))
@test gradtest(x -> logsoftmax(x).*(1:3), 3)
@test gradtest(x -> logsoftmax(x).*(1:3), (3,5))

@test gradtest(conv, rand(10, 3, 2), randn(Float64,2, 3, 2))
@test gradtest(conv, rand(10, 10, 3, 2), randn(Float64,2, 2, 3, 2))
@test gradtest(conv, rand(10, 10, 10, 3, 2), randn(Float64,2, 2, 2, 3, 2))

@test gradtest(x -> maxpool(x, (2,2)), rand(10, 10, 3, 2))
@test gradtest(x -> maxpool(x, (2,2,2)), rand(10, 10, 10, 3, 2))

@test gradtest(x -> meanpool(x, (2,2)), rand(10, 10, 3, 2))
@test gradtest(x -> meanpool(x, (2,2,2)), rand(5, 5, 5, 3, 2))

@test gradtest(x -> permutedims(x, [3,1,2]), rand(4,5,6))

@test gradtest(x -> repeat(x; inner=2), rand(5))
@test gradtest(x -> repeat(x; inner=2, outer=3), rand(5))
@test gradtest(x -> repeat(x; inner=(2,2,1), outer=(1,1,3)), rand(5,4,3))

@testset "dot" begin
  rng = MersenneTwister(123456)
  @test gradtest((x, y)->dot(x[1], y[1]), [randn(rng)], [randn(rng)])
  @test gradtest(dot, randn(rng, 10), randn(rng, 10))
  @test gradtest(dot, randn(rng, 10, 3), randn(rng, 10, 3))
end

@test gradtest(kron, rand(5), rand(3))
@test gradtest(kron, rand(5), rand(3), rand(8))
@test gradtest(kron, rand(5,1), rand(3,1))
@test gradtest(kron, rand(5,1), rand(3,1), rand(8,1))
@test gradtest(kron, rand(5,2), rand(3,2), rand(8,2))

@testset "maximum" begin
  @test gradtest(maximum, rand(2, 3))

  @test gradtest(x -> maximum(x, dims=1), rand(2, 3))
  @test gradtest(x -> maximum(x, dims=2), rand(2, 3))
  @test gradtest(x -> maximum(x, dims=3), rand(2, 3, 4))

  @test gradtest(x -> maximum(x, dims=[1, 2]), rand(2, 3, 4))
end

@testset "minimum" begin
  @test gradtest(minimum, rand(2, 3))

  @test gradtest(x -> minimum(x, dims=1), rand(2, 3))
  @test gradtest(x -> minimum(x, dims=2), rand(2, 3))
  @test gradtest(x -> minimum(x, dims=3), rand(2, 3, 4))

  @test gradtest(x -> minimum(x, dims=[1, 2]), rand(2, 3, 4))
end

@testset "backsolve" begin
  rng, P, Q = MersenneTwister(123456), 10, 9
  X, Y, y = randn(rng, P, P), randn(rng, P, Q), randn(rng, P)

  # \
  @test gradtest(X -> X \ Y, X)
  @test gradtest(Y -> X \ Y, Y)
  @test gradtest(X -> X \ y, X)
  @test gradtest(y -> X \ y, y)

  # /
  @test gradtest(X -> Y' / X, X)
  @test gradtest(Y -> Y' / X, Y)
  @test gradtest(X -> y' / X, X)
  @test gradtest(y -> y' / X, y)

  @testset "Cholesky" begin

    # Check that the forwards pass computes the correct thing.
    @test Zygote.forward(X->cholesky(X * X' + I) \ Y, X)[1] == cholesky(X * X' + I) \ Y
    @test gradtest(X->cholesky(X * X' + I) \ Y, X)
    @test gradtest(Y->cholesky(X * X' + I) \ Y, Y)
    @test gradtest(X->cholesky(X * X' + I) \ y, X)
    @test gradtest(y->cholesky(X * X' + I) \ y, y)
  end
end

@testset "Symmetric" begin
  rng, P = MersenneTwister(123456), 7
  A = randn(rng, P, P)
  @test gradtest(Symmetric, A)
end

@testset "diag" begin
  rng, P = MersenneTwister(123456), 10
  A = randn(rng, P, P)
  @test gradtest(diag, A)
end

@testset "dense + UniformScaling" begin
  rng = MersenneTwister(123456)
  A, λ = randn(rng, 10, 10), randn(rng)
  @test gradtest(A->A + 5I, A)
  @test gradtest(λ->A + λ[1] * I, [λ])
end

@testset "cholesky" begin
  rng, N = MersenneTwister(123456), 5
  A = randn(rng, N, N)
  @test gradtest(A->logdet(cholesky(A' * A + 1e-6I)), A)
  @testset "cholesky - scalar" begin
    y, back = Zygote.forward(cholesky, 5.0 * ones(1, 1))
    y′, back′ = Zygote.forward(cholesky, 5.0)
    C̄ = randn(rng, 1, 1)
    @test back′((factors=C̄,))[1] isa Real
    @test back′((factors=C̄,))[1] ≈ back((factors=C̄,))[1][1, 1]
  end
end

using Distances

Zygote.refresh()

@testset "distances" begin
  rng, P, Q, D = MersenneTwister(123456), 10, 9, 8

   # Check sqeuclidean.
  let
    x, y = randn(rng, D), randn(rng, D)
    @test gradtest(x->sqeuclidean(x, y), x)
    @test gradtest(y->sqeuclidean(x, y), y)
  end

   # Check binary colwise.
  let
    X, Y = randn(rng, D, P), randn(rng, D, P)
    @test gradtest(X->colwise(SqEuclidean(), X, Y), X)
    @test gradtest(Y->colwise(SqEuclidean(), X, Y), Y)
  end

   # Check binary pairwise.
  let
    X, Y = randn(rng, D, P), randn(rng, D, Q)
    @test gradtest(X->pairwise(SqEuclidean(), X, Y), X)
    @test gradtest(Y->pairwise(SqEuclidean(), X, Y), Y)
  end

   # Check unary pairwise.
  @test gradtest(X->pairwise(SqEuclidean(), X), randn(rng, D, P))
end

function cat_test(f, A::Union{AbstractVector, AbstractMatrix}...)
  @test gradtest(f, A...)
  Z, back = Zygote.forward(f, A...)
  Ā = back(randn(size(Z)))
  @test all(map((a, ā)->ā isa typeof(a), A, Ā))
end

@testset "vcat" begin

  # Vector-only.
  cat_test(vcat, randn(1))
  cat_test(vcat, randn(10))

  cat_test(vcat, randn(1), randn(1))
  cat_test(vcat, randn(5), randn(1))
  cat_test(vcat, randn(1), randn(6))
  cat_test(vcat, randn(5), randn(6))

  # Matrix-only.
  for c in [1, 2], r1 in [1, 2], r2 in [1, 2]
    cat_test(vcat, randn(r1, c))
    cat_test(vcat, randn(r1, c), randn(r2, c))
  end

  # Matrix-Vector / Vector-Matrix / Vector-Matrix-Vector.
  for r in [1, 2]
    cat_test(vcat, randn(r, 1), randn(r))
    cat_test(vcat, randn(r), randn(r, 1))
    cat_test(vcat, randn(r), randn(r, 1), randn(r))
  end
end

@testset "hcat" begin

  # Vector-only.
  for r in [1, 2]
    cat_test(hcat, randn(r))
    cat_test(hcat, randn(r), randn(r))
  end

  # Matrix-only.
  for r in [1, 2], c1 in [1, 2], c2 in [1, 2]
    cat_test(hcat, randn(r, c1), randn(r, c2))
  end

  # Matrix-Vector / Vector-Matrix / Vector-Matrix-Vector.
  for r in [1, 2], c in [1, 2]
    cat_test(hcat, randn(r, c), randn(r))
    cat_test(hcat, randn(r), randn(r, c))
    cat_test(hcat, randn(r), randn(r, c), randn(r))
  end
end

@testset "one(s) and zero(s)" begin
  @test Zygote.gradient(x->sum(ones(size(x))), randn(5))[1] isa Nothing
  @test Zygote.gradient(x->sum(one(x)), randn(3, 3))[1] isa Nothing
  @test Zygote.gradient(x->sum(zeros(size(x))), randn(7))[1] isa Nothing
  @test Zygote.gradient(x->sum(zero(x)), randn(3))[1] isa Nothing
end

import StatsFuns

Zygote.refresh()

@testset "xlogx" begin
  @test gradcheck(x->2.5 * StatsFuns.xlogx(x[1]), [1.0])
  @test gradcheck(x->2.5 * StatsFuns.xlogx(x[1]), [2.45])
end

@testset "logistic" begin
  @test gradcheck(x->3.0 * StatsFuns.logistic(x[1]), [-5.0])
  @test gradcheck(x->3.0 * StatsFuns.logistic(x[1]), [-1.0])
  @test gradcheck(x->3.0 * StatsFuns.logistic(x[1]), [-eps()])
  @test gradcheck(x->3.0 * StatsFuns.logistic(x[1]), [0.0])
  @test gradcheck(x->3.0 * StatsFuns.logistic(x[1]), [eps()])
  @test gradcheck(x->3.0 * StatsFuns.logistic(x[1]), [1.0])
  @test gradcheck(x->3.0 * StatsFuns.logistic(x[1]), [5.0])
end

@testset "logit" begin
  @test gradcheck(x->5.0 * StatsFuns.logit(x[1]), [0.1])
  @test gradcheck(x->5.0 * StatsFuns.logit(x[1]), [0.3])
  @test gradcheck(x->5.0 * StatsFuns.logit(x[1]), [0.5])
  @test gradcheck(x->5.0 * StatsFuns.logit(x[1]), [0.7])
  @test gradcheck(x->5.0 * StatsFuns.logit(x[1]), [0.9])
end

function test_log1pexp(T, xs)
  y = T(4.3)
  for x in xs
    @test gradcheck(x->y * StatsFuns.log1pexp(x[1]), [x])
  end
end

@testset "log1pexp" begin
  @testset "Float64" begin
    @testset "x ∈ (-∞, 18.0)" begin
      test_log1pexp(Float64, [-1000.0, -50.0, -25.0, -10.0, 0.0, 10.0, 18.0 - eps()])
    end
    @testset "x ∈ [18.0, 33.3)" begin
      test_log1pexp(Float64, [18.0, 18.0 + eps(), 33.3 - eps()])
    end
    @testset "x ∈ [33.3, ∞)" begin
      test_log1pexp(Float64, [33.3, 33.3 + eps(), 100.0])
    end
  end
end

@testset "log1psq" begin
  rng = MersenneTwister(123456)
  @testset "Float64" begin
    for x in [-10.0, -5.0, -1.0, -eps(), 0.0, eps(), 1.0, 5.0, 10.0]
      @test gradcheck(x->5.1 * StatsFuns.log1psq(x[1]), [x])
    end
  end
end

@testset "logsumexp" begin
  rng = MersenneTwister(123456)
  @testset "Float64" begin
    @test gradtest(StatsFuns.logsumexp, randn(rng, 1))
    @test gradtest(StatsFuns.logsumexp, randn(rng, 1, 1))
    @test gradtest(StatsFuns.logsumexp, randn(rng, 3))
    @test gradtest(StatsFuns.logsumexp, randn(rng, 3, 4, 5))
  end
end

@testset "* sizing" begin
  @test size(Zygote.gradient((x, y)->sum(x * y), randn(1, 1), randn(1, 10))[1]) == (1, 1)
  @test size(Zygote.gradient((x, y)->sum(x * y), randn(1, 1), randn(1, 10))[2]) == (1, 10)
end


ref_line = ReferenceLine()
@test dim(ref_line) == 1

ref_triangle = ReferenceTriangle()
@test dim(ref_triangle) == 2

ref_square = ReferenceSquare()
@test dim(ref_square) == 2

ref_tetrahedron = ReferenceTetrahedron()
@test dim(ref_tetrahedron) == 3

ref_cube = ReferenceCube()
@test dim(ref_cube) == 3

for p = 2:5
    qr = make_quadrule(ref_line, p)
    for n = [2*p - 1, 2*p - 2]
        analytic = 1 / (n+1) * (1. + (-1.)^n)
        @test_approx_eq integrate(qr, x -> x[1]^n) analytic
    end

    # http://www.m-hikari.com/ijma/ijma-2011/ijma-1-4-2011/venkateshIJMA1-4-2011.pdf
end


using FeedbackParticleFilters, Random, Statistics, LinearAlgebra, Distributions, FillArrays

println("Testing ppFPF.jl:")

@testset "ppFPF.jl" begin
    # helper function for keeping particles between 0 and 2*pi
    function mod2pi!(x::AbstractArray)
        for i in eachindex(x)
            x[i] = mod2pi(x[i])
        end
    end

    print("  inner constructor for ppFPFState")
    ens = UnweightedParticleEnsemble(2π .* randn(1,10))
    function h(x)
        20*exp.(10*cos.(x[1] .- 2*pi*collect(0:3)/4))/exp(10.)
    end
    h(x::AbstractMatrix) = mapslices(h, x, dims=1)
    eq_dt  = PoissonEquation(x -> -sum(h(x)), ens)
    eq_dN  = PoissonEquation(x -> log.(h(x)), ens)
    state = ppFPFState(ens, eq_dN, eq_dt)
    print("-")
    @test state.ensemble == ens
    print("-")
    @test state.eq_dt == eq_dt
    print("-")
    @test state.eq_dN == eq_dN
    println("DONE")

    print("  inner constructor for ppFPF")
    st_mod = BrownianMotionCircle()
    ob_mod = CountingObservationModel{Float64, Int, typeof(h)}(1, 4, h)
    filt_prob = FilteringProblem(st_mod, ob_mod)
    g_method = DifferentialRKHSMethodS1(0.1, 0.01)
    f_method = DeterministicFlow(100, g_method, mod2pi!)
    fpf = ppFPF(st_mod, ob_mod, g_method, f_method, 10)
    print("-")
    @test fpf.state_model == st_mod
    print("-")
    @test fpf.obs_model == ob_mod
    print("-")
    @test fpf.gain_method == g_method
    print("-")
    @test fpf.flow_method == f_method
    print("-")
    @test fpf.N == 10
    println("DONE")

    print("  method initial_condition")
    print("-")
    @test initial_condition(fpf) == Uniform(0,2π)
    println("DONE")

    print("  method no_of_particles")
    print("-")
    @test no_of_particles(fpf) == 10
    print("-")
    @test no_of_particles(state) == 10
    println("DONE")

    print("  method state_model")
    print("-")
    @test state_model(fpf) == st_mod
    println("DONE")

    print("  method obs_model")
    print("-")
    @test obs_model(fpf) == ob_mod
    println("DONE")

    print("  method gain_estimation_method")
    print("-")
    @test gain_estimation_method(fpf) == g_method
    println("DONE")

    print("  method gain_estimation_method")
    print("-")
    @test flow_method(fpf) == f_method
    println("DONE")

    print("  method for Statistics.mean")
    print("-")
    @test Statistics.mean(state) ≈ mean(ens)
    println("DONE")

    print("  method for Statistics.cov")
    print("-")
    @test Statistics.cov(state) ≈ cov(ens)
    println("DONE")

    print("  method for Statistics.var")
    print("-")
    @test Statistics.var(state) ≈ var(ens)
    println("DONE")

    print("  outer constructors for ppFPFState")
    state2 = ppFPFState(fpf)
    print("-")
    @test no_of_particles(state2) == 10
    print("-")
    @test all(0 .<= state2.ensemble.positions .<= 2π)
    filt_prob = FilteringProblem(st_mod, ob_mod)
    state3 = ppFPFState(filt_prob, 20)
    print("-")
    @test no_of_particles(state3) == 20
    println("DONE")

    print("  outer constructors for ppFPF")
    fpf2 = ppFPF(filt_prob, g_method, f_method, 100)
    print("-")
    @test no_of_particles(fpf2) == 100
    println("DONE")

    print("  method initialize")
    state4 = initialize(fpf)
    print("-")
    @test state4 isa ppFPFState
    print("-")
    @test all(0 .<= state4.ensemble.positions .<= 2π)
    println("DONE")

    print("  method assimilate!")
    oldstate = deepcopy(state)
    assimilate!([0, 1, 0, 1], state, fpf, 0.01)
    print("-")
    @test state.eq_dt.gain != oldstate.eq_dt.gain
    solve!(oldstate.eq_dt, g_method)
    @test state.eq_dt.gain ≈ oldstate.eq_dt.gain

    print("-")
    @test state.eq_dt.positions ≈ state.ensemble.positions
    print("-")
    @test state.ensemble != oldstate.ensemble
    println("DONE")

    print("  method propagate!")
    oldstate = deepcopy(state)
    propagate!(state, fpf, 0.01)
    print("-")
    @test all(state.ensemble.positions .!= oldstate.ensemble.positions)
    println("DONE")

    print("  method update!")
    fpf = ppFPF(st_mod, ob_mod, g_method, f_method, 100)
    state1 = initialize(fpf)
    state2 = deepcopy(state1)
    Random.seed!(0)
    update!(state1, fpf, [1, 1, 0, 0], 0.01)
    Random.seed!(0)
    propagate!(state2, fpf, 0.01)
    assimilate!([1, 1, 0, 0], state2, fpf, 0.01)
    print("-")
    @test state1.ensemble.positions ≈ state2.ensemble.positions
    print("-")
    @test state1.eq_dt.gain ≈ state2.eq_dt.gain
    println("DONE")
end; #ppFPF.jl


using Pkg
current_env = Base.load_path()[1]
Pkg.activate(@__DIR__)

using RadialBasisFunctionModels
using Test
using ForwardDiff
using RadialBasisFunctionModels.StaticArrays
using InteractiveUtils: subtypes

#%%
@testset "1D-Data" begin
	for T in subtypes(AbstractFloat)
		φ = Cubic()	# is good at interpolating ill-conditioned data
		features = rand(T, 5)
		labels = rand(T, 5)

		rbf = RBFModel( features, labels, φ; )

		@test typeof(rbf).parameters[1] == true
		@test rbf(features[1]) isa Vector{T}
		if T != Float16
			for (f,l) in zip(features, labels)
				@test isapprox(rbf(f)[1], l; atol = 0.01)
			end
		end
		@test rbf.rbf.kernels[1].c isa Vector{T}

		rbf = RBFModel( features, labels; vector_output = false )
		@test typeof(rbf).parameters[1] == false
		@test rbf(features[1]) isa T
		@test rbf.rbf.kernels[1].c isa Vector{T}

		rbf = RBFModel( features, labels; vector_output = false )
		@test typeof(rbf).parameters[1] == false
		@test rbf(features[1]) isa T
		@test rbf.rbf.kernels[1].c isa Vector{T}
	end
end

#%%

@testset "NonStaticData" begin

	for T in subtypes(AbstractFloat)
		for φ in [Gaussian(), InverseMultiquadric(), Multiquadric(), Cubic(), ThinPlateSpline(1)]
			n_in = rand(1:10)
			n_out = rand(1:10)
			num_data = n_in + 1
			features = [ rand(T, n_in) for i = 1 : num_data ]
			labels = [ rand(T, n_out) for i = 1 : num_data ]

			rbf = RBFModel( features, labels, φ, 1 )

			T == Float16 || @test all( rbf( f ) ≈ l for (f,l) in zip(features,labels) )

			x = features[1]
			y = labels[1]

			@test rbf(x) isa Vector{T}

			for (x,y) in zip(features, labels)
				for ℓ = 1 : n_out
					G = grad(rbf, x, ℓ)
					aG = auto_grad(rbf, x, ℓ)
					@test G isa Vector{T}
					@test aG isa Vector{T}
					T == Float16 || @test G ≈ ForwardDiff.gradient( ξ -> rbf(ξ, ℓ), x )
					T == Float16 || @test G ≈ aG
				end
			end
		end
	end
end

#%%
@testset "StaticArraysAndTypes" begin

	FTypes = Iterators.Stateful(Iterators.cycle(subtypes(AbstractFloat)))
	Rads = Iterators.Stateful(Iterators.cycle([Cubic(),Gaussian(),Multiquadric(),InverseMultiquadric(),ThinPlateSpline(1)]))

	SizedTypes = [ SizedVector, MVector, SVector ]

	for FT in SizedTypes, LT in SizedTypes, CT in SizedTypes
		n_in = rand(1:10)
		n_out = rand(1:10)
		num_data = n_in + 1
		T = popfirst!(FTypes)
		φ = popfirst!(Rads)

		sized_features = [ FT{n_in}(rand(T,n_in)) for i = 1 : num_data ]
		sized_labels = [ LT{n_out}(rand(T, n_out)) for i = 1 : num_data ]

		rbf = RBFModel(sized_features, sized_labels, φ)

		@test length( rbf.rbf.kernels ) == num_data
		@test rbf.rbf.kernels[1].c isa FT
		@test !(rbf.rbf.weights isa StaticArray)	# no sized array of sized centers was provided

		@test !(rbf.psum.weights isa StaticArray)	# no sized array of sized centers was provided

		# check if _type_guard works
		x = Vector(sized_features[1])
		@test rbf.rbf(x) isa Vector
		@test rbf.psum(x) isa Vector
		@test rbf(x) isa Vector
		for XT in SizedTypes
			ξ = XT{n_in}(x)
			@test rbf.rbf(ξ) isa XT
			@test rbf.psum(ξ) isa XT
			@test rbf(ξ) isa XT
		end
		## if we provide length information on the number of centers, we should get sized output
		sized_scenters = SizedVector{num_data}( sized_features )
		rbf = RBFModel( sized_features, sized_labels, φ; centers = sized_scenters )

		@test rbf.rbf.weights isa StaticArray
		@test rbf.psum.weights isa StaticArray

		@test rbf.rbf(x) isa Vector
		@test rbf.psum(x) isa Vector
		@test rbf(x) isa Vector
		for XT in SizedTypes
			ξ = XT{n_in}(x)
			@test rbf.rbf(ξ) isa XT
			@test rbf.psum(ξ) isa XT
			@test rbf(ξ) isa XT
		end
	end
end

#%%
Pkg.activate(current_env)


VERTEX=Int
EDGE=Array{VERTEX,1}
FACE=Array{EDGE,1}
CELL=Array{FACE,1}

function vertices(edge::EDGE)
    return edge
end
function vertices(face::FACE)
    f=copy(face)
    v=union(vcat(f...))
    w=[v[1]]
    for j ∈ 1:(length(v)-1)
        i=findfirst(e->w[end] ∈ e, f)
        push!(w,filter(v->v≠w[end],f[i])[1])
        deleteat!(f,i)
    end
    return w
end
function vertices(cell::CELL)
    return unique(vcat(vcat(cell...)...))
end
function edges(face::FACE)
    return face
end
function edges(cell::CELL)
    return unique(sort.(vcat(cell...)))
end
function faces(cell::CELL)
    return cell
end

import Base.∈
function ∈(vertex::VERTEX, face::FACE)
    return vertex ∈ vertices(face)
end
function ∈(vertex::VERTEX, cell::CELL)
    return vertex ∈ vertices(cell)
end
function ∈(edge::EDGE, cell::CELL)
    return edge ∈ edges(cell)
end

function DualRegularPlolyhedron(POINTS,c)
    fs=copy(c)
    es=edges(fs)
    vs=vertices(fs)
    POINTS_=normalize.([+(POINTS[vertices(f)]...) for f ∈ fs])
    vs_=collect(1:length(faces(fs)))
    es_=[findall(f->e ∈ f, fs) for e ∈ es]
    fs_=[es_[findall(e->issubset(e,findall(f->v ∈ f, c)),es_)] for v ∈ vs]
    return POINTS_, fs_
end

# F₄
𝒑₁=[1,1,1]
𝒑₂=[1,-1,-1]
𝒑₃=[-1,1,-1]
𝒑₄=[-1,-1,1]
POINTS=[𝒑₁,𝒑₂,𝒑₃,𝒑₄]
v₁=1
v₂=2
v₃=3
v₄=4
e₁=[v₁,v₂]
e₂=[v₁,v₃]
e₃=[v₁,v₄]
e₄=[v₂,v₃]
e₅=[v₂,v₄]
e₆=[v₃,v₄]

f₁=[e₄,e₅,e₆]
f₂=[e₂,e₃,e₆]
f₃=[e₁,e₃,e₅]
f₄=[e₁,e₂,e₄]
c₁=[f₁,f₂,f₃,f₄]
F₄=normalize.(POINTS), c₁

# F₆
𝒑₁=[1,1,1]
𝒑₂=[1,1,-1]
𝒑₃=[1,-1,1]
𝒑₄=[1,-1,-1]
𝒑₅=[-1,1,1]
𝒑₆=[-1,1,-1]
𝒑₇=[-1,-1,1]
𝒑₈=[-1,-1,-1]
POINTS=[𝒑₁,𝒑₂,𝒑₃,𝒑₄,𝒑₅,𝒑₆,𝒑₇,𝒑₈]
v₁=1
v₂=2
v₃=3
v₄=4
v₅=5
v₆=6
v₇=7
v₈=8
e₁=[v₁,v₂]
e₂=[v₁,v₃]
e₃=[v₁,v₅]
e₄=[v₂,v₄]
e₅=[v₂,v₆]
e₆=[v₃,v₄]
e₇=[v₃,v₇]
e₈=[v₄,v₈]
e₉=[v₅,v₆]
e₁₀=[v₅,v₇]
e₁₁=[v₆,v₈]
e₁₂=[v₇,v₈]
f₁=[e₁,e₂,e₄,e₆]
f₂=[e₁,e₃,e₅,e₉]
f₃=[e₂,e₃,e₇,e₁₀]
f₄=[e₄,e₅,e₈,e₁₁]
f₅=[e₆,e₇,e₈,e₁₂]
f₆=[e₉,e₁₀,e₁₁,e₁₂]
c₁=[f₁,f₂,f₃,f₄,f₅,f₆]
F₆=normalize.(POINTS), c₁

F₈=DualRegularPlolyhedron(F₆...)

# F₁₂
𝒑₁=[1,1,1]
𝒑₂=[1,1,-1]
𝒑₃=[1,-1,1]
𝒑₄=[1,-1,-1]
𝒑₅=[-1,1,1]
𝒑₆=[-1,1,-1]
𝒑₇=[-1,-1,1]
𝒑₈=[-1,-1,-1]
𝒑₉=[0,1/φ,φ]
𝒑₁₀=[0,1/φ,-φ]
𝒑₁₁=[0,-1/φ,φ]
𝒑₁₂=[0,-1/φ,-φ]
𝒑₁₃=[φ,0,1/φ]
𝒑₁₄=[φ,0,-1/φ]
𝒑₁₅=[-φ,0,1/φ]
𝒑₁₆=[-φ,0,-1/φ]
𝒑₁₇=[1/φ,φ,0]
𝒑₁₈=[1/φ,-φ,0]
𝒑₁₉=[-1/φ,φ,0]
𝒑₂₀=[-1/φ,-φ,0]
POINTS=[𝒑₁,𝒑₂,𝒑₃,𝒑₄,𝒑₅,𝒑₆,𝒑₇,𝒑₈,𝒑₉,𝒑₁₀,𝒑₁₁,𝒑₁₂,𝒑₁₃,𝒑₁₄,𝒑₁₅,𝒑₁₆,𝒑₁₇,𝒑₁₈,𝒑₁₉,𝒑₂₀]
v₁=1
v₂=2
v₃=3
v₄=4
v₅=5
v₆=6
v₇=7
v₈=8
v₉=9
v₁₀=10
v₁₁=11
v₁₂=12
v₁₃=13
v₁₄=14
v₁₅=15
v₁₆=16
v₁₇=17
v₁₈=18
v₁₉=19
v₂₀=20
e₁=[v₁,v₉]
e₂=[v₅,v₉]
e₃=[v₃,v₁₁]
e₄=[v₇,v₁₁]
e₅=[v₉,v₁₁]
e₆=[v₂,v₁₀]
e₇=[v₆,v₁₀]
e₈=[v₄,v₁₂]
e₉=[v₈,v₁₂]
e₁₀=[v₁₀,v₁₂]
e₁₁=[v₁,v₁₃]
e₁₂=[v₃,v₁₃]
e₁₃=[v₂,v₁₄]
e₁₄=[v₄,v₁₄]
e₁₅=[v₁₃,v₁₄]
e₁₆=[v₅,v₁₅]
e₁₇=[v₇,v₁₅]
e₁₈=[v₆,v₁₆]
e₁₉=[v₈,v₁₆]
e₂₀=[v₁₅,v₁₆]
e₂₁=[v₁,v₁₇]
e₂₂=[v₂,v₁₇]
e₂₃=[v₅,v₁₉]
e₂₄=[v₆,v₁₉]
e₂₅=[v₁₇,v₁₉]
e₂₆=[v₃,v₁₈]
e₂₇=[v₄,v₁₈]
e₂₈=[v₇,v₂₀]
e₂₉=[v₈,v₂₀]
e₃₀=[v₁₈,v₂₀]
f₁=[e₁,e₃,e₅,e₁₁,e₁₂]
f₂=[e₂,e₄,e₅,e₁₆,e₁₇]
f₃=[e₆,e₈,e₁₀,e₁₃,e₁₄]
f₄=[e₇,e₉,e₁₀,e₁₈,e₁₉]
f₅=[e₁₁,e₁₃,e₁₅,e₂₁,e₂₂]
f₆=[e₁₂,e₁₄,e₁₅,e₂₆,e₂₇]
f₇=[e₁₆,e₁₈,e₂₀,e₂₃,e₂₄]
f₈=[e₁₇,e₁₉,e₂₀,e₂₈,e₂₉]
f₉=[e₁,e₂,e₂₁,e₂₃,e₂₅]
f₁₀=[e₆,e₇,e₂₂,e₂₄,e₂₅]
f₁₁=[e₃,e₄,e₂₆,e₂₈,e₃₀]
f₁₂=[e₈,e₉,e₂₇,e₂₉,e₃₀]
c₁=[f₁,f₂,f₃,f₄,f₅,f₆,f₇,f₈,f₉,f₁₀,f₁₁,f₁₂]
F₁₂=normalize.(POINTS), c₁

F₂₀=DualRegularPlolyhedron(F₁₂...)


# Adapted from https://github.com/tpapp/TransformVariables.jl/blob/master/src/special_arrays.jl

export CorrCholesky

"""
    CorrCholesky(n)
Cholesky factor of a correlation matrix of size `n`.
Transforms ``n(n-1)/2`` real numbers to an ``n×n`` lower-triangular matrix `L`, such that
`L*L'` is a correlation matrix (positive definite, with unit diagonal).

# Notes
If
- `z` is a vector of `n` IID standard normal variates,
- `σ` is an `n`-element vector of standard deviations,
- `C` is obtained from `CorrCholesky(n)`,
then `Diagonal(σ) * C.L * z` is a zero-centered multivariate normal variate with the standard deviations `σ` and
correlation matrix `C.L * C.U`.
"""
struct CorrCholesky{N} <: TV.VectorTransform
    n::N
end

TV.dimension(t::CorrCholesky{Int}) = TV.dimension(CorrCholeskyUpper(t.n))

TV.dimension(t::CorrCholesky{StaticInt{n}}) where {n} = TV.dimension(CorrCholeskyUpper(t.n))

# TODO: For now we just transpose the CorrCholeskyUpper result. We should
# consider whether it can help performance to implement this directly for the
# lower triangular case
function TV.transform_with(flag::TV.LogJacFlag, t::CorrCholesky{StaticInt{n}}, x::AbstractVector, index) where {n}
    @nospecialize t
    U, ℓ, index = TV.transform_with(flag, CorrCholeskyUpper(n), x, index)
    U = UpperTriangular(SizedMatrix{n,n}(U.data))
    return Cholesky(U,'U', 0), ℓ, index
end

function TV.transform_with(flag::TV.LogJacFlag, t::CorrCholesky, x::AbstractVector, index)
    n = t.n
    U, ℓ, index = TV.transform_with(flag, CorrCholeskyUpper(n), x, index)
    return Cholesky(U,'U', 0), ℓ, index
end

TV.inverse_eltype(t::CorrCholesky, x::AbstractMatrix) = TV.extended_eltype(x)

function TV.inverse_at!(x::AbstractVector, index, t::CorrCholesky, L::LowerTriangular)
    return TV.inverse_at!(x, index, CorrCholeskyUpper(t.n), L')
end

function TV.inverse_at!(x::AbstractVector, index, t::CorrCholesky, U::UpperTriangular)
    return TV.inverse_at!(x, index, CorrCholeskyUpper(t.n), U)
end

function TV.inverse_at!(x::AbstractVector, index, t::CorrCholesky, C::Cholesky)
    return TV.inverse_at!(x, index, t, C.UL)
end


abstract AbstractResult

abstract SpectralResult <: AbstractResult

typealias Projection{T <: Real} AbstractMatrix{T}

type Diffmap{T <: Real}
    d::Int
    t::Int
    K::AbstractMatrix{T}
    Y::AbstractMatrix{T}
end

function Base.show(io::IO, res::Diffmap)
    println(io, "Dimensionality:")
    show(io, res.d)
    print(io, "\n\n")
    println(io, "Timesteps:")
    show(io, res.t)
    print(io, "\n\n")
    println(io, "Kernel:")
    show(io, res.K)
    print(io, "\n\n")
    println(io, "Embedding:")
    show(io, res.Y)
end


using InterpretableValidation
using Test
using Distributions
using Random

## Test construction of ConstrainedTimeseriesDistribution
x = collect(1:100.)
uniform = ConstrainedTimeseriesDistribution(IID(x, Uniform(2,3)))
v = rand(uniform)
@test all(v.>2) && all(v.<3)

normal = ConstrainedTimeseriesDistribution(IID(x, Normal(0.5, 1.)))
normal.lb = uniform.lb
v = rand(normal)
@test all(v.>2)

categorical = ConstrainedTimeseriesDistribution(IID(x, Categorical(5)))
v = rand(categorical)
@test all([v[i] in collect(1:5) for i=1:length(v)])

bernoulli = ConstrainedTimeseriesDistribution(IID(x, Bernoulli(0.1)))
v = rand(bernoulli)
@test sum(v) > 0
@test sum(v) < 20

gp = ConstrainedTimeseriesDistribution(GaussianProcess(m = (x) -> 0, k = squared_exp_kernel(l=4), x = x))
gp.lb = -1*ones(100)
v = rand(gp)
@test all(v .> -1)

@test N_pts(gp) == 100

## Test discrete vs. continuous
@test !isdiscrete(uniform)
@test isdiscrete(categorical)
@test !iscontinuous(categorical)
@test isdiscrete(bernoulli)
@test iscontinuous(gp)

## Test feasible check
@test isfeasible(uniform)
@test isfeasible(categorical)

old = categorical.feasible[10]
categorical.feasible[10] = Int64[]
@test !isfeasible(categorical)
categorical.feasible[10] = old

old = uniform.lb[1]
uniform.lb[1]=100
@test !isfeasible(uniform)
uniform.lb[1]=old

## Test construction of MvTimeseriesDistribution
mvts = MvTimeseriesDistribution(:x => uniform, :y =>normal, :z => categorical, :gp =>gp, :b => bernoulli)
@test N_pts(mvts) == 100

v = rand(mvts)
@test length(v) == 5

## Test the application of constraints
uniform.lb = -Inf*ones(100)
uniform.ub = Inf*ones(100)
greaterthan!(uniform, 3., [fill(true, 50)..., fill(false, 50)...])
@test all(uniform.lb[1:50] .== 3.0)
@test all(uniform.ub[51:100] .== 3.0)

greaterthan!(uniform, 2., [fill(true, 50)..., fill(false, 50)...])
@test all(uniform.lb[1:50] .== 3.0)
@test all(uniform.ub[51:100] .== 2.0)

greaterthan!(uniform, 4., [fill(true, 50)..., fill(false, 50)...])
@test all(uniform.lb[1:50] .== 4.0)
@test all(uniform.ub[51:100] .== 2.0)

greaterthan!(uniform, 5., [fill(true, 50)..., fill(:anybool, 50)...])
@test all(uniform.lb[1:50] .== 5.0)
@test all(uniform.ub[51:100] .== 2.0)

lessthan!(uniform, 4., [fill(true, 50)..., fill(false, 50)...])
@test all(uniform.ub[1:50] .== 4.0)
@test all(uniform.lb[51:100] .== 4.0)

uniform.lb = -Inf*ones(100)
uniform.ub = Inf*ones(100)
continuous_equality!(uniform, 3., [fill(true, 50)..., fill(:anybool, 50)...])
@test all(uniform.lb[1:50] .== uniform.ub[1:50])
@test all(rand(uniform)[1:50] .== 3.)

discrete_equality!(categorical, 3, [:anybool, fill(true, 50)..., fill(false, 49)...])
@test categorical.feasible[1] == [1,2,3,4,5]
@test all([categorical.feasible[i] == [3] for i=2:51])
@test all([!(3 in categorical.feasible[i]) for i in 52:100])

discrete_equality!(bernoulli, 1, [:anybool, fill(true, 50)..., fill(false, 49)...])
@test bernoulli.feasible[1] == [0,1]
@test all([bernoulli.feasible[i] == [1] for i=2:51])
@test all([bernoulli.feasible[i] == [0] for i=52:100])

## Test constraint on the time series
constrain_timeseries!(mvts, Meta.parse("x .<= 1."), Array{Any}(fill(true, 100)))
@test mvts[:x].ub == ones(100)

constraints = [(Meta.parse("y .<= 1."), Array{Any}(fill(true, 100)))]
constrain_timeseries!(mvts, constraints)
@test mvts[:y].ub == ones(100)

## Test the sampling based on expression
for (key, val) in mvts
    if iscontinuous(val)
        val.lb = -Inf*ones(100)
        val.ub = Inf*ones(100)
    elseif isdiscrete(val)
        val.feasible = fill([1,2,3,4,5], 100)
    end
end
v = Base.rand(Random.GLOBAL_RNG, Meta.parse("all(x .>= 1.) && all(y .>= 1.) && all(z .== 2)"), mvts)
@test all(v[:x] .> 1.)
@test all(v[:y] .> 1.)
@test all(v[:z] .== 2)

@test_throws InfeasibleConstraint Base.rand(Random.GLOBAL_RNG, Meta.parse("all(x .>= 1.) && all(x .<= 0.5)"), mvts)

mvts = MvTimeseriesDistribution(:x => IID(x, Uniform()), :y => IID(x, Normal()), :z => IID(x, Categorical(5)))

function target_expr(data::Dict{Symbol, Array})
    truth_vals = ((data[:x] .> 0.75) .& (data[:y] .< -0.75)) .| ((data[:x] .< 0.25) .& (data[:y] .> -0.25))
    truth_vals[1] = truth_vals[1] &&  (data[:z][1] == 5)
    return (length(truth_vals) - sum(truth_vals)) / length(truth_vals)
end

good_expr = Meta.parse("all(((x .>= 0.75) .& (y .<= -0.75)) .| ((x .<= 0.25) .& (y .>= -0.25)))")
good_vals = rand(Random.GLOBAL_RNG, good_expr, mvts)
@test target_expr(good_vals) <= 0.01

expr = Meta.parse("all_between((x .== 2.881512049363103) .| (x .<= 2.3141385919362447), 59, 14)")
constraints = sample_constraints(expr, 100, Random.GLOBAL_RNG)
mvts = MvTimeseriesDistribution(:x => IID(x, Uniform(2,3)))
good_vals = rand(Random.GLOBAL_RNG, expr, mvts)
x = good_vals[:x]
@test all_between((x .== 2.881512049363103) .| (x .<= 2.3141385919362447), 59, 14)


#
# Perform multiple trials to find gloal minimum
#
function find_best_fit(model, X, Y, np, options, lower, upper)
  local best_fit
  local fit
  best = +Inf
  nbest = 0
  ntrial = 1
  p0 = Vector{Float64}(undef,np)
  while nbest <= options.nbest && ntrial <= options.maxtrials
    ntrial += 1
    try
      initP!(p0,options,lower,upper)
      fit = curve_fit(model, X, Y, p0, lower=lower, upper=upper)
      sum_residues = sum(fit.resid.^2)
      if abs(sum_residues - best) < options.besttol
        nbest = nbest + 1
        if sum_residues < best
          best = sum_residues
          best_fit = deepcopy(fit)
        end
      elseif sum_residues < best
        nbest = 1
        best = sum_residues
        best_fit = deepcopy(fit)
      end
    catch msg
      if options.debug
        error("ERROR: $msg \n $fit")
      end
    end
  end
  if nbest == 0
error(" Could not obtain any successful fit, probably the data is not well posed.
        Further information can be obtained by adding Option(debug=true) to the input.")
  end
  return best_fit
end


# Copyright (c) Microsoft Corporation.
# Licensed under the MIT license.

module ZygoteGMM
include("../../shared/load.jl")
using ADPerfTest
using GMMData
using Zygote
using SpecialFunctions
using LinearAlgebra

export get_gmm_test

"Computes logsumexp. Input should be 1 dimensional"
function logsumexp(x)
    mx = maximum(x)
    log(sum(exp.(x .- mx))) + mx
end

function ltri_unpack(D, LT)
    d = length(D)
    make_col(r::Int, L) = vcat(zeros(r - 1), D[r], reshape([L[i] for i=1:d-r], d - r))
    col_start(r::Int) = (r - 1) * (2d - r) ÷ 2
    inds(r) = col_start(r) .+ (1:d-r)
    hcat([make_col(r, LT[inds(r)]) for r=1:d]...)
end

function get_Q(d, icf)
    ltri_unpack((icf[1:d]), icf[d+1:end])
end

function log_gamma_distrib(a, p)
    out = 0.25 * p * (p - 1) * 1.1447298858494002 #convert(Float64, log(pi))
    out += sum(j -> loggamma(a + 0.5 * (1 - j)), 1:p)
    out
end

Zygote.@adjoint function loggamma(x)
    loggamma(x), Δ -> (Δ * digamma(x),)
end

function log_wishart_prior(wishart::Wishart, sum_qs, Qs, k)
    p = size(Qs, 1)
    n = p + wishart.m + 1
    C = n * p * (log(wishart.gamma) - 0.5 * log(2)) - log_gamma_distrib(0.5 * n, p)

    frobenius = sum(abs2, Qs)
	0.5 * wishart.gamma^2 * frobenius - wishart.m * sum(sum_qs) - k * C
end

function diagsums(Qs)
    mapslices(slice -> sum(diag(slice)), Qs; dims=[1, 2])
end

Zygote.@adjoint function diagsums(Qs)
    diagsums(Qs),
    function (Δ)
        Δ′ = zero(Qs)
        for (i, δ) in enumerate(Δ)
            for j in 1:size(Qs, 1)
                Δ′[j, j, i] = δ
            end
        end
        (Δ′,)
    end
end

function expdiags(Qs)
    mapslices(Qs; dims=[1, 2]) do slice
        slice[diagind(slice)] .= exp.(slice[diagind(slice)])
        slice
    end
end

Zygote.@adjoint function expdiags(Qs)
    expdiags(Qs),
    function (Δ)
        Δ′ = zero(Qs)
        Δ′ .= Δ
        for i in 1:size(Qs, 3)
            for j in 1:size(Qs, 1)
                Δ′[j, j, i] *= exp(Qs[j, j, i])
            end
        end
        (Δ′,)
    end
end

function unzip(tuples)
    map(1:length(first(tuples))) do i
        map(tuple -> tuple[i], tuples)
    end
end

Zygote.@adjoint function map(f, args...)
    ys_and_backs = map((args...) -> Zygote._forward(__context__, f, args...), args...)
    ys, backs = unzip(ys_and_backs)
    ys, function (Δ)
        Δf_and_args_zipped = map((f, δ) -> f(δ), backs, Δ)
        Δf_and_args = unzip(Δf_and_args_zipped)
        Δf = reduce(Zygote.accum, Δf_and_args[1])
        (Δf, Δf_and_args[2:end]...)
    end
end

Base.:*(::Float64, ::Nothing) = nothing

function gmm_objective(alphas, means, Qs, x, wishart::Wishart)
    d = size(x, 1)
    n = size(x, 2)
    k = size(means, 2)
    CONSTANT = -n * d * 0.5 * log(2 * pi)
    sum_qs = reshape(diagsums(Qs), 1, size(Qs, 3))
    Qs = expdiags(Qs)

    slse = 0.
    for ix=1:n
        formula(ik) = -0.5 * sum(abs2, Qs[:, :, ik] * (x[:,ix] .- means[:, ik])) + alphas[ik] + sum_qs[ik]
        terms = map(formula, 1:k)
        slse += logsumexp(terms)
    end

    CONSTANT + slse - n * logsumexp(alphas) + log_wishart_prior(wishart, sum_qs, Qs, k)
end

function zygote_J_to_packed_J(J, k, d)
    alphas = reshape(J[1], :)
    means = reshape(J[2], :)
    icf_unpacked = map(1:k) do Q_idx
        Q = J[3][:, :, Q_idx]
        lt_cols = map(1:d-1) do col
            Q[col+1:d, col]
        end
        vcat(diag(Q), lt_cols...)
    end
    icf = collect(Iterators.flatten(icf_unpacked))
    packed_J = vcat(alphas, means, icf)
    packed_J
  end

mutable struct ZygoteGMMContext
    input::Union{GMMInput, Nothing}
    Qs
    wrapper_gmm_objective::Union{Function, Nothing}
    objective::Float64
    zygote_J
end

function zygote_gmm_prepare!(ctx::ZygoteGMMContext, input::GMMInput)
    # Running Zygote.gradient with test input, because the first invocation
    # of gradient on a given function is very long.
    # Using test input ensures that all computations related to the actual input
    # are done in calculate_jacobian!
    # Test input taken from ../../../../data/gmm/test.txt
    testinput = GMMInput([-0.649014 1.181166 -0.758453], [0.092339 0.345561 0.538817; 0.18626 0.396767 0.419195], [0.586443 -1.509405 0.166813; -0.851887 0.875874 -1.965419; 0.800321 -0.24279 -1.270071], [1.175171 2.02916]', Wishart(1.0, 0))
    testd = size(testinput.x, 1)
    testk = size(testinput.means, 2)
    testQs = cat([get_Q(testd, testinput.icfs[:, ik]) for ik in 1:testk]...; dims=[3])
    test_wrapper_gmm_objective = (alphas, means, Qs) -> gmm_objective(alphas, means, Qs, testinput.x, testinput.wishart)
    Zygote.gradient(test_wrapper_gmm_objective, testinput.alphas, testinput.means, testQs)

    ctx.input = input
    d = size(input.x, 1)
    k = size(input.means, 2)
    ctx.Qs = cat([get_Q(d, input.icfs[:, ik]) for ik in 1:k]...; dims=[3])
    ctx.wrapper_gmm_objective = (alphas, means, Qs) -> gmm_objective(alphas, means, Qs, input.x, input.wishart)
end

function zygote_gmm_calculate_objective!(ctx::ZygoteGMMContext, times)
    for i in 1:times
        ctx.objective = gmm_objective(ctx.input.alphas, ctx.input.means, ctx.Qs, ctx.input.x, ctx.input.wishart)
    end
end

function zygote_gmm_calculate_jacobian!(ctx::ZygoteGMMContext, times)
    for i in 1:times
        ctx.zygote_J = Zygote.gradient(ctx.wrapper_gmm_objective, ctx.input.alphas, ctx.input.means, ctx.Qs)
    end
end

function zygote_gmm_output!(out::GMMOutput, ctx::ZygoteGMMContext)
    out.objective = ctx.objective
    d = size(ctx.input.x, 1)
    k = size(ctx.input.means, 2)
    out.gradient = ctx.zygote_J === nothing ? [] : zygote_J_to_packed_J(ctx.zygote_J, k, d)
end

get_gmm_test() = Test{GMMInput, GMMOutput}(
    ZygoteGMMContext(nothing, nothing, nothing, 0.0, nothing),
    zygote_gmm_prepare!,
    zygote_gmm_calculate_objective!,
    zygote_gmm_calculate_jacobian!,
    zygote_gmm_output!
)

end


## cholesky.jl : Gaussian random field generator based on the Cholesky decomposition of the covariance matrix

"""
    Cholesky()

Retuns a [`GaussianRandomFieldGenerator`](@ref) based on a Cholesky factorization of the covariance matrix.

# Examples
```jldoctest
julia> cov = CovarianceFunction(2, Matern(.3, 1))
2d Matérn covariance function (λ=0.3, ν=1.0, σ=1.0, p=2.0)

julia> pts = range(0, stop=1, length=51)
0.0:0.02:1.0

julia> grf = GaussianRandomField(cov, Cholesky(), pts, pts)
Gaussian random field with 2d Matérn covariance function (λ=0.3, ν=1.0, σ=1.0, p=2.0) on a 51×51 structured grid, using a Cholesky decomposition

julia> heatmap(grf)
[...]

```

!!! warning

    The Cholesky factorization requires the covariance matrix to be symmetric and positive definite. If the covariance matrix is not positive definite, an error will be thrown. Try using the `Spectral` method in that case.

See also: [`Spectral`](@ref), [`KarhunenLoeve`](@ref), [`CirculantEmbedding`](@ref)
"""
struct Cholesky <: GaussianRandomFieldGenerator end

# compose a GaussianRandomField using Cholesky factorization
function _GaussianRandomField(mean, cov::CovarianceFunction, method::Cholesky, pts...)
    C = apply(cov, pts...)

    # compute Cholesky factorization
    isposdef!(Symmetric(C)) || throw(ArgumentError("to use a Cholesky factorization, the covariance matrix must be positive definite"))
    L = LowerTriangular(C')

    GaussianRandomField{Cholesky,typeof(cov),typeof(pts),typeof(mean),typeof(L)}(mean,cov,pts,L)
end

# returns the required dimension of the random points
randdim(grf::GaussianRandomField{Cholesky}) = size(grf.data, 1)

# sample from the GaussianRandomField using Cholesky factorization
function _sample(grf::GaussianRandomField{Cholesky}, xi)
	grf.mean + std(grf.cov) * reshape(grf.data * xi, size(grf.mean))
end

Base.show(io::IO, ::Cholesky) = print(io, "Cholesky decomposition")


@inline function _sampleSignedBernoulli(rng)
  return 2*(rand(rng) < 0.5) - 1
end

## A modified version of the algorithm (attributed to S. Vempala) in
## Wästlund, J., 1999. Functions arising by coin flipping. Technical Report,
## KTH, Stockholm.
##
## Specifically, I modified the algorithm so that the calls to f() and the
## sampling of the Bernoulli r.v.s occurs in tandem, so as to avoid the
## infinite expected running time of the initial pass of the original algorithm.
##
## This is also more efficient in terms of calls to f() than the version
## described in
## Flajolet, P., Pelletier, M. and Soria, M., 2011, January. On Buffon machines
## and numbers. In Proc. 22nd ACM-SIAM symposium on Discrete algorithms.
function _wastlundSqrt(f::F, rng::RNG) where
  {F<:Function, RNG<:AbstractRNG}
  flips::Int64 = 0
  x::Bool = f(); flips += 1
  x && (return true, flips)
  s::Int64 = _sampleSignedBernoulli(rng)
  while s > 0
    x = f(); flips += 1
    x && (return true, flips)
    s += _sampleSignedBernoulli(rng) + _sampleSignedBernoulli(rng)
  end
  return false, flips
end


using PtFEM

data = Dict(
  # GenericSolid(ndim, nst, nels, nn, nip, finite_element(nod, nodof), axisymmentrix)
  :struc_el => GenericSolid(3, 6, 6, 8, 1, Tetrahedron(4, 3), false),
  :properties => [100.0 0.3 0.0;],
  :g_coord => [
    0.0 0.0 0.0; 1.0 0.0 0.0; 0.0 0.0 -1.0; 1.0 0.0 -1.0;
    0.0 1.0 0.0; 1.0 1.0 0.0; 0.0 1.0 -1.0; 1.0 1.0 -1.0],
  :g_num => [
    1, 3, 4, 7,
    1, 4, 2, 7,
    1, 2, 5, 7,
    6, 4, 8, 7,
    6, 2, 4, 7,
    6, 5, 2, 7
  ],
  :support => [
    (1, [0 0 1]), (2, [1 0 1]), (3, [0 0 0]), (4, [1 0 0]),
    (5, [0 1 1]), (7, [0 1 0]), (8, [1 1 0])
  ],
  :loaded_nodes => [
    ( 1, [0.0 0.0 -0.1667]),
    ( 2, [0.0 0.0 -0.3333]),
    ( 5, [0.0 0.0 -0.3333]),
    ( 6, [0.0 0.0 -0.1667])
  ]
)

data |> display
println()

@time m, dis_df, fm_df = p54(data)
println()

dis_df |> display
println()

println("\nThe integration point (nip = $(data[:struc_el].nip)) stresses are:\n")

fm_df |> display
println()


# This file is a part of Julia. License is MIT: https://julialang.org/license

# Real
function realeigtest(n, iter)
    A = rand(n,n)
    d = Vector{eltype(A)}
    v = similar(A)
    for i = 1:iter
        d,v = eig(A)
    end
    d,v
end

# Symmetric
function symeigtest(n, iter)
    A = rand(n,n)
    A = A + adjoint(A)
    d = Vector{eltype(A)}
    v = similar(A)
    for i = 1:iter
        d,v = eig(A)
    end
    d,v
end

# Hermitian
function hermitianeigtest(n, iter)
    A = rand(n,n) + im*rand(n,n)
    A = A + adjoint(A)
    d = Vector{eltype(A)}
    v = similar(A)
    for i = 1:iter
        d,v = eig(A)
    end
    d,v
end

problemsizes = [(2, 10^4, "tiny"), (2^4, 10^3, "small"), (2^6, 10^2, "medium"), (2^8, 5, "large"), (2^10, 1, "huge")]
testdata = [(realeigtest, "realeig", "Real matrix eigenfactorization", problemsizes),
            (symeigtest, "symeig", "Symmetric matrix eigenfactorization", problemsizes),
            (hermitianeigtest, "hermitianeig", "Hermitian matrix eigenfactorization", problemsizes)]
include("../perfgeneric.jl")


"""
    degree_out(N::AbstractEcologicalNetwork)
"""
function degree_out(N::AbstractEcologicalNetwork)
  d_o = vec(sum(N.edges, dims=2))
  return Dict(zip(species(N; dims=1), d_o))
end

"""
    degree_in(N::AbstractEcologicalNetwork)
"""
function degree_in(N::AbstractEcologicalNetwork)
  d_i = vec(sum(N.edges, dims=1))
  return Dict(zip(species(N; dims=2), d_i))
end

"""
    degree(N::AbstractEcologicalNetwork; dims::Union{Nothing,Integer}=nothing)

Returns the degrees of nodes in a network; `dims` can be `1` for out degree, or
`2` for in degree.

#### References

Delmas, E., Besson, M., Brice, M.-H., Burkle, L.A., Dalla Riva, G.V., Fortin,
M.-J., Gravel, D., Guimarães, P.R., Hembry, D.H., Newman, E.A., Olesen, J.M.,
Pires, M.M., Yeakel, J.D., Poisot, T., 2018. Analysing ecological networks of
species interactions. Biological Reviews 112540.
https://doi.org/10.1111/brv.12433

Poisot, T., Cirtwill, A.R., Cazelles, K., Gravel, D., Fortin, M.-J., Stouffer,
D.B., 2016. The structure of probabilistic networks. Methods in Ecology and
Evolution 7, 303–312. https://doi.org/10.1111/2041-210X.12468

Williams, R.J., 2011. Biology, Methodology or Chance? The Degree Distributions
of Bipartite Ecological Networks. PLoS One 6, e17645.
https://doi.org/10.1371/journal.pone.0017645
"""
function degree(N::AbstractEcologicalNetwork; dims::Union{Nothing,Integer}=nothing)
  dims == 1 && return degree_out(N)
  dims == 2 && return degree_in(N)
  if isnothing(dims)
    if typeof(N) <: AbstractBipartiteNetwork
      return merge(degree_out(N), degree_in(N))
    else
      din = degree_in(N)
      dout = degree_out(N)
      for k in keys(din)
        din[k] += dout[k]
      end
      return din
    end
  end
  throw(ArgumentError("dims can only be 1 (out-degre) or 2 (in-degree) or `nothing` (both), you used $(dims)"))
end

function degree(N::QuantitativeNetwork; dims::Union{Nothing,Integer}=nothing)
  degree(convert(BinaryNetwork, N), dims=dims)
end

"""
**Variance in the outgoing degree**

    degree_out_var(N::ProbabilisticNetwork)
"""
function degree_out_var(N::ProbabilisticNetwork)
  d_o_v = vec(mapslices(_additive_bernoulli_variance, convert(Matrix, N.edges); dims=2))
  return Dict(zip(species(N; dims=1), d_o_v))
end

"""
**Variance in the ingoing degree**

    degree_in_var(N::ProbabilisticNetwork)
"""
function degree_in_var(N::ProbabilisticNetwork)
  d_i_v = vec(mapslices(_additive_bernoulli_variance, convert(Matrix, N.edges); dims=1))
  return Dict(zip(species(N; dims=2), d_i_v))
end

"""
    degree_var(N::ProbabilisticNetwork; dims::Union{Nothing,Integer}=nothing)

Variance in the degree of species in a probabilistic network.
"""
function degree_var(N::ProbabilisticNetwork; dims::Union{Nothing,Integer}=nothing)
  dims == 1 && return degree_out_var(N)
  dims == 2 && return degree_in_var(N)
  if isnothing(dims)
    if typeof(N) <: AbstractBipartiteNetwork
      return merge(degree_out_var(N), degree_in_var(N))
    else
      din = degree_in_var(N)
      dout = degree_out_var(N)
      for k in keys(din)
        din[k] += dout[k]
      end
      return din
    end
  end
  throw(ArgumentError("dims can only be 1 (out-degre) or 2 (in-degree) or `nothing` (both), you used $(dims)"))
end


using TSne
using Test
using Random, Statistics, Distances
using RDatasets, MLDatasets

include("test_tsne.jl")


using Lazy

cfrac(n::Real) = let m = floor(n)
    if (n-m)==0
        @lazy Integer(m)
    else
        @lazy Integer(m):cfrac(1/(n-m))
    end
end

restore_cfrac(cf::List) = begin
    ans = Inf
    for x in reverse(cf)
        ans = x + 1/ans
    end
    ans
end


module CRT

using JFVM
using PyPlot

export CRT_SNIA, CRT_SIA, CRT_GIA_eps, CRT_GIA_kin_eq, CRT_GIA_kin_fb, plot_concentrations

#Define functions
function epsilon(C_A, C_B, C_C, nu_A, nu_B, nu_C, K_eq)
    # A + B <-> C
    return 1/(2*nu_A*nu_B*K_eq)*(nu_A*C_B*K_eq+nu_B*C_A*K_eq+nu_C-sqrt(nu_A^2*C_B^2*K_eq^2-2*nu_A*nu_B*C_A*C_B*K_eq^2+nu_B^2*C_A^2*K_eq^2+4*nu_A*nu_B*C_C*K_eq+2*nu_A*nu_C*C_B*K_eq+2*nu_B*nu_C*C_A*K_eq+nu_C^2))
end

function epsilon_A(C_A, C_B, C_C, nu_A, nu_B, nu_C, K_eq)
  return 1/(2*nu_A*nu_B*K_eq)*(nu_B*K_eq-(C_A*K_eq^2*nu_B^2-C_B*K_eq^2*nu_A*nu_B+K_eq*nu_B*nu_C)/sqrt(nu_A^2*C_B^2*K_eq^2-2*nu_A*nu_B*C_A*C_B*K_eq^2+nu_B^2*C_A^2*K_eq^2+4*nu_A*nu_B*C_C*K_eq+2*nu_A*nu_C*C_B*K_eq+2*nu_B*nu_C*C_A*K_eq+nu_C^2))
end

function epsilon_AA(C_A, C_B, C_C, nu_A, nu_B, nu_C, K_eq)
  return 1/(2*nu_A*nu_B*K_eq)*(-(K_eq^2*nu_B^2)/sqrt(nu_A^2*C_B^2*K_eq^2-2*nu_A*nu_B*C_A*C_B*K_eq^2+nu_B^2*C_A^2*K_eq^2+4*nu_A*nu_B*C_C*K_eq+2*nu_A*nu_C*C_B*K_eq+2*nu_B*nu_C*C_A*K_eq+nu_C^2)+((C_A*K_eq^2*nu_B^2-C_B*K_eq^2*nu_A*nu_B+K_eq*nu_B*nu_C)*(2*C_A*K_eq^2*nu_B^2-2*C_B*K_eq^2*nu_A*nu_B+2*K_eq*nu_B*nu_C))/(2*(nu_A^2*C_B^2*K_eq^2-2*nu_A*nu_B*C_A*C_B*K_eq^2+nu_B^2*C_A^2*K_eq^2+4*nu_A*nu_B*C_C*K_eq+2*nu_A*nu_C*C_B*K_eq+2*nu_B*nu_C*C_A*K_eq+nu_C^2)^(3/2)))
end

function epsilon_B(C_A, C_B, C_C, nu_A, nu_B, nu_C, K_eq)
  return 1/(2*nu_A*nu_B*K_eq)*(nu_A*K_eq-(C_B*K_eq^2*nu_A^2-C_A*K_eq^2*nu_A*nu_B+K_eq*nu_A*nu_C)/sqrt(nu_A^2*C_B^2*K_eq^2-2*nu_A*nu_B*C_A*C_B*K_eq^2+nu_B^2*C_A^2*K_eq^2+4*nu_A*nu_B*C_C*K_eq+2*nu_A*nu_C*C_B*K_eq+2*nu_B*nu_C*C_A*K_eq+nu_C^2))
end

function epsilon_BB(C_A, C_B, C_C, nu_A, nu_B, nu_C, K_eq)
  return 1/(2*nu_A*nu_B*K_eq)*(-(K_eq^2*nu_A^2)/sqrt(nu_A^2*C_B^2*K_eq^2-2*nu_A*nu_B*C_A*C_B*K_eq^2+nu_B^2*C_A^2*K_eq^2+4*nu_A*nu_B*C_C*K_eq+2*nu_A*nu_C*C_B*K_eq+2*nu_B*nu_C*C_A*K_eq+nu_C^2)+((-C_A*K_eq^2*nu_A*nu_B+C_B*K_eq^2*nu_A+K_eq*nu_A*nu_C)*(-2*C_A*K_eq^2*nu_A*nu_B+2*C_B*K_eq^2*nu_A^2+2*K_eq*nu_A*nu_C))/(2*(nu_A^2*C_B^2*K_eq^2-2*nu_A*nu_B*C_A*C_B*K_eq^2+nu_B^2*C_A^2*K_eq^2+4*nu_A*nu_B*C_C*K_eq+2*nu_A*nu_C*C_B*K_eq+2*nu_B*nu_C*C_A*K_eq+nu_C^2)^(3/2)))
end

function epsilon_C(C_A, C_B, C_C, nu_A, nu_B, nu_C, K_eq)
  return -1/sqrt(nu_A^2*C_B^2*K_eq^2-2*nu_A*nu_B*C_A*C_B*K_eq^2+nu_B^2*C_A^2*K_eq^2+4*nu_A*nu_B*C_C*K_eq+2*nu_A*nu_C*C_B*K_eq+2*nu_B*nu_C*C_A*K_eq+nu_C^2)
end

function epsilon_CC(C_A, C_B, C_C, nu_A, nu_B, nu_C, K_eq)
  return (2*K_eq*nu_A*nu_B)/((nu_A^2*C_B^2*K_eq^2-2*nu_A*nu_B*C_A*C_B*K_eq^2+nu_B^2*C_A^2*K_eq^2+4*nu_A*nu_B*C_C*K_eq+2*nu_A*nu_C*C_B*K_eq+2*nu_B*nu_C*C_A*K_eq+nu_C^2)^(3/2))
end

include("CRT_SNIA.jl")
include("CRT_SIA.jl")
include("CRT_GIA_eps.jl")
include("CRT_GIA_kin_eq.jl")
include("CRT_GIA_kin_fb.jl")
include("plot_concentrations.jl")

end


abstract type MeanFunction end

"""
    ZeroMean{T<:Real} <: MeanFunction

Returns `zero(T)` everywhere.
"""
struct ZeroMean{T<:Real} <: MeanFunction end

Base.map(::ZeroMean{T}, x::AbstractVector) where T = zeros(T, length(x))

ZeroMean() = ZeroMean{Float64}()

"""
    ConstMean{T<:Real} <: MeanFunction

Returns `c` everywhere.
"""
struct ConstMean{T<:Real} <: MeanFunction
    c::T
end

Base.map(m::ConstMean, x::AbstractVector) = fill(m.c, length(x))

"""
    CustomMean{Tf} <: MeanFunction

A wrapper around whatever unary function you fancy. Must be able to be mapped over an
`AbstractVector` of inputs.
"""
struct CustomMean{Tf} <: MeanFunction
    f::Tf
end

Base.map(f::CustomMean, x::AbstractVector) = map(f.f, x)


"""
    ConstantStepsize <: Stepsize

A functor that always returns a fixed step size.

# Fields
* `length` – constant value for the step size.

# Constructors

    ConstantStepsize(s)

initialize the stepsize to a constant `s` (deprecated)

    ConstantStepsize(M::AbstractManifold=DefaultManifold(2); stepsize=injectivity_radius(M)/2)

initialize the stepsize to a constant `stepsize`, which by default is half the injectivity
radius, unless the radius is infinity, then the default step size is `1`.
"""
mutable struct ConstantStepsize{T} <: Stepsize
    length::T
end
@deprecate ConstantStepsize(s::Real) ConstantStepsize(; stepsize=s)
function ConstantStepsize(
    M::AbstractManifold=DefaultManifold(2);
    stepsize=isinf(injectivity_radius(M)) ? 1.0 : injectivity_radius(M) / 2,
)
    return ConstantStepsize{typeof(stepsize)}(stepsize)
end
function (s::ConstantStepsize)(
    p::P, o::O, i::Int, args...; kwargs...
) where {P<:Problem,O<:Options}
    return s.length
end
get_initial_stepsize(s::ConstantStepsize) = s.length

@doc raw"""
    DecreasingStepsize()

A functor that represents several decreasing step sizes

# Fields
* `length` – (`1`) the initial step size ``l``.
* `factor` – (`1`) a value ``f`` to multiply the initial step size with every iteration
* `subtrahend` – (`0`) a value ``a`` that is subtracted every iteration
* `exponent` – (`1`) a value ``e`` the current iteration numbers ``e``th exponential
  is taken of
* `shift` – (`0`) shift the denominator iterator ``i`` by ``s```.

In total the complete formulae reads for the ``i``th iterate as

````math
s_i = \frac{(l - i a)f^i}{(i+s)^e}
````

and hence the default simplifies to just ``s_i = \frac{l}{i}``

# Constructor

    DecreasingStepsize(l=1,f=1,a=0,e=1,s=0)

Alternatively one can also use the following keyword.

    DecreasingStepsize(
        M::AbstractManifold=DefaultManifold(3);
        length=injectivity_radius(M)/2, multiplier=1.0, subtrahend=0.0, exponent=1.0, shift=0)

initialiszes all fields above, where none of them is mandatory and the length is set to
half and to $1$ if the injectivity radius is infinite.
"""
mutable struct DecreasingStepsize <: Stepsize
    length::Float64
    factor::Float64
    subtrahend::Float64
    exponent::Float64
    shift::Int
    function DecreasingStepsize(l::Real, f::Real=1.0, a::Real=0.0, e::Real=1.0, s::Int=0)
        return new(l, f, a, e, s)
    end
end
function DecreasingStepsize(
    M::AbstractManifold=DefaultManifold(3);
    length=isinf(manifold_dimension(M)) ? 1.0 : manifold_dimension(M) / 2,
    factor=1.0,
    subtrahend=0.0,
    exponent=1.0,
    shift=0,
)
    return DecreasingStepsize(length, factor, subtrahend, exponent, shift)
end
function (s::DecreasingStepsize)(
    ::P, ::O, i::Int, args...; kwargs...
) where {P<:Problem,O<:Options}
    return (s.length - i * s.subtrahend) * (s.factor^i) / ((i + s.shift)^(s.exponent))
end
get_initial_stepsize(s::DecreasingStepsize) = s.length

"""
    Linesearch <: Stepsize

An abstract functor to represent line search type step size deteminations, see
[`Stepsize`](@ref) for details. One example is the [`ArmijoLinesearch`](@ref)
functor.

Compared to simple step sizes, the linesearch functors provide an interface of
the form `(p,o,i,η) -> s` with an additional (but optional) fourth parameter to
provide a search direction; this should default to something reasonable, e.g.
the negative gradient.
"""
abstract type Linesearch <: Stepsize end

@doc raw"""
    ArmijoLinesearch <: Linesearch

A functor representing Armijo line search including the last runs state, i.e. a
last step size.

# Fields
* `initialStepsize` – (`1.0`) and initial step size
* `retraction_method` – (`ExponentialRetraction()`) the rectraction to use, defaults to
  the exponential map
* `contractionFactor` – (`0.95`) exponent for line search reduction
* `sufficientDecrease` – (`0.1`) gain within Armijo's rule
* `last_stepsize` – (`initialstepsize`) the last step size we start the search with
* `linesearch_stopsize` - (`0.0`) a safeguard when to stop the line search
    before the step is numerically zero. This should be combined with [`StopWhenStepSizeLess`](@ref)

# Constructor

    ArmijoLineSearch()

with the Fields above in their order as optional arguments (deprecated).

    ArmijoLineSearch(M)

with the Fields above as keyword arguments and the retraction is set to the default retraciton on `M`.

The constructors return the functor to perform Armijo line search, where two inter
faces are available:
* based on a tuple `(p,o,i)` of a [`GradientProblem`](@ref) `p`, [`Options`](@ref) `o`
  and a current iterate `i`.
* with `(M, x, F, gradFx[,η=-gradFx]) -> s` where [Manifold](https://juliamanifolds.github.io/Manifolds.jl/stable/interface.html#Manifold) `M`, a current
  point `x` a function `F`, that maps from the manifold to the reals,
  its gradient (a tangent vector) `gradFx```=\operatorname{grad}F(x)`` at  `x` and an optional
  search direction tangent vector `η=-gradFx` are the arguments.
"""
mutable struct ArmijoLinesearch{TRM<:AbstractRetractionMethod} <: Linesearch
    initialStepsize::Float64
    retraction_method::TRM
    contractionFactor::Float64
    sufficientDecrease::Float64
    last_stepsize::Float64
    linesearch_stopsize::Float64
    @deprecate ArmijoLinesearch(
        s::Float64=1.0,
        r::AbstractRetractionMethod=ExponentialRetraction(),
        contractionFactor::Float64=0.95,
        sufficientDecrease::Float64=0.1,
        linesearch_stopsize::Float64=0.0,
    ) ArmijoLinesearch(
        DefaultManifold(2);
        initial_stepsize=s,
        retraction_method=r,
        contractionFactor=contractionFactor,
        sufficientDecrease=sufficientDecrease,
        linesearch_stopsize=linesearch_stopsize,
    )
    function ArmijoLinesearch(
        M;
        initial_stepsize::Float64=1.0,
        retraction_method::AbstractRetractionMethod=default_retraction_method(M),
        contractionFactor::Float64=0.95,
        sufficientDecrease::Float64=0.1,
        linesearch_stopsize::Float64=0.0,
    )
        return new{typeof(retraction_method)}(
            initial_stepsize,
            retraction_method,
            contractionFactor,
            sufficientDecrease,
            initial_stepsize,
            linesearch_stopsize,
        )
    end
end
function (a::ArmijoLinesearch)(
    p::GradientProblem, o::Options, ::Int, η=-get_gradient(p, o.x); kwargs...
)
    a.last_stepsize = linesearch_backtrack(
        p.M,
        x -> p.cost(p.M, x),
        o.x,
        get_gradient!(p, o.gradient, o.x),
        a.last_stepsize,
        a.sufficientDecrease,
        a.contractionFactor,
        a.retraction_method,
        η;
        stop_step=a.linesearch_stopsize,
    )
    return a.last_stepsize
end
get_initial_stepsize(a::ArmijoLinesearch) = a.initialStepsize

@doc raw"""
    linesearch_backtrack(M, F, x, gradFx, s, decrease, contract, retr, η = -gradFx, f0 = F(x); stop_step=0.)

perform a linesearch for
* a manifold `M`
* a cost function `F`,
* an iterate `x`
* the gradient ``\operatorname{grad}F(x)``
* an initial stepsize `s` usually called ``γ``
* a sufficient `decrease`
* a `contract`ion factor ``σ``
* a `retr`action, which defaults to the `ExponentialRetraction()`
* a search direction ``η = -\operatorname{grad}F(x)``
* an offset, ``f_0 = F(x)``
* a keyword `stop_step` as a minimal step size when to stop
"""
function linesearch_backtrack(
    M::AbstractManifold,
    F::TF,
    x,
    gradFx::T,
    s,
    decrease,
    contract,
    retr::AbstractRetractionMethod=ExponentialRetraction(),
    η::T=-gradFx,
    f0=F(x);
    stop_step=0.0,
) where {TF,T}
    xNew = retract(M, x, s * η, retr)
    fNew = F(xNew)
    search_dir_inner = inner(M, x, η, gradFx)
    extended = false
    while fNew < f0 + decrease * s * search_dir_inner # increase
        extended = true
        s = s / contract
        retract!(M, xNew, x, s * η, retr)
        fNew = F(xNew)
    end
    if extended
        s *= contract  # undo last increase
        retract!(M, xNew, x, s * η, retr)
        fNew = F(xNew)
    end
    while fNew > f0 + decrease * s * search_dir_inner # decrease
        s = contract * s
        retract!(M, xNew, x, s * η, retr)
        fNew = F(xNew)
        (s < stop_step) && break
    end
    return s
end

@doc raw"""
    NonmonotoneLinesearch <: Linesearch

A functor representing a nonmonotone line search using the Barzilai-Borwein step size[^Iannazzo2018]. Together with a gradient descent algorithm
this line search represents the Riemannian Barzilai-Borwein with nonmonotone line-search (RBBNMLS) algorithm. We shifted the order of the algorithm steps from the paper
by Iannazzo and Porcelli so that in each iteration we first find

```math
y_{k} = \operatorname{grad}F(x_{k}) - \operatorname{T}_{x_{k-1} → x_k}(\operatorname{grad}F(x_{k-1}))
```

and

```math
s_{k} = - α_{k-1} * \operatorname{T}_{x_{k-1} → x_k}(\operatorname{grad}F(x_{k-1})),
```

where ``α_{k-1}`` is the step size computed in the last iteration and ``\operatorname{T}`` is a vector transport.
We then find the Barzilai–Borwein step size

```math
α_k^{\text{BB}} = \begin{cases}
\min(α_{\text{max}}, \max(α_{\text{min}}, τ_{k})),  & \text{if } ⟨s_{k}, y_{k}⟩_{x_k} > 0,\\
α_{\text{max}}, & \text{else,}
\end{cases}
```

where

```math
τ_{k} = \frac{⟨s_{k}, s_{k}⟩_{x_k}}{⟨s_{k}, y_{k}⟩_{x_k}},
```

if the direct strategy is chosen,

```math
τ_{k} = \frac{⟨s_{k}, y_{k}⟩_{x_k}}{⟨y_{k}, y_{k}⟩_{x_k}},
```

in case of the inverse strategy and an alternation between the two in case of the
alternating strategy. Then we find the smallest ``h = 0, 1, 2, …`` such that

```math
F(\operatorname{retr}_{x_k}(- σ^h α_k^{\text{BB}} \operatorname{grad}F(x_k)))
\leq
\max_{1 ≤ j ≤ \min(k+1,m)} F(x_{k+1-j}) - γ σ^h α_k^{\text{BB}} ⟨\operatorname{grad}F(x_k), \operatorname{grad}F(x_k)⟩_{x_k},
```

where ``σ`` is a step length reduction factor ``∈ (0,1)``, ``m`` is the number of iterations
after which the function value has to be lower than the current one
and ``γ`` is the sufficient decrease parameter ``∈(0,1)``. We can then find the new stepsize by

```math
α_k = σ^h α_k^{\text{BB}}.
```

[^Iannazzo2018]:
    > B. Iannazzo, M. Porcelli, __The Riemannian Barzilai–Borwein Method with Nonmonotone Line Search and the Matrix Geometric Mean Computation__,
    > In: IMA Journal of Numerical Analysis. Volume 38, Issue 1, January 2018, Pages 495–517,
    > doi [10.1093/imanum/drx015](https://doi.org/10.1093/imanum/drx015)

# Fields
* `initial_stepsize` – (`1.0`) the step size we start the search with
* `linesearch_stopsize` - (`0.0`) a safeguard when to stop the line search
    before the step is numerically zero. This should be combined with [`StopWhenStepSizeLess`](@ref)
* `memory_size` – (`10`) number of iterations after which the cost value needs to be lower than the current one
* `min_stepsize` – (`1e-3`) lower bound for the Barzilai-Borwein step size greater than zero
* `max_stepsize` – (`1e3`) upper bound for the Barzilai-Borwein step size greater than min_stepsize
* `retraction_method` – (`ExponentialRetraction()`) the rectraction to use
* `strategy` – (`direct`) defines if the new step size is computed using the direct, indirect or alternating strategy
* `storage` – (`x`, `gradient`) a [`StoreOptionsAction`](@ref) to store `old_x` and `old_gradient`, the x-value and corresponding gradient of the previous iteration
* `stepsize_reduction` – (`0.5`) step size reduction factor contained in the interval (0,1)
* `sufficient_decrease` – (`1e-4`) sufficient decrease parameter contained in the interval (0,1)
* `vector_transport_method` – (`ParallelTransport()`) the vector transport method to use

# Constructor

    NonmonotoneLinesearch()

with the Fields above in their order as optional arguments (deprecated).

    NonmonotoneLinesearch(M)

with the Fields above in their order as keyword arguments and where the retraction
and vector transport are set to the default ones on `M`, repsectively.

The constructors return the functor to perform nonmonotone line search.
"""
mutable struct NonmonotoneLinesearch{
    TRM<:AbstractRetractionMethod,VTM<:AbstractVectorTransportMethod,T<:AbstractVector
} <: Linesearch
    retraction_method::TRM
    vector_transport_method::VTM
    stepsize_reduction::Float64
    sufficient_decrease::Float64
    min_stepsize::Float64
    max_stepsize::Float64
    initial_stepsize::Float64
    old_costs::T
    strategy::Symbol
    storage::StoreOptionsAction
    linesearch_stopsize::Float64
    function NonmonotoneLinesearch(
        M::AbstractManifold=DefaultManifold(2);
        initial_stepsize::Float64=1.0,
        retraction_method::AbstractRetractionMethod=default_retraction_method(M),
        vector_transport_method::AbstractVectorTransportMethod=default_vector_transport_method(
            M
        ),
        stepsize_reduction::Float64=0.5,
        sufficient_decrease::Float64=1e-4,
        memory_size::Int=10,
        min_stepsize::Float64=1e-3,
        max_stepsize::Float64=1e3,
        strategy::Symbol=:direct,
        storage::StoreOptionsAction=StoreOptionsAction((:x, :gradient)),
        linesearch_stopsize::Float64=0.0,
    )
        if strategy ∉ [:direct, :inverse, :alternating]
            @warn string(
                "The strategy '",
                strategy,
                "' is not defined. The 'direct' strategy is used instead.",
            )
            strategy = :direct
        end
        if min_stepsize <= 0.0
            throw(
                DomainError(
                    min_stepsize,
                    "The lower bound for the step size min_stepsize has to be greater than zero.",
                ),
            )
        end
        if max_stepsize <= min_stepsize
            throw(
                DomainError(
                    max_stepsize,
                    "The upper bound for the step size max_stepsize has to be greater its lower bound min_stepsize.",
                ),
            )
        end
        if memory_size <= 0
            throw(DomainError(memory_size, "The memory_size has to be greater than zero."))
        end
        return new{
            typeof(retraction_method),typeof(vector_transport_method),Vector{Float64}
        }(
            retraction_method,
            vector_transport_method,
            stepsize_reduction,
            sufficient_decrease,
            min_stepsize,
            max_stepsize,
            initial_stepsize,
            zeros(memory_size),
            strategy,
            storage,
            linesearch_stopsize,
        )
    end
    @deprecate NonmonotoneLinesearch(
        initial_stepsize::Float64,
        retraction_method::AbstractRetractionMethod=ExponentialRetraction(),
        vector_transport_method::AbstractVectorTransportMethod=ParallelTransport(),
        stepsize_reduction::Float64=0.5,
        sufficient_decrease::Float64=1e-4,
        memory_size::Int=10,
        min_stepsize::Float64=1e-3,
        max_stepsize::Float64=1e3,
        strategy::Symbol=:direct,
        storage::StoreOptionsAction=StoreOptionsAction((:x, :gradient)),
        linesearch_stopsize::Float64=0.0,
    ) NonmonotoneLinesearch(
        DefaultManifold(3);
        initial_stepsize=initial_stepsize,
        retraction_method=retraction_method,
        vector_transport_method=vector_transport_method,
        stepsize_reduction=stepsize_reduction,
        sufficient_decrease=sufficient_decrease,
        memory_size=memory_size,
        min_stepsize=min_stepsize,
        max_stepsize=max_stepsize,
        strategy=strategy,
        storage=storage,
        linesearch_stopsize=linesearch_stopsize,
    )
end
function (a::NonmonotoneLinesearch)(
    p::GradientProblem, o::Options, i::Int, η=-get_gradient(p, o.x); kwargs...
)
    if !all(has_storage.(Ref(a.storage), [:x, :gradient]))
        old_x = o.x
        old_gradient = get_gradient(p, o.x)
    else
        old_x, old_gradient = get_storage.(Ref(a.storage), [:x, :gradient])
    end
    update_storage!(a.storage, o)
    return a(p.M, o.x, x -> get_cost(p, x), get_gradient(p, o.x), η, old_x, old_gradient, i)
end
function (a::NonmonotoneLinesearch)(
    M::mT, x, F::TF, gradFx::T, η::T, old_x, old_gradient, iter::Int; kwargs...
) where {mT<:AbstractManifold,TF,T}
    #find the difference between the current and previous gardient after the previous gradient is transported to the current tangent space
    grad_diff =
        gradFx - vector_transport_to(M, old_x, old_gradient, x, a.vector_transport_method)
    #transport the previous step into the tangent space of the current manifold point
    x_diff =
        -a.initial_stepsize *
        vector_transport_to(M, old_x, old_gradient, x, a.vector_transport_method)

    #compute the new Barzilai-Borwein step size
    s1 = inner(M, x, x_diff, grad_diff)
    s2 = inner(M, x, grad_diff, grad_diff)
    s2 = s2 == 0 ? 1.0 : s2
    s3 = inner(M, x, x_diff, x_diff)
    #indirect strategy
    if a.strategy == :inverse
        if s1 > 0
            BarzilaiBorwein_stepsize = min(a.max_stepsize, max(a.min_stepsize, s1 / s2))
        else
            BarzilaiBorwein_stepsize = a.max_stepsize
        end
        #alternating strategy
    elseif a.strategy == :alternating
        if s1 > 0
            if iter % 2 == 0
                BarzilaiBorwein_stepsize = min(a.max_stepsize, max(a.min_stepsize, s1 / s2))
            else
                BarzilaiBorwein_stepsize = min(a.max_stepsize, max(a.min_stepsize, s3 / s1))
            end
        else
            BarzilaiBorwein_stepsize = a.max_stepsize
        end
        #direct strategy
    else
        if s1 > 0
            BarzilaiBorwein_stepsize = min(a.max_stepsize, max(a.min_stepsize, s2 / s1))
        else
            BarzilaiBorwein_stepsize = a.max_stepsize
        end
    end

    memory_size = length(a.old_costs)
    if iter <= memory_size
        a.old_costs[iter] = F(x)
    else
        a.old_costs[1:(memory_size - 1)] = a.old_costs[2:memory_size]
        a.old_costs[memory_size] = F(x)
    end

    #compute the new step size with the help of the Barzilai-Borwein step size
    a.initial_stepsize = linesearch_backtrack(
        M,
        F,
        x,
        gradFx,
        BarzilaiBorwein_stepsize,
        a.sufficient_decrease,
        a.stepsize_reduction,
        a.retraction_method,
        η,
        maximum([a.old_costs[j] for j in 1:min(iter, memory_size)]);
        stop_step=a.linesearch_stopsize,
    )
    return a.initial_stepsize
end

@doc raw"""
    WolfePowellLinesearch <: Linesearch

Do a backtracking linesearch to find a step size ``α`` that fulfils the
Wolfe conditions along a search direction ``η`` starting from ``x``, i.e.

```math
f\bigl( \operatorname{retr}_x(αη) \bigr) ≤ f(x_k) + c_1 α_k ⟨\operatorname{grad}f(x), η⟩_x
\quad\text{and}\quad
\frac{\mathrm{d}}{\mathrm{d}t} f\bigr(\operatorname{retr}_x(tη)\bigr)
\Big\vert_{t=α}
≥ c_2 \frac{\mathrm{d}}{\mathrm{d}t} f\bigl(\operatorname{retr}_x(tη)\bigr)\Big\vert_{t=0}.
```

# Constructors

There exist two constructors, where, when prodivind the manifold `M` as a first (optional)
parameter, its default retraction and vector transport are the default.
In this case the retraction and the vector transport are also keyword arguments for ease of use.
The other constructor is kept for backward compatibility.
Note that the `linesearch_stopsize` to stop for too small stepsizes is only available in the
new signature including `M`.
For the old (deprecated) signature the `linesearch_stopsize` is set to the old hard-coded default of  `1e-12`

    WolfePowellLinesearch(
        retr::AbstractRetractionMethod=ExponentialRetraction(),
        vtr::AbstractVectorTransportMethod=ParallelTransport(),
        c1::Float64=10^(-4),
        c2::Float64=0.999
    )

    WolfePowellLinesearch(
        M,
        c1::Float64=10^(-4),
        c2::Float64=0.999;
        retraction_method = default_retraction_method(M),
        vector_transport_method = default_vector_transport(M),
        linesearch_stopsize = 0.0
    )
"""
mutable struct WolfePowellLinesearch <: Linesearch
    retraction_method::AbstractRetractionMethod
    vector_transport_method::AbstractVectorTransportMethod
    c1::Float64
    c2::Float64
    last_stepsize::Float64
    linesearch_stopsize::Float64

    @deprecate WolfePowellLinesearch(
        retr::AbstractRetractionMethod=ExponentialRetraction(),
        vtr::AbstractVectorTransportMethod=ParallelTransport(),
        c1::Float64=10^(-4),
        c2::Float64=0.999,
    ) WolfePowellLinesearch(
        DefaultManifold(3),
        c1,
        c2,
        retraction_method=retr,
        vector_transport_method=vtr,
        linesearch_stopsize=1e-12,
    )
    function WolfePowellLinesearch(
        M::AbstractManifold,
        c1::Float64=10^(-4),
        c2::Float64=0.999;
        retraction_method::AbstractRetractionMethod=default_retraction_method(M),
        vector_transport_method::AbstractVectorTransportMethod=default_vector_transport_method(
            M
        ),
        linesearch_stopsize::Float64=0.0,
    )
        return new(
            retraction_method, vector_transport_method, c1, c2, 0.0, linesearch_stopsize
        )
    end
end

function (a::WolfePowellLinesearch)(
    p::P, o::O, ::Int, η=-get_gradient(p, o.x); kwargs...
) where {P<:GradientProblem{T,mT} where {T,mT<:AbstractManifold},O<:Options}
    s = 1.0
    s_plus = 1.0
    s_minus = 1.0
    f0 = get_cost(p, o.x)
    xNew = retract(p.M, o.x, s * η, a.retraction_method)
    fNew = get_cost(p, xNew)
    η_xNew = vector_transport_to(p.M, o.x, η, xNew, a.vector_transport_method)
    if fNew > f0 + a.c1 * s * inner(p.M, o.x, η, o.gradient)
        while (fNew > f0 + a.c1 * s * inner(p.M, o.x, η, o.gradient)) && (s_minus > 10^(-9)) # decrease
            s_minus = s_minus * 0.5
            s = s_minus
            retract!(p.M, xNew, o.x, s * η, a.retraction_method)
            fNew = get_cost(p, xNew)
        end
        s_plus = 2.0 * s_minus
    else
        vector_transport_to!(p.M, η_xNew, o.x, η, xNew, a.vector_transport_method)
        if inner(p.M, xNew, get_gradient(p, xNew), η_xNew) <
            a.c2 * inner(p.M, o.x, η, o.gradient)
            while fNew <= f0 + a.c1 * s * inner(p.M, o.x, η, o.gradient) &&
                (s_plus < 10^(9))# increase
                s_plus = s_plus * 2.0
                s = s_plus
                retract!(p.M, xNew, o.x, s * η, a.retraction_method)
                fNew = get_cost(p, xNew)
            end
            s_minus = s_plus / 2.0
        end
    end
    retract!(p.M, xNew, o.x, s_minus * η, a.retraction_method)
    vector_transport_to!(p.M, η_xNew, o.x, η, xNew, a.vector_transport_method)
    while inner(p.M, xNew, get_gradient(p, xNew), η_xNew) <
          a.c2 * inner(p.M, o.x, η, o.gradient)
        s = (s_minus + s_plus) / 2
        retract!(p.M, xNew, o.x, s * η, a.retraction_method)
        fNew = get_cost(p, xNew)
        if fNew <= f0 + a.c1 * s * inner(p.M, o.x, η, o.gradient)
            s_minus = s
        else
            s_plus = s
        end
        abs(s_plus - s_minus) <= a.linesearch_stopsize && break
        retract!(p.M, xNew, o.x, s_minus * η, a.retraction_method)
        vector_transport_to!(p.M, η_xNew, o.x, η, xNew, a.vector_transport_method)
    end
    s = s_minus
    a.last_stepsize = s
    return s
end

@doc raw"""
    WolfePowellBinaryLinesearch <: Linesearch

A [`Linesearch`](@ref) method that determines a step size `t` fulfilling the Wolfe conditions

based on a binary chop. Let ``η`` be a search direction and ``c1,c_2>0`` be two constants.
Then with

```math
A(t) = f(x_+) ≤ c1 t ⟨\operatorname{grad}f(x), η⟩_{x}
\quad\text{and}\quad
W(t) = ⟨\operatorname{grad}f(x_+), \text{V}_{x_+\gets x}η⟩_{x_+} ≥ c_2 ⟨η, \operatorname{grad}f(x)⟩_x,
```

where ``x_+ = \operatorname{retr}_x(tη)`` is the current trial point, and ``\text{V}`` is a
vector transport, we perform the following Algorithm similar to Algorithm 7 from [^Huang2014]
1. set ``α=0``, ``β=∞`` and ``t=1``.
2. While either ``A(t)`` does not hold or ``W(t)`` does not hold do steps 3-5.
3. If ``A(t)`` fails, set ``β=t``.
4. If ``A(t)`` holds but ``W(t)`` fails, set ``α=t``.
5. If ``β<∞`` set ``t=\frac{α+β}{2}``, otherwise set ``t=2α``.

# Constructors

There exist two constructors, where, when prodivind the manifold `M` as a first (optional)
parameter, its default retraction and vector transport are the default.
In this case the retraction and the vector transport are also keyword arguments for ease of use.
The other constructor is kept for backward compatibility.
Note that the `linesearch_stopsize` to stop for too small stepsizes is only available in the
new signature including `M`, for the first it is set to the old default of `1e-9`.

    WolfePowellBinaryLinesearch(
        retr::AbstractRetractionMethod=ExponentialRetraction(),
        vtr::AbstractVectorTransportMethod=ParallelTransport(),
        c1::Float64=10^(-4),
        c2::Float64=0.999
    )

    WolfePowellLinesearch(
        M,
        c1::Float64=10^(-4),
        c2::Float64=0.999;
        retraction_method = default_retraction_method(M),
        vector_transport_method = default_vector_transport(M),
        linesearch_stopsize = 0.0
    )

[^Huang2014]:
    > Huang, W.: _Optimization algorithms on Riemannian manifolds with applications_,
    > Dissertation, Flordia State University, 2014.
    > [pdf](https://www.math.fsu.edu/~whuang2/pdf/Huang_W_Dissertation_2013.pdf)
"""
mutable struct WolfePowellBinaryLinesearch <: Linesearch
    retraction_method::AbstractRetractionMethod
    vector_transport_method::AbstractVectorTransportMethod
    c1::Float64
    c2::Float64
    last_stepsize::Float64
    linesearch_stopsize::Float64

    @deprecate WolfePowellBinaryLinesearch(
        retr::AbstractRetractionMethod=ExponentialRetraction(),
        vtr::AbstractVectorTransportMethod=ParallelTransport(),
        c1::Float64=10^(-4),
        c2::Float64=0.999,
    ) WolfePowellBinaryLinesearch(
        DefaultManifold(3),
        c1,
        c2,
        retraction_method=retr,
        vector_transport_method=vtr,
        linesearch_stopsize=1e-9,
    )
    function WolfePowellBinaryLinesearch(
        M::AbstractManifold,
        c1::Float64=10^(-4),
        c2::Float64=0.999;
        retraction_method::AbstractRetractionMethod=default_retraction_method(M),
        vector_transport_method::AbstractVectorTransportMethod=default_vector_transport_method(
            M
        ),
        linesearch_stopsize::Float64=0.0,
    )
        return new(
            retraction_method, vector_transport_method, c1, c2, 0.0, linesearch_stopsize
        )
    end
end

function (a::WolfePowellBinaryLinesearch)(
    p::GradientProblem, o::Options, ::Int, η=-get_gradient(p, o.x); kwargs...
)
    α = 0.0
    β = Inf
    t = 1.0
    f0 = get_cost(p, o.x)
    xNew = retract(p.M, o.x, t * η, a.retraction_method)
    fNew = get_cost(p, xNew)
    η_xNew = vector_transport_to(p.M, o.x, η, xNew, a.vector_transport_method)
    gradient_new = get_gradient(p, xNew)
    nAt = fNew > f0 + a.c1 * t * inner(p.M, o.x, η, o.gradient)
    nWt = inner(p.M, xNew, gradient_new, η_xNew) < a.c2 * inner(p.M, o.x, η, o.gradient)
    while (nAt || nWt) &&
              (t > a.linesearch_stopsize) &&
              ((α + β) / 2 - 1 > a.linesearch_stopsize)
        nAt && (β = t)            # A(t) fails
        (!nAt && nWt) && (α = t)  # A(t) holds but W(t) fails
        t = isinf(β) ? 2 * α : (α + β) / 2
        # Update trial point
        retract!(p.M, xNew, o.x, t * η, a.retraction_method)
        fNew = get_cost(p, xNew)
        gradient_new = get_gradient(p, xNew)
        vector_transport_to!(p.M, η_xNew, o.x, η, xNew, a.vector_transport_method)
        # Update conditions
        nAt = fNew > f0 + a.c1 * t * inner(p.M, o.x, η, o.gradient)
        nWt = inner(p.M, xNew, gradient_new, η_xNew) < a.c2 * inner(p.M, o.x, η, o.gradient)
    end
    a.last_stepsize = t
    return t
end

@doc raw"""
    get_stepsize(p::Problem, o::Options, vars...)

return the stepsize stored within [`Options`](@ref) `o` when solving [`Problem`](@ref) `p`.
This method also works for decorated options and the [`Stepsize`](@ref) function within
the options, by default stored in `o.stepsize`.
"""
function get_stepsize(p::Problem, o::Options, vars...; kwargs...)
    return get_stepsize(p, o, dispatch_options_decorator(o), vars...; kwargs...)
end
function get_stepsize(p::Problem, o::Options, ::Val{true}, vars...; kwargs...)
    return get_stepsize(p, o.options, vars...; kwargs...)
end
function get_stepsize(p::Problem, o::Options, ::Val{false}, vars...; kwargs...)
    return o.stepsize(p, o, vars...; kwargs...)
end

function get_initial_stepsize(p::Problem, o::Options, vars...; kwargs...)
    return get_initial_stepsize(
        p::Problem, o::Options, dispatch_options_decorator(o), vars...; kwargs...
    )
end
function get_initial_stepsize(p::Problem, o::Options, ::Val{true}, vars...)
    return get_initial_stepsize(p, o.options)
end
function get_initial_stepsize(p::Problem, o::Options, ::Val{false}, vars...)
    return get_initial_stepsize(o.stepsize)
end

function get_last_stepsize(p::Problem, o::Options, vars...)
    return get_last_stepsize(p, o, dispatch_options_decorator(o), vars...)
end
function get_last_stepsize(p::Problem, o::Options, ::Val{true}, vars...)
    return get_last_stepsize(p, o.options, vars...)
end
function get_last_stepsize(p::Problem, o::Options, ::Val{false}, vars...)
    return get_last_stepsize(p, o, o.stepsize, vars...)
end
#
# dispatch on stepsize
get_last_stepsize(p::Problem, o::Options, s::Stepsize, vars...) = s(p, o, vars...)
get_last_stepsize(::Problem, ::Options, s::ArmijoLinesearch, ::Any...) = s.last_stepsize
function get_last_stepsize(::Problem, ::Options, s::WolfePowellLinesearch, ::Any...)
    return s.last_stepsize
end
function get_last_stepsize(::Problem, ::Options, s::WolfePowellBinaryLinesearch, ::Any...)
    return s.last_stepsize
end