#####################################
#     Basic interface functions     #
#####################################

"""
    cost,i1,i2 = dtw(seq1, seq2, [dist=SqEuclidean, postprocess=nothing])
    cost,i1,i2 = dtw(seq1, seq2, dist, i2min, i2max)

Perform dynamic-time warping to measure the distance between two sequences.

Find a set of indices (`i1`,`i2`) that align two series (`seq1`,`seq2`) by
dynamic axis warping. Also returns the distance (after warping) according to
the SemiMetric `dist`, which defaults to squared Euclidean distance (see
Distances.jl). If `seq1` and `seq2` are matrices, each column is considered
an observation.

If `i2min/max` are provided, do DTW to align `seq1` and `seq2` confined to a window. Vectors `i2min` and
`i2max` specify (inclusive) lower and upper bounds for `seq2` for each index in
`seq1`. Thus, `i2min` and `i2max` are required to be the same length as `seq1`.

If `filternernel::AbstractMatrix` is provided, it's used to filter the cost matrix. Create a suitable kerlen using, e.g., `ImageFiltering.Kernel.gaussian(3)`. The filtering of the cost matrix makes the warping smoother, effectively penalizing small-scale warping.

See also [`dtw_cost`](@ref) and [`dtwnn`](@ref).
"""
function dtw(args...; kwargs...)
    D = dtw_cost_matrix(args...; kwargs...)
    return trackback(D)
end

##############################
#  Cost matrix computations  #
##############################

Distances.pairwise(d::PreMetric, s1::AbstractVector, s2::AbstractVector; dims=2) = evaluate.(Ref(d), s1, transpose(s2))

function Distances.pairwise(d::PreMetric, s1::AbstractArray, s2::AbstractArray; dims=2)
    [evaluate(d, s1[!,i], s2[!,j]) for i in 1:lastlength(s1), j in lastlength(s2)]
end

@inbounds function dtw_cost_matrix(seq1::AbstractArray{T}, seq2::AbstractArray{T}, dist::SemiMetric = SqEuclidean();
    transportcost=1,
    postprocess = nothing) where T
    # Build the cost matrix
    m = lastlength(seq2)
    n = lastlength(seq1)

    # Initialize first column and first row
    D = pairwise(dist, seq2, seq1, dims=2)
    @assert size(D) == (m,n)

    for r=2:m
        D[r,1] += D[r-1,1]
    end
    for c=2:n
        D[1,c] += D[1,c-1]
    end

    # Complete the cost matrix
    for c = 2:n
        for r = 2:m
            best_neighbor_cost = min(transportcost*D[r-1, c], D[r-1, c-1], transportcost*D[r, c-1])
            D[r, c] += best_neighbor_cost
        end
    end

    if postprocess !== nothing
        D = postprocess(D)
    end

    return D
end

Base.@propagate_inbounds function dtw_cost_matrix(
    seq1::AbstractArray{T},
    seq2::AbstractArray{T},
    dist::SemiMetric,
    i2min::AbstractVector{U},
    i2max::AbstractVector{U};
    transportcost = 1
) where {T,U<:Integer}
    n = lastlength(seq1) # of columns in cost matrix
    m = lastlength(seq2) # of rows in cost matrix
    Base.@boundscheck begin
        n == length(i2min) || throw(ArgumentError("i2min does not match length of seq1."))
        n == length(i2max) || throw(ArgumentError("i2max does not match length of seq1."))
        1 == i2min[1]      || throw(ArgumentError("i2min must start at 1."))
        m == i2max[end]    || throw(ArgumentError("i2max must end at length(seq2), was $(i2max[end]) ≂̸ $(m)"))
    end

    # Build the (n x m) cost matrix into a WindowedMatrix, because it's ragged.
    # That type gives efficient storage with convenient [r,c] indexing and returns
    # Inf when accessed outside the window.
    D = WindowedMatrix(i2min, i2max, Inf)

    # First column first
    D[1, 1] = evaluate(dist, seq1[!, 1], seq2[!, 1])
    for r = 2:i2max[1]
        D[r, 1] = D[r-1, 1] + evaluate(dist, seq1[!, 1], seq2[!, r])
    end

    # Complete the cost matrix from columns 2 to m.
    for c = 2:n
        for r = i2min[c]:i2max[c]
            best_neighbor_cost = min(transportcost*D[r-1, c], D[r-1, c-1], transportcost*D[r, c-1])
            D[r, c] = best_neighbor_cost + evaluate(dist, seq1[!, c], seq2[!, r])
        end
    end

    return D
end

########################################
#  Find Best Path through Cost Matrix  #
########################################

"""
    cost,cols,rows = trackback(D::Matrix)

Given the cost matrix `D`, computes the optimal track from end to beginning.
Returns `cols` and `rows` which are vectors respectively holding the track.
"""
function trackback(D::AbstractMatrix{T}) where {T<:Number}

    # initialize trackback throught rows/columns
    r, c       = size(D)
    rows, cols = Int[r], Int[c]

    # estimate that we'll need N⋅logN elements
    N  = max(r, c)
    sz = 2 * N
    sizehint!(rows, sz)
    sizehint!(cols, sz)

    # do trackback
    @inbounds while r > 1 && c > 1
        tb, r, c = indmin3(D[r-1, c-1], D[r-1, c], D[r, c-1], r, c)
        push!(rows, r)
        push!(cols, c)
    end
    # Possibly either r>1 or c>1 at this point (but not both).
    # Add the unfinished part of the track to reach [1,1]
    for r = r-1:-1:1
        push!(rows, r)
        push!(cols, 1)
    end
    for c = c-1:-1:1
        push!(rows, 1)
        push!(cols, c)
    end
    return D[end, end], reverse(cols), reverse(rows)
end

"""
    dtw_cost(a::AbstractArray, b::AbstractArray, dist::Distances.SemiMetric, r::Int; best_so_far = Inf, cumulative_bound = Zeros(length(a)))

Perform dynamic time warping to measure the distance between two sequences.

Calculate the DTW cost between `a` and `b` with maximum warping radius `r`. You may provide values of `best_so_far` and `cumulative_bound` in order to enable early stopping.

# Keyword arguments:
- `best_so_far`: The best cost value obtained so far (optional)
- `cumulative_bound`: A vector the same length as a and b (optional)
- `s1`: Optional storage vector of length 2r+1, can be used to save allocations.
- `s2`: Optional storage vector of length 2r+1, can be used to save allocations.

Providing the two vectors `s1, s2` does not save very much time, but it makes the function completely allocation free. Can be useful in a threaded context.

See also [`dtw`](@ref) and [`dtwnn`](@ref).
This code was inspired by https://www.cs.ucr.edu/~eamonn/UCRsuite.html
"""
function dtw_cost(
    a::AbstractArray{QT},
    b::AbstractArray,
    dist::F,
    r::Int;
    transportcost = 1,
    best_so_far = typemax(floattype(QT)),
    cumulative_bound = Zeros(lastlength(a)),
    s1 = fill(typemax(floattype(QT)), 2r + 1),
    s2 = fill(typemax(floattype(QT)), 2r + 1),
    kwargs...
) where {QT, F <: Union{Distances.SemiMetric, Function}}

    T = floattype(QT)
    cost      = s1 # just change the name of these variables
    cost_prev = s2

    # Instead of using matrix of size O(m^2) or O(mr), we will reuse two array of size O(r).
    m, mb = lastlength(a), lastlength(b)
    mb == m || throw(ArgumentError("a and b must have the same length, got $m and $mb. To compare two series of different lengths, use function dtw"))
    length(cumulative_bound) == m || throw(ArgumentError("cumulative_bound and a must have the same length."))
    length(s1) == 2r+1 || throw(ArgumentError("s1 must be length 2r+1."))
    length(s2) == 2r+1 || throw(ArgumentError("s2 must be length 2r+1."))

    local k

    for i = 0:m-1
        k = max(0, r - i)
        min_cost = typemax(T)

        for j = max(0, i - r):min(m - 1, i + r)
            if j == 0 && i == 0
                cost[k+1] = dist(a[!,1], b[!,1]; kwargs...)
                min_cost = cost[k+1]
                k += 1
                continue
            end
            y = (j - 1 < 0) || (k - 1 < 0)     ? typemax(T) : cost[k]
            x = (i - 1 < 0) || (k + 1 > 2 * r) ? typemax(T) : transportcost*cost_prev[k+2]
            z = (i - 1 < 0) || (j - 1 < 0)     ? typemax(T) : transportcost*cost_prev[k+1]

            cost[k+1] = min(x, y, z) + dist(a[!,i+1], b[!,j+1]; kwargs...)

            # Find minimum cost in row for early stopping
            if cost[k+1] < min_cost
                min_cost = cost[k+1]
            end
            k += 1
        end

        # We can abandon early if the current cumulative distace with lower bound together are larger than best_so_far
        if ((i + r) < (m - 1)) && (min_cost + cumulative_bound[i+r+1] >= best_so_far)
            return min_cost + cumulative_bound[i+r+1]
        end

        cost_prev, cost = cost, cost_prev
    end

    # the DTW distance is in the last cell in the matrix of size O(m^2) or at the middle of our array.
    final_dtw = cost_prev[k]
    return T(final_dtw)
end

@inbounds function soft_dtw_cost_matrix(seq1::AbstractArray, seq2::AbstractArray, dist::SemiMetric = SqEuclidean(); γ = 1,
    transportcost=1, radius=nothing)
    # Build the cost matrix
    m = lastlength(seq2)
    n = lastlength(seq1)

    # Initialize first column and first row
    D = pairwise(dist, seq2, seq1, dims=2)
    @assert size(D) == (m,n)

    for r=2:m
        D[r,1] += D[r-1,1]
    end
    for c=2:n
        D[1,c] += D[1,c-1]
    end

    # Complete the cost matrix
    if radius === nothing
        for c = 2:n
            for r = 2:m
                D[r, c] += softmin(transportcost*D[r-1, c], D[r-1, c-1], transportcost*D[r, c-1], γ)
            end
        end
    else
        for c = 2:n
            for r = 2:m
                if abs(c-r) > radius
                    D[r, c] += 1/γ
                    # continue
                end
                D[r, c] += softmin(transportcost*D[r-1, c], D[r-1, c-1], transportcost*D[r, c-1], γ)
            end
        end
    end

    return D
end

"""
    soft_dtw_cost(args...; γ = 1, kwargs...)

Perform Soft DTW. This is a differentiable version of DTW. The "distance" returned by this function is quite far from a true distance and can be negative. A smaller value of `γ` makes the distance closer to the standard DTW distance.

To differentiate w.r.t. the first argument, try
```julia
using ReverseDiff
da = ReverseDiff.gradient(a->soft_dtw_cost(a,b), a)
```

Ref: "Soft-DTW: a Differentiable Loss Function for Time-Series" https://arxiv.org/pdf/1703.01541.pdf

#Arguments:
- `args`: same as for [`dtw`](@ref)
- `γ`: The smoothing factor. A small value means less smoothing and a result closer to [`dtw_cost`](@ref)
- `kwargs`: same as for [`dtw`](@ref)
"""
function soft_dtw_cost(args...; γ = 1, kwargs...)
    D = soft_dtw_cost_matrix(args...; γ = γ, kwargs...)
    D[end,end]
end

@fastmath @inline function softmin(a, b, c, γ)
    γ = -γ
    a,b,c = a/γ, b/γ, c/γ
    maxv = max(a,b,c)
    ae,be,ce = exp(a - maxv), exp(b - maxv), exp(c - maxv)
    γ*(log(ae+be+ce) + maxv)
end

const LVB = LoopVectorization.VectorizationBase
@inline function softmin(a::T, b::T, c::T, γ) where T <: Union{Float64, Float32}
    γ = -γ
    ninvγ = one(T) / γ
    v = LVB.Vec{4,T}(a, b, c, typemax(T))
    v = v * ninvγ
    @fastmath maxv = min(a,b,c) * ninvγ
    ve = exp(v - maxv) * LVB.Vec{4,T}(one(T), one(T), one(T), zero(T))
    γ*(log(LVB.vsum(ve)) + maxv)
end


using DelimitedFiles
using Test

"""
    count_chars(data)

Count the characters according to part 1's rules.
"""
function count_chars(data)::Int
    total = 0
    for (r, t, s) in eachrow(data)
        a, b = [parse(Int, ri) for ri in split(r, "-")]
        n = length(collect(m.match for m in eachmatch(Regex("$(t[1])"), s)))
        if a <= n <= b
            total += 1
        end
    end
    return total
end

test = readdlm("./data/day02_test.txt")
@test count_chars(test) == 2

data = readdlm("./data/day02.txt")
println("Part 1 count: $(count_chars(data))")

"""
    count_chars_again(data)

Count according to the new rules.
"""
function count_chars_again(data)::Int
    total = 0
    for (r, t, s) in eachrow(data)
        a, b = [parse(Int, ri) for ri in split(r, "-")]
        n = (s[a] == t[1]) + (s[b] == t[1])
        if n == 1
            total += 1
        end
    end
    return total
end

@test count_chars_again(test) == 1

println("Part 2 count: $(count_chars_again(data))")


export divides

"""
    divides(t1::AbstractTermLike, t2::AbstractTermLike)

Returns whether the monomial of t1 divides the monomial of t2.

### Examples

Calling `divides(2x^2y, 3xy)` should return false because `x^2y` does not divide `xy` since `x` has a degree 2 in `x^2y` which is greater than the degree of `x` on `xy`.
However, calling `divides(3xy, 2x^2y)` should return true.
"""
function divides(t1::AbstractTermLike, t2::AbstractTermLike)
    divides(monomial(t1), monomial(t2))
end
divides(t1::AbstractVariable, t2::AbstractVariable) = t1 == t2

Base.gcd(m1::AbstractMonomialLike, m2::AbstractMonomialLike) = mapexponents(min, m1, m2)
Base.lcm(m1::AbstractMonomialLike, m2::AbstractMonomialLike) = mapexponents(max, m1, m2)

# _div(a, b) assumes that b divides a
_div(m1::AbstractMonomialLike, m2::AbstractMonomialLike) = mapexponents(-, m1, m2)
function _div(t::AbstractTerm, m::AbstractMonomial)
    coefficient(t) * _div(monomial(t), m)
end
function _div(t1::AbstractTermLike, t2::AbstractTermLike)
    (coefficient(t1) / coefficient(t2)) * _div(monomial(t1), monomial(t2))
end

Base.div(f::APL, g::Union{APL, AbstractVector{<:APL}}; kwargs...) = divrem(f, g; kwargs...)[1]
Base.rem(f::APL, g::Union{APL, AbstractVector{<:APL}}; kwargs...) = divrem(f, g; kwargs...)[2]

proddiff(x, y) = x/y - x/y
function Base.divrem(f::APL{T}, g::APL{S}; kwargs...) where {T, S}
    rf = convert(polynomialtype(f, Base.promote_op(proddiff, T, S)), f)
    q = r = zero(rf)
    lt = leadingterm(g)
    rg = removeleadingterm(g)
    lm = monomial(lt)
    while !iszero(rf)
        ltf = leadingterm(rf)
        if isapproxzero(ltf; kwargs...)
            rf = removeleadingterm(rf)
        elseif divides(lm, ltf)
            qt = _div(ltf, lt)
            q += qt
            rf = removeleadingterm(rf) - qt * rg
        elseif lm > monomial(ltf)
            # Since the monomials are sorted in decreasing order,
            # lm is larger than all of them hence it cannot divide any of them
            r += rf
            break
        else
            r += ltf
            rf = removeleadingterm(rf)
        end
    end
    q, r
end
function Base.divrem(f::APL{T}, g::AbstractVector{<:APL{S}}; kwargs...) where {T, S}
    rf = convert(polynomialtype(f, Base.promote_op(proddiff, T, S)), f)
    r = zero(rf)
    q = similar(g, typeof(rf))
    for i in eachindex(q)
        q[i] = zero(rf)
    end
    lt = leadingterm.(g)
    rg = removeleadingterm.(g)
    lm = monomial.(lt)
    useful = BitSet(eachindex(g))
    while !iszero(rf)
        ltf = leadingterm(rf)
        if isapproxzero(ltf; kwargs...)
            rf = removeleadingterm(rf)
            continue
        end
        divisionoccured = false
        for i in useful
            if divides(lm[i], ltf)
                qt = _div(ltf, lt[i])
                q[i] += qt
                rf = removeleadingterm(rf) - qt * rg[i]
                divisionoccured = true
                break
            elseif lm[i] > monomial(ltf)
                # Since the monomials are sorted in decreasing order,
                # lm is larger than all of them hence it cannot divide any of them
                delete!(useful, i)
            end
        end
        if !divisionoccured
            if isempty(useful)
                r += rf
                break
            else
                r += ltf
                rf = removeleadingterm(rf)
            end
        end
    end
    q, r
end

function Base.gcd(p::AbstractPolynomialLike{T}, q::AbstractPolynomialLike{S}) where {T, S}
    if isapproxzero(q)
        convert(polynomialtype(p, Base.promote_op(proddiff, T, S)), p)
    else
        gcd(q, rem(p, q))
    end
end
Base.lcm(p::AbstractPolynomialLike, q::AbstractPolynomialLike) = p * div(q, gcd(p, q))


# by Lazaro Alonso
# using GLMakie # HIDE
using GLMakie
GLMakie.activate!() # HIDE
let
    x = 1:10
    y = 1:10
    z = 1:10
    f(x,y,z) = x^2 + y^2 + z^2
    positions = vec([(i, j, k) for i in x,j in y, k in z])
    vals = [f(ix,iy,iz) for ix in x, iy in y, iz in z]
    fig, ax, pltobj = meshscatter(positions, color = vec(vals),
        marker = FRect3D(Vec3f0(0), Vec3f0(10)), # here, if you use less than 10, you will see smaller squares.
        colormap = :Spectral_11, colorrange = (minimum(vals), maximum(vals)),
        transparency = true, # set to false, if you don't want the transparency.
        shading= false,
        figure = (; resolution = (800,800)),
        axis=(; type=Axis3, perspectiveness = 0.5,  azimuth = 7.19, elevation = 0.57,
            xlabel = "x label", ylabel = "y label", zlabel = "z label",
            aspect = (1,1,1)))
    cbar = Colorbar(fig, pltobj, label = "f values", height = Relative(0.5))
    xlims!(ax,-1,11)
    ylims!(ax,-1,11)
    zlims!(ax,-1,11)
    fig[1,2] = cbar
    fig
    #save("fileName.png", fig) # here, you save your figure.
    save(joinpath(@__DIR__, "output", "volumeScatters.png"), fig, px_per_unit = 2.0) # HIDE
end
using Pkg # HIDE
Pkg.status(["GLMakie"]) # HIDE


"""
```
t = find_threshold(Balanced(), histogram, edges)
```

Balanced histogram thresholding weighs a histogram and compares the overall weight
of each side of the histogram. Each iteration weight is removed from the heavier
side and the start/middle/end points are recalculated. The agorithm continues
untill the start and end points have converged to meet the middle point.

# Output

Returns a real number `t` that specifies the threshold.

# Details

If after the start and end points have converged the final position is at the
initial start or end point then the histogram must have a single peak and has
failed to find a threshold. In this case the algorithm will fall back to using
the `UnimodalRosin` method to select a threshold.

# Arguments

The function arguments are described in more detail below.

##  `histogram`

An `AbstractArray` storing the frequency distribution.

##  `edges`

An `AbstractRange` specifying how the intervals for the frequency distribution
are divided.

# Example

Compute the threshold for the "cameraman" image in the `TestImages` package.

```julia

using TestImages, ImageContrastAdjustment, HistogramThresholding

img = testimage("cameraman")
edges, counts = build_histogram(img, 256)
#=
  The `counts` array stores at index 0 the frequencies that were below the
  first bin edge. Since we are seeking a threshold over the interval
  partitioned by `edges` we need to discard the first bin in `counts`
  so that the dimensions of `edges` and `counts` match.
=#
t = find_threshold(Balanced(), counts[1:end], edges)
```

# Reference

1. "BI-LEVEL IMAGE THRESHOLDING - A Fast Method", Proceedings of the First International Conference on Bio-inspired Systems and Signal Processing, 2008. Available: [10.5220/0001064300700076](https://doi.org/10.5220/0001064300700076)
"""
function find_threshold(algorithm::Balanced, histogram::AbstractArray, edges::AbstractRange)
    # set initial start/middle/end points and weigths
    Iₛ = 1
    Iₑ = length(histogram)
    Iₘ = round(Int, (Iₛ + Iₑ) / 2)
    Wₗ = 0
    Wᵣ = 0
    for i = 1:Iₘ
        Wₗ += histogram[i]
    end
    for i = Iₘ:Iₑ
        Wᵣ += histogram[i]
    end
    while Iₛ < Iₑ
        if Wₗ < Wᵣ
            Wᵣ -= histogram[Iₑ]
            Iₑ -= 1
            if (Iₛ + Iₑ) / 2 < Iₘ
                Wₗ -= histogram[Iₘ]
                Wᵣ += histogram[Iₘ]
                Iₘ -= 1
            end
        else
            Wₗ -= histogram[Iₛ]
            Iₛ += 1
            if (Iₛ + Iₑ) / 2 > Iₘ
                Wₗ += histogram[Iₘ + 1]
                Wᵣ -= histogram[Iₘ + 1]
                Iₘ += 1
            end
        end
    end
    if Iₘ == 1 || Iₘ == length(histogram)
        @warn "Failed to threshold. Falling back to `UnimodalRosin` method."
        return find_threshold(UnimodalRosin(), histogram, edges)
    else
        return edges[Iₘ]
    end
end


# """
# History: Based on Octave code samin.cc, by Michael Creel,
# which was originally based on Gauss code by E.G. Tsionas. A source
# for the Gauss code is http://web.stanford.edu/~doubleh/otherpapers/sa.txt
# The original Fortran code by W. Goffe is at
# http://www.degruyter.com/view/j/snde.1996.1.3/snde.1996.1.3.1020/snde.1996.1.3.1020.xml?format=INT
# Tsionas and Goffe agreed to MIT licensing of samin.jl in email
# messages to Creel.
#
# This Julia code uses the same names for control variables,
# for the most part. A notable difference is that the initial
# temperature can be found automatically to ensure that the active
# bounds when the temperature begins to reduce cover the entire
# parameter space (defined as a n-dimensional rectangle that is the
# Cartesian product of the(lb_i, ub_i), i = 1,2,..n. The code also
# allows for parameters to be restricted, by setting lb_i = ub_i,
# for the appropriate i.

"""
# SAMIN
## Constructor
```julia
SAMIN(; nt::Int = 5     # reduce temperature every nt*ns*dim(x_init) evaluations
        ns::Int = 5     # adjust bounds every ns*dim(x_init) evaluations
        rt::T = 0.9     # geometric temperature reduction factor: when temp changes, new temp is t=rt*t
        neps::Int = 5   # number of previous best values the final result is compared to
        f_tol::T = 1e-12 # the required tolerance level for function value comparisons
        x_tol::T = 1e-6 # the required tolerance level for x
        coverage_ok::Bool = false, # if false, increase temperature until initial parameter space is covered
        verbosity::Int = 1) # scalar: 0, 1, 2 or 3 (default = 1).
```
## Description
The `SAMIN` method implements the Simulated Annealing algorithm for problems with
bounds constrains a described in Goffe et. al. (1994) and Goffe (1996). The
algorithm

## References
 - Goffe, et. al. (1994) "Global Optimization of Statistical Functions with Simulated Annealing", Journal of Econometrics, V. 60, N. 1/2.
 - Goffe, William L. (1996) "SIMANN: A Global Optimization Algorithm using Simulated Annealing " Studies in Nonlinear Dynamics & Econometrics, Oct96, Vol. 1 Issue 3.
"""
@with_kw struct SAMIN{T}<:AbstractConstrainedOptimizer
    nt::Int = 5 # reduce temperature every nt*ns*dim(x_init) evaluations
    ns::Int = 5 # adjust bounds every ns*dim(x_init) evaluations
    rt::T = 0.9 # geometric temperature reduction factor: when temp changes, new temp is t=rt*t
    neps::Int = 5 # number of previous best values the final result is compared to
    f_tol::T = 1e-12 # the required tolerance level for function value comparisons
    x_tol::T = 1e-6 # the required tolerance level for x
    coverage_ok::Bool = false # if false, increase temperature until initial parameter space is covered
    verbosity::Int = 1 # scalar: 0, 1, 2 or 3 (default = 1: see final results).
end
# * verbosity: scalar: 0, 1, 2 or 3 (default = 1).
#     * 0 = no screen output
#     * 1 = only final results to screen
#     * 2 = summary every temperature change, without param values
#     * 3 = summary every temperature change, with param values
#         covered by the trial values. 1: start decreasing temperature immediately
Base.summary(::SAMIN) = "SAMIN"

function optimize(obj_fn, lb::AbstractArray, ub::AbstractArray, x::AbstractArray{Tx}, method::SAMIN, options::Options = Options()) where Tx

    t0 = time() # Initial time stamp used to control early stopping by options.time_limit

    hline = "="^80
    d = NonDifferentiable(obj_fn, x)

    tr = OptimizationTrace{typeof(value(d)), typeof(method)}()
    tracing = options.store_trace || options.show_trace || options.extended_trace || options.callback !== nothing

    @unpack nt, ns, rt, neps, f_tol, x_tol, coverage_ok, verbosity = method
    verbose = verbosity > 0

    x0 = copy(x)
    n = size(x,1) # dimension of parameter
    #  Set initial values
    nacc = 0 # total accepted trials
    t = 2.0 # temperature - will initially rise or fall to cover parameter space. Then it will fall
    converge = 0 # convergence indicator 0 (failure), 1 (normal success), or 2 (convergence but near bounds)
    # most recent values, to compare to when checking convergend
    fstar = typemax(Float64)*ones(neps)
    # Initial obj_value
    xopt = copy(x)
    f_old = value!(d, x)
    fopt = copy(f_old) # give it something to compare to
    details = [f_calls(d) t fopt xopt']
    bounds = ub - lb
    # check for out-of-bounds starting values
    for i = 1:n
        if(( x[i] > ub[i]) || (x[i] < lb[i]))
            error("samin: initial parameter $(i) out of bounds")
        end
    end

    options.show_trace && print_header(method)
    iteration = 0
    _time = time()
    trace!(tr, d, (x=xopt, iteration=iteration), iteration, method, options, _time-t0)

    stopped_by_callback = false

    # main loop, first increase temperature until parameter space covered, then reduce until convergence
    while converge==0
        # statistics to report at each temp change, set back to zero
        nup = 0
        nrej = 0
        nnew = 0
        ndown = 0
        lnobds = 0
        # repeat nt times then adjust temperature
        for m = 1:nt
            # repeat ns times, then adjust bounds
            nacp = zeros(n)
            for j = 1:ns
                # generate new point by taking last and adding a random value
                # to each of elements, in turn
                for h = 1:n
                    iteration += 1
                    # new Sept 2011, if bounds are same, skip the search for that vbl.
                    # Allows restrictions without complicated programming
                    if (lb[h] != ub[h])
                        xp = copy(x)
                        xp[h] += (Tx(2.0) * rand(Tx) - Tx(1.0)) * bounds[h]
                        if (xp[h] < lb[h]) || (xp[h] > ub[h])
                            xp[h] = lb[h] + (ub[h] - lb[h]) * rand(Tx)
                            lnobds += 1
                        end
                        # Evaluate function at new point
                        f_proposal = value(d, xp)
                        #  Accept the new point if the function value decreases
                        if (f_proposal <= f_old)
                            x = copy(xp)
                            f_old = f_proposal
                            nacc += 1 # total number of acceptances
                            nacp[h] += 1 # acceptances for this parameter
                            nup += 1
                            #  If lower than any other point, record as new optimum
                            if f_proposal < fopt
                                xopt = copy(xp)
                                fopt = f_proposal
                                d.F = f_proposal
                                nnew +=1
                                details = [details; [f_calls(d) t f_proposal xp']]
                            end
                        # If the point is higher, use the Metropolis criteria to decide on
                        # acceptance or rejection.
                        else
                            p = exp(-(f_proposal - f_old) / t)
                            if rand(Tx) < p
                                x = copy(xp)
                                f_old = copy(f_proposal)
                                d.F = f_proposal
                                nacc += 1
                                nacp[h] += 1
                                ndown += 1
                            else
                                nrej += 1
                            end
                        end
                    end

                    if tracing
                        # update trace; callbacks can stop routine early by returning true
                        stopped_by_callback =  trace!(tr, d, (x=xopt,iteration=iteration), iteration, method, options, time()-t0)
                    end

                    # If options.iterations exceeded, terminate the algorithm
                    _time = time()
                    if f_calls(d) >= options.iterations || _time-t0 > options.time_limit || stopped_by_callback

                        if verbose
                            println(hline)
                            println("SAMIN results")
                            println("NO CONVERGENCE: MAXEVALS exceeded")
                            @printf("\n     Obj. value:  %16.5f\n\n", fopt)
                            println("       parameter      search width")
                            for i=1:n
                                @printf("%16.5f  %16.5f \n", xopt[i], bounds[i])
                            end
                            println(hline)
                        end
                        converge = 0

                        return MultivariateOptimizationResults(method,
                                                                x0,# initial_x,
                                                                xopt, #pick_best_x(f_incr_pick, state),
                                                                fopt, # pick_best_f(f_incr_pick, state, d),
                                                                f_calls(d), #iteration,
                                                                f_calls(d) >= options.iterations, #iteration == options.iterations,
                                                                false, # x_converged,
                                                                0.0,#T(options.x_tol),
                                                                0.0,#T(options.x_tol),
                                                                NaN,# x_abschange(state),
                                                                NaN,# x_abschange(state),
                                                                false,# f_converged,
                                                                0.0,#T(options.f_tol),
                                                                0.0,#T(options.f_tol),
                                                                NaN,#f_abschange(d, state),
                                                                NaN,#f_abschange(d, state),
                                                                false,#g_converged,
                                                                0.0,#T(options.g_tol),
                                                                NaN,#g_residual(d),
                                                                false, #f_increased,
                                                                tr,
                                                                f_calls(d),
                                                                g_calls(d),
                                                                h_calls(d),
                                                                true,
                                                                options.time_limit,
                                                                _time-t0,NamedTuple())
                    end
                end
            end
            #  Adjust bounds so that approximately half of all evaluations are accepted
            test = 0
            for i = 1:n
                if (lb[i] != ub[i])
                    ratio = nacp[i] / ns
                    if(ratio > 0.6) bounds[i] = bounds[i] * (1.0 + 2.0 * (ratio - 0.6) / 0.4) end
                    if(ratio < .4) bounds[i] = bounds[i] / (1.0 + 2.0 * ((0.4 - ratio) / 0.4)) end
                    # keep within initial bounds
                    if(bounds[i] > (ub[i] - lb[i]))
                        bounds[i] = ub[i] - lb[i]
                        test += 1
                    end
                else
                    test += 1 # make sure coverage check passes for the fixed parameters
                end
            end
            nacp = 0 # set back to zero
            # check if we cover parameter space, if we have yet to do so
            if !coverage_ok
                coverage_ok = (test == n)
            end
        end

        # intermediate output, if desired
        if verbosity > 1
            println(hline)
            println("samin: intermediate results before next temperature change")
            println("temperature: ", round(t, digits=5))
            println("current best function value: ", round(fopt, digits=5))
            println("total evaluations so far: ", f_calls(d))
            println("total moves since last temperature reduction: ", nup + ndown + nrej)
            println("downhill: ", nup)
            println("accepted uphill: ", ndown)
            println("rejected uphill: ", nrej)
            println("out of bounds trials: ", lnobds)
            println("new minima this temperature: ", nnew)
            println()
            println("       parameter      search width")
            for i=1:n
                @printf("%16.5f  %16.5f \n", xopt[i], bounds[i])
            end
            println(hline*"\n")
        end
        # Check for convergence, if we have covered the parameter space
        if coverage_ok
            # last value close enough to last neps values?
            fstar[1] = f_old
            test = 0
            for i=1:neps
                test += (abs(f_old - fstar[i]) > f_tol)
            end
            test = (test > 0) # if different from zero, function conv. has failed
            # last value close enough to overall best?
            if (((fopt - f_old) <= f_tol) && (!test))
                # check for bound narrow enough for parameter convergence
                for i = 1:n
                    if (bounds[i] > x_tol)
                        converge = 0 # no conv. if bounds too wide
                        break
                    else
                        converge = 1
                    end
                end
            end
            # check if optimal point is near boundary of parameter space, and change message if so
            if (converge == 1) && (lnobds > 0)
                converge = 2
            end
            # Like to see the final results?
            if (converge > 0)
                if verbose
                    println(hline)
                    println("SAMIN results")
                    if (converge == 1)
                        println("==> Normal convergence <==")
                    end
                    if (converge == 2)
                        printstyled("==> WARNING <==\n", color=:red)
                        println("Last point satisfies convergence criteria, but is near")
                        println("boundary of parameter space.")
                        println(lnobds, " out of  ", (nup+ndown+nrej), " evaluations were out of bounds in the last round.")
                        println("Expand bounds and re-run, unless this is a constrained minimization.")
                    end
                    println("total number of objective function evaluations: ", f_calls(d))
                    @printf("\n     Obj. value:  %16.10f\n\n", fopt)
                    println("       parameter      search width")
                    for i=1:n
                        @printf("%16.5f  %16.5f \n", xopt[i], bounds[i])
                    end
                    println(hline*"\n")
                end
            end
            # Reduce temperature, record current function value in the
            # list of last "neps" values, and loop again
            t *= rt
            for i = neps:-1:2
                fstar[i] = fstar[i-1]
            end
            f_old = copy(fopt)
            x = copy(xopt)
        else  # coverage not ok - increase temperature quickly to expand search area
            t *= 10.0
            for i = neps:-1:2
                fstar[i] = fstar[i-1]
            end
            f_old = fopt
            x = xopt
        end
    end
    return MultivariateOptimizationResults(method,
                                            x0,# initial_x,
                                            xopt, #pick_best_x(f_incr_pick, state),
                                            fopt, # pick_best_f(f_incr_pick, state, d),
                                            f_calls(d), #iteration,
                                            f_calls(d) >= options.iterations, #iteration == options.iterations,
                                            false, # x_converged,
                                            0.0,#T(options.x_tol),
                                            0.0,#T(options.x_tol),
                                            NaN,# x_abschange(state),
                                            NaN,# x_abschange(state),
                                            false,# f_converged,
                                            0.0,#T(options.f_tol),
                                            0.0,#T(options.f_tol),
                                            NaN,#f_abschange(d, state),
                                            NaN,#f_abschange(d, state),
                                            false,#g_converged,
                                            0.0,#T(options.g_tol),
                                            NaN,#g_residual(d),
                                            false, #f_increased,
                                            tr,
                                            f_calls(d),
                                            g_calls(d),
                                            h_calls(d),
                                            true,
                                            options.time_limit,
                                            _time-t0,
                                            NamedTuple())

end

# TODO
# Handle traces
# * details: a px3 matrix. p is the number of times improvements were found.
#            The columns record information at the time an improvement was found
#            * first: cumulative number of function evaluations
#            * second: temperature
#            * third: function value
#
# Add doc entry


using Gadfly

p = plot(x=randn(2000), Geom.histogram(bincount=100))

plot(x=rand(100), y=rand(100))

plot(x=rand(20), y=rand(20), Geom.point, Geom.line)

plot(x=1:20, y=3.^rand(20),
     Scale.y_sqrt, Geom.point, Geom.smooth,
     Guide.xlabel("This is the X label"),
     Guide.ylabel("This is the y label"),
     Guide.title("This is the title"))

?SVGJS

?Gadfly.plot

using RDatasets

plot(dataset("datasets", "iris"),
    x="SepalLength", y="SepalWidth", Geom.point)

plot(dataset("car", "SLID"),
    x="Wages", color="Language", Geom.histogram)

plot(dataset("mlmRev","Gcsemv"),
    x="Course", color="Gender", Geom.histogram)

dd  =  plot(x =  rand(10),  y = rand(10));
draw(SVG("random-pts.svg",  15cm, 12cm) , dd);

set_default_plot_size(15cm, 9cm);
mlmf = dataset("mlmRev","Gcsemv")
df = mlmf[complete_cases(mlmf), :]
names(df)
plot(df, x="Course", y="Written", color="Gender")

plot((x,y) -> x*exp(-(x-int(x))^3-y^2), -10., 10, -3., 3)

plot([sin, cos], 0, 30)

x = [1:100;];
y1 = 1 - 2*rand(100);
y2 = randn(100);
plot(
 layer(x=x,y=y1,Geom.line,Theme(default_color=color("red"))),
 layer(x=x,y=y2,Geom.line,Theme(default_color=color("blue")))
)

p1 = plot(x=rand(1,1,10), y=rand(1,1,10))
p2 = plot(x=rand(10,1,20), y=rand(10,1,20))
draw(PNG("p1and2.png", 6inch, 6inch), vstack(p1,p2))
p3 = plot(x=rand(20,1,30), y=rand(20,1,30))
p4 = plot(x=rand(30,1,40), y=rand(30,1,40))
draw(PNG("p1to4.png", 6inch, 9inch), vstack(hstack(p1,p2),hstack(p3,p4)))

shapes = Gadfly.compose(Gadfly.context(), fill("cornflowerblue"),
          (Gadfly.context( 0.1, 0.1, 0.15, 0.1 ),  Gadfly.circle()),
          (Gadfly.context( 0.35, 0.06, 0.2, 0.18 ),
          Gadfly.rectangle(), Gadfly.fill("red")),
          (Gadfly.context( 0.6, 0.05, 0.2, 0.2), Gadfly.fill("magenta3"),
          Gadfly.polygon([(1, 1), (0.3, 1), (0.5, 0)]) ));
img = SVG("shapes.svg", 10cm, 8.66cm)
draw(img,shapes)

using Compose
function sierpinski(n)
  if n == 0
    compose(context(), polygon([(1,1), (0,1), (1/2, 0)]));
  else
    t = sierpinski(n - 1);
    compose( context(), (context( 1/4, 0, 1/2, 1/2), t),
                        (context( 0, 1/2, 1/2, 1/2), t),
                        (context( 1/2, 1/2, 1/2, 1/2), t));
  end
end

cx1 = compose(sierpinski(1), linewidth(0.2mm),
  fill(nothing), stroke("black"));
img = SVG("sierp1.svg", 10cm, 8.66cm); draw(img,cx1)

cx3 = compose(sierpinski(3), linewidth(0.2mm),
  fill(nothing), stroke("black"));
img = SVG("sierp3.svg", 10cm, 8.66cm); draw(img,cx3)

cx5 = compose(sierpinski(5), linewidth(0.2mm),
  fill(nothing), stroke("black"));
img = SVG("sierp5.svg", 10cm, 8.66cm); draw(img,cx5)

plot(x=rand(30), y=rand(30), Stat.step, Geom.line)

using Distributions
plot(x=rand(Normal(), 150), y=rand(Normal(), 150), Stat.qq, Geom.point)
plot(x=rand(Normal(), 150), y=Normal(), Stat.qq, Geom.point)

# Providing a fixed set of ticks
plot(x=rand(20), y=rand(20),
     Stat.xticks(ticks=[0.0, 0.2, 0.8, 1.0]),
     Stat.yticks(ticks=[0.0, 0.1, 0.9, 1.0]),
     Geom.point)

plot(dataset("lattice", "singer"),
    x="VoicePart", y="Height", Geom.boxplot)

plot(dataset("ggplot2", "diamonds"),
    x="Price", color="Cut", Geom.density)

plot(dataset("HistData", "ChestSizes"),
    x="Chest", y="Count", Geom.bar)

# Explicitly setting the number of bins
plot(dataset("car", "UN"), x="GDP", y="InfantMortality",
     Scale.x_log10, Scale.y_log10,
     Geom.histogram2d(xbincount=20, ybincount=20))

x_data = 0.0:0.1:3.0
y_data = x_data.^2 + rand(length(x_data))
plot(x=x_data, y=y_data,
    Geom.point,
    Geom.smooth(method=:loess,smoothing=0.9))

set_default_plot_size(20cm, 7.5cm)
plot(dataset("datasets", "OrchardSprays"),
     xgroup="Treatment", x="ColPos", y="RowPos", color="Decrease",
     Geom.subplot_grid(Geom.point))

using DataFrames
set_default_plot_size(8cm, 12cm)
widedf = DataFrame(x = [1:10], var1 = [1:10], var2 = [1:10].^2)
longdf = stack(widedf, [:var1, :var2])
plot(longdf, ygroup="variable", x="x", y="value",
    Geom.subplot_grid(Geom.point, free_y_axis=true))

set_default_plot_size(15cm, 10cm)

plot(dataset("datasets", "iris"), x="SepalLength", y="SepalWidth",
     yintercept=[2.5, 4.0], Geom.point, Geom.hline,
     xintercept=[5.0, 7.0], Geom.point, Geom.vline)

xs = 0:0.1:20

df_cos = DataFrame(
    x=xs,
    y=cos(xs),
    ymin=cos(xs) .- 0.5,
    ymax=cos(xs) .+ 0.5,
    f="cos"
)

df_sin = DataFrame(
    x=xs,
    y=sin(xs),
    ymin=sin(xs) .- 0.5,
    ymax=sin(xs) .+ 0.5,
    f="sin"
)

df = vcat(df_cos, df_sin)
p = plot(df, x=:x, y=:y, ymin=:ymin, ymax=:ymax, color=:f, Geom.line, Geom.ribbon)

plot(dataset("lattice", "singer"),
    x="VoicePart", y="Height", Geom.violin)

# Binding categorial data to x
plot(dataset("lattice", "singer"),
    x="VoicePart", y="Height", Geom.beeswarm)

# Transform both dimensions
plot(x=rand(100), y=rand(100),
    Scale.x_log, Scale.y_log)

# Treat numerical y data as categories
plot(x=rand(20), y=rand(20),
    Scale.x_discrete)

using Colors
x = repeat([1:10], inner=[10])
y = repeat([1:10], outer=[10])
plot(x=x,y=y,color=x+y, Geom.rectbin,
     Scale.ContinuousColorScale(Scale.lab_gradient(colorant"green",
                                                   colorant"white",
                                                   colorant"red")))

plot(sin, 0, 2pi,
     Guide.annotation(
       compose(context(), circle([pi/2, 3*pi/2], [1.0, -1.0], [2mm]), fill(nothing),
       stroke(colorant"orange"))))


using SpectralKit
using Test
import ForwardDiff

####
#### utility functions for tests
####

function test_roots(family, N; atol = 1e-13)
    r = roots(family, N)
    @test r isa Vector{Float64}
    @test length(r) == N
    @test all(abs.(evaluate.(family, N, r, Val(0))) .≤ atol)
end

function test_augmented_extrema(family, N; atol = 1e-13)
    a = augmented_extrema(family, N)
    @test a isa Vector{Float64}
    @test length(a) == N
    @test all(abs.(evaluate.(family, N - 1, a[2:(end-1)], Val(1))) .≤ atol)
    mi, ma = domain_extrema(family)
    â = sort(a)                 # non-increasing transformations switch signs
    @test mi ≈ â[1] atol = 1e-16
    @test ma ≈ â[end] atol = 1e-16
end

function test_endpoint_continuity(family, expected_extrema, ns;
                                  eps_step = eps(), inf_proxy = 1e6, atol = eps()^0.25)
    mi, ma = domain_extrema(family)
    @test mi == expected_extrema[1]
    @test ma == expected_extrema[2]
    for n in ns
        ev(x, order) = evaluate(family, n, x, order)
        # at domain minimum
        f_mi, f′_mi = @inferred ev(mi, Val(0:1))
        mi_approx = isfinite(mi) ? mi + eps() : -inf_proxy
        @test f_mi isa Float64
        @test f′_mi isa Float64
        @test f_mi ≈ ev(mi, Val(0))
        @test f_mi ≈ ev(mi_approx, Val(0)) atol = atol
        @test f′_mi ≈ ev(mi, Val(1))
        @test f′_mi ≈ ev(mi_approx, Val(1)) atol = atol

        # at domain maximum
        f_ma, f′_ma = @inferred ev(ma, Val(0:1))
        ma_approx = isfinite(ma) ? ma - eps() : inf_proxy
        @test f_ma isa Float64
        @test f′_ma isa Float64
        @test f_ma ≈ ev(ma, Val(0))
        @test f_ma ≈ ev(ma_approx, Val(0)) atol = atol
        @test f′_ma ≈ ev(ma, Val(1))
        @test f′_ma ≈ ev(ma_approx, Val(1)) atol = atol
    end
end

function test_derivatives(family, generator, ns; M = 100)
    for n in ns
        for _ in 1:M
            x = generator()
            f, f′ = @inferred evaluate(family, n, x, Val(0:1))
            @test f ≈ @inferred evaluate(family, n, x, Val(0))
            @test f′ ≈ @inferred evaluate(family, n, x, Val(1))
            @test f′ ≈ ForwardDiff.derivative(x -> evaluate(family, n, x, Val(0)), x)
        end
    end
end

####
#### tests
####

@testset "Chebyshev" begin
    F = Chebyshev()

    test_roots(F, 11)
    @test roots(F, 11)[6] == 0  # precise 0

    test_augmented_extrema(F, 11)
    @test augmented_extrema(F, 11)[6] == 0 # precise 0

    test_endpoint_continuity(F, (-1, 1), 0:10)

    test_derivatives(F, () -> rand() * 2 - 1, 0:10)
end

@testset "ChebyshevSemiInf" begin
    F = ChebyshevSemiInf(2.0, 4.7)

    test_roots(F, 9)

    test_augmented_extrema(F, 10)

    test_endpoint_continuity(F, (2.0, Inf), 0:10; atol = 1e-3)

    test_derivatives(F, () -> 2.0 + abs2(randn()), 0:10)

    F = ChebyshevSemiInf(3.0, -1.9)

    test_roots(F, 11; atol = 1e-13)

    test_augmented_extrema(F, 7; atol = 1e-10)

    test_endpoint_continuity(F, (-Inf, 3.0), 0:10; atol = 1e-3)

    test_derivatives(F, () -> 3.0 - abs2(randn()), 0:10)

    @test_throws ArgumentError ChebyshevSemiInf(0.0, 0.0)
end

@testset "ChebyshevInf" begin
    F = ChebyshevInf(0.0, 1.0)

    test_roots(F, 11)
    @test roots(F, 11)[6] == 0  # precise 0

    test_augmented_extrema(F, 11)
    @test augmented_extrema(F, 11)[6] == 0 # precise 0

    test_endpoint_continuity(F, (-Inf, Inf), 0:10)

    test_derivatives(F, randn, 0:10)

    @test_throws ArgumentError ChebyshevInf(0.0, -3.0)
    @test_throws ArgumentError ChebyshevInf(0.0, 0.0)
end

@testset "ChebyshevInterval" begin
    F = ChebyshevInterval(2.0, 5)

    test_roots(F, 11)

    test_augmented_extrema(F, 11)

    test_endpoint_continuity(F, (2.0, 5.0), 0:10)

    test_derivatives(F, () -> rand() * 3 + 2, 0:10)

    @test_throws ArgumentError ChebyshevInterval(2.0, 1.0)
    @test_throws ArgumentError ChebyshevInterval(2.0, 2)
    @test_throws ArgumentError ChebyshevInterval(-Inf, Inf)
end


@testset "centering" begin

    p1 = Point([4.831961989402771, 45.75764678012361])
    l1 = LineString([[
        4.86020565032959,
        45.76884015325622
      ],
      [
        4.85994815826416,
        45.749558161214516
      ]])

    poly = Polygon([[[
          4.8250579833984375,
          45.79398056386735
        ],
        [
          4.882392883300781,
          45.79254427435898
        ],
        [
          4.910373687744141,
          45.76081677972451
        ],
        [
          4.894924163818359,
          45.7271539426975
        ],
        [
          4.824199676513671,
          45.71337148333104
        ],
        [
          4.773387908935547,
          45.74021417890731
        ],
        [
          4.778022766113281,
          45.778418789239055
        ],
        [
          4.8250579833984375,
          45.79398056386735
        ]]])

    @test centroid(p1).coordinates == [4.831961989402771, 45.75764678012361]
    @test centroid(l1).coordinates == [4.860076904296875,45.75919915723537]
    @test centroid(poly).coordinates == [4.839177131652832,45.76256007199914]

    poly2 = Polygon([[[0, 0], [0, 0], [0, 0], [0, 0]]])

    @test meancenter(poly).coordinates ≈ [4.839177,45.76256]
    @test meancenter(poly2).coordinates == [0, 0]
    @test meancenter(l1).coordinates ≈ [4.860077,45.759199]
    @test meancenter(p1).coordinates == p1.coordinates

    @test masscenter(poly2).coordinates == [0, 0]
    @test masscenter(poly).coordinates == [4.840728965137111,45.75581209996416]

    coll = FeatureCollection([Feature(Point([0, 0])),
      Feature(Point([1, 0])), Feature(Point([0, 1])),
      Feature(Point([5, 8]))])

    coll2 = FeatureCollection([Feature(Point([0, 0])),Feature(Point([9, 9])), Feature(Point([9.25, 9.25])),
      Feature(Point([9.5, 9.5])), Feature(Point([9.75, 9.75])), Feature(Point([10, 10]))])

    @test mediancenter(coll).coordinates ≈ [0.383876,0.616989] atol=0.0001
    @test mediancenter(coll2).coordinates ≈ [9.254246,9.254246] atol=0.0001

    fc = FeatureCollection([Feature(Point([4.833351373672485,45.760809294695534])),
      Feature(Point([4.8331475257873535,45.760296567821456])), Feature(Point([4.833984374999999,45.76073818687033])),
      Feature(Point([4.834005832672119,45.76022171678877]))])

    @test centroid(fc).coordinates ≈ [4.8336222767829895,45.76051644154402] atol=0.001
    @test center(fc).coordinates ≈ centroid(fc).coordinates atol=0.01
end


using InfiniteOpt, Ipopt, Plots
using KNITRO

# Parmeters
A = 1
B = 1
C = 4
# Time span
t0 = 0
tf = 1
# Initial values
x1 = 1

# Model
model = InfiniteModel(Ipopt.Optimizer)

# infinite_parameter
@infinite_parameter(model, t ∈ [t0, tf], num_supports = 101)

# variable
@variable(model, x, Infinite(t))

@variable(model, u, Infinite(t), start = 0.2)

# objective
@objective(model, Max, ∫(A * x - B * u^2, t))

# constraint
@constraint(model, x(0) == x1)
@constraint(model, x_constr, ∂(x, t) == -0.5 * x^2 + C * u)

# Optimization

print(model)

optimize!(model)

# Visulization
ts = value(t)
u = value(u)
x = value(x)

display(plot(ts, u))
display(plot(ts, x))


function solve(
    i::MinimumBudget{ElementaryPathInstance{T, Maximise}, T},
    ::DynamicProgramming,
) where {T}
    return solve(i, BellmanFordAlgorithm())
end

function solve(
    i::MinimumBudget{ElementaryPathInstance{T, Maximise}, T},
    ::BellmanFordAlgorithm,
) where {T}
    V = Dict{Tuple{T, Int}, Float64}()
    S = Dict{Tuple{T, Int}, Vector{Edge{T}}}()

    # Initialise. For β = 0, this is exactly Bellman-Ford algorithm.
    # Otherwise, use the same initialisation as Bellman-Ford.
    β0 = solve(i.instance, BellmanFordAlgorithm())
    for v in vertices(i.instance.graph)
        S[v, 0] = β0.solutions[v]
        V[v, 0] =
            length(S[v, 0]) == 0 ? 0 :
            sum(i.instance.rewards[e] for e in S[v, 0])
    end

    for β in 1:(i.min_budget)
        for v in vertices(i.instance.graph)
            V[v, β] = -Inf
            S[v, β] = Edge{T}[]
        end

        V[i.instance.src, β] = 0.0
    end

    # Dynamic part.
    for β in 1:(i.min_budget)
        # Loop needed when at least a weight is equal to zero. TODO: remove it when all weights are nonzero?
        while true
            changed = false

            for v in vertices(i.instance.graph)
                for w in inneighbors(i.instance.graph, v)
                    # Compute the remaining part of the budget still to use.
                    remaining_budget = max(0, β - i.weights[Edge(w, v)])

                    # If the explored subproblem has no solution, skip it.
                    if V[w, remaining_budget] == -Inf
                        continue
                    end

                    # If using the solution to the currently explored subproblem would
                    # lead to a cycle, skip it.
                    if any(src(e) == v for e in S[w, remaining_budget])
                        continue
                    end

                    # Compute the amount of budget already used by the solution to the currently explored subproblem.
                    used_budget = 0
                    if length(S[w, remaining_budget]) > 0
                        used_budget =
                            sum(i.weights[e] for e in S[w, remaining_budget])
                    end
                    if used_budget < remaining_budget
                        continue
                    end

                    # Compute the maximum: is passing through w advantageous?
                    if V[w, remaining_budget] + i.instance.rewards[Edge(w, v)] >
                       V[v, β] && used_budget >= remaining_budget
                        changed = true

                        V[v, β] =
                            V[w, remaining_budget] +
                            i.instance.rewards[Edge(w, v)]
                        S[v, β] = vcat(S[w, remaining_budget], Edge(w, v))
                    end
                end
            end

            if !changed
                break
            end
        end
    end

    S_refined = Dict(β => S[i.instance.dst, β] for β in 0:(i.min_budget))
    return BudgetedElementaryPathSolution(
        i,
        S[i.instance.dst, i.min_budget],
        S_refined,
    )
end


#=
# 3D Poisson, Dirichlet bc
# Uses Dendro imported as a custom gen target
=#
if !@isdefined(Femshop)
    include("../Femshop.jl");
    using .Femshop
end
init_femshop("poisson3dcustom");

useLog("poisson3dcustomlog", level=3)

# default values (max_depth=6, wavelet_tol = 0.1, partition_tol = 0.3, solve_tol = 1e-6, max_iters = 100)
generateFor("target_dendro_cg.jl", params=(5, 1, 0.3, 0.000001, 100))

domain(3)
functionSpace(order=1)

u = variable("u")
testSymbol("v")

boundary(u, 1, DIRICHLET, 0)

# Write the weak form
coefficient("f", "-14*pi*pi*sin(3*pi*x)*sin(2*pi*y)*sin(pi*z)")
weakForm(u, "-dot(grad(u),grad(v)) - f*v")

solve(u);

finalize_femshop()


@doc """
    struct LaserGeometry

The geometrical aspects of the laser can be described through its orientation
with respect to a given reference system and by its propagation.
"""
LaserGeometry

struct LaserGeometry{O,R}
    orientation::O
    rotation_matrix::R
end

function LaserGeometry(orientation; isversor=false)
    v₁ = !isversor ? normalize(orientation[1]) : orientation[1]
    v₃ = !isversor ? normalize(orientation[2]) : orientation[2]
    v₂ = v₃ × v₁

    # original basis
    e₁ = SVector{3}(1,0,0)
    e₂ = SVector{3}(0,1,0)
    e₃ = SVector{3}(0,0,1)

    # original basis matrix
    E = [e₁ e₂ e₃]

    # desired direction matrix
    V = [v₁ v₂ v₃]

    # rotation matrix
    R = E * transpose(V)

    LaserGeometry(orientation, R)
end

function sym2vec(dir::Symbol)
    if dir == :x
        SVector{3}(1,0,0)
    elseif dir == :y
        SVector{3}(0,1,0)
    else
        SVector{3}(0,0,1)
    end
end

function LaserGeometry(orientation::Tuple{Symbol,Symbol})
    if orientation[1] == :x && orientation[2] == :z
        return LaserGeometry((:x, :z), I)
    end
    v₁ = sym2vec(orientation[1])
    v₃ = sym2vec(orientation[2])

    LaserGeometry((v₁, v₃), isversor=true)
end


include("linear_elasticity.jl")

E = Variable(1.0)
ν = Variable(0.0)
H11 = E*(1. -ν)/((1+ν)*(1. -2. *ν));
H12 = H11*ν/(1-ν);
H21 = H12;
H22 = H11;
H33 = H11*0.5*(1. -2. *ν)/(1. -ν);
H = tensor(
  [H11 H12 0.0
  H21 H22 0.0
  0.0 0.0 H33]
)
d, v, a= ExplicitSolver(globaldata, domain, d0, v0, a0, Δt, NT, H, Fext, ubd, abd)
idx = [1;domain.nnodes+1]
loss = sum((d[:, idx] - d_[:, idx])^2)
sess = Session(); init(sess)
BFGS!(sess, loss, var_to_bounds=Dict(E=>(0.0,100.0), ν=>(-0.5,0.49)))
run(sess, [E, ν])


import GreyDecision.Utility: makeminmax
import GreyDecision.GreyNumbers: GreyNumber
import GreyDecision.Aras: aras

@testset "Aras" begin
    tol = 0.0001

    K1 = [105000.0, 120000, 150000, 115000, 135000]
    K2 = [105.0, 110, 120, 105, 115]
    K3 = [10.0, 15, 12, 20, 15]
    K4 = [4.0, 4, 3, 4, 5]
    K5 = [300.0, 500, 550, 600, 400]
    K6 = [10.0, 8, 12, 9, 9]

    decisionMat = hcat(K1, K2, K3, K4, K5, K6)
    functionlist = [minimum, maximum, minimum, maximum, maximum, minimum]
    w = [0.05, 0.20, 0.10, 0.15, 0.10, 0.40]
    row, col = size(decisionMat)
    greydec = Array{GreyNumber{Float64}, 2}(undef, row, col)
    for i in 1:row
        for j in 1:col
            greydec[i, j] = GreyNumber(Float64(decisionMat[i, j]))
        end
    end

    result = aras(greydec, w, functionlist)

    @test result[:bestIndex] == 2
    @test result[:orderings] == [3, 1, 4, 5, 2]
    @test isapprox(result[:scores][1].a, 0.81424068, atol = tol)
    @test isapprox(result[:scores][2].a, 0.89288620, atol = tol)
    @test isapprox(result[:scores][3].a, 0.76415790, atol = tol)
    @test isapprox(result[:scores][4].a, 0.84225462, atol = tol)
    @test isapprox(result[:scores][5].a, 0.86540635, atol = tol)

end


import Statistics
import Graphs

include("STOCK.jl")

# 1. compute stock price daily changes
X = STOCK.Closes - STOCK.Opens

# 2. compute *squared* correlation matrix
Xcorr = Statistics.cor(X).^2

# 3. compute *maximum* spanning tree
edges = Graphs.kruskal_mst(Graphs.SimpleGraphs.complete_graph(size(Xcorr, 1)), Xcorr, minimize=false)
T = Graphs.SimpleGraphFromIterator(edges)

# 4. write a graphviz file for the graph, see Makefile for generating the pdf figure
open("stocks.graphviz", "w") do file
  write(file, "graph G {\n")

  for (i, name) in enumerate(STOCK.names)
    write(file, "\t\"$(name)\" [style=filled fillcolor=$(STOCK.colors[STOCK.industries[i]])];\n")
  end

  for (i, pa) in enumerate(Graphs.dfs_tree(T, 1).badjlist)
    for p in pa
      write(file, "\t\"$(STOCK.names[i])\" -- \"$(STOCK.names[p])\";\n")
    end
  end
  write(file, "}")
end


########################################
## File Name: belief_types.jl
## Author: Haruki Nishimura (hnishimura@stanford.edu)
## Date Created: 2020/05/12
## Description: Belief Type Definitions for SACBP
########################################

import Distributions: Normal, MvNormal
import Plots.Plot
using LinearAlgebra

abstract type Belief end
BelState = Union{Belief,Array{<:Belief}};

# MultiVariate Gaussian Belief.
@auto_hash_equals struct BelMvNormal{T<:Real} <:Belief
    t::Float64
    params::Vector{T} #[μ;vcat(Σ...)];
    dim::Int64
    function BelMvNormal{T}(t,params,dim) where {T<:Real}
        if length(params) != dim*(dim+1)
            error(ArgumentError("Invalid parameter vector length."))
#        elseif !isposdef(reshape(params[dim+1:end],dim,dim))
#            error(ArgumentError("Σ must be positive definite."))
        else
            return new(t,params,dim)
        end
    end
end
BelMvNormal(t::Real,params::Vector{T},dim::Int64) where {T<:Real} = BelMvNormal{T}(t,params,dim);
function BelMvNormal(t::Real,params::Vector{T}) where {T<:Real}
    l = length(params)
    try
        dim = Int64((-1+sqrt(1+4l))/2) # Speculate dimension from the parameter vector length.
        return BelMvNormal(t,params,dim)
    catch
        error(ArgumentError("Invalid parameter vector length."))
    end
end
function BelMvNormal(t::Real,μ::Vector{<:Real},Σ::Matrix{<:Real})
    if size(Σ,1) != size(Σ,2) || size(Σ,2) != length(μ)
        error(ArgumentError("Invalid parameter vector length."))
    else
        return BelMvNormal(t,[μ;vcat(Σ...)],length(μ));
    end
end
function Distributions.MvNormal(b::BelMvNormal)
    μ = b.params[1:b.dim];
    Σ = reshape(b.params[b.dim+1:end],b.dim,b.dim);
    return Distributions.MvNormal(μ,Σ)
end
function plot_e_ellipse!(b::BelMvNormal,probability::Float64,plt::Plots.Plot)
    d = Distributions.MvNormal(b);
    μ = d.μ;
    Σ = Matrix(d.Σ);
    ϵ = (1. - probability)/(2*pi*sqrt(det(Σ)));
    theta = range(0.,stop=2.0*pi, length=100);
    radius = sqrt(-2.0*log(2.0*pi) - logdet(Σ) - 2*log(ϵ));
    x = zeros(length(theta));
    y = zeros(length(theta));
    for jj = 1:length(theta)
        pos = sqrt(Σ)*[radius*cos(theta[jj]);radius*sin(theta[jj])] + μ;
        x[jj] = pos[1];
        y[jj] = pos[2];
    end
    plt = plot!(x,y,color=:aquamarine, fill=true, fillalpha=0.3,label="")
end;

VecBelMvNormal{T<:Real} = Vector{BelMvNormal{T}};
function VecBelMvNormal(bvec::VecBelMvNormal)
    if !all([isequal(bvec[1].t,b.t) for b in bvec])
        error(ArgumentError("Beliefs have inconsistent time parameters."))
    else
        return bvec
    end
end


### A Pluto.jl notebook ###
# v0.14.3

using Markdown
using InteractiveUtils

# This Pluto notebook uses @bind for interactivity. When running this notebook outside of Pluto, the following 'mock version' of @bind gives bound variables a default value (instead of an error).
macro bind(def, element)
    quote
        local el = $(esc(element))
        global $(esc(def)) = Core.applicable(Base.get, el) ? Base.get(el) : missing
        el
    end
end

# ╔═╡ 2d89ea8d-697c-4278-b467-d5ed7e123dd7
using Plots, Latexify, LaTeXStrings, PlutoUI

# ╔═╡ 751e5150-a132-11eb-2a66-5d81e11face8
md""" # Mínimos quadrados e ortogonalidade

Vamos ilustrar o processo algébrico da aproximação por mínimos quadrados """

# ╔═╡ 6c41c486-376e-42b1-9d0f-89178e1020a2
md" Primeiro vamos construir os vetores de dados, vamos dividir o intervalo $[0,10]$ em $n$ partes "

# ╔═╡ 3fc129c6-b277-4af5-b114-1b33a9f6ebcf
md" n = $(@bind n Slider(1:100, default=10, show_value=true))"

# ╔═╡ ec6d7196-ef3a-4c6b-a673-127c22a44aeb
x=range(0,10,step=10/n)

# ╔═╡ 0e86ddce-1b8c-4158-9713-9bc209089091
y=(10*(0.5 .- rand(n+1))) + 2*x + ones(n+1) + x.^2

# ╔═╡ d1982bed-31dc-4cbc-a11b-8bafa9295c9c

# ╔═╡ 34ed10fa-7241-475e-9611-402931dd3784
scatter(x,y, label=false)

# ╔═╡ b2e7caf3-3e6d-4fe7-a063-5820e86de6e6
md""" Agora quero saber qual é a melhor reta $y=ax+b$ que melhor se ajusta à essa massa de dados que minimiza $\sum_{i=0}^n (ax_i-b -y_i)^2= \| A\begin{bmatrix} a \\ b \end{bmatrix} - y\|^2$.

Pela teoria que acabamos de ver:
$\begin{bmatrix} a \\ b \end{bmatrix} = (A^tA)^{-1}A^ty$

"""

# ╔═╡ ce16bd9e-0753-43bd-af54-c2494b09eb46

# ╔═╡ 50886dfa-68a6-4c2a-8583-f5e6240c7364
begin
	um=ones(n+1);
	A=[x um];
	latexstring(latexify(A))
end

# ╔═╡ c5570f33-d189-43e2-b1af-2e892067e9d3
md"""
```math
A^tA
=
``` $(latexstring(latexify(A'*A)))
"""

# ╔═╡ 6566ec1b-b9b8-4210-8e43-efa816db54cc
md"""
```math
A^ty
=
``` $(latexstring(latexify(A'*y)))
"""

# ╔═╡ 42230d2b-cc08-414d-bf0f-610344cd4890
md""" ```math
\begin{bmatrix} a \\ b \end{bmatrix} =
``` $(latexstring(latexify(inv(A'*A)*(A'*y))))
"""

# ╔═╡ bd08d6d3-726d-47c5-aab4-d4a86606e99d
begin
	m=inv(A'*A)*(A'*y);
	r= m[1]*x+ m[2]*um;
end

# ╔═╡ 9b683dd1-cc8f-4129-ad44-2f94817e632e
begin
	plot(x,r,label=false)
	scatter!(x,y,label=false)
end

# ╔═╡ 82eb86ba-47aa-43bd-8b9f-7a42e0f944bf
md"""

 **y** = $(m[1])**x** + $(m[2])

é a equação da reta que melhor se ajusta aos dados
"""

# ╔═╡ 2a1ef996-4516-4fbc-8b6a-8b5fdc12f0ba
md" Para o ajuste de uma parábola: "

# ╔═╡ 30a7bba5-9105-4564-8f61-0ff49c21801e
x2=x.^2

# ╔═╡ 48382d7f-a66e-43a2-8cee-3ba1f79d2690
begin
	A2 =[x2 x um]
	N=A2'*A2
	p=inv(N)*A2'*y
	q=p[1]*x2 + p[2]*x+p[3]*um
end

# ╔═╡ 41271278-dc7f-42ee-a565-eb48afdf3dda
begin
	plot(x,q,label=false)
	plot!(x,r,label=false)
	scatter!(x,y,label=false)
end

# ╔═╡ Cell order:
# ╟─751e5150-a132-11eb-2a66-5d81e11face8
# ╟─2d89ea8d-697c-4278-b467-d5ed7e123dd7
# ╟─6c41c486-376e-42b1-9d0f-89178e1020a2
# ╟─3fc129c6-b277-4af5-b114-1b33a9f6ebcf
# ╟─ec6d7196-ef3a-4c6b-a673-127c22a44aeb
# ╟─0e86ddce-1b8c-4158-9713-9bc209089091
# ╟─d1982bed-31dc-4cbc-a11b-8bafa9295c9c
# ╟─34ed10fa-7241-475e-9611-402931dd3784
# ╟─b2e7caf3-3e6d-4fe7-a063-5820e86de6e6
# ╠═ce16bd9e-0753-43bd-af54-c2494b09eb46
# ╟─50886dfa-68a6-4c2a-8583-f5e6240c7364
# ╟─c5570f33-d189-43e2-b1af-2e892067e9d3
# ╟─6566ec1b-b9b8-4210-8e43-efa816db54cc
# ╟─42230d2b-cc08-414d-bf0f-610344cd4890
# ╟─bd08d6d3-726d-47c5-aab4-d4a86606e99d
# ╟─9b683dd1-cc8f-4129-ad44-2f94817e632e
# ╟─82eb86ba-47aa-43bd-8b9f-7a42e0f944bf
# ╟─2a1ef996-4516-4fbc-8b6a-8b5fdc12f0ba
# ╟─30a7bba5-9105-4564-8f61-0ff49c21801e
# ╟─48382d7f-a66e-43a2-8cee-3ba1f79d2690
# ╟─41271278-dc7f-42ee-a565-eb48afdf3dda


export relu

doc"""
    relu(x::Var)

Rectified Linear Unit.

```math
f(x) = \max(0, x)
```
"""
relu(x::Var) = Var(relu(x.data), ∇relu!, (x,))
relu(x::T) where T<:AbstractFloat = max(x, zero(T))
relu(x::Array) = relu.(x)
relu(x::CuArray) = CUDNN.relu(x)
relu(x::Node) = Node(relu, (x,))

function ∇relu!(y::Var, x::Var)
    isnothing(x.grad) && return
    ∇relu!(y.data, y.grad, x.data, x.grad)
end

function ∇relu!(y::Array{T}, gy::Array{T}, x::Array{T}, gx::Array{T}) where T
    @inbounds for i = 1:length(x)
        gx[i] += x[i] > T(0) ? gy[i] : T(0)
    end
end

∇relu!(y::CuArray, gy::CuArray, x::CuArray, gx::CuArray) = CUDNN.∇relu!(y, gy, x, gx)


#==============================================================================

  Lifcycle Model based on code from Ben Moll's website:
      www.princeton.edu/~moll/HACTproject.htm

==============================================================================#

using LinearAlgebra, SparseArrays, Plots

# First set up model parameters
γ = 2 #CRRA utility parameter
σ_sq = (0.8)^2 # for the processes for productivity
corr = exp(-.9) # persistence for productivity
ρ = 0.05 # discount rate
r = 0.035 #interest rate
w =1 #wages

zmean = exp(σ_sq/2) # mean productivity
θ = -log(corr) # parameter for OU process for productivity

J =15 # number of points in gridspace for productivity
zmin = 0.75
zmax = 2.5

H = 300 #number of points in gridspace for assets
amin = 0
amax=100

T = 75 # maximum age of agents
N = 300 # number of age steps
dt = T/N

maxit =1000 #max number of iterations for HJB
crit = 10^(-10) # convergence criterion for HJB
convergence_criterion=10^(-5) #criterion for the KF and HJB?

# Set up some key variables
a = LinRange(amin,amax, H)
da = (amax-amin)/(H-1)
aa = a*ones(1,J)

z = LinRange(zmin,zmax,J)
dz = (zmax-zmin)/(J-1)
dz_sq = dz^2
zz=z*ones(1,H)

μ = -θ.*z.*log.(z)+σ_sq/2*z # drift for z
Σ_sq = σ_sq.*z.^2 #variance from Ito's lemma

# Create the matrix that summarizes the evolution of our system in the z dimension
 yy = (-Σ_sq/dz_sq - μ/dz)
 χ = Σ_sq/(2*dz_sq)
 ζ = μ/dz + Σ_sq/(2*dz_sq)

 # Define the Diagonals of this matrix
 updiag_z = zeros(H,1)
 	for j = 1:J
		global updiag_z =[updiag_z; repeat([ζ[j]], H, 1)]
	end

 centerdiag_z=repeat([χ[1]+yy[1]],H,1)
	for j = 2:J-1
		global centerdiag_z = [centerdiag_z; repeat([yy[j]], H, 1)]
	end
 centerdiag_z=[centerdiag_z; repeat([yy[J]+ζ[J]], H, 1)]

lowdiag_z = repeat([χ[2]], H, 1)
	for j=3:J
		global lowdiag_z = [lowdiag_z; repeat([χ[j]],H,1)]
	end
# spdiags in Matlab allows for automatic trimming/adding of zeros
    # spdiagm does not do this

Aswitch =  spdiagm(0=>centerdiag_z[:], -H=>lowdiag_z[:], H=>updiag_z[H+1:end-H])

# Set terminal condition, i.e. value of death ≈ 0
small_number=10^(-8)
v_terminal = small_number*(small_number.+aa).^(1-γ)/(1-γ)

global V=v_terminal

# Set up spaces for variables from the finite difference loop
Vaf, Vab, c = [zeros(H,J) for i in 1:3]

v = zeros(H,J,N)
gg =[]
A_t=[]
c_t=[]
ss_t=[]

#================ Main loop =================================================#

for n in N:-1:1
	global V=V
    v[:,:,n]=V
    #forward difference
    Vaf[1:H-1,:] = (V[2:H,:]-V[1:H-1,:])/da
    Vaf[H,:] = (w*z .+ r.*amax).^(-γ) #will never be used, but impose state constraint a<=amax just in case
    #backward difference
    Vab[2:H,:] = (V[2:H,:]-V[1:H-1,:])/da
    Vab[1,:] = (w*z .+ r.*amin).^(-γ) #state constraint boundary condition

    #I_concave = Vab .> Vaf #indicator whether value function is concave (problems arise if this is not the case)

    # consumption and savings with forward difference
    cf = Vaf.^(-1/γ)
    ssf = zz' + r.*aa - cf
    # consumption and savings with backward difference
    cb = Vab.^(-1/γ)
    ssb = zz' + r.*aa - cb
    # consumption and derivative of value function at steady state
    c0 = zz' + r.*aa
    V0 = c0.^(-γ)

    #= dV_upwind makes a choice of forward or backward differences based on
     the sign of the drift    =#
    If = ssf .> 0 #positive drift → forward difference
    Ib = ssb .< 0 #negative drift → backward difference
    I0 = (1.0 .-If-Ib) #at steady state

    dV_Upwind = Vaf.*If + Vab.*Ib + V0.*I0  #important to include third term
    c = dV_Upwind.^(-1/γ)
    u = c.^(1-γ)/(1-γ)

    # CONSTRUCT MATRIX
    X = - min.(ssb,0)/da
    Y = - max.(ssf,0)/da + min.(ssb,0)/da
    Z = max.(ssf,0)/da

	# Define the Diagonals of this transition matrix
	updiag = 0
       for j = 1:J
           updiag =[updiag; Z[1:H-1,j]; 0]
       end
    updiag =(updiag[:])

    centerdiag=reshape(Y, H*J, 1)
    centerdiag = (centerdiag[:]) # for tuples

   lowdiag = X[2:H, 1]
       for j = 2:J
           lowdiag = [lowdiag; 0; X[2:H,j]]
       end
   lowdiag=(lowdiag)

   # spdiags in Matlab allows for automatic trimming/adding of zeros
       # spdiagm does not do this
  	AA = sparse(Diagonal(centerdiag))+ [zeros(1, H*J); sparse(Diagonal(lowdiag)) zeros(H*J-1,1)] + sparse(Diagonal(updiag))[2:end, 1:(H*J)] # trim first element
	A = AA + Aswitch

    push!(A_t,A)
    B = (1/dt + ρ)*sparse(I, H*J, H*J) - A

    u_stacked = reshape(u, H*J,1)
    V_stacked = reshape(V, H*J,1)

    b = u_stacked + V_stacked/dt
    V_stacked = B\b #SOLVE SYSTEM OF EQUATIONS

    global V = reshape(V_stacked, H,J)

    push!(ss_t,zz'+ r.*aa -c)
	push!(c_t, c)
end

# Plots

#inital consumption
plot(a, c_t[end], legend=false)
#inital savings
plot(a, ss_t[end], legend=false)
plot!(a, zeros(1,H))

# Create plots from lifecycle.pdf
dt_inv = convert(Int64, 1/dt)
dt_inv40 = convert(Int64, 40/dt)
dt_inv70 = convert(Int64, 70/dt)

# Policy function for consumption

p1=plot(a, c_t[300-dt_inv][:,1], label="Age 1, Lowest Income", ylims=(0,5), xlims=(0,80))
plot!(a, c_t[300-dt_inv][:,J], label="Age 1, Highest Income", xlabel="Wealth")
plot!(a, c_t[300-dt_inv40][:,1], label="Age 40 Lowest Income", ylabel="Consumption" )
plot!(a, c_t[300-dt_inv40][:,J], label="Age 40 Highest Income", legend=:bottomright)
plot!(a, c_t[300-dt_inv70][:,1], label="Age 70 Lowest Income")
plot!(a, c_t[300-dt_inv70][:,J], label="Age 70 Highest Income")

# Policy function for savings
p2= plot(a, ss_t[300-dt_inv][:,1], label="Age 1, Lowest Income", ylims=(-5,2), xlims=(0,100))
plot!(a, ss_t[300-dt_inv][:,J], label="Age 1, Highest Income", xlabel="Wealth")
plot!(a, ss_t[300-dt_inv40][:,1], label="Age 40 Lowest Income", ylabel="Savings" )
plot!(a, ss_t[300-dt_inv40][:,J], label="Age 40 Highest Income", legend=:bottomright)
plot!(a, ss_t[300-dt_inv70][:,1], label="Age 70 Lowest Income" )
plot!(a, ss_t[300-dt_inv70][:,J], label="Age 70 Highest Income")
plot!(a, zeros(H,1), line=:dash, color=:black, label="")

plot(p1,p2,legend=:bottomright,layout = (1,2))


"""
This example learns to classify hand-written digits from the
[MNIST](http://yann.lecun.com/exdb/mnist) dataset.  There are 60000
training and 10000 test examples. Each input x consists of 784 pixels
representing a 28x28 image.  The pixel values are normalized to
[0,1]. Each output y is converted to a ten-dimensional one-hot vector
(a vector that has a single non-zero component) indicating the correct
class (0-9) for a given image.  10 is used to represent 0.
You can run the demo using `julia mnist.jl` on the command line or
`julia> include(KnetLayers.dir("examples/mnist.jl"))` at the Julia prompt.
"""

using Knet, KnetLayers
import Knet: Data
#Data
include(Knet.dir("data","mnist.jl"))
dtrn,dtst = mnistdata(xsize=(784,:)); # dtrn and dtst = [ (x1,y1), (x2,y2), ... ] where xi,yi are

#Model
(m::MLP)(x,y) = nll(m(x),y)
(m::MLP)(d::Data) = mean(m(x,y) for (x,y) in d)
HIDDEN_SIZES = [100,50]
model = MLP(784,HIDDEN_SIZES...,10)

#Train
EPOCH=10
progress!(sgd(model,repeat(dtrn,EPOCH)))

#Test
@show 100accuracy(model, dtst)


"""

    hs93initialset(setting)

Perform the Hadi & Simonoff (1993) algorithm's first part for a given regression setting.
The returned array of indices are indices of clean subset length of p + 1
where p is the number of regression parameters.

# Arguments
- `setting::RegressionSetting`: RegressionSetting object with a formula and dataset.

# Examples
```julia-repl
julia> reg0001 = createRegressionSetting(@formula(calls ~ year), phones);
julia> hs93initialset(reg0001)
3-element Array{Int64,1}:
 4
 3
 5
```

# References
Hadi, Ali S., and Jeffrey S. Simonoff. "Procedures for the identification of
multiple outliers in linear models." Journal of the American Statistical
Association 88.424 (1993): 1264-1272.
"""
function hs93initialset(setting::RegressionSetting)::Array{Int,1}
    X, y = @extractRegressionSetting setting
    return hs93initialset(X, y)
end

function hs93initialset(X::Array{Float64,2}, y::Array{Float64,1})::Array{Int,1}
    n, p = size(X)
    s = p + 1
    dfs = abs.(dffit(X, y))
    sortedindices = sortperm(dfs)
    basicsetindices = sortedindices[1:s]
    return basicsetindices
end

"""

    hs93basicsubset(setting, initialindices)

Perform the Hadi & Simonoff (1993) algorithm's second part for a given regression setting.
The returned array of indices are indices of clean subset of length h
where h is at least the half of the number of observations. h is set to
integer part of (n + p - 1) / 2.

# Arguments
- `setting::RegressionSetting`: RegressionSetting object with a formula and dataset.
- `initialindices::Array{Int, 1}`: (p + 1) subset of clean observations.

# Examples
```julia-repl
julia> reg0001 = createRegressionSetting(@formula(calls ~ year), phones);
julia> initialsetindices = hs93initialset(reg0001)
3-element Array{Int64,1}:
 4
 3
 5
 julia> hs93basicsubset(reg0001, initialsetindices)
12-element Array{Int64,1}:
  5
  9
 10
  3
  6
  4
  7
 22
 11
  8
 12
 13
```

# References
Hadi, Ali S., and Jeffrey S. Simonoff. "Procedures for the identification of
multiple outliers in linear models." Journal of the American Statistical
Association 88.424 (1993): 1264-1272.
"""
function hs93basicsubset(setting::RegressionSetting, initialindices::Array{Int,1})::Array{Int,1}
    X = designMatrix(setting)
    y = responseVector(setting)
    return hs93basicsubset(X, y, initialindices)
end

function hs93basicsubset(X::Array{Float64,2}, y::Array{Float64,1}, initialindices::Array{Int,1})::Array{Int,1}
    n, p = size(X)
    h = floor((n + p - 1) / 2)
    s = length(initialindices)
    indices = initialindices
    for i in range(s + 1, stop=h)
        olsreg = ols(X[indices,:], y[indices])
        betas = coef(olsreg)
        d = zeros(Float64, n)
        XM = X[indices,:]
        for j in 1:n
            xxxx = X[j,:]' * inv(XM'XM) * X[j,:]
            if j in indices
                @inbounds d[j] = abs.(y[j] - sum(X[j,:] .* betas)) / sqrt(1 - xxxx)
            else
                @inbounds d[j] = abs.(y[j] - sum(X[j,:] .* betas)) / sqrt(1 + xxxx)
            end
        end
        orderingd = sortperm(abs.(d))
        indices = orderingd[1:Int(i)]
    end
    return indices
end

"""

    hs93(setting; alpha = 0.05, basicsubsetindices = nothing)

Perform the Hadi & Simonoff (1993) algorithm for the given regression setting.

# Arguments
- `setting::RegressionSetting`: RegressionSetting object with a formula and dataset.
- `alpha::Float64`: Optional argument of the probability of rejecting the null hypothesis.
- `basicsubsetindices::Array{Int, 1}`: Initial basic subset, by default, the algorithm creates an initial set of clean observations.

# Examples
```julia-repl
julia> reg0001 = createRegressionSetting(@formula(calls ~ year), phones);
julia> hs93(reg0001)
Dict{Any,Any} with 3 entries:
  "outliers" => [14, 15, 16, 17, 18, 19, 20, 21]
  "t"        => -3.59263
  "d"        => [2.04474, 1.14495, -0.0633255, 0.0632934, -0.354349, -0.766818, -1.06862, -1.47638, -0.7…
```

# References
Hadi, Ali S., and Jeffrey S. Simonoff. "Procedures for the identification of
multiple outliers in linear models." Journal of the American Statistical
Association 88.424 (1993): 1264-1272.
"""
function hs93(setting::RegressionSetting; alpha=0.05, basicsubsetindices=nothing)
    X = designMatrix(setting)
    y = responseVector(setting)
    return hs93(X, y, alpha=alpha, basicsubsetindices=basicsubsetindices)
end

function hs93(X::Array{Float64,2}, y::Array{Float64,1}; alpha=0.05, basicsubsetindices=nothing)
    if basicsubsetindices === nothing
        initialsetindices = hs93initialset(X, y)
        basicsubsetindices = hs93basicsubset(X, y, initialsetindices)
    end
    indices = basicsubsetindices
    n, p = size(X)
    s = length(indices)
    while s < n
        olsreg = ols(X[indices, :], y[indices])
        betas = coef(olsreg)
        resids = residuals(olsreg)
        sigma = sqrt(sum(resids.^2.0) / (length(resids) - p))
        d = zeros(Float64, n)
        XM = @inbounds X[indices,:]
        iXmXm = inv(XM'XM)
        for j in 1:n
            @inbounds xMMx = X[j,:]' * iXmXm * X[j,:]
            if j in indices
                @inbounds d[j] = (y[j] - sum(X[j,:] .* betas)) / (sigma * sqrt(1.0 - xMMx))
            else
                @inbounds d[j] = (y[j] - sum(X[j,:] .* betas)) / (sigma * sqrt(1.0 + xMMx))
            end
        end
        orderingd = sortperm(abs.(d))
        tdist = TDist(s - p)
        tcalc = quantile(tdist,  alpha / (2 * (s + 1)))
        if abs(d[orderingd][s + 1]) > abs(tcalc)
            result = Dict()
            result["d"] = d
            result["t"] = tcalc
            result["outliers"] = filter(x -> abs(d[x]) > abs(tcalc), 1:n)
            return result
        end
        s += 1
        indices = orderingd[1:s]
    end
    return []
end


export laplacian_loss, edge_loss

"""
    laplacian_loss(m::TriMesh)

Computes the laplacian smoothning metrics for TriMesh `m`

"""
function laplacian_loss(m::TriMesh)
    L = @ignore get_laplacian_packed(m)
    verts = get_verts_packed(m)
    L = L * cpu(transpose(verts))
    L = _norm(L; dims = 2)
    return mean(L)
end

"""
    edge_loss(m::TriMesh, target_length::Number = 0.0)

Computes mean edge length in TriMesh.

`target_length` is the optional arguments for computing edge length.
"""
function edge_loss(m::TriMesh, target_length::Number = 0.0)
    verts = get_verts_packed(m)
    edges = @ignore get_edges_packed(m)
    v1 = verts[:, edges[:, 1]]
    v2 = verts[:, edges[:, 2]]
    el = (_norm(v1 - v2; dims = 1) .- Float32(target_length)) .^ 2
    loss = mean(el)
    return loss
end

function chamfer_distance(
    m1::TriMesh,
    m2::TriMesh,
    num_samples::Int = 5000;
    w1::Number = 1.0,
    w2::Number = 1.0,
)
    A = sample_points(m1, num_samples)
    B = sample_points(m2, num_samples)
    return _chamfer_distance(A, B, Float32(w1), Float32(w2))
end


# Latafat, Themelis, Patrinos, "Block-coordinate and incremental aggregated
# proximal gradient methods for nonsmooth nonconvex problems."
# arXiv:1906.10053 (2019).
#

using LinearAlgebra
using ProximalOperators
using ProximalAlgorithms.IterationTools
using Printf
using Base.Iterators
using Random
using StatsBase: sample

export solution

include("ProShI_basic.jl")

struct Proshi{R<:Real}
    γ::Maybe{Union{Array{R},R}}
    sweeping::Int8
    minibatch::Tuple{Bool,Int}
    maxit::Int
    verbose::Bool
    freq::Int
    α::R
    function Proshi{R}(;
        γ::Maybe{Union{Array{R},R}} = nothing,
        sweeping = 1,
        minibatch::Tuple{Bool,Int} = (false, 1),
        maxit::Int = 10000,
        verbose::Bool = false,
        freq::Int = 10000,
        α::R = R(0.999),
    ) where {R}
        @assert γ === nothing || minimum(γ) > 0
        @assert maxit > 0
        @assert freq > 0
        new(γ, sweeping, minibatch, maxit, verbose, freq, α)
    end
end

function (solver::Proshi{R})(
    x0::AbstractArray{C};
    F = nothing,
    g = ProximalOperators.Zero(),
    L = nothing,
    N = N,
) where {R,C<:RealOrComplex{R}}

    stop(state) = false

    disp(it, state) = @printf "%5d | %.3e  \n" it state.hat_γ

    F === nothing && (F = fill(ProximalOperators.Zero(), (N,)))
    # dispatching the iterator
    iter = Proshi_basic_iterable(
        F,
        g,
        x0,
        N,
        L,
        solver.γ,
        solver.sweeping,
        solver.minibatch[2],
        solver.α,
    )

    iter = take(halt(iter, stop), solver.maxit)
    iter = enumerate(iter)

    num_iters, state_final = nothing, nothing
    for (it_, state_) in iter  # unrolling the iterator
        # see https://docs.julialang.org/en/v1/manual/interfaces/index.html
        if solver.verbose && mod(it_, solver.freq) == 0
            disp(it_, state_)
        end
        num_iters, state_final = it_, state_
    end
    if solver.verbose && mod(num_iters, solver.freq) !== 0
        disp(num_iters, state_final)
    end # for the final iteration
    return solution(state_final), num_iters
end

"""
    Proshi([γ, sweeping, LProshi, adaptive, minibatch, maxit, verbose, freq, tol, tol_b])

Instantiate the Proshi algorithm for solving fully nonconvex optimization problems of the form

    minimize 1/N sum_{i=1}^N f_i(x) + g(x)

where `f_i` are smooth and `g` is possibly nonsmooth, all of which may be nonconvex.

If `solver = Proshi(args...)`, then the above problem is solved with

	solver(x0, [F, g, N, L])

where F is an array containing f_i's, x0 is the initial point, and L is an array of
smoothness moduli of f_i's; it is optional in the adaptive mode or if γ is provided.

Optional keyword arguments are:
* `γ`: an array of N stepsizes for each coordinate
* `sweeping::Int` 1 for uniform randomized (default), 2 for cyclic, 3 for shuffled
* `LProshi::Bool` low memory variant of the Proshi/MISO algorithm
* `adaptive::Bool` to activate adaptive smoothness parameter computation
* `minibatch::(Bool,Int)` to use batchs of a given size
* `maxit::Integer` (default: `10000`), maximum number of iterations to perform.
* `verbose::Bool` (default: `true`), whether or not to print information during the iterations.
* `freq::Integer` (default: `10000`), frequency of verbosity.
* `α::R` parameter where γ_i = αN/L_i
* `tol::Real` (default: `1e-8`), absolute tolerance for the adaptive case
* `tol_b::R` tolerance for the backtrack (default: `1e-9`)
"""

Proshi(::Type{R}; kwargs...) where {R} = Proshi{R}(; kwargs...)
Proshi(; kwargs...) = Proshi(Float64; kwargs...)

"""
If `solver = Proshi(args...)`, then

    itr = iterator(solver, x0, [F, g, N, L])

is an iterable object. Note that [maxit, verbose, freq] fields of the solver are ignored here.

The solution at any given state can be obtained using solution(state), e.g.,
for state in Iterators.take(itr, maxit)
    # do something using solution(state)
end

See https://docs.julialang.org/en/v1/manual/interfaces/index.html
and https://docs.julialang.org/en/v1/base/iterators/ for a list of iteration utilities
"""

function iterator(
    solver::Proshi{R},
    x0::AbstractArray{C};
    F = nothing,
    g = ProximalOperators.Zero(),
    L = nothing,
    N = N,
) where {R,C<:RealOrComplex{R}}
    F === nothing && (F = fill(ProximalOperators.Zero(), (N,)))
    # dispatching the iterator
    iter = Proshi_basic_iterable(
        F,
        g,
        x0,
        N,
        L,
        solver.γ,
        solver.sweeping,
        solver.minibatch[2],
        solver.α,
    )
    return iter
end