# The Bellman Ford algorithm for single-source shortest path

###################################################################
#
#   The type that capsulates the states of Bellman Ford algorithm
#
###################################################################

type NegativeCycleError <: Exception end

type BellmanFordStates{V,D<:Number}
    parents::Vector{V}
    dists::Vector{D}
end

# create Bellman Ford states

function create_bellman_ford_states{V,D<:Number}(g::AbstractGraph{V}, ::Type{D})
    n = num_vertices(g)
    parents = Array(V, n)
    dists = fill(typemax(D), n)

    BellmanFordStates(parents, dists)
end

function bellman_ford_shortest_paths!{V,D}(
    graph::AbstractGraph{V},
    edge_dists::AbstractEdgePropertyInspector{D},
    sources::AbstractVector{V},
    state::BellmanFordStates{V,D})

    @graph_requires graph incidence_list vertex_map vertex_list
    edge_property_requirement(edge_dists, graph)

    active = Set{V}()
    for v in sources
        i = vertex_index(v, graph)
        state.dists[i] = 0
        state.parents[i] = v
        push!(active, v)
    end
    no_changes = false
    for i in 1:num_vertices(graph)
        no_changes = true
        new_active = Set{V}()
        for u in active
            uind = vertex_index(u, graph)
            for e in out_edges(u, graph)
                v = target(e, graph)
                vind = vertex_index(v, graph)
                edist = edge_property(edge_dists, e, graph)
                if state.dists[vind] > state.dists[uind] + edist
                    state.dists[vind] = state.dists[uind] + edist
                    state.parents[vind] = u
                    no_changes = false
                    push!(new_active, v)
                end
            end
        end
        if no_changes
            break
        end
        active = new_active
    end
    if !no_changes
        throw(NegativeCycleError())
    end
    state
end

function bellman_ford_shortest_paths{V,D}(
    graph::AbstractGraph{V},
    edge_dists::AbstractEdgePropertyInspector{D},
    sources::AbstractVector{V})
    state = create_bellman_ford_states(graph, D)
    bellman_ford_shortest_paths!(graph, edge_dists, sources, state)
end

function bellman_ford_shortest_paths{V,D}(
    graph::AbstractGraph{V},
    edge_dists::Vector{D},
    sources::AbstractVector{V})
    edge_inspector = VectorEdgePropertyInspector{D}(edge_dists)
    bellman_ford_shortest_paths(graph, edge_inspector, sources)
end

function has_negative_edge_cycle{V, D}(
    graph::AbstractGraph{V},
    edge_dists::AbstractEdgePropertyInspector{D})
    try
        bellman_ford_shortest_paths(graph, edge_dists, vertices(graph))
    catch e
        if isa(e, NegativeCycleError)
            return true
        end
    end
    return false
end

function has_negative_edge_cycle{V, D}(
    graph::AbstractGraph{V},
    edge_dists::Vector{D})
    edge_inspector = VectorEdgePropertyInspector{D}(edge_dists)
    has_negative_edge_cycle(graph, edge_inspector)
end


module DarcyFEOperatorsTests

##
using Test
using Gridap
import Gridap: ∇

import LinearAlgebra: det
using Base: transpose
##

# model = CartesianDiscreteModel(domain=(0.0,1.0,0.0,1.0), partition=(2,2))
# model.cgrid
# ps = points(model.cgrid)
# typeof(ps)
# collect(ps)
#
#
#
# f(x) = Point(x[1],5*x[2])
#
# broadcast(f,ps)
#
#
# struct MappedCartesianGrid <: CartesianGrid
#   map
#
# end
# with mapping
#
#
# trian = Triangulation(model)
#
#
# phi = CellGeomap(trian)
# jac = gradient(phi)
# jact = transpose(jac)
#
#
# piola_map = inv(jact)
#
#
# graph = GridGraph(model)
# pols = CellPolytopes(Grid(model))
# pt = pols.value
#
# order = 1
# _reffe = NedelecRefFE(pt,Float64,order)
#
# _labels = FaceLabels(model)
#
# # V = ConformingFESpace(_reffe,trian,graph,_labels,[5,6])
# V = CurlConformingFESpace(_reffe,trian,graph,_labels,[5,6])
##

u(x) = VectorValue(x[1]*x[2], -0.5*x[2]^2)
∇u(x) = TensorValue(x[2],0.0,x[1],-x[2])
divu(x) = 0.0
∇(::typeof(u)) = ∇u

p(x) = x[1] + x[2]
∇p(x) = VectorValue(1.0,1.0)
∇(::typeof(p)) = ∇p

f(x) = divu(x)
g(x) = p(x)

const kinv_1 = TensorValue(1.0,0.0,0.0,1.0)
r(x) = kinv_1*u(x) + ∇p(x)

gmap(x) = 2*x
model = CartesianDiscreteModel(domain=(0.0,1.0,0.0,1.0), partition=(50,50), map = gmap)
order = 2
V = FESpace( reffe=:RaviartThomas, conformity=:HDiv, order=2, model=model, diritags = [5,6])
_Q = FESpace( reffe=:QLagrangian, conformity=:L2, valuetype = Float64, order = 1,
                    model = model)
# If all faces of the domain are zero values, we must fix zero mean value
# _Q = FESpace( reffe=:QLagrangian, conformity=:L2, valuetype = Float64, order = 1,
                    # model = model, constraint = :zeromean)

V_0 = TestFESpace(V)
Q = TestFESpace(_Q)
Y = [V_0, Q]

V_g = TrialFESpace(V,u)
X = [V_g, Q]

trian = Triangulation(model)
quad = CellQuadrature(trian,degree=2)

@law σ(x,u) = kinv_1*u

function a(y,x)
  v, q = y
  u, p = x
  inner(v,σ(u)) - inner(div(v),p) + inner(q,div(u))
end

function b_Ω(y)
  v, q = y
  inner(v,r) + inner(q,f)
end

t_Ω = AffineFETerm(a,b_Ω,trian,quad)

neumanntags = [7,8]
btrian = BoundaryTriangulation(model,neumanntags)
bquad = CellQuadrature(btrian,degree=order*2)
nb = NormalVector(btrian)

function b_Γ(y)
  v, q = y
  - inner(v*nb,g)
end

t_Γ = FESource(b_Γ,btrian,bquad)

op = LinearFEOperator(Y,X,t_Ω,t_Γ)

uh, ph = solve(op)

e1 = u - uh
e2 = p - ph

# Define norms to measure the error
l2(u) = inner(u,u)
hdiv(u) = inner(div(u),div(u)) + l2(u)

# Compute errors
e1l2 = sqrt(sum( integrate(l2(e1),trian,quad) ))
e1hdiv = sqrt(sum( integrate(hdiv(e1),trian,quad) ))
e2l2 = sqrt(sum( integrate(l2(e2),trian,quad) ))

# writevtk(trian,"/home/santiago/github-repos/Gridap/tmp/darcyresults",
         # cellfields=["uh"=>uh,"ph"=>ph, "euh"=> e1, "eph"=> e2])

@test e1l2 < 1.e-8
@test e1hdiv < 1.e-8
@test e2l2 < 1.e-8
##
end # module


# TRAJECTORY CALCULATION #

const DEFAULT_DT = 0.001

"""
   compute_trajectory(u,v,X₀::Vector,Tmax[,Δt=0.001])

Calculate the trajectory of a tracer particle with initial location(s) `X₀`, which
can be specified as either a single vector `[x0,y0]` or a vector of vectors
for multiple tracer particles. The arguments
`u` and `v` are interpolated velocity field components, `Tmax` is the final
integration time, and `Δt` is the time step size. The output is the solution
structure for the `OrdinaryDiffEq` package (or, for multiple particles, a vector
of such solution structures).
"""
function compute_trajectory(ufield::AbstractInterpolation{T,2},
                            vfield::AbstractInterpolation{T,2},
                            X₀::Vector{S},Tmax::Real;Δt::Real=DEFAULT_DT) where {T,S<:Real}

  vfcn!(dR,R,p,t) = _vfcn_autonomous!(dR,R,p,t,ufield,vfield)

  sol = _solve_trajectory(vfcn!,X₀,Tmax,Δt)
  return sol

end

function compute_trajectory(ufield::AbstractInterpolation{T,2},vfield::AbstractInterpolation{T,2},
   pts::Vector{Vector{S}},Tmax;Δt=DEFAULT_DT) where {T,S<:Real}

  sol_array = ODESolution[]
  for X₀ in pts
    sol = compute_trajectory(ufield,vfield,X₀,Tmax,Δt=Δt)
    push!(sol_array,sol)
  end
  return sol_array

end

function _solve_trajectory(vfcn!,u0,Tmax,Δt)
  Path = ODEProblem(vfcn!,u0,(0.0,Tmax))
  sol = solve(Path,Tsit5(), dt = Δt, maxiters = 1e8, adaptive = false, dense = false)
end

function _vfcn_autonomous!(dR,R,p,t,u,v)
 dR[1] = u(R[1],R[2])
 dR[2] = v(R[1],R[2])

 return dR
end


# ------------------------------------------------------------------
# Licensed under the ISC License. See LICENSE in the project root.
# ------------------------------------------------------------------

"""
    BlockWeighter(sides)
    BlockWeighter(side₁, side₂, ...)

A weighting method that assigns weights to points in spatial object
based on blocks of given `sides`. The number `n` of points inside a
block determines the weights `1/n` of these points.
"""
struct BlockWeighter{T,N} <: AbstractWeighter
  sides::SVector{N,T}
end

BlockWeighter(sides::NTuple{N,T}) where {N,T} =
  BlockWeighter{T,N}(sides)

BlockWeighter(sides::Vararg{T,N}) where {N,T} =
  BlockWeighter(sides)

function weight(sdomain::AbstractDomain, weighter::BlockWeighter)
  p = partition(sdomain, BlockPartitioner(weighter.sides))

  weights = Vector{Float64}(undef, npoints(sdomain))
  for s in subsets(p)
    n = length(s)
    weights[s] .= 1/n
  end

  SpatialWeights(sdomain, weights)
end

weight(sdata::AbstractData, weighter::BlockWeighter) =
  weight(domain(sdata), weighter)


"""
# Description
Runs the Binary Segmentation algorithm using a specified cost function for a given penalty to find the position and number of changepoints

# Usage
BS(cost_function(data, distribtion), length(data), penalty = log(n))

# Arguments
* `segment_cost::Function`: Calculates cost between two specified indices
* `n::Int`: Length of time series
* `pen::Float64`: Penalty of changepoints

# Returns
* `CP::Vector{Int}`: Vector of indices of detected changepoints
* `cost::Float64`: Cost of optimal segmentation

# Example
```
# Sample Normal time series with changing mean
n = 1000
λ = 100
μ, σ = Normal(0.0, 10.0), 1.0
sample, cps = @changepoint_sampler n λ Normal(μ, σ)
# Run binary segmentation
BS_cps, BS_cost = @BS sample Normal(?, σ)
```

# See also
@segment_cost

# References
Scott, A.J. and Knott, M. (1974) A Cluster Analysis Method for Grouping Means in the Analysis of Variance, Biometrics 30(3), 507 - 512
"""
function BS( segment_cost::Function , n::Int64; pen::Float64 = log(n) )
    tau = Tuple{Int, Int}[] # Segmentations to test
    CP = Array{Int64}(undef,0)

    push!(tau, (0, n))
    push!(CP, 0)

    # keep adding segments until none contain a changepoint

    while length(tau) > 0
        a, b = pop!(tau)
        x = Array{Float64}(undef, 0)
        for j in (a+1):(b - 1)
            push!(x , segment_cost(a,j) + segment_cost(j,b) + pen)
        end
        minval , ind = findmin(x)
        if minval - segment_cost(a,b) < 0
            # significant so add seg to tau
            chpt = ind + a
            push!(CP,chpt)
            if chpt != a + 1;
                push!(tau, (a, chpt))
            end
            if chpt != b - 1
                push!(tau, (chpt,b))
            end
        end
    end

    cost = 0.0

    CP = sort(CP)

    for j in 1:(length(CP)-1)
        cost = cost + segment_cost(CP[j]+1,CP[j+1]) + pen
    end

    cost = cost + segment_cost(CP[end]+1,n)

    return CP, cost
end


include("laubloomis.jl")

using TaylorModels
using TaylorModels: validated_integ

# Equations of motion
# We write the function such that the operations are either unary or binary:
@taylorize function laubloomis!(t, x, dx)
    dx[1] = 1.4*x[3] - 0.9*x[1]
    dx[2] = 2.5*x[5] - 1.5*x[2]
    dx[3] = 0.6*x[7] - 0.8*(x[2]*x[3])
    dx[4] = 2 - 1.3*(x[3]*x[4])
    dx[5] = 0.7*x[1] - (x[4]*x[5])
    dx[6] = 0.3*x[1] - 3.1*x[6]
    dx[7] = 1.8*x[6] - 1.6*(x[2]*x[7])
    return dx
end

"""
    laubloomis_TMJets(; [t0], [T], [W], [abs_tol], [orderT], [orderQ],
                        [maxsteps], [property])

Build and run the Laub-Loomis model.

### Input

- `t0`       -- (optional, default: `0.0`) initial time
- `T`        -- (optional, default: `20.0`) time horizon
- `W`        -- (optional, default: `0.01`) width of the initial states
- `abs_tol`  -- (optional, default: `1e-10`) absolute tolerance used for time step
- `orderT`   -- (optional, default: `7`) order of the Taylor model in t
- `orderQ`   -- (optional, default: `2`) order of the Taylor model for Jet transport
                variables
- `maxsteps` -- (optional, default: `200`) use this maximum number of steps in
                the validated integration
- `property` -- (optional, default: `(t,x) -> x[4] < 4.5`) safe states property
"""
function laubloomis_TMJets(; t0=0.0, T=20.0, W=0.01, abs_tol=1e-10,
                             orderT=7, orderQ=2, maxsteps=1000,
                             property=(t,x) -> x[4] < 4.5)

    # center of initial conditions
    q0 = [1.2, 1.05, 1.5, 2.4, 1.0, 0.1, 0.45]

    # initial box (around `q0`) of the initial conditions
    δq0 = IntervalBox(-W..W, 7)

    # set variables
    set_variables("δ", numvars=length(q0), order=2orderQ)

    # TODO: wrap as a Reachability algorithm
    tTM, xTM = validated_integ(laubloomis!, q0, δq0, t0, T, orderQ, orderT,
                               abs_tol, maxsteps=maxsteps,
                               check_property=property)
    return tTM, xTM
end

#=
# return `true` if the specification is specification is satisfied and `false`
# otherwise
function check_property(xTM)
    satisfied = true
    for ind in eachindex(xTM[:])
        # check if solution crosses unsafe set (x4 ≥ 4.5)
        if (xTM[ind][4] >= 4.5)
            satisfied = false
            break
        end
    end
    return satisfied
end

using Plots

# plot solution and spec
function plot_solution(xTM)
    p = plot(xTM[:], color=:lightblue, legend=:false)
    plot!(p, [-2.5, 2.5], [2.75, 2.75], color=:red)
    xlims!(p, -2.5,2.5)
    ylims!(p, -3,3)
    return p
end

julia> data = [IntervalBox(IntervalArithmetic.Interval(tTM[i], tTM[i+1]), xTM[i][4]) for i in 1:length(tTM)-1];
julia> plot(data[:], legend=false, xlab="time", ylab="x4")

=#


## test_circulant_embedding.jl : test circulant embedding for GRF generation

@testset "circulant embedding      " begin

## 1d Exponential ##
cov = CovarianceFunction(1,Exponential(0.1))
pts = 0:1//1001:1
grf = GaussianRandomField(cov,CirculantEmbedding(),pts)
@test isa(grf,GaussianRandomField)
@test isa(grf.cov,CovarianceFunction)
@test isa(grf.cov.cov,Exponential)
@test ndims(grf.cov) == 1
@test isa(grf,GaussianRandomField{C,CirculantEmbedding} where {C})
@test length(grf.pts[1]) == length(pts)
@test length(sample(grf)) == length(pts)

## 1d Matern with padding ##
m = CovarianceFunction(1,Matern(1.0,1.0))
grf = GaussianRandomField(m,CirculantEmbedding(),pts,padding=8)
@test isa(grf,GaussianRandomField)
@test isa(grf.cov,CovarianceFunction)
@test isa(grf.cov.cov,Matern)
@test ndims(grf.cov) == 1
@test isa(grf,GaussianRandomField{C,CirculantEmbedding} where {C})
@test length(grf.pts[1]) == length(pts)
@test length(sample(grf)) == length(pts)

## 2d Anisotropic ##
A = [1000 0; 0 1000]
a = AnisotropicExponential(A)
c = CovarianceFunction(2,a)
pts1 = range(0,stop = 1,length = 128)
pts2 = range(0,stop = 1,length = 256)
@suppress grf = GaussianRandomField(c,CirculantEmbedding(),pts1,pts2,padding=4)
@test isa(grf,GaussianRandomField)
@test isa(grf.cov,CovarianceFunction)
@test isa(grf.cov.cov,AnisotropicExponential)
@test ndims(grf.cov) == 2
@test isa(grf,GaussianRandomField{C,CirculantEmbedding} where {C})
@test length(grf.pts) == 2
@test length(grf.pts[1]) == length(pts1)
@test length(grf.pts[2]) == length(pts2)
@test size(sample(grf),1) == length(pts1)
@test size(sample(grf),2) == length(pts2)

## 3d AnisotropicExponential ##
A = [100 0 0; 0 100 0; 0 0 100]
a = AnisotropicExponential(A)
c = CovarianceFunction(3,a)
pts = 0:0.025:1
@suppress grf = GaussianRandomField(c,CirculantEmbedding(),pts,pts,pts,padding=1,measure=false)
@test isa(grf,GaussianRandomField)
@test isa(grf.cov,CovarianceFunction)
@test isa(grf.cov.cov,AnisotropicExponential)
@test ndims(grf.cov) == 3
@test isa(grf,GaussianRandomField{C,CirculantEmbedding} where {C})
@test length(grf.pts) == 3
@test length(grf.pts[1]) == length(pts)
@test length(grf.pts[2]) == length(pts)
@test length(grf.pts[3]) == length(pts)
@test size(sample(grf),1) == length(pts)
@test size(sample(grf),2) == length(pts)
@test size(sample(grf),3) == length(pts)

## Larger domain ##
A = [1 0.8; 0.8 1]
m = AnisotropicExponential(A)
c = CovarianceFunction(2,m)
pts1 = range(-5,stop = 5,length = 128)
pts2 = range(10,stop = 0,length = 128)
@suppress grf = GaussianRandomField(c,CirculantEmbedding(),pts1,pts2,padding=2)
@test isa(grf,GaussianRandomField)
@test isa(grf.cov,CovarianceFunction)
@test isa(grf.cov.cov,AnisotropicExponential)
@test ndims(grf.cov) == 2
@test isa(grf,GaussianRandomField{C,CirculantEmbedding} where {C})
@test length(grf.pts) == 2
@test length(grf.pts[1]) == length(pts1)
@test length(grf.pts[2]) == length(pts2)
@test size(sample(grf),1) == length(pts1)
@test size(sample(grf),2) == length(pts2)

## Dedicated plotting commands ##
m = Matern(0.5,2)
c = CovarianceFunction(3,m)
pts1 = range(0,stop = 1,length = 16)
pts2 = range(0,stop = 1,length = 8)
pts3 = range(0,stop = 1,length = 4)
#plot_covariance_matrix(c,pts1,pts2,pts3); close()
m = Matern(0.1,1)
c = CovarianceFunction(2,m)
pts = range(0,stop = 1,length = 512)
g = GaussianRandomField(c,CirculantEmbedding(),pts,pts,padding=4)
#contourf(g); close()
#plot_eigenvalues(g); close()

end


module GradientTests

using Base.Test, Interpolations, DualNumbers

nx = 10
f1(x) = sin((x-3)*2pi/(nx-1) - 1)
g1(x) = 2pi/(nx-1) * cos((x-3)*2pi/(nx-1) - 1)

# Gradient of Constant should always be 0
itp1 = interpolate(Float64[f1(x) for x in 1:nx-1],
            BSpline(Constant()), OnGrid())

g = Array(Float64, 1)

for x in 1:nx
    @test gradient(itp1, x)[1] == 0
    @test gradient!(g, itp1, x)[1] == 0
    @test g[1] == 0
end

# Since Linear is OnGrid in the domain, check the gradients between grid points
itp1 = interpolate(Float64[f1(x) for x in 1:nx-1],
            BSpline(Linear()), OnGrid())
itp2 = interpolate((1:nx-1,), Float64[f1(x) for x in 1:nx-1],
            Gridded(Linear()))
for itp in (itp1, itp2)
    for x in 2.5:nx-1.5
        @test_approx_eq_eps g1(x) gradient(itp, x)[1] abs(.1*g1(x))
        @test_approx_eq_eps g1(x) gradient!(g, itp, x)[1] abs(.1*g1(x))
        @test_approx_eq_eps g1(x) g[1] abs(.1*g1(x))
    end

    for i = 1:10
        x = rand()*(nx-2)+1.5
        gtmp = gradient(itp, x)[1]
        xd = dual(x, 1)
        @test_approx_eq epsilon(itp[xd]) gtmp
    end
end

# test gridded on a non-uniform grid
knots = (1.0:0.3:nx-1,)
itp_grid = interpolate(knots, Float64[f1(x) for x in knots[1]],
                       Gridded(Linear()))

for x in 1.5:0.5:nx-1.5
    @test_approx_eq_eps g1(x) gradient(itp_grid, x)[1] abs(.5*g1(x))
    @test_approx_eq_eps g1(x) gradient!(g, itp_grid, x)[1] abs(.5*g1(x))
    @test_approx_eq_eps g1(x) g[1] abs(.5*g1(x))
end

# Since Quadratic is OnCell in the domain, check gradients at grid points
itp1 = interpolate(Float64[f1(x) for x in 1:nx-1],
            BSpline(Quadratic(Periodic())), OnCell())
for x in 2:nx-1
    @test_approx_eq_eps g1(x) gradient(itp1, x)[1] abs(.05*g1(x))
    @test_approx_eq_eps g1(x) gradient!(g, itp1, x)[1] abs(.05*g1(x))
    @test_approx_eq_eps g1(x) g[1] abs(.1*g1(x))
end

for i = 1:10
    x = rand()*(nx-2)+1.5
    gtmp = gradient(itp1, x)[1]
    xd = dual(x, 1)
    @test_approx_eq epsilon(itp1[xd]) gtmp
end

# For a quadratic function and quadratic interpolation, we expect an
# "exact" answer
# 1d
c = 2.3
a = 8.1
o = 1.6
qfunc = x -> a*(x-c).^2 + o
dqfunc = x -> 2*a*(x-c)
xg = Float64[1:5;]
y = qfunc(xg)

iq = interpolate(y, BSpline(Quadratic(Free())), OnCell())
x = 1.8
@test_approx_eq iq[x] qfunc(x)
@test_approx_eq gradient(iq, x)[1] dqfunc(x)

# 2d (biquadratic)
p = [(x-1.75)^2 for x = 1:7]
A = p*p'
iq = interpolate(A, BSpline(Quadratic(Free())), OnCell())
@test_approx_eq iq[4,4] (4-1.75)^4
@test_approx_eq iq[4,3] (4-1.75)^2*(3-1.75)^2
g = gradient(iq, 4, 3)
@test_approx_eq g[1] 2*(4-1.75)*(3-1.75)^2
@test_approx_eq g[2] 2*(4-1.75)^2*(3-1.75)

iq = interpolate!(copy(A), BSpline(Quadratic(InPlace())), OnCell())
@test_approx_eq iq[4,4] (4-1.75)^4
@test_approx_eq iq[4,3] (4-1.75)^2*(3-1.75)^2
g = gradient(iq, 4, 3)
@test_approx_eq_eps g[1] 2*(4-1.75)*(3-1.75)^2 0.03
@test_approx_eq_eps g[2] 2*(4-1.75)^2*(3-1.75) 0.2

# InPlaceQ is exact for an underlying quadratic
iq = interpolate!(copy(A), BSpline(Quadratic(InPlaceQ())), OnCell())
@test_approx_eq iq[4,4] (4-1.75)^4
@test_approx_eq iq[4,3] (4-1.75)^2*(3-1.75)^2
g = gradient(iq, 4, 3)
@test_approx_eq g[1] 2*(4-1.75)*(3-1.75)^2
@test_approx_eq g[2] 2*(4-1.75)^2*(3-1.75)

A2 = rand(Float64, nx, nx) * 100
for BC in (Flat,Line,Free,Periodic,Reflect,Natural), GT in (OnGrid, OnCell)
    itp_a = interpolate(A2, (BSpline(Linear()), BSpline(Quadratic(BC()))), GT())
    itp_b = interpolate(A2, (BSpline(Quadratic(BC())), BSpline(Linear())), GT())
    itp_c = interpolate(A2, (NoInterp(), BSpline(Quadratic(BC()))), GT())
    itp_d = interpolate(A2, (BSpline(Quadratic(BC())), NoInterp()), GT())

    for i = 1:10
        x = rand()*(nx-2)+1.5
        y = rand()*(nx-2)+1.5
        xd = dual(x, 1)
        yd = dual(y, 1)
        gtmp = gradient(itp_a, x, y)
        @test length(gtmp) == 2
        @test_approx_eq epsilon(itp_a[xd, y]) gtmp[1]
        @test_approx_eq epsilon(itp_a[x, yd]) gtmp[2]
        gtmp = gradient(itp_b, x, y)
        @test length(gtmp) == 2
        @test_approx_eq epsilon(itp_b[xd, y]) gtmp[1]
        @test_approx_eq epsilon(itp_b[x, yd]) gtmp[2]
        ix, iy = round(Int, x), round(Int, y)
        gtmp = gradient(itp_c, ix, y)
        @test length(gtmp) == 1
        @test_approx_eq epsilon(itp_c[ix, yd]) gtmp[1]
        gtmp = gradient(itp_d, x, iy)
        @test length(gtmp) == 1
        @test_approx_eq epsilon(itp_d[xd, iy]) gtmp[1]
    end
end

end


function test_recall(truth::Array{Int,1}, pred::Array{Int,1}, k::Int)
    actual = measure(Recall(), truth, pred, k)
    expected = 0.333
    eps = 0.001

    @test abs(actual - expected) < eps
end

function test_precision(truth::Array{Int,1}, pred::Array{Int,1}, k::Int)
    actual = measure(Precision(), truth, pred, k)
    expected = 0.5
    eps = 0.001

    @test abs(actual - expected) < eps
end

function test_map(truth::Array{Int,1}, pred::Array{Int,1})
    actual = measure(MAP(), truth, pred)
    expected = 0.756
    eps = 0.001

    @test abs(actual - expected) < eps
end

function test_auc(truth::Array{Int,1}, pred::Array{Int,1})
    actual = measure(AUC(), truth, pred)
    expected = 0.667
    eps = 0.001

    @test abs(actual - expected) < eps
end

function test_mrr(truth::Array{Int,1}, pred::Array{Int,1})
    actual = measure(ReciprocalRank(), truth, pred)
    expected = 1.0
    eps = 0.001

    @test abs(actual - expected) < eps
end

function test_mpr(truth::Array{Int,1}, pred::Array{Int,1})
    actual = measure(MPR(), truth, pred)
    expected = 33.333
    eps = 0.001

    @test abs(actual - expected) < eps
end

function test_ndcg(truth::Array{Int,1}, pred::Array{Int,1}, k::Int)
    actual = measure(NDCG(), truth, pred, k)
    expected = 0.613
    eps = 0.001

    @test abs(actual - expected) < eps
end

println("-- Testing ranking metrics")

truth = [1, 2, 4]
pred = [1, 3, 2, 6, 4, 5]
k = 2

test_recall(truth, pred, k)
test_precision(truth, pred, k)
test_map(truth, pred)
test_auc(truth, pred)
test_mrr(truth, pred)
test_mpr(truth, pred)
test_ndcg(truth, pred, k)


# write your own tests here
N = 10
ph = discrete_phs2(N, 0, 1);
num_freq = frequencies(ph)

@test abs(num_freq[1]- 3.51602) < 1e-3

N = 10;
K = 7;
a = 0;
b = 1;
ad = 0;
bd = 0.1;
phport = discrete_phs2_distports(N,K,a,b,ad,bd)
Bd = phport.Bd*ones(K,1)
@test abs(Bd[2] - 110.799) < 1e-2


using PyPlot
using Gen

include("poisson_process.jl")
include("load_data.jl")

# Example from Section 4 of Reversible jump Markov chain Monte Carlo
# computation and Bayesian model determination

# minimum of k draws from uniform_continuous(lower, upper)

# we can sequentially sample the order statistics of a collection of K uniform
# continuous samples on the interval [a, b], by:
# x1 ~ min_uniform_continuous(a, b, K)
# x2 | x1 ~ min_uniform_continuous(x1, b, K-1)
# ..
# xK | x1 .. x_{K-1} ~ min_uniform_continuous(x_{K-1}, b, 1)

struct MinUniformContinuous <: Distribution{Float64} end
const min_uniform_continuous = MinUniformContinuous()

function Gen.logpdf(::MinUniformContinuous, x::Float64, lower::T, upper::U, k::Int) where {T<:Real,U<:Real}
    if x > lower && x < upper
        (k-1) * log(upper - x) + log(k) - k * log(upper - lower)
    else
        -Inf
    end
end

function Gen.random(::MinUniformContinuous, lower::T, upper::U, k::Int) where {T<:Real,U<:Real}
    # inverse CDF method
    p = rand()
    upper - (upper - lower) * (1. - p)^(1. / k)
end

#########
# model #
#########

const K = :k
const EVENTS = :events
const CHANGEPT = :changept
const RATE = :rate

@gen function model(T::Float64)

    # prior on number of change points
    k = @trace(poisson(3.), K)

    # prior on the location of (sorted) change points
    change_pts = Vector{Float64}(undef, k)
    lower = 0.
    for i=1:k
        cp = @trace(min_uniform_continuous(lower, T, k-i+1), (CHANGEPT, i))
        change_pts[i] = cp
        lower = cp
    end

    # k + 1 rate values
    # h$i is the rate for cp$(i-1) to cp$i where cp0 := 0 and where cp$(k+1) := T
    alpha = 1.
    beta = 200.
    rates = Float64[@trace(Gen.gamma(alpha, 1. / beta), (RATE, i)) for i=1:k+1]

    # poisson process
    bounds = vcat([0.], change_pts, [T])
    @trace(piecewise_poisson_process(bounds, rates), EVENTS)
end

function render(trace; ymax=0.02)
    T = get_args(trace)[1]
    k = trace[:k]
    bounds = vcat([0.], sort([trace[(CHANGEPT, i)] for i=1:k]), [T])
    rates = [trace[(RATE, i)] for i=1:k+1]
    for i=1:length(rates)
        lower = bounds[i]
        upper = bounds[i+1]
        rate = rates[i]
        plot([lower, upper], [rate, rate], color="black", linewidth=2)
    end
    points = trace[EVENTS]
    scatter(points, -rand(length(points)) * (ymax/5.), color="black", s=5)
    ax = gca()
    xlim = [0., T]
    plot(xlim, [0., 0.], "--")
    ax[:set_xlim](xlim)
    ax[:set_ylim](-ymax/5., ymax)
end

function show_prior_samples()
    figure(figsize=(16,16))
    T = 40000.
    for i=1:16
        println("simulating $i")
        subplot(4, 4, i)
        (trace, ) = generate(model, (T,), EmptyChoiceMap())
        render(trace; ymax=0.015)
    end
    tight_layout(pad=0)
    savefig("prior_samples.pdf")
end

#############
# rate move #
#############

@gen function rate_proposal(trace)

    # pick a random segment whose rate to change
    i = @trace(uniform_discrete(1, trace[K]+1), :i)

    # propose new value for the rate
    cur_rate = trace[(RATE, i)]
    @trace(uniform_continuous(cur_rate/2., cur_rate*2.), :new_rate)

    nothing
end

# it is an involution because it:
# - maintains i = fwd_choices[:i] constant
# - swaps choices[(RATE, i)] with fwd_choices[:new_rate]

@transform rate_involution() (model_in, aux_in) to (model_out, aux_out) begin
    i = @read(aux_in[:i], :discrete)
    @write(aux_out[:i], i, :discrete)
    new_rate = @read(aux_in[:new_rate], :continuous)
    @write(model_out[(RATE, i)], new_rate, :continuous)
    prev_rate = @read(model_in[(RATE, i)], :continuous)
    @write(aux_out[:new_rate], prev_rate, :continuous)
end

is_involution!(rate_involution)

rate_move(trace) = metropolis_hastings(trace, rate_proposal, (), rate_involution, check=true)

#################
# position move #
#################

@gen function position_proposal(trace)
    k = trace[K]
    @assert k > 0

    # pick a random changepoint to change
    i = @trace(uniform_discrete(1, k), :i)

    lower = (i == 1) ? 0. : trace[(CHANGEPT, i-1)]
    upper = (i == k) ? T : trace[(CHANGEPT, i+1)]
    @trace(uniform_continuous(lower, upper), :new_changept)

    i
end

# it is an involution because it:
# - maintains i = fwd_choices[:i] constant
# - swaps choices[(CHANGEPT, i)] with fwd_choices[:new_changept]

@transform position_involution (model_in, aux_in) to (model_out, aux_out) begin
    i = @read(aux_in[:i], :discrete)
    @write(aux_out[:i], i, :discrete)
    @copy(model_in[(CHANGEPT, i)], aux_out[:new_changept])
    @copy(aux_in[:new_changept], model_out[(CHANGEPT, i)])
end

is_involution!(position_involution)

position_move(trace) = metropolis_hastings(trace, position_proposal, (), position_involution, check=true)

######################
# birth / death move #
######################

const CHOSEN = :chosen
const IS_BIRTH = :is_birth
const NEW_CHANGEPT = :new_changept
const U = :u

@gen function birth_death_proposal(trace)
    T = get_args(trace)[1]
    k = trace[K]

    # if k = 0, then always do a birth move
    # if k > 0, then randomly choose a birth or death move
    isbirth = (k == 0) ? true : @trace(bernoulli(0.5), IS_BIRTH)

    if isbirth
        # pick the segment in which to insert the new changepoint
        # changepoints before move:  | 1     2    3 |
        # new changepoint (i = 2):   |    *         |
        # changepoints after move:   | 1  2  3    4 |
        i = @trace(uniform_discrete(1, k+1), CHOSEN)
        lower = (i == 1) ? 0. : trace[(CHANGEPT, i-1)]
        upper = (i == k+1) ? T : trace[(CHANGEPT, i)]
        @trace(uniform_continuous(lower, upper), NEW_CHANGEPT)
        @trace(uniform_continuous(0., 1.), U)
    else
        # pick the changepoint to be deleted
        # changepoints before move:     | 1  2  3    4 |
        # deleted changepoint (i = 2):  |    *         |
        # changepoints after move:      | 1     2    3 |
        @trace(uniform_discrete(1, k), CHOSEN)
    end
    nothing
end

function new_rates(cur_rate, u, cur_cp, prev_cp, next_cp)
    d_prev = cur_cp - prev_cp
    d_next = next_cp - cur_cp
    @assert d_prev > 0
    @assert d_next > 0
    d_total = d_prev + d_next
    log_cur_rate = log(cur_rate)
    log_ratio = log(1 - u) - log(u)
    prev_rate = exp(log_cur_rate - (d_next / d_total) * log_ratio)
    next_rate = exp(log_cur_rate + (d_prev / d_total) * log_ratio)
    @assert prev_rate > 0.
    @assert next_rate > 0.
    (prev_rate, next_rate)
end

function new_rates_inverse(prev_rate, next_rate, cur_cp, prev_cp, next_cp)
    d_prev = cur_cp - prev_cp
    d_next = next_cp - cur_cp
    @assert d_prev > 0
    @assert d_next > 0
    d_total = d_prev + d_next
    log_prev_rate = log(prev_rate)
    log_next_rate = log(next_rate)
    cur_rate = exp((d_prev / d_total) * log_prev_rate + (d_next / d_total) * log_next_rate)
    u = prev_rate / (prev_rate + next_rate)
    @assert cur_rate > 0.
    (cur_rate, u)
end

# it is an involution because:
# - it switches back and forth between birth move and death move
# - it maintains fwd_choices[CHOSEN] constant (applying it twice will first insert a new
#   changepoint and then remove that same changepoint)
# - new_rates, curried on cp_new, cp_prev, and cp_next, is the inverse of new_rates_inverse.

@transform birth_death_involution (model_in, aux_in) to (model_out, aux_out) begin
    T = get_args(model_in)[1]

    # current number of changepoints
    k = @read(model_in[K], :discrete)

    # if k == 0, then we can only do a birth move
    isbirth = (k == 0) || @read(aux_in[IS_BIRTH], :discrete)

    # if we are a birth move, the inverse is a death move
    if k > 1 || isbirth
        @write(aux_out[IS_BIRTH], !isbirth, :discrete)
    end

    # the changepoint to be added or deleted
    i = @read(aux_in[CHOSEN], :discrete)
    @copy(aux_in[CHOSEN], aux_out[CHOSEN])

    if isbirth
        @tcall(birth(k, i))
    else
        @tcall(death(k, i))
    end
end

@transform birth(k::Int, i::Int) (model_in, aux_in) to (model_out, aux_out) begin
    @write(model_out[K], k+1, :discrete)

    cp_new = @read(aux_in[NEW_CHANGEPT], :continuous)
    cp_prev = (i == 1) ? 0. : @read(model_in[(CHANGEPT, i-1)], :continuous)
    cp_next = (i == k+1) ? T : @read(model_in[(CHANGEPT, i)], :continuous)

    # set new changepoint
    @copy(aux_in[NEW_CHANGEPT], model_out[(CHANGEPT, i)])

    # shift up changepoints
    for j=i+1:k+1
        @copy(model_in[(CHANGEPT, j-1)], model_out[(CHANGEPT, j)])
    end

    # compute new rates
    h_cur = @read(model_in[(RATE, i)], :continuous)
    u = @read(aux_in[U], :continuous)
    (h_prev, h_next) = new_rates(h_cur, u, cp_new, cp_prev, cp_next)

    # set new rates
    @write(model_out[(RATE, i)], h_prev, :continuous)
    @write(model_out[(RATE, i+1)], h_next, :continuous)

    # shift up rates
    for j=i+2:k+2
        @copy(model_in[(RATE, j-1)], model_out[(RATE, j)])
    end
end

@transform death(k::Int, i::Int) (model_in, aux_in) to (model_out, aux_out) begin
    @write(model_out[K], k-1, :discrete)

    cp_deleted = @read(model_in[(CHANGEPT, i)], :continuous)
    cp_prev = (i == 1) ? 0. : @read(model_in[(CHANGEPT, i-1)], :continuous)
    cp_next = (i == k) ? T : @read(model_in[(CHANGEPT, i+1)], :continuous)
    @copy(model_in[(CHANGEPT, i)], aux_out[NEW_CHANGEPT])

    # shift down changepoints
    for j=i:k-1
        @copy(model_in[(CHANGEPT, j+1)], model_out[(CHANGEPT, j)])
    end

    # compute cur rate and u
    h_prev = @read(model_in[(RATE, i)], :continuous)
    h_next = @read(model_in[(RATE, i+1)], :continuous)
    (h_cur, u) = new_rates_inverse(h_prev, h_next, cp_deleted, cp_prev, cp_next)
    @write(aux_out[U], u, :continuous)

    # set cur rate
    @write(model_out[(RATE, i)], h_cur, :continuous)

    # shift down rates
    for j=i+1:k
        @copy(model_in[(RATE, j+1)], model_out[(RATE, j)])
    end
end

is_involution!(birth_death_involution)

birth_death_move(trace) = metropolis_hastings(trace, birth_death_proposal, (), birth_death_involution, check=true)

function mcmc_step(trace)
    (trace, _) = rate_move(trace)
    if trace[K] > 0
        (trace, _) = position_move(trace)
    end
    (trace, _) = birth_death_move(trace)
    trace
end

function simple_mcmc_step(trace)
    (trace, _) = rate_move(trace)
    if trace[K] > 0
        (trace, _) = position_move(trace)
    end
    (trace, _) = metropolis_hastings(trace, k_selection)
    trace
end

function do_mcmc(T, num_steps::Int)
    (trace, _) = generate(model, (T,), observations)
    for iter=1:num_steps
        k = trace[K]
        if iter % 1000 == 0
            println("iter $iter of $num_steps, k: $k")
        end
        trace = mcmc_step(trace)
    end
    trace
end

const k_selection = select(K)

function do_simple_mcmc(T, num_steps::Int)
    (trace, _) = generate(model, (T,), observations)
    for iter=1:num_steps
        k = trace[K]
        if iter % 1000 == 0
            println("iter $iter of $num_steps, k: $k")
        end
        trace = simple_mcmc_step(trace)
    end
    trace
end

########################
# inference experiment #
########################

import Random
Random.seed!(1)

const points = load_data_set()
const T = maximum(points)
const observations = choicemap()
observations[EVENTS] = points

function show_posterior_samples()
    figure(figsize=(16,16))
    for i=1:16
        println("replicate $i")
        subplot(4, 4, i)
        trace = do_mcmc(T, 200)
        render(trace; ymax=0.015)
    end
    tight_layout(pad=0)
    savefig("posterior_samples.pdf")

    figure(figsize=(16,16))
    for i=1:16
        println("replicate $i")
        subplot(4, 4, i)
        trace = do_simple_mcmc(T, 200)
        render(trace; ymax=0.015)
    end
    tight_layout(pad=0)
    savefig("posterior_samples_simple.pdf")
end

function get_rate_vector(trace, test_points)
    k = trace[K]
    cps = [trace[(CHANGEPT, i)] for i=1:k]
    hs = [trace[(RATE, i)] for i=1:k+1]
    rate = Vector{Float64}()
    cur_h_idx = 1
    cur_h = hs[cur_h_idx]
    next_cp_idx = 1
    upper = (next_cp_idx == k + 1) ? T : cps[next_cp_idx]
    for x in test_points
        while x > upper
            next_cp_idx += 1
            upper = (next_cp_idx == k + 1) ? T : cps[next_cp_idx]
            cur_h_idx += 1
            cur_h = hs[cur_h_idx]
        end
        push!(rate, cur_h)
    end
    rate
end

# compute posterior mean rate curve

function plot_posterior_mean_rate()
    test_points = collect(1.0:10.0:T)
    rates = Vector{Vector{Float64}}()
    num_samples = 0
    num_steps = 8000
    for reps=1:20
        (trace, _) = generate(model, (T,), observations)
        for iter=1:num_steps
            if iter % 1000 == 0
                println("iter $iter of $num_steps, k: $(trace[K])")
            end
            trace = mcmc_step(trace)
            if iter > 4000
                num_samples += 1
                rate_vector = get_rate_vector(trace, test_points)
                @assert length(rate_vector) == length(test_points)
                push!(rates, rate_vector)
            end
        end
    end
    posterior_mean_rate = zeros(length(test_points))
    for rate in rates
        posterior_mean_rate += rate / Float64(num_samples)
    end
    ymax = 0.010
    figure()
    plot(test_points, posterior_mean_rate, color="black")
    scatter(points, -rand(length(points)) * (ymax/6.), color="black", s=5)
    ax = gca()
    xlim = [0., T]
    plot(xlim, [0., 0.], "--")
    ax[:set_xlim](xlim)
    ax[:set_ylim](-ymax/5., ymax)
    savefig("posterior_mean_rate.pdf")
end

function plot_trace_plot()
    figure(figsize=(8, 4))

    # reversible jump
    (trace, _) = generate(model, (T,), observations)
    rate1 = Float64[]
    num_clusters_vec = Int[]
    burn_in = 0
    for iter=1:burn_in + 1000
        trace = mcmc_step(trace)
        if iter > burn_in
            push!(num_clusters_vec, trace[K])
        end
    end
    subplot(2, 1, 1)
    plot(num_clusters_vec, "b")

    # simple MCMC
    (trace, _) = generate(model, (T,), observations)
    rate1 = Float64[]
    num_clusters_vec = Int[]
    burn_in = 0
    for iter=1:burn_in + 1000
        trace = simple_mcmc_step(trace)
        if iter > burn_in
            push!(num_clusters_vec, trace[K])
        end
    end
    subplot(2, 1, 2)
    plot(num_clusters_vec, "b")

    savefig("trace_plot.pdf")
end

println("showing prior samples...")
show_prior_samples()

println("showing posterior samples...")
show_posterior_samples()

println("estimating posterior mean rate...")
plot_posterior_mean_rate()

println("making trace plot...")
plot_trace_plot()


using NLSolvers, SparseArrays, LinearAlgebra
using Test
@testset "optimization preconditioning" begin
debug_print = true
for N in (10, 50, 100, 150, 250)
    if debug_print
        println("N = ", N)
    end
    initial_x = OPT_PROBS["laplacian"]["array"]["x0(n)"](N)
    Plap = precond(initial_x)
    ID = nothing
    iter = []
    mino = []
    for optimizer in (p->LineSearch(GradientDescent(Direct(), p)), p->LineSearch(ConjugateGradient(HZ(), p), HZAW()), p->LineSearch(ConjugateGradient(HZ(), p), Backtracking()), p->LineSearch(LBFGS(Inverse(), NLSolvers.TwoLoop(), 5, p)), p->LineSearch(LBFGS(Inverse(), NLSolvers.TwoLoop(), 5, p), HZAW()))
        for (P, wwo) in zip((nothing, (x, P=nothing)->inv(Array(precond(N)))), (" WITHOUT", " WITH"))
            if debug_print
                println(summary(optimizer(P)))
            end
            results = solve(OPT_PROBS["laplacian"]["array"]["mutating"], copy(initial_x), optimizer(P), OptimizationOptions(g_abstol=1e-6))
            push!(mino, results.info.minimum)
            push!(iter, results.info.iter)
        end
        if debug_print
            println("Iterations without precon: $(iter[end-1])")
            println("Iterations with    precon: $(iter[end])")
            println("Minimum without precon:    $(mino[end-1])")
            println("Minimum with    precon:    $(mino[end])")
            println()
        end
        if !(N == 10 && optimizer(nothing).linesearcher isa HZAW && optimizer(nothing).scheme isa LBFGS)
            @test iter[end-1] >= iter[end]
        end
    end
end
end


module Models

  using ITensors

  export Model, critical_point, local_boltzmann_weight

  struct Model{model} end
  Model(s::AbstractString) = Model{Symbol(s)}()
  macro Model_str(s)
    :(Model{$(Expr(:quote, Symbol(s)))})
  end

  #mpo_itensor(m::String, args...; kwargs...) =
  #  mpo_itensor(Model(m), args...; kwargs...)

  local_boltzmann_weight(m::String, args...; kwargs...) =
    local_boltzmann_weight(Model(m), args...; kwargs...)

  f(λ₊, λ₋) = [
    (λ₊ + λ₋)/2 (λ₊ - λ₋)/2
    (λ₊ - λ₋)/2 (λ₊ + λ₋)/2
  ]

  function sqrt_bond_matrix(; β::Real, J::Real=1.0)
    # Alternative method
    #Q = [exp(β * J) exp(-β * J); exp(-β * J) exp(β * J)]
    #return √Q
    λ₊ = √(exp(β * J) + exp(-β * J))
    λ₋ = √(exp(β * J) - exp(-β * J))
    return f(λ₊, λ₋)
  end

  # The local Boltzmann weight for the Ising
  # model in 1 dimension
  function local_boltzmann_weight(
    ::Model"ising",
    ::Val{1};
    β::Real,
    J::Real=1.0,
    sz::Bool=false,
  )
    d = 2 # local dimension of the Ising local Boltzmann factor
    s, s′ = Index.((d, d))
    T = ITensor(s, s′)
    for i in 1:d
      T[i, i] = 1.0
    end
    if sz
      T[1, 1] = -T[1, 1]
    end
    s̃, s̃′ = sim.((s, s′))
    T̃ = T * δ(s, s̃) * δ(s′, s̃′)
    sqrtQ = sqrt_bond_matrix(; β=β, J=J)
    @show sqrtQ
    X = itensor(vec(sqrtQ), s̃, s)
    X′ = itensor(vec(sqrtQ), s̃′, s′)
    return array(permute(T̃ * X′ * X, s, s′))
  end

  # The local Boltzmann weight for the Ising
  # model in 2 dimensions
  function local_boltzmann_weight(
    ::Model"ising",
    ::Val{2};
    β::Real,
    J::Real=1.0,
    sz::Bool=false,
  )
    d = 2 # local dimension of the Ising local Boltzmann factor
    sₕ, sₕ′ = Index.((d, d))
    sᵥ, sᵥ′ = Index.((d, d))
    @assert dim(sₕ) == dim(sᵥ)
    d = dim(sₕ)
    T = ITensor(sₕ, sₕ′, sᵥ, sᵥ′)
    for i in 1:d
      T[i, i, i, i] = 1.0
    end
    if sz
      T[1, 1, 1, 1] = -T[1, 1, 1, 1]
    end
    s̃ₕ, s̃ₕ′, s̃ᵥ, s̃ᵥ′ = sim.((sₕ, sₕ′, sᵥ, sᵥ′))
    T̃ = T * δ(sₕ, s̃ₕ) * δ(sₕ′, s̃ₕ′) * δ(sᵥ, s̃ᵥ) * δ(sᵥ′, s̃ᵥ′)
    X = sqrt_bond_matrix(; β, J)
    Xₕ = itensor(vec(X), s̃ₕ, sₕ)
    Xₕ′ = itensor(vec(X), s̃ₕ′, sₕ′)
    Xᵥ = itensor(vec(X), s̃ᵥ, sᵥ)
    Xᵥ′ = itensor(vec(X), s̃ᵥ′, sᵥ′)
    return array(permute(T̃ * Xₕ′ * Xᵥ′ * Xₕ * Xᵥ, sₕ, sₕ′, sᵥ, sᵥ′))
  end

  function mpo_itensor(
    m::Model,
    pair_sₕ::Pair{<:Index,<:Index},
    pair_sᵥ::Pair{<:Index,<:Index};
    kwargs...
  )
    sₕ, sₕ′ = pair_sₕ
    sᵥ, sᵥ′ = pair_sᵥ
    return itensor(mpo_array(m; kwargs...), sₕ, sₕ′, sᵥ, sᵥ′)
  end

  ## function ising_mpo(sₕ::Index, sᵥ::Index, args...; kwargs...)
  ##   return ising_mpo(sₕ => sₕ', sᵥ => sᵥ', args...; kwargs...)
  ## end

  critical_point(::Model"ising") = 0.5 * log(√2 + 1)
  #const βc = critical_point(Model("ising"))

  function free_energy(::Model"ising"; β::Real, J::Real=1.0)
    k = β * J
    c = cosh(2 * k)
    s = sinh(2 * k)
    xmin = 0.0
    xmax = π
    integrand(x) = log(c^2 + √(s^4 + 1 - 2 * s^2 * cos(x)))
    integral, err = quadgk(integrand, xmin, xmax)::Tuple{Float64,Float64}
    return -(log(2) + integral / π) / (2 * β)
  end

  function magnetization(::Model"ising"; β::Real)
    β > βc && return (1 - sinh(2 * β)^(-4))^(1 / 8)
    return 0.0
  end

end


#=
    Computes and array of magnitudes for 2D or 3D vector fields. Each component of
    the vectors field is provided in a separate array:
        magnitudemap( U, V ) or magnitudemap( U, V, W )
=#
function magnitudemap( arrays::Array{T,N}... ; typ=Float32 ) where {T,N}
    magnitudes = zeros( typ, size( arrays[1] ) )
    magnitudemap!( magnitudes, arrays... )
    return magnitudes
end

function magnitudemap!( magnitudes::Array{U,N}, arrays::Array{T,N}... ) where {T,U,N}
    numcomponents = length( arrays );
    @inbounds for e in 1:length( arrays[1] )
        sum2 = 0.0;
        @simd for c in 1:numcomponents
            sum2 += arrays[c][e]*arrays[c][e]
        end
        magnitudes[e] = convert( U, sqrt( sum2 ) )
    end
end

#=
    Computes a boolean mask, where vectors whose magnitude is > n * std are set to
    true. Each component of the vectors field is provided in a separate array.
=#
function stdmask( n, arrays::Array{T,N}... ) where {T,N}
    mask = zeros( Bool, size( arrays[1] ) )
    stdmask!( n, mask, arrays... );
    return mask
end

function stdmask!( n, mask::Array{Bool,N}, arrays::Array{T,N}... ) where {T,N}
    magnitudes = magnitudemap( arrays... );
    mean = Statistics.mean( magnitudes );
    std  = Statistics.std( magnitudes, mean=mean ) ;
    @simd for e in 1:length( magnitudes )
        @inbounds mask[ e ] = abs( magnitudes[e] - mean ) > n*std
    end
end

#=
    Replacement functions accept a coordinate and a radius argument ( a part from
    the vector field components ). zeroRepalce does not use the coordinate or
    radius, but mean and median require them.
=#
function zeroReplace( coords::NTuple{N,Int64}, radius::Int64,
                      arrays::Array{T,N}... ) where {T,N}
    return zeros( T, length(arrays) )
end

function medianReplace( coords::NTuple{N,Int64}, radius::Int64,
                        arrays::Array{T,N}... ) where {T,N}
    mincoords = max.( 1, coords .- radius );
    maxcoords = min.( size(arrays[1]), coords .+ radius );
    medians   = zeros( T, length( arrays ) );

    for c in 1:length( arrays )
        medians[ c ] = Statistics.median( arrays[c][ UnitRange.( mincoords, maxcoords )... ] );
    end
    return medians
end

function meanReplace( coords::NTuple{N,Int64}, radius::Int64,
                      arrays::Array{T,N}... ) where {T,N}
    mincoords = max.( 1, coords .- radius );
    maxcoords = min.( size(arrays[1]), coords .+ radius );
    means     = zeros( T, length( arrays ) );

    for c in 1:length( arrays )
        means[ c ] = Statistics.mean( arrays[c][ UnitRange.( mincoords, maxcoords )... ] )
    end
    return means
end

"""
    Replaces vectors whose magnitude is > n * std. The replacement function can be
    specified, as well as the radius of the neighbouring vectors that are used to
    compute median and mean vectors.

    Implemented replacement functions are.
        PIV3D.zeroReplace ----> replaces outlier vectors by a vector of zeros
        PIV3D.medianReplace --> replaces outlider vector by the median vector
        PIV3D.meanReplace ----> replaces outlier vectors by the mean vector
"""
function stdFilter( n, arrays::Array{T,N}...;
                    replace::Function=zeroReplace, radius=0 ) where {T,N}

    filtered = [ copy( arrays[idx] ) for idx in 1:length(arrays) ];

    if n > 0
        mask   = stdmask( n, arrays... );
        coords = CartesianIndices( mask );

        for e in 1:length(mask)
            if mask[e]
                replacement = replace( Tuple(coords[e]), radius, arrays... )
                for c in 1:length(arrays)
                    filtered[c][e] = replacement[c]
                end
            end
        end
    end

    return filtered
end

"""
    Replaces vectors whose magnitude is > n * std. The replacement function can be
    specified, as well as the radius of the neighbouring vectors that are used to
    compute median and mean vectors.

    Implemented replacement functions are.
        PIV3D.zeroReplace ----> replaces outlier vectors by a vector of zeros
        PIV3D.medianReplace --> replaces outlider vector by the median vector
        PIV3D.meanReplace ----> replaces outlier vectors by the mean vector
"""
function magnitudeFilter( threshold, u::Array{T,N}, v::Array{T,N} ) where {T<:Real,N}
    return magnitudeFilter( >, threshold, u, v )
end

function magnitudeFilter( f, threshold, arrays::Array{T,N}...;
                          replace::Function=zeroReplace, radius=0 ) where {T,N}

    magnitudes = magnitudemap( arrays... );
    coords     = CartesianIndices( magnitudes );
    filtered   = [ copy( arrays[idx] ) for idx in 1:length(arrays) ];

    for e in 1:length(magnitudes)
        if f( magnitudes[e], threshold )
            replacement = replace( Tuple(coords[e]), radius, arrays... )
            for c in 1:length(arrays)
                filtered[c][e] = replacement[c]
            end
        end
    end
    return filtered
end

""" Spatial averaging for 2D vector fields"""
function spaceAveraging( avg_radius::Int64, u::Array{T,2}, v::Array{T,2};
                         th=0.1 ) where {T<:Real}
    u_avg = zeros( T, size(u) );
    v_avg = zeros( T, size(v) );
    h, w  = size( u )

    for col in 1:w
        cmin = max( 1, col-avg_radius );
        cmax = min( w, col+avg_radius );

        for row in 1:h

            mag = sqrt( u[ row, col ]^2 + v[ row, col ]^2 );
            if mag < th
                continue
            end

            rmin = max( 1, row-avg_radius );
            rmax = min( h, row+avg_radius );
            N = length(cmin:cmax)*length(rmin:rmax);

            mean_u = 0.0;
            mean_v = 0.0;
            @inbounds for x in cmin:cmax
                @simd for y in rmin:rmax
                    mean_u += u[y,x]
                    mean_v += v[y,x]
                end
            end

            u_avg[ row, col ] = mean_u/N;
            v_avg[ row, col ] = mean_v/N;
        end
    end
    return u_avg, v_avg
end

""" Spatial averaging for 3D vector fields"""
function spaceAveraging( avg_radius::Int64, U::Array{T,3}, V::Array{T,3}, W::Array{T,3} ) where {T<:Real}

    u_avg   = zeros( T, size(U) );
    v_avg   = zeros( T, size(V) );
    w_avg   = zeros( T, size(W) );
    h, w, d = size( U )

    for zet in 1:d
        zmin = max( 1, zet-avg_radius );
        zmax = min( d, zet+avg_radius );

        for col in 1:w
            cmin = max( 1, col-avg_radius );
            cmax = min( w, col+avg_radius );

            for row in 1:h

                mag = sqrt( U[row,col,zet]^2 + V[row,col,zet]^2 + W[row,col,zet]^2 )
                if mag == 0
                    continue
                end

                rmin = max( 1, row-avg_radius );
                rmax = min( h, row+avg_radius );
                N = length(rmin:rmax)*length(cmin:cmax)*length(zmin:zmax);

                mean_u = 0.0;
                mean_v = 0.0;
                mean_w = 0.0;

                # Looping over neighbours
                for z in zmin:zmax
                    for x in cmin:cmax
                        for y in rmin:rmax
                            mean_u += U[y,x,z]
                            mean_v += V[y,x,z]
                            mean_w += W[y,x,z]
                        end
                    end
                end

                u_avg[ row, col, zet ] = mean_u/N;
                v_avg[ row, col, zet ] = mean_v/N;
                w_avg[ row, col, zet ] = mean_w/N;
            end
        end
    end

    return u_avg, v_avg, w_avg
end

""" space-time averaging """ # fancier circular mask in notes_16Oct20
function spaceTimeAveraging( avg_rs::Int64, avg_rt::Int64, u::Array{T,3}, v::Array{T,3}; th=0.0 )  where {T<:Real}

    u_avg = zeros( T, size(u) );
    v_avg = zeros( T, size(v) );

    h, w, time = size( u )

    for t in 1:time
        tmin = max(   1 , t - avg_rt );
        tmax = min( time, t + avg_rt );

        for c in 1:w
            cmin = max( 1, c - avg_rs );
            cmax = min( w, c + avg_rs );

            for r in 1:h

                magnitude = sqrt( u[r,c,t]^2 + v[r,c,t]^2 )
                if  magnitude < th
                    continue
                end

                rmin = max( 1, r - avg_rs );
                rmax = min( h, r + avg_rs );
                N = length(tmin:tmax) + length(rmin:rmax)*length(cmin:cmax) - 1

                # central vector is counted twice, so we account for it by substracting it once
                mean_u = -u[r,c,t];
                mean_v = -v[r,c,t];

                # spatial averaging
                for x in cmin:cmax
                    for y in rmin:rmax
                        mean_u += u[y,x,t]
                        mean_v += v[y,x,t]
                    end
                end

                # temporal averaging
                for tp in tmin:tmax
                    mean_u += u[r,c,tp]
                    mean_v += v[r,c,tp]
                end

                u_avg[r,c,t] = mean_u/N;
                v_avg[r,c,t] = mean_v/N;
            end
        end
    end

    return u_avg, v_avg
end

""" space-time averaging """ # fancier circular mask in notes_16Oct20
function spaceTimeAveraging( avg_rs::Int64, avg_rt::Int64, u::Array{T,4}, v::Array{T,4}, w::Array{T,4}; th=0.0 )  where {T<:Real}

    u_avg = zeros( T, size(u) );
    v_avg = zeros( T, size(v) );
    w_avg = zeros( T, size(w) );

    h, w, d, time = size( u )

    for t in 1:time
        tmin = max(   1 , t - avg_rt );
        tmax = min( time, t + avg_rt );

        for s in 1:d
            smin = max( 1, s - avg_rs );
            smax = min( d, s + avg_rs );

            for c in 1:w
                cmin = max( 1, c - avg_rs );
                cmax = min( w, c + avg_rs );

                for r in 1:h

                    magnitude = sqrt( u[r,c,s,t]^2 + v[r,c,s,t]^2 + w[r,c,s,t]^2 )
                    if  magnitude < th
                        continue
                    end

                    rmin = max( 1, r - avg_rs );
                    rmax = min( h, r + avg_rs );
                    N = length(tmin:tmax) + length(smin:smax)*length(cmin:cmax)*length(rmin:rmax)

                    # central vector is counted twice, so we account for it by substracting it once
                    mean_u = -u[ r, c, s, t ];
                    mean_v = -v[ r, c, s, t ];
                    mean_w = -w[ r, c, s, t ];

                    # spatial averaging, t is constant
                    for z in smin:smax
                        for x in cmin:cmax
                            for y in rmin:rmax
                                mean_u += u[y,x,z,t]
                                mean_v += v[y,x,z,t]
                                mean_w += w[y,x,z,t]
                            end
                        end
                    end

                    # temporal averaging, (r,c,s) are constant
                    for tp in tmin:tmax
                        mean_u += u[r,c,s,tp]
                        mean_v += v[r,c,s,tp]
                        mean_w += w[r,c,s,tp]
                    end

                    u_avg[ r,c,s,t ] = mean_u/N;
                    v_avg[ r,c,s,t ] = mean_v/N;
                    w_avg[ r,c,s,t ] = mean_w/N;
                end
            end
        end
    end

    return u_avg, v_avg, w_avg
end

""" Spatial averaging + similarity thresholding combo """
function similarityAveraging3D( avg_radius::Int64, U::Array{T,3}, V::Array{T,3}, W::Array{T,3};
                                st=0.0 ) where {T<:Real}

    u_avg   = zeros( T, size(U) );
    v_avg   = zeros( T, size(V) );
    w_avg   = zeros( T, size(W) );
    h, w, d = size( U )

    for zet in 1:d
        zmin = max( 1, zet-avg_radius );
        zmax = min( d, zet+avg_radius );

        for col in 1:w
            cmin = max( 1, col-avg_radius );
            cmax = min( w, col+avg_radius );

            for row in 1:h

                u1 = U[row,col,zet]
                v1 = V[row,col,zet]
                w1 = W[row,col,zet]
                mag = sqrt( u1*u1 + v1*v1 + w1*w1 )
                if mag == 0
                    continue
                end

                nu1 = U[row,col,zet]/mag
                nv1 = V[row,col,zet]/mag
                nw1 = W[row,col,zet]/mag

                rmin = max( 1, row-avg_radius );
                rmax = min( h, row+avg_radius );

                len  = length(rmin:rmax)*length(cmin:cmax)*length(zmin:zmax);

                # variables to hold the averaged vector component
                mean_u = 0.0;
                mean_v = 0.0;
                mean_w = 0.0;

                # counter of similar neighbouring vectors
                n  = 0;

                for z in zmin:zmax
                    for x in cmin:cmax
                        for y in rmin:rmax

                            mag2 = sqrt( U[y,x,z]*U[y,x,z] + V[y,x,z]*V[y,x,z] + W[y,x,z]*W[y,x,z] )
                            nu2  = U[y,x,z]/mag2
                            nv2  = V[y,x,z]/mag2
                            nw2  = W[y,x,z]/mag2
                            dot  = nu1*nu2 + nv1*nv2 + nw1*nw2

                            if dot > st
                                mean_u += U[y,x,z]
                                mean_v += V[y,x,z]
                                mean_w += W[y,x,z]
                                n += 1;
                            end
                        end
                    end
                end

                sim  = (n/len)^2;
                mmag = sqrt( mean_u*mean_u + mean_v*mean_v + mean_w*mean_w );

                u_avg[ row, col, zet ] = mean_u/mmag * sim;
                v_avg[ row, col, zet ] = mean_v/mmag * sim;
                w_avg[ row, col, zet ] = mean_w/mmag * sim;
            end
        end
    end

    return u_avg, v_avg, w_avg
end

""" Spatial averaging """
function similarityMap( avg_radius::Int64, U::Array{T,3}, V::Array{T,3}, W::Array{T,3};
                        st=0.0 ) where {T<:Real}

    similty = zeros( Float32, size(U) );
    h, w, d = size( U )

    for zet in 1:d
        zmin = max( 1, zet-avg_radius );
        zmax = min( d, zet+avg_radius );

        for col in 1:w
            cmin = max( 1, col-avg_radius );
            cmax = min( w, col+avg_radius );

            for row in 1:h

                mag = sqrt( U[row,col,zet]^2 + V[row,col,zet]^2 + W[row,col,zet]^2 )
                if mag == 0
                    continue
                end

                nu1 = U[row,col,zet]/mag
                nv1 = V[row,col,zet]/mag
                nw1 = W[row,col,zet]/mag

                rmin = max( 1, row-avg_radius );
                rmax = min( h, row+avg_radius );

                len  = length(rmin:rmax)*length(cmin:cmax)*length(zmin:zmax);

				len  = avg_radius*avg_radius*avg_radius;

                n = 0;
                for z in zmin:zmax
                    for x in cmin:cmax
                        for y in rmin:rmax
                            mag2 = sqrt( U[y,x,z]^2 + V[y,x,z]^2 + W[y,x,z]^2 )
                            nu2  = U[y,x,z]/mag2
                            nv2  = V[y,x,z]/mag2
                            nw2  = W[y,x,z]/mag2
                            dot  = nu1*nu2 + nv1*nv2 + nw1*nw2
                            if dot > st
                                n += 1;
                            end
                        end
                    end
                end

                similty[row,col,zet] = (n/len);
            end
        end
    end

    return similty
end

function velocityMap( U::Array{T,3}, V::Array{T,3}, W::Array{T,3} ) where {T<:Real}

    mags = zeros( Float32, size( U ) )
    for li in 1:length( U )
        mags[li] = sqrt( U[li]*U[li] + V[li]*V[li] + W[li]*W[li] )
    end
    return mags
end

# 2D
function sink( radius::Int64, pos::II )
    return [ [ (pos[1] - y )/sqrt( (pos[1]-y)^2 + (pos[2]-x)^2 ),
               (pos[2] - x )/sqrt( (pos[1]-y)^2 + (pos[2]-x)^2 ) ] for y in pos[1]-radius:pos[1]+radius,
                                                                       x in pos[2]-radius:pos[2]+radius ]
end

# 3D
function sink( radius::Int64, pos::III )
    return [ [ (pos[1] - y )/sqrt( (pos[1]-y)^2 + (pos[2]-x)^2 + (pos[3]-z)^2 ),
               (pos[2] - x )/sqrt( (pos[1]-y)^2 + (pos[2]-x)^2 + (pos[3]-z)^2 ),
               (pos[3] - z )/sqrt( (pos[1]-y)^2 + (pos[2]-x)^2 + (pos[3]-z)^2 ) ] for y in pos[1]-radius:pos[1]+radius,
                                                                                x in pos[2]-radius:pos[2]+radius,
                                                                                z in pos[3]-radius:pos[3]+radius ]
end

function crossCorrelateSink!( u, v, w, sink, corr )

    hi, wi, di = size(sink) # interrogation area
    hs, ws, ds = size(u)    # search area

    @inbounds begin
    for z in di:-1:1
        for c in wi:-1:1
            for r in hi:-1:1
                px = sink[ r, c, z ];
                iy = hi-r+1;
                ix = wi-c+1;
                iz = di-z+1;
                corr[ iy:(iy-1+hs),
                      ix:(ix-1+ws),
                      iz:(iz-1+ds) ] .+= px[1] .* u .+ px[2] .* v .+ px[3] .* w
            end
        end
    end
    end # @inbounds
end

function divergenceMap( r::Int64, U::Array{T,3}, V::Array{T,3}, W::Array{T,3}  ) where {T<:Real}

    interSink = sink( r, (0,0,0) );

    for idx in 1:length(interSink)
        isnan(interSink[idx][1]) && ( interSink[idx][1] = 0.0 );
        isnan(interSink[idx][2]) && ( interSink[idx][2] = 0.0 );
        isnan(interSink[idx][3]) && ( interSink[idx][3] = 0.0 );
    end

    # normalizing vector fields
    NU = zeros( T, size( U ) );
    NV = zeros( T, size( V ) );
    NW = zeros( T, size( W ) );

    for e in 1:length( U )
        mag   = sqrt( U[e]*U[e] + V[e]*V[e] + W[e]*W[e] )
        NU[e] = ( mag == 0 ) ? 0.0 : U[e]/mag
        NV[e] = ( mag == 0 ) ? 0.0 : V[e]/mag
        NW[e] = ( mag == 0 ) ? 0.0 : W[e]/mag
    end

    cmatrix = zeros( Float64, size(NU) .+ 2*r )

    crossCorrelateSink!( NU, NV, NW, interSink, cmatrix )

    return cmatrix
end

""" PIV Trajectories, work started by Michelle Gottlieb  """

function PIVtrajectories( U::Array{P,4}, V::Array{P,4}, W::Array{P,4},
                          T0, T1, numpoints; facF=1,
				          subr=( -1: -1, -1: -1, -1: -1 ) ) where {P<:Real}
    return PIVtrajectories( U, V, W, T0, T1, numpoints, 1:size(U,1), 1:size(U,2), 1:size(U,3), subr=subr, facF=facF )
end

function PIVtrajectories( U::Array{P,4}, V::Array{P,4}, W::Array{P,4},
                          T0, T1, numpoints, vrange, lrange, drange;
                          subr=( -1: -1, -1: -1, -1: -1 ), facF=1 ) where {P<:Real}

    # Vertical, lateral and depth axes.
    VV = collect( vrange );
    LL = collect( lrange );
    DD = collect( drange );

    dims = ( length(VV), length(LL), length(DD) );

    # I am still not sure of what these scaling factors do
    endV = VV[ end ]
    endH = LL[ end ]
    endD = DD[ end ]
    facV = 1 # endV/dims[1]
    facL = 1 # endH/dims[2]
    facD = 1 # endD/dims[3]

    numT = T1 - T0;
    TrajectoriesV = zeros( Float64, numT, numpoints )
    TrajectoriesL = zeros( Float64, numT, numpoints )
    TrajectoriesD = zeros( Float64, numT, numpoints )

    for pidx in 1:numpoints
        # Each iteration in this loop corresponds to a new particle
        # A particle is nothing but a pair of coordinates
        # Random coordinates of the particle inside VV, HH and DD
        c1 = rand( subr[1].start == -1 ? (2:dims[1]-1) : subr[1] )
        c2 = rand( subr[2].start == -1 ? (2:dims[2]-1) : subr[2] )
        c3 = rand( subr[3].start == -1 ? (2:dims[3]-1) : subr[3] )

        # Trajectory starts at the ( VV, HH ) position
        TrajectoriesV[ 1, pidx ] = VV[ c1 ] + 0.0
        TrajectoriesL[ 1, pidx ] = LL[ c2 ] + 0.0
        TrajectoriesD[ 1, pidx ] = DD[ c3 ] + 0.0

        # Translation converted into indices on VV and HH
        Ut = facF * U[ c1, c2, c3, T0 ]
        Vt = facF * V[ c1, c2, c3, T0 ]
        Wt = facF * W[ c1, c2, c3, T0 ]

        # Time-iteration
        for t in 2:numT

            TrajectoriesV[t,pidx] = TrajectoriesV[t-1, pidx] + Ut
            TrajectoriesL[t,pidx] = TrajectoriesL[t-1, pidx] + Vt
            TrajectoriesD[t,pidx] = TrajectoriesD[t-1, pidx] + Wt

            c1 = round( Int64, TrajectoriesV[t,pidx] / facV )
            c2 = round( Int64, TrajectoriesL[t,pidx] / facL )
            c3 = round( Int64, TrajectoriesD[t,pidx] / facD )

            # If new position is out of the coordinate space, stop
            if c1 > dims[1] || c1 < 1 ||
               c2 > dims[2] || c2 < 1 ||
               c3 > dims[3] || c3 < 1
                TrajectoriesV[ t:end, pidx ] .= TrajectoriesV[ t-1, pidx ]
                TrajectoriesL[ t:end, pidx ] .= TrajectoriesL[ t-1, pidx ]
                TrajectoriesD[ t:end, pidx ] .= TrajectoriesD[ t-1, pidx ]
                break
            end

            # Using nextV and nextH to sample the translation from U and V
            Ut = facF * U[ c1, c2, c3, T0+t-1 ]
            Vt = facF * V[ c1, c2, c3, T0+t-1 ]
            Wt = facF * W[ c1, c2, c3, T0+t-1 ]

        end # Time-iteration

    end

    return TrajectoriesV, TrajectoriesL, TrajectoriesD
end


function Euler1(N::Int64)

	sum = 0;
	for i in 1:(N-1)
		if (mod(i,3)==0) || (mod(i,5)==0)
			sum+=i
			println(i)
		end
	end

	return sum
end

@time Euler1(1000)
println(Euler1(1000))


# Ring information and interpolation functions

export RingInfo

@doc raw"""
    RingInfo

Information about a ring of pixels, i.e., the set of pixels on a Healpix map
which have the same colatitude. The type is "mutable", so that one object can begin
reused many times without further memory allocations.

The list of fields defined in this structure is the following:

- `ring`: an integer index, running from

- `firstPixIdx`: index of the first pixel (using the `RING` scheme)
  belonging to this ring

- `numOfPixels`: number of consecutive pixels within the ring

- `colatitude_rad`: value of the colatitude for this ring (in radians)

- `shifted`: Boolean flag; it is `true` if the longitude of the first
  pixel in the ring is not zero.

# References

See also [`getringinfo!`](@ref) and [`getringinfo`](@ref).

# Example

```julia
import Healpix
res = Healpix.Resolution(256)

# Show information about ring #10
print(getringinfo(res, 10))
```
"""
mutable struct RingInfo
    ring::Int
    firstPixIdx::Int
    numOfPixels::Int
    colatitude_rad::Float64
    shifted::Bool
end

even(x) = (x & 1) == 0

@doc raw"""
    getringinfo!(resol::Resolution, ring, ringinfo::RingInfo; full=true) :: RingInfo

Fill the RingInfo structure with information about the specified ring.
If `full` is `false`, the field `colatitude_rad` (the most expensive in
terms of computation) is set to `NaN`.
"""
function getringinfo!(resol::Resolution, ring, ringinfo::RingInfo; full=true)
    # In the body of this code, firstidx is zero-based (we switch to 1-based
    # index just in the last statement)

    # "northring" is always in the Northern emisphere
    northring = (ring > resol.nsideTimesTwo) ? (resol.nsideTimesFour - ring) : ring
    if northring < resol.nside
        θ = if full
            tmp = northring^2 * resol.fact2
            atan(sqrt(tmp * (2 - tmp)), 1 - tmp)
        else
            NaN
        end
        numOfPixels = 4 * northring
        shifted = true
        firstPixIdx = 2 * northring * (northring - 1)
    else
        θ = full ? acos((resol.nsideTimesTwo - northring) * resol.fact1) : NaN
        numOfPixels = resol.nsideTimesFour
        shifted = even(northring - resol.nside)
        firstPixIdx = resol.ncap + (northring - resol.nside) * numOfPixels
    end

    if northring != ring
        # Southern emisphere
        full && (θ = π - θ)
        firstPixIdx = resol.numOfPixels - firstPixIdx - numOfPixels
    end

    ringinfo.ring = ring
    ringinfo.firstPixIdx = firstPixIdx + 1
    ringinfo.numOfPixels = numOfPixels
    ringinfo.colatitude_rad = θ
    ringinfo.shifted = shifted
end

@doc raw"""
    getringinfo(resol::Resolution, ring; kwargs...) :: RingInfo

Return a RingInfo structure containing information about
the specified ring. For the list of accepted keyword arguments,
see getringinfo!.
"""
function getringinfo(resol::Resolution, ring; full=true)
    ringinfo = RingInfo(0, 0, 0, 0.0, true)
    getringinfo!(resol, ring, ringinfo, full=full)
    ringinfo
end

function ringabove(resol::Resolution, z)
    az = abs(z)
    az <= 2 // 3 && return round(Int, resol.nside * (2 - 1.5 * z), RoundToZero)

    iring = round(Int, resol.nside * sqrt(3 * (1 - az)), RoundToZero)
    if z > 0
        iring
    else
        resol.nsideTimesFour - iring - 1
    end
end

function ring2idxw(ringinfo::RingInfo, ϕ)
    shift = ringinfo.shifted ? 0.5 : 0
    # Angle subtended by each pixel along this ring
    δϕ = 2π / ringinfo.numOfPixels
    tmp = ϕ / δϕ - shift
    i1 = (tmp < 0) ? round(Int, tmp, RoundToZero) - 1 : round(Int, tmp, RoundToZero)
    w1 = (ϕ - (i1 + shift) * δϕ) / δϕ
    i2 = i1 + 1
    i1 < 0 && (i1 += ringinfo.numOfPixels)
    i2 >= ringinfo.numOfPixels && (i2 -= ringinfo.numOfPixels)

    ([ringinfo.firstPixIdx + i1, ringinfo.firstPixIdx + i2], [1 - w1, w1])
end

@doc raw"""
    getinterpolRing(resol::Resolution, θ, ϕ) :: (Array{Int,1}, Array{Float64, 1})

Return the indices and the weights of the four neighbour pixels for the given
direction (θ, ϕ) in a map with the specified resolution.
"""
function getinterpolRing(resol::Resolution, θ, ϕ)
    @assert (0 ≤ θ ≤ π)

    z = cos(θ)
    ir1 = ringabove(resol, z)
    ir2 = ir1 + 1

    pix = Array{Int}(undef, 4) # Zero-based till the end
    wgt = Array{Float64}(undef, 4)
    if ir1 > 0
        ring1 = getringinfo(resol, ir1)
        pix1, w1 = ring2idxw(ring1, ϕ)
        pix[1:2] = pix1 .- 1 # Switch from 1-based to 0-based
        wgt[1:2] = w1
    end

    if ir2 < resol.nsideTimesFour
        ring2 = getringinfo(resol, ir2)
        pix2, w2 = ring2idxw(ring2, ϕ)
        pix[3:4] = pix2 .- 1 # Switch from 1-based to 0-based
        wgt[3:4] = w2
    end

    if ir1 == 0
        wθ = θ / ring2.colatitude_rad
        wgt[3:4] .*= wθ
        fac = (1 - wθ) / 4
        wgt[:] = [fac, fac, wgt[3] + fac, wgt[4] + fac]
        pix[1] = (pix[3] + 2) & 3
        pix[2] = (pix[4] + 2) & 3
    elseif ir2 == resol.nsideTimesFour
        wθ = (θ - ring1.colatitude_rad) / (π - ring1.colatitude_rad)
        wgt[1:2] .*= 1 - wθ
        fac = wθ / 4
        wgt[:] = [wgt[1] + fac, wgt[2] + fac, fac, fac]
        pix[3] = ((pix[1] + 2) & 3) + resol.numOfPixels - 4
        pix[4] = ((pix[2] + 2) & 3) + resol.numOfPixels - 4
    else
        wθ = (θ - ring1.colatitude_rad) / (ring2.colatitude_rad - ring1.colatitude_rad)
        wgt[1:2] .*= (1 - wθ)
        wgt[3:4] .*= wθ
    end

    (pix .+ 1, wgt)
end


using GalacticOptim, Optim, Test

rosenbrock(x, p) =  (p[1] - x[1])^2 + p[2] * (x[2] - x[1]^2)^2
x0 = zeros(2)
_p  = [1.0, 100.0]

l1 = rosenbrock(x0, _p)
prob = OptimizationProblem(rosenbrock, x0, p=_p)
sol = solve(prob, SimulatedAnnealing())
@test 10*sol.minimum < l1

prob = OptimizationProblem(rosenbrock, x0, p=_p, lb=[-1.0, -1.0], ub=[0.8, 0.8])
sol = solve(prob, SAMIN())
@test 10*sol.minimum < l1

rosenbrock(x, p=nothing) =  (1 - x[1])^2 + 100 * (x[2] - x[1]^2)^2

l1 = rosenbrock(x0)
prob = OptimizationProblem(rosenbrock, x0)
sol = solve(prob, NelderMead())
@test 10*sol.minimum < l1

optprob = OptimizationFunction(rosenbrock, x0, GalacticOptim.AutoZygote())

prob = OptimizationProblem(optprob, x0)
sol = solve(prob, BFGS())
@test 10*sol.minimum < l1

prob = OptimizationProblem(optprob, x0)
sol = solve(prob, Newton())
@test 10*sol.minimum < l1

prob = OptimizationProblem(optprob, x0)
sol = solve(prob, Optim.KrylovTrustRegion())
@test 10*sol.minimum < l1

prob = OptimizationProblem(optprob, x0, lb=[-1.0, -1.0], ub=[0.8, 0.8])
sol = solve(prob, Fminbox())
@test 10*sol.minimum < l1

prob = OptimizationProblem(optprob, x0, lb=[-1.0, -1.0], ub=[0.8, 0.8])
sol = solve(prob, SAMIN())
@test 10*sol.minimum < l1

using NLopt
prob = OptimizationProblem(optprob, x0)
sol = solve(prob, Opt(:LN_BOBYQA, 2))
@test 10*sol.minimum < l1

sol = solve(prob, Opt(:LD_LBFGS, 2))
@test 10*sol.minimum < l1

prob = OptimizationProblem(optprob, x0, lb=[-1.0, -1.0], ub=[0.8, 0.8])
sol = solve(prob, Opt(:G_MLSL_LDS, 2), nstart=5, local_method = Opt(:LD_LBFGS, 2))
@test 10*sol.minimum < l1

# using MultistartOptimization
# sol = solve(prob, MultistartOptimization.TikTak(100), local_method = NLopt.LD_LBFGS)
# @test 10*sol.minimum < l1

# using QuadDIRECT
# sol = solve(prob, QuadDirect(); splits = ([-0.5, 0.0, 0.5],[-0.5, 0.0, 0.5]))
# @test 10*sol.minimum < l1

using Evolutionary
sol = solve(prob, CMAES(μ =40 , λ = 100),abstol=1e-15)
@test 10*sol.minimum < l1

using BlackBoxOptim
prob = GalacticOptim.OptimizationProblem(optprob, x0, lb=[-1.0, -1.0], ub=[0.8, 0.8])
sol = solve(prob, BBO())
@test 10*sol.minimum < l1


"""
    γ = opnorm1(op)

Compute `γ`, the induced `1`-norm of the linear operator `op`, as the maximum of `1`-norm of the
columns of the associated `m x n` matrix ``M`` `= Matrix(op)`:
```math
\\gamma = \\|op\\|_1 := \\max_{1 ≤ j ≤ n} \\|M_j\\|_1
```
with ``M_j`` the `j`-th column of ``M``. This function is not recommended to be used for large order operators.

# Examples
```jldoctest
julia> A = [-6. -2. 1.; 5. 1. -1; -4. -2. -1.]
3×3 Array{Float64,2}:
 -6.0  -2.0   1.0
  5.0   1.0  -1.0
 -4.0  -2.0  -1.0

julia> opnorm1(lyapop(A))
30.0

julia> opnorm1(invlyapop(A))
3.7666666666666706
```
"""
function opnorm1(op::AbstractLinearOperator)
  (m, n) = size(op)
  T = eltype(op)
  Tnorm = real(T)
  Tsum = promote_type(Float64, Tnorm)
  nrm::Tsum = 0
  for j = 1 : n
      ej = zeros(Tsum, n)
      ej[j] = 1
      try
         nrm = max(nrm,norm(op*ej,1))
      catch err
         # if isnothing(findfirst("SingularException",string(err))) &&
         #    isnothing(findfirst("LAPACKException",string(err)))
         findfirst("SingularException",string(err)) === nothing &&
         findfirst("LAPACKException",string(err)) === nothing ? rethrow() : (return Inf)
      end
  end
  return convert(Tnorm, nrm)
end
"""
    γ = opnorm1est(op)

Compute `γ`, a lower bound of the `1`-norm of the square linear operator `op`, using
reverse communication based computations to evaluate `op * x` and `op' * x`.
It is expected that in most cases ``\\gamma > \\|op\\|_1/10``, which is usually
acceptable for estimating the condition numbers of linear operators.

# Examples
```jldoctest
julia> A = [-6. -2. 1.; 5. 1. -1; -4. -2. -1.]
3×3 Array{Float64,2}:
 -6.0  -2.0   1.0
  5.0   1.0  -1.0
 -4.0  -2.0  -1.0

julia> opnorm1est(lyapop(A))
18.0

julia> opnorm1est(invlyapop(A))
3.76666666666667
```
"""
function opnorm1est(op::AbstractLinearOperator)
  m, n = size(op)
  m == n || throw(DimensionMismatch("The operator op must be square"))
  T = promote_type(Float64,eltype(op))
  TR = real(T)
  TR == Float64 ? SMLNUM = reinterpret(Float64, 0x2000000000000000) : SMLNUM = reinterpret(Float32, 0x20000000)
  BIGNUM = 2*eps(TR) / SMLNUM
  cmplx = T<:Complex
  V = Array{T,1}(undef,n)
  X = Array{T,1}(undef,n)
  cmplx ? ISGN = Array{Int,1}(undef,1) : ISGN = Array{Int,1}(undef,n)
  ISAVE = Array{Int,1}(undef,3)
  ANORM = zero(TR)
  KASE = 0
  finish = false
  while !finish
     if cmplx
        ANORM, KASE = LapackUtil.lacn2!(V, X, ANORM, KASE, ISAVE )
     else
        ANORM, KASE = LapackUtil.lacn2!(V, X, ISGN, ANORM, KASE, ISAVE )
     end
     (isfinite(ANORM) && ANORM < BIGNUM) || (return Inf)
     if KASE != 0
        try
           KASE == 1 ? X = op*X : X = op'*X
        catch err
         #   if isnothing(findfirst("SingularException",string(err))) &&
         #      isnothing(findfirst("LAPACKException",string(err)))
           findfirst("SingularException",string(err)) === nothing &&
           findfirst("LAPACKException",string(err)) === nothing ? rethrow() : (return Inf)
        end
      else
        finish = true
     end
  end
  return ANORM
end

"""
    sep = opsepest(opinv; exact = false)

Compute `sep`, an estimation of the `1`-norm separation of a linear operator
`op`, where `opinv` is the inverse operator `inv(op)`. The estimate is computed as
``\\text{sep}  = 1 / \\|opinv\\|_1`` , using an estimate of the `1`-norm, if `exact = false`, or
the computed exact value of the `1`-norm, if `exact = true`.
The `exact = true` option is not recommended for large order operators.

The separation of the operator `op` is defined as
```math
\\text{sep} = \\displaystyle\\min_{X\\neq 0} \\frac{\\|op*X\\|}{\\|X\\|}.
```

An estimate of the reciprocal condition number of `op` can be computed as ``\\text{sep}/\\|op\\|_1``.

# Example
```jldoctest
julia> A = [-6. -2. 1.; 5. 1. -1; -4. -2. -1.]
3×3 Array{Float64,2}:
 -6.0  -2.0   1.0
  5.0   1.0  -1.0
 -4.0  -2.0  -1.0

julia> opsepest(invlyapop(A))
0.26548672566371656

julia> 1/opnorm1est(invlyapop(A))
0.26548672566371656

julia> opsepest(invlyapop(A),exact = true)
0.26548672566371656

julia> 1/opnorm1(invlyapop(A))
0.26548672566371656
```
"""
function opsepest(opinv::AbstractLinearOperator; exact = false)
   T = promote_type(Float64,eltype(opinv))
   TR = real(T)
   TR == Float64 ? SMLNUM = reinterpret(Float64, 0x2000000000000000) : SMLNUM = reinterpret(Float32, 0x20000000)
   BIGNUM = 2*eps(TR) / SMLNUM
   ZERO = zero(TR)
   exact ? opinvnrm1 = opnorm1(opinv) : opinvnrm1 = opnorm1est(opinv)
   if opinvnrm1 >= BIGNUM
      return ZERO
   end
   return one(TR)/opinvnrm1
end
"""
    rcond = oprcondest(op, opinv; exact = false)

Compute `rcond`, an estimation of the `1`-norm reciprocal condition number
of a linear operator `op`, where `opinv` is the inverse operator `inv(op)`. The estimate is computed as
``\\text{rcond} = 1 / (\\|op\\|_1\\|opinv\\|_1)``, using estimates of the `1`-norm, if `exact = false`, or
computed exact values of the `1`-norm, if `exact = true`.
The `exact = true` option is not recommended for large order operators.

_Note:_ No check is performed to verify that `opinv = inv(op)`.

# Examples
```jldoctest
julia> A = [-6. -2. 1.; 5. 1. -1; -4. -2. -1.]
3×3 Array{Float64,2}:
 -6.0  -2.0   1.0
  5.0   1.0  -1.0
 -4.0  -2.0  -1.0

julia> oprcondest(lyapop(A),invlyapop(A))
0.014749262536873142

julia> 1/opnorm1est(lyapop(A))/opnorm1est(invlyapop(A))
0.014749262536873142

julia> oprcondest(lyapop(A),invlyapop(A),exact = true)
0.008849557522123885

julia> 1/opnorm1(lyapop(A))/opnorm1(invlyapop(A))
0.008849557522123885
```
"""
function oprcondest(op:: LinearOperator, opinv::LinearOperator; exact = false)
   return opsepest(op, exact = exact)*opsepest(opinv, exact = exact)
end


### Forward-Backward Algorithm
function forward_backward(hmm::HMM,o::Vector; scaling=true)
	if scaling
		alpha, log_p_obs, coeff = forward(hmm,o; scaling=true)
		beta = backward(hmm,o; scale_coeff=coeff)
	else
		alpha, p_obs = forward(hmm,o; scaling=false)
		log_p_obs = log(p_obs)
	    beta = backward(hmm,o)
	end
	return alpha, beta, log_p_obs
end

function forward(hmm::HMM, o::Vector; scaling=true)
	n_obs = length(o)

	# alpha[t,i] = probability of being in state 'i' given o[1:t]
	alpha = zeros(n_obs, hmm.n)

	# base case (initialize at start)
	for i = 1:hmm.n
		alpha[1,i] = hmm.p[i] * pdf(hmm.B[i],o[1])
	end

	if scaling
		c = (Float64)[] # scaling coefficients
		push!(c,1./sum(alpha[1,:]))
		alpha[1,:] *= c[end]
	end

	# induction step
	for t = 2:n_obs
		for j = 1:hmm.n
			for i = 1:hmm.n
				alpha[t,j] += hmm.A[i,j] * alpha[t-1,i]
			end
			alpha[t,j] *= pdf(hmm.B[j],o[t])
		end
		if scaling
			push!(c,1./sum(alpha[t,:]))
			alpha[t,:] *= c[end]
		end
	end

	# Calculate likelihood (or log-likelihood) of observed sequence
	if scaling
		log_p_obs = -sum(log(c)) # see Rabiner (1989), eqn 103
		return (alpha,log_p_obs,c)
	else
		p_obs = sum(alpha[end,:])
		return (alpha,p_obs)
	end
end

function backward(hmm::HMM, o::Vector; scale_coeff=nothing)
	# scale_coeff are 1/sum(alpha[t,:]) calculated by forward algorithm
	n_obs = length(o)

	# beta[t,i] = probability of being in state 'i' and then obseverving o[t+1:end]
	beta = zeros(n_obs, hmm.n)

	# base case (initialize at end)
	if scale_coeff == nothing
		beta[end,:] += 1
	else
		if length(scale_coeff) != n_obs
			error("scale_coeff is improperly defined (wrong length)")
		end
		beta[end,:] += scale_coeff[end]
	end

	# induction step
	for t = reverse(1:n_obs-1)
		for i = 1:hmm.n
			for j = 1:hmm.n
				beta[t,i] += hmm.A[i,j] * pdf(hmm.B[j],o[t+1]) * beta[t+1,j]
			end
		end
		if scale_coeff != nothing
			beta[t,:] *= scale_coeff[t]
		end
	end

	return beta
end

### Viterbi

function viterbi(hmm::HMM, o::Vector)
	n_obs = length(o)

	# delta[i,j] = highest probability of state sequence ending in state j on step i
	# psi[i,j] = most likely state on step i-1 given state j on step i (argmax of deltas)
	delta = zeros(n_obs, hmm.n)
	psi = ones(Int, n_obs, hmm.n)

	# base case, psi[:,1] is ignored so don't initialize
	for i = 1:hmm.n
		delta[1,i] = hmm.p[i] .* pdf(hmm.B[i],o[1])
	end

	# induction step
	for t = 2:n_obs
		for j = 1:hmm.n
			delta[t,j],psi[t,j] = findmax(hmm.A[:,j].*delta[t-1,:]')
			delta[t,j] *= pdf(hmm.B[j],o[t])
		end
	end

	# backtrack to uncover the most likely path / state sequence
	q = zeros(Int,n_obs) # vector holding state sequence
	q[end] = indmax(delta[end,:])

	# backtrack recursively
	for t = reverse(1:n_obs-1)
		q[t] = psi[t+1,q[t+1]]
	end
	return q
end

### Smoothed state sequence based on forward-backward algorithm
function smoothed_forward_backward(hmm::HMM, o::Vector; scaling=true)
	log_p_obs,alpha,beta,x,g = calc_stats(hmm,o;scaling=scaling)
	return g,log_p_obs
end


# # titanic
# Titanic is one of many competitions available through the [Kaggle](https://www.kaggle.com/c/titanic) platform.
# Its goal is to train a classifier which recognizes people who survived the Titanic catastrophe based on some known characteristics, such as age, gender, etc.
# This package provides some possible methods which (at least up to some precision) solve this problem.
#
# ## Input data
# Before we describe the used methods, we need to prepare the data.

# First we need to load the package

using titanic

# Then we can load training data from provided datasets. (can be downloaded on [Kaggle](https://www.kaggle.com/c/titanic/data))

traindata, trainlabels = loadtrain("../data/train.csv")

# Note that this function call performs normalization on the data, so for some classificators later (adaboost), we may turn this procedure off with `normalize=false` keyword parameter.

# Now we do almost the same for testing data.
# Here, Kaggle does not provide labels for the testing dataset and the classificator predictions need to be uploaded to their webpage to receive evaluation.

testdata = loadtest("../data/test.csv")

# The datasets were loaded from `.csv` files into inner `Dataset` structures.
# These provide an useful abstraction, so one does not have to operate on matrices and vectors, but it usually suffices to pass these structures.
# As was said before, the constructor of this structure automacially normalizes the data so they are more suitable for most classifiers.
# Also, many of these methods need to add so-called 'bias' term into the feature vector.
# This is conducted in this structure as well by adding $1$ to the beginning of each point in the data matrix.
#
# The same can be said about `trainlabels`, which are stored inside `Labels` structure.
#
# Both of these structures can return their main field when called as functions (or functors).

traindata()
#-
trainlabels()

# ## Logistic regression
# As first and most basic classifier, we use Logistic regression.
#
# This classifier searches for a vector `w` which determines a hyperplane in the feature space that optimally separates the data points.
# However, the data are most likely not linearly separable, so there will be some some misclassified points.
#
# This search occurs in high-level function

w = regression(traindata, trainlabels; epsilon=1e-4, step=1.0)

# The keyword parameters `epsilon` and `step` are optional and default to values in the example.
# `epsilon` indicates precision for the underlying optimizer and `step` determines coefficient of initial step.
#
# As the method for finding the optimal `w` was implemented gradient descent with adaptive stepsize.
# In each iteration, the gradient descent computes so called *negative log-likelihood*
# $$ E(w) = \sum_{x}{\log(1 + e^{-kxw})} $$
# , where $k$ is the class of $x$, and its derivation
# $$ \frac{\partial E(w)}{\partial w} = - \sum_x\frac{kx}{1 + e^{kxw}} $$
# Using this derivation we find the w with minimal *negative log-likelihood* (or *cross entropy*).
#
# The adaptive step size behaves in a way that when the current step size finds better solution, we double the step in the next iteration.
# On the other hand, when the solution is worse (meaning it "jumped" over the optimum), we use half of it in next iteration (this iteration's result is not even used).
# The speed of convergence could be improved by using some more advanced step size adaptation, like the *Armijo conditions*, but the solution would be (almost) the same.

# When we obtain the optimal `w`, we are ready to clasify the test labels.
#
# Before that we can check classification error on the train dataset.

trainprediction = classify(traindata, w)
#-
using titanic: classificationerror
classificationerror(trainlabels, trainprediction)

# We can see that even on the training set, the error is not zero, so we can conclude that the data are indeed linearly unserparable.

# Then we can predict classes for the test dataset.
# For this we cannot get error so straightforwardly, because we have to upload it to Kaggle website.
# So, the package provides a function for exporting `Labels` into `.csv` file.

testprediction = classify(testdata, w)
#-
savepredictions("./logreg.csv", testprediction)

# From the website we obtain slightly worse error than on the training set, but this is expected and the difference is not that huge.
#
# | training error | test error |
# | :------------: | :--------: |
# | 0.19978        | 0.22967    |
#
# These results are not awesome, but for a basic linear classifier not that bad.

# ## Suppor Vector Machines
# Second, we showcase more complicated linear classifier and that is **SVM**.
# We implemented *soft-margin* variant with different kernel possibilities in function

# *Soft-margin* is used because the data are not linearly separable and thus we introduce penalty $C$ for misclassified points.
# $C$ is optional keyword parameter and defaults to $10$.

# The support vector machine in its base form again searches for some optimal vector `w` which defines the hyperplane.
# However, for optimization it is more convenient to optimize its dual problem
# \begin{align*}
# \alpha = \text{argmax}\quad&\sum_{i = 1}^{N}\alpha_i - \frac{1}{2}\sum_{i,j = 1}^{N} \alpha_i \alpha_j y_i y_j K(x_i, x_j) \\
# \text{subject to}\quad&\sum_{i=1}^N \alpha_i y_i = 0 \\
# & 0 \leq \alpha_i \leq C
# \end{align*}

alpha = svm(traindata, trainlabels; C=10.0)

# Kernels on the other hand are supposed to handle the non-separability as they modify the feature space and possibly making the data more separable.
# They usually take some form of a dot product, as in vanilla svm.
# More generally, each symetric positive semidefinite matrix is a kernel function.
#
# Used kernels:
# - Linear kernel: $$ K(x_i, x_j) = x_i \cdot x_j $$
# - Polynomial kernel of power $p$: $$ K(x_i, x_j) = (1 + x_i \cdot x_j)^p $$
# - Gaussian kernel with variance $\sigma^2$: $$ K(x_i, x_j) = e^{\frac{-||x_i - x_j||_2^2}{2\sigma^2}} $$

# Kernels can be changed with setting the kernel function to one of these functions:

titanic.kernel(xi::Vector{<:Real}, xj::Vector{<:Real}) = titanic.linearkernel(xi, xj)
linearalpha = svm(traindata, trainlabels)
#-
titanic.kernel(xi::Vector{<:Real}, xj::Vector{<:Real}) = titanic.polynomialkernel(xi, xj; degree=2)
polynomialalpha = svm(traindata, trainlabels)
#-
titanic.kernel(xi::Vector{<:Real}, xj::Vector{<:Real}) = titanic.gaussiankernel(xi, xj; variance=1.0)
gaussianalpha = svm(traindata, trainlabels)

# When we obtain alpha from the dual problem, the classification then goes as follows:
# $$ \text{class}(x) = \text{sign}(\sum_{i=1}^N \alpha_i y_i K(x_i, x)) $$

# Again, we can compute errors on the training set

titanic.kernel(xi::Vector{<:Real}, xj::Vector{<:Real}) = titanic.linearkernel(xi, xj)
classificationerror(trainlabels, classify(traindata, traindata, trainlabels, linearalpha))
#-
titanic.kernel(xi::Vector{<:Real}, xj::Vector{<:Real}) = titanic.polynomialkernel(xi, xj; degree=2)
classificationerror(trainlabels, classify(traindata, traindata, trainlabels, polynomialalpha))
#-
titanic.kernel(xi::Vector{<:Real}, xj::Vector{<:Real}) = titanic.gaussiankernel(xi, xj; variance=1.0)
classificationerror(trainlabels, classify(traindata, traindata, trainlabels, gaussianalpha))

# As in logistic regression, for getting test error on test data, we need to upload predictions to the Kaggle page.
titanic.kernel(xi::Vector{<:Real}, xj::Vector{<:Real}) = titanic.linearkernel(xi, xj)
linearpredictions = classify(testdata, traindata, trainlabels, linearalpha)
savepredictions("./linearsvm.csv", linearpredictions)
#-
titanic.kernel(xi::Vector{<:Real}, xj::Vector{<:Real}) = titanic.polynomialkernel(xi, xj; degree=2)
polynomialpredictions = classify(testdata, traindata, trainlabels, polynomialalpha)
savepredictions("./polynomialsvm.csv", polynomialpredictions)
#-
titanic.kernel(xi::Vector{<:Real}, xj::Vector{<:Real}) = titanic.gaussiankernel(xi, xj; variance=1.0)
gaussianpredictions = classify(testdata, traindata, trainlabels, gaussianalpha)
savepredictions("./gaussiansvm.csv", gaussianpredictions)

# Results on training and test data are summarized in table below
#
# | kernel     | training error | test error |
# | :--------- | :------------: | :--------: |
# | linear     | 0.21324        | 0.23445    |
# | polynomial | 0.16498        | 0.24642    |
# | gaussian   | 0.12233        | 0.23924    |
#
# Interestingly, we can observe that while polynomial and gaussian kernels have significantly smaller error on the training set, they perform worse than the linear kernel on the test set.

# ## Adaboost
#
# Last implemented classifier is adaboost.
# Adaboost is actually a multitude of *weak (simple) classifiers* united into one so-called *strong classifier*.
# Each of these weak classifiers classifies a point based on some criterion and returns its decision to the strong classifier, which decides based on weighted sum of all these decisions.
# Weights are computed from classification errors of the weak classifiers.
# Simultaneously, it assigns different weights to data samples based on how hard it is for the weak classifiers to classify them correctly.
# Also, thanks to partial errors of the weak classifiers we can compute upper bound of the classification error, which can be convenient.
#
# Pseudocode for adaboost follows
#
# 1. initialize data weights $$D_1(i) = \frac{1}{N} \qquad \forall i = 1, \dots, N$$

# 2. for t in $1,\dots,T$
#
#     3. find the weak classifier $h_t$ with lowest error $$ h_t = \text{argmin }\epsilon(h) \qquad \epsilon(h) = \sum_{i=1}^N D_t(i)\llbracket y_i \neq  h_t(x_i)\rrbracket $$
#     4. if $\epsilon(h_t) \geq 0.5$ then stop
#     5. compute classifer weight $$\alpha_t = \frac{1}{2}\log(\frac{1 - \epsilon_t}{\epsilon_t}) $$
#     6. adjust data weights $$D_{t+1}(i) = \frac{1}{Z_t}D_t(i) e^{-\alpha_t y_i h_t(x_i)} \quad \forall i = 1, \dots, N \qquad Z_t = \sum_{i=1}^N D_t(i) e^{-\alpha_t y_i h_t(x_i)} $$
#
# $Z_t$ normalizes the weights so they sum to $1$ and also is the already mentioned upper bound on classification error.

# This choice seemed sensible as the features used in data samples are usually not related and adaboost gives the option to classify based on some feature only.
# With this thought, we designed class of weak classifiers that selects one data feature and finds threshold that misclassifies the least amount of samples based only on this feature.
# For each pair feature and threshold there are two weak classifiers, one that classifies all featuers $<$ than threhsold as $1$ and others as $-1$, and second that classifies all features $\geq$ than threshold as $1$ and others as $-1$.

# As this classifier operates on individual features, the data normalization is not needed.
# Thus, the data can be loaded like this

traindata, trainlabels = loadtrain("../data/train.csv"; normalize=false)
#-
testdata = loadtest("../data/test.csv"; normalize=false)

# Training of our classifier is performed by

H = boost(traindata, trainlabels; limit=100)

# The keyword parameter `limit` corresponds to $T$ in the pseudocode and limits the number of weak classifiers constructing the strong one.

# Classification of a point then proceeds as follows
# $$ H(x) = \text{sign}(\sum_{t=1}^T \alpha_t h_t(x)) $$
# Again, we compare errors on train and test datasets.

trainprediction = classify(traindata, H)
classificationerror(trainlabels, trainprediction)
#-
testprediction = classify(testdata, H)
savepredictions("./adaboost.csv", testprediction)

# Again, the training error is quite low, but the test error is very similar to results from other classifiers.
# It is surprising, that it did not work as expected.
# It is possible, that some features are present more in the training set and are more discriminative than in the test set.
# Perhaps, some experiments with what features use to decide would find an answer to this possibility and the algorithm could be improved.
#
# | training error | test error |
# | :------------: | :--------: |
# | 0.16835        | 0.23924    |

# The implementation of the algorithm is sped up by presorting feature spaces, so it is not necessary to iterate over all values in range, but only until the error starts increasing again
# If this feature was removed, we would get the same results but it would be a lot slower.
#
# Another speedup could be achieved by decreasing the `limit`.
# When we reduce it to $10$, the learning is approximately $10$ times faster, but the error on training set increased only by $0.2$.
# If speed was prefered to accuracy, this is another area to take advantage of.

# ## Conclusion
#
# | method              | training error | test error |
# | :------------------ | :------------: | :--------: |
# | logistic regression | 0.19978        | 0.22967    |
# | kernel svm          | 0.21324        | 0.23445    |
# | adaboost            | 0.16835        | 0.23924    |
#
# From the results we can see, that logistic regression is the best of the algorithms on test dataset, even though it does not have the lowest error on training data.
# On the other hand, adaboost has the lowest training error, but is the worst on testing data.
# This could mean, that it is overfitted or that the test data are more different from the training set than expected.
# The kernel svm generalizes quite well as the training error and test error are similar.
# It is disappointing that the kernels with lower training error performed worse on the test data.
# Possibly, it could be improved by finding better exponent or variance.
#
# In terms of speed, the worst is probably adaboost, because it does not have an easily optimizable form as gradient descent or quadratic optimization problem.
# We need to search for the best classifier over all features and that makes it slower.
#
# In conclusion, the results cannot be considered as a success as all the methods classified with quite high errors on both training and testing datasets.
# Some more powerful classifiers, such as neural nets, would probably work better or some smarter data preprocessing could be investigated.
# For example, some features could be omitted as unimportant and better results could be achieved.