### Linear Interpolation
struct LinearInterpolation{uType,tType,FT,T} <: AbstractInterpolation{FT,T}
  u::uType
  t::tType
  LinearInterpolation{FT}(u,t) where FT = new{typeof(u),typeof(t),FT,eltype(u)}(u,t)
end

function LinearInterpolation(u,t)
  u, t = munge_data(u, t)
  LinearInterpolation{true}(u,t)
end

### Quadratic Interpolation
struct QuadraticInterpolation{uType,tType,FT,T} <: AbstractInterpolation{FT,T}
  u::uType
  t::tType
  QuadraticInterpolation{FT}(u,t) where FT = new{typeof(u),typeof(t),FT,eltype(u)}(u,t)
end

function QuadraticInterpolation(u,t)
  u, t = munge_data(u, t)
  QuadraticInterpolation{true}(u,t)
end

### Lagrange Interpolation
struct LagrangeInterpolation{uType,tType,FT,T} <: AbstractInterpolation{FT,T}
  u::uType
  t::tType
  n::Int
  LagrangeInterpolation{FT}(u,t,n) where FT = new{typeof(u),typeof(t),FT,eltype(u)}(u,t,n)
end

function LagrangeInterpolation(u,t,n=nothing)
  u, t = munge_data(u, t)
  if n == nothing
    n = length(t) - 1 # degree
  end
  if n != length(t) - 1
    error("Currently only n=length(t) - 1 is supported")
  end
  LagrangeInterpolation{true}(u,t,n)
end

### Akima Interpolation
struct AkimaInterpolation{uType,tType,bType,cType,dType,FT,T} <: AbstractInterpolation{FT,T}
  u::uType
  t::tType
  b::bType
  c::cType
  d::dType
  AkimaInterpolation{FT}(u,t,b,c,d) where FT = new{typeof(u),typeof(t),typeof(b),typeof(c),
                                                     typeof(d),FT,eltype(u)}(u,t,b,c,d)
end

function AkimaInterpolation(u, t)
  u, t = munge_data(u, t)
  n = length(t)
  dt = diff(t)
  m = Array{eltype(u)}(undef, n + 3)
  m[3:end-2] = diff(u) ./ dt
  m[2] = 2m[3] - m[4]
  m[1] = 2m[2] - m[3]
  m[end-1] = 2m[end-2] - m[end-3]
  m[end] = 2m[end-1] - m[end-2]

  b = 0.5 .* (m[4:end] .+ m[1:end-3])
  dm = abs.(diff(m))
  f1 = dm[3:n+2]
  f2 = dm[1:n]
  f12 = f1 + f2
  ind = findall(f12 .> 1e-9 * maximum(f12))
  b[ind] = (f1[ind] .* m[ind .+ 1] .+
            f2[ind] .* m[ind .+ 2]) ./ f12[ind]
  c = (3.0 .* m[3:end-2] .- 2.0 .* b[1:end-1] .- b[2:end]) ./ dt
  d = (b[1:end-1] .+ b[2:end] .- 2.0 .* m[3:end-2]) ./ dt.^2

  AkimaInterpolation{true}(u,t,b,c,d)
end

### ConstantInterpolation Interpolation
struct ConstantInterpolation{uType,tType,dirType,FT,T} <: AbstractInterpolation{FT,T}
  u::uType
  t::tType
  dir::Symbol # indicates if value to the $dir should be used for the interpolation
  ConstantInterpolation{FT}(u,t,dir) where FT = new{typeof(u),typeof(t),typeof(dir),FT,eltype(u)}(u,t,dir)
end

function ConstantInterpolation(u,t;dir=:left)
  u, t = munge_data(u, t)
  ConstantInterpolation{true}(u,t,dir)
end

Base.@deprecate_binding ZeroSpline ConstantInterpolation

### QuadraticSpline Interpolation
struct QuadraticSpline{uType,tType,tAType,dType,zType,FT,T} <: AbstractInterpolation{FT,T}
  u::uType
  t::tType
  tA::tAType
  d::dType
  z::zType
  QuadraticSpline{FT}(u,t,tA,d,z) where FT = new{typeof(u),typeof(t),typeof(tA),
                                                  typeof(d),typeof(z),FT,eltype(u)}(u,t,tA,d,z)
end

function QuadraticSpline(u,t)
  u, t = munge_data(u, t)
  s = length(t)
  dl = ones(eltype(t),s-1)
  d_tmp = ones(eltype(t),s)
  du = zeros(eltype(t),s-1)
  tA = Tridiagonal(dl,d_tmp,du)
  d = map(i -> i == 1 ? 0 : 2//1 * (u[i] - u[i-1])/(t[i] - t[i-1]), 1:s)
  z = tA\d
  QuadraticSpline{true}(u,t,tA,d,z)
end

# Cubic Spline Interpolation
struct CubicSpline{uType,tType,hType,zType,FT,T} <: AbstractInterpolation{FT,T}
  u::uType
  t::tType
  h::hType
  z::zType
  CubicSpline{FT}(u,t,h,z) where FT = new{typeof(u),typeof(t),typeof(h),typeof(z),FT,eltype(u)}(u,t,h,z)
end

function CubicSpline(u,t)
  u, t = munge_data(u, t)
  n = length(t) - 1
  h = vcat(0, map(k -> t[k+1] - t[k], 1:length(t)-1), 0)
  dl = h[2:n+1]
  d_tmp = 2 .* (h[1:n+1] .+ h[2:n+2])
  du = h[2:n+1]
  tA = LinearAlgebra.Tridiagonal(dl,d_tmp,du)
  d = map(i -> i == 1 || i == n + 1 ? 0 : 6(u[i+1] - u[i]) / h[i+1] - 6(u[i] - u[i-1]) / h[i], 1:n+1)
  z = tA\d
  CubicSpline{true}(u,t,h[1:n+1],z)
end

### BSpline Curve Interpolation
struct BSplineInterpolation{uType,tType,pType,kType,cType,FT,T} <: AbstractInterpolation{FT,T}
  u::uType
  t::tType
  d::Int    # degree
  p::pType  # params vector
  k::kType  # knot vector
  c::cType  # control points
  pVecType::Symbol
  knotVecType::Symbol
  BSplineInterpolation{FT}(u,t,d,p,k,c,pVecType,knotVecType) where FT =  new{typeof(u),typeof(t),typeof(p),typeof(k),typeof(c),FT,eltype(u)}(u,t,d,p,k,c,pVecType,knotVecType)
end

function BSplineInterpolation(u,t,d,pVecType,knotVecType)
  u, t = munge_data(u, t)
  n = length(t)
  s = zero(eltype(u))
  p = zero(t)
  k = zeros(eltype(t),n+d+1)
  l = zeros(eltype(u),n-1)
  p[1] = zero(eltype(t))
  p[end] = one(eltype(t))

  for i = 2:n
    s += √((t[i] - t[i-1])^2 + (u[i] - u[i-1])^2)
    l[i-1] = s
  end
  if pVecType == :Uniform
    for i = 2:(n-1)
      p[i] = p[1] + (i-1)*(p[end]-p[1])/(n-1)
    end
  elseif pVecType == :ArcLen
    for i = 2:(n-1)
      p[i] = p[1] + l[i-1]/s * (p[end]-p[1])
    end
  end

  lidx = 1
  ridx = length(k)
  while lidx <= (d+1) && ridx >= (length(k)-d)
    k[lidx] = p[1]
    k[ridx] = p[end]
    lidx += 1
    ridx -= 1
  end

  ps = zeros(eltype(t),n-2)
  s = zero(eltype(t))
  for i = 2:n-1
    s += p[i]
    ps[i-1] = s
  end

  if knotVecType == :Uniform
    # uniformly spaced knot vector
    # this method is not recommended because, if it is used with the chord length method for global interpolation,
    # the system of linear equations would be singular.
    for i = (d+2):n
      k[i] = k[1] + (i-d-1)//(n-d) * (k[end]-k[1])
    end
  elseif knotVecType == :Average
    # average spaced knot vector
    idx = 1
    if d+2 <= n
      k[d+2] = 1//d * ps[d]
    end
    for i = (d+3):n
      k[i] = 1//d * (ps[idx+d] - ps[idx])
      idx += 1
    end
  end
  # control points
  N = spline_coefficients(n,d,k,p)
  c = vec(N\u[:,:])
  BSplineInterpolation{true}(u,t,d,p,k,c,pVecType,knotVecType)
end

### BSpline Curve Approx
struct BSplineApprox{uType,tType,pType,kType,cType,FT,T} <: AbstractInterpolation{FT,T}
  u::uType
  t::tType
  d::Int    # degree
  h::Int    # number of control points (n => h >= d >= 1)
  p::pType  # params vector
  k::kType  # knot vector
  c::cType  # control points
  pVecType::Symbol
  knotVecType::Symbol
  BSplineApprox{FT}(u,t,d,h,p,k,c,pVecType,knotVecType) where FT =  new{typeof(u),typeof(t),typeof(p),typeof(k),typeof(c),FT,eltype(u)}(u,t,d,h,p,k,c,pVecType,knotVecType)
end

function BSplineApprox(u,t,d,h,pVecType,knotVecType)
  u, t = munge_data(u, t)
  n = length(t)
  s = zero(eltype(u))
  p = zero(t)
  k = zeros(eltype(t),h+d+1)
  l = zeros(eltype(u),n-1)
  p[1] = zero(eltype(t))
  p[end] = one(eltype(t))

  for i = 2:n
    s += √((t[i] - t[i-1])^2 + (u[i] - u[i-1])^2)
    l[i-1] = s
  end
  if pVecType == :Uniform
    for i = 2:(n-1)
      p[i] = p[1] + (i-1)*(p[end]-p[1])/(n-1)
    end
  elseif pVecType == :ArcLen
    for i = 2:(n-1)
      p[i] = p[1] + l[i-1]/s * (p[end]-p[1])
    end
  end

  lidx = 1
  ridx = length(k)
  while lidx <= (d+1) && ridx >= (length(k)-d)
    k[lidx] = p[1]
    k[ridx] = p[end]
    lidx += 1
    ridx -= 1
  end

  ps = zeros(eltype(t),n-2)
  s = zero(eltype(t))
  for i = 2:n-1
    s += p[i]
    ps[i-1] = s
  end

  if knotVecType == :Uniform
    # uniformly spaced knot vector
    # this method is not recommended because, if it is used with the chord length method for global interpolation,
    # the system of linear equations would be singular.
    for i = (d+2):h
      k[i] = k[1] + (i-d-1)//(h-d) * (k[end]-k[1])
    end
  elseif knotVecType == :Average
    # NOTE: verify that average method can be applied when size of k is less than size of p
    # average spaced knot vector
    idx = 1
    if d+2 <= h
      k[d+2] = 1//d * ps[d]
    end
    for i = (d+3):h
      k[i] = 1//d * (ps[idx+d] - ps[idx])
      idx += 1
    end
  end
  # control points
  c = zeros(eltype(u), h)
  c[1] = u[1]
  c[end] = u[end]
  q = zeros(eltype(u), n)
  N = zeros(eltype(t), n, h)
  for i = 1:n
    N[i, :] .= spline_coefficients(h,d,k,p[i])
  end
  for k = 2:n-1
    q[k] = u[k] - N[k, 1]*u[1] - N[k, h]*u[end]
  end
  Q = Matrix{eltype(u)}(undef, h-2, 1)
  for i = 2:h-1
    s = 0.0
    for k = 2:n-1
      s += N[k, i]*q[k]
    end
    Q[i-1] = s
  end
  N = N[2:end-1,2:h-1]
  M = transpose(N) * N
  P = M\Q
  c[2:end-1] .= vec(P)
  BSplineApprox{true}(u,t,d,h,p,k,c,pVecType,knotVecType)
end

### Curvefit
struct CurvefitCache{uType,tType,mType,p0Type,ubType,lbType,algType,pminType,FT,T} <: AbstractInterpolation{FT,T}
  u::uType
  t::tType
  m::mType        # model type
  p0::p0Type      # intial params
  ub::ubType      # upper bound of params
  lb::lbType      # lower bound of params
  alg::algType    # alg to optimize cost function
  pmin::pminType  # optimized params
  CurvefitCache{FT}(u,t,m,p0,ub,lb,alg,pmin) where FT = new{typeof(u),typeof(t),typeof(m),
                                      typeof(p0),typeof(ub),typeof(lb),
                                      typeof(alg),typeof(pmin),FT,eltype(u)}(u,t,m,p0,ub,lb,alg,pmin)
end

function Curvefit(u, t, model, p0, alg, box=false, lb=nothing, ub=nothing)
  u, t = munge_data(u, t)
  errfun(t,u,p) = sum(abs2.(u .- model(t,p)))
  if box == false
      mfit = optimize(p->errfun(t,u,p), p0, alg)
  else
      if lb == nothing || ub == nothing
          error("lower or upper bound should not be nothing")
      end
      od = OnceDifferentiable(p->errfun(t,u,p), p0, autodiff=:finite)
      mfit = optimize(od, lb, ub, p0, Fminbox(alg))
  end
  pmin = Optim.minimizer(mfit)
  CurvefitCache{true}(u,t,model,p0,ub,lb,alg,pmin)
end


using LibCEED, Printf

include("common.jl")

function transform_mesh_coords!(dim, mesh_size, mesh_coords)
    @witharray coords = mesh_coords begin
        if dim == 1
            for i = 1:mesh_size
                # map [0,1] to [0,1] varying the mesh density
                coords[i] = 0.5 + 1.0/sqrt(3.0)*sin((2.0/3.0)*pi*(coords[i] - 0.5))
            end
            exact_volume = 1.0
        else
            num_nodes = mesh_size÷dim
            @inbounds @simd for i = 1:num_nodes
                # map (x,y) from [0,1]x[0,1] to the quarter annulus with polar
                # coordinates, (r,phi) in [1,2]x[0,pi/2] with area = 3/4*pi
                u = coords[i]
                v = coords[i+num_nodes]
                u = 1.0 + u
                v = pi/2*v
                coords[i] = u*cos(v)
                coords[i+num_nodes] = u*sin(v)
            end
            exact_volume = 3.0/4.0*pi
        end
        return exact_volume
    end
end

function run_ex1(; ceed_spec, dim, mesh_order, sol_order, num_qpts, prob_size, gallery)
    ncompx = dim
    prob_size < 0 && (prob_size = 256*1024)

    ceed = Ceed(ceed_spec)
    mesh_basis =
        create_tensor_h1_lagrange_basis(ceed, dim, ncompx, mesh_order + 1, num_qpts, GAUSS)
    sol_basis =
        create_tensor_h1_lagrange_basis(ceed, dim, 1, sol_order + 1, num_qpts, GAUSS)

    # Determine the mesh size based on the given approximate problem size.
    nxyz = get_cartesian_mesh_size(dim, sol_order, prob_size)
    println("Mesh size: ", nxyz)

    # Build CeedElemRestriction objects describing the mesh and solution discrete
    # representations.
    mesh_size, mesh_restr, _ =
        build_cartesian_restriction(ceed, dim, nxyz, mesh_order, ncompx, num_qpts)
    sol_size, sol_restr, sol_restr_i = build_cartesian_restriction(
        ceed,
        dim,
        nxyz,
        sol_order,
        1,
        num_qpts,
        mode=RestrictionAndStrided,
    )
    println("Number of mesh nodes     : ", div(mesh_size, dim))
    println("Number of solution nodes : ", sol_size)

    # Create a CeedVector with the mesh coordinates.
    mesh_coords = CeedVector(ceed, mesh_size)
    set_cartesian_mesh_coords!(dim, nxyz, mesh_order, mesh_coords)
    # Apply a transformation to the mesh.
    exact_vol = transform_mesh_coords!(dim, mesh_size, mesh_coords)

    # Create the Q-function that builds the mass operator (i.e. computes its
    # quadrature data) and set its context data.
    if !gallery
        @interior_qf build_qfunc = (
            ceed,
            dim=dim,
            (J, :in, EVAL_GRAD, dim, dim),
            (w, :in, EVAL_WEIGHT),
            (qdata, :out, EVAL_NONE),
            begin
                qdata .= w*det(J)
            end,
        )
    else
        build_qfunc = create_interior_qfunction(ceed, "Mass$(dim)DBuild")
    end

    # Create the operator that builds the quadrature data for the mass operator.
    build_oper = Operator(
        ceed,
        qf=build_qfunc,
        fields=[
            (gallery ? :dx : :J, mesh_restr, mesh_basis, CeedVectorActive()),
            (gallery ? :weights : :w, ElemRestrictionNone(), mesh_basis, CeedVectorNone()),
            (:qdata, sol_restr_i, BasisCollocated(), CeedVectorActive()),
        ],
    )

    # Compute the quadrature data for the mass operator.
    elem_qpts = num_qpts^dim
    num_elem = prod(nxyz)
    qdata = CeedVector(ceed, num_elem*elem_qpts)

    print("Computing the quadrature data for the mass operator ...")
    flush(stdout)
    apply!(build_oper, mesh_coords, qdata)
    println(" done.")

    # Create the Q-function that defines the action of the mass operator.
    if !gallery
        @interior_qf apply_qfunc = (
            ceed,
            (u, :in, EVAL_INTERP),
            (qdata, :in, EVAL_NONE),
            (v, :out, EVAL_INTERP),
            begin
                v .= qdata*u
            end,
        )
    else
        apply_qfunc = create_interior_qfunction(ceed, "MassApply")
    end

    # Create the mass operator.
    oper = Operator(
        ceed,
        qf=apply_qfunc,
        fields=[
            (:u, sol_restr, sol_basis, CeedVectorActive()),
            (:qdata, sol_restr_i, BasisCollocated(), qdata),
            (:v, sol_restr, sol_basis, CeedVectorActive()),
        ],
    )

    # Compute the mesh volume using the mass operator: vol = 1^T \cdot M \cdot 1
    print("Computing the mesh volume using the formula: vol = 1^T.M.1 ...")
    flush(stdout)
    # Create auxiliary solution-size vectors.
    u = CeedVector(ceed, sol_size)
    v = CeedVector(ceed, sol_size)
    # Initialize 'u' with ones.
    u[] = 1.0
    # Apply the mass operator: 'u' -> 'v'.
    apply!(oper, u, v)
    # Compute and print the sum of the entries of 'v' giving the mesh volume.
    vol = witharray_read(sum, v, MEM_HOST)

    println(" done.")
    @printf("Exact mesh volume    : % .14g\n", exact_vol)
    @printf("Computed mesh volume : % .14g\n", vol)
    @printf("Volume error         : % .14g\n", vol - exact_vol)
end

run_ex1(
    ceed_spec="/cpu/self",
    dim=3,
    mesh_order=4,
    sol_order=4,
    num_qpts=4 + 2,
    prob_size=-1,
    gallery=false,
)


# marginalize:
## Normal (Distribution), NormalSample -> Normal distribution
## Normal (Distribution), DiscretizedNormalSample -> MCEBHistogram ...(?)
## PriorConvexClass, DiscretizedNormalSample -> ...

function marginalize(prior::Normal, Z::EBayes.AbstractNormalSample)
    prior_var = var(prior)
    prior_μ = mean(prior)
    likelihood_var = var(Z)
    marginal_σ = sqrt(likelihood_var + prior_var)
    Normal(prior_μ, marginal_σ)
end

function marginalize(prior::MixtureModel, Z::EBayes.AbstractNormalSample)
    prior_probs = probs(prior)
    marginal_components = marginalize.(components(prior), Ref(Z))
    MixtureModel(marginal_components, prior_probs)
end

const NormalOrNormalMixture = Union{Normal,
    MixtureModel{Univariate, Continuous, Normal}}

function marginalize(prior::NormalOrNormalMixture, Z::DiscretizedStandardNormalSamples)
    marginal_normal = marginalize(prior, StandardNormalSample(NaN))

    grid = Z.mhist.grid
    hist = deepcopy(Z.mhist.hist)
    hist = @set hist.weights = zeros(Float64, length(hist.weights))

    hist.weights[1] = cdf(marginal_normal,  first(grid))
    hist.weights[end] = ccdf(marginal_normal, last(grid))

    for i=2:length(grid)
        hist.weights[i] = cdf(marginal_normal, grid[i]) -  cdf(marginal_normal, grid[i-1])
        #TODO: add sanity check that this is approx. h*density
    end

    MCEBHistogram(grid, hist)
end

# Normal: MarginalDensityTarget

function (target::MarginalDensityTarget{<:StandardNormalSample})(prior::Normal)
    x = response(location(target)) #ok this notation is not nice...
    pdf(marginalize(prior, StandardNormalSample(0.0)), x)
end

## Posterior of Normal DBN

function _normal_normal_posterior(prior::Normal, Z::EBayes.AbstractNormalSample)
    x = response(Z)
    sigma_squared = var(Z)
    prior_mu = mean(prior)
    prior_A = var(prior)

    post_mean = x*(prior_A)/(prior_A + sigma_squared) + sigma_squared/(prior_A + sigma_squared)*prior_mu
    post_var = prior_A * sigma_squared / (prior_A + sigma_squared)
    Normal(post_mean, sqrt(post_var))
end

# Normal: PosteriorMeanTarget

function (target::MinimaxCalibratedEBayes.PosteriorMeanNumerator{<:StandardNormalSample})(prior::Normal)
     Z = location(target)
     _post = _normal_normal_posterior(prior, Z)
     ratio = mean(_post)
     denom = MarginalDensityTarget(Z)(prior)
     ratio*denom
end

# Normal: LocalFalseSignRate

function (target::MinimaxCalibratedEBayes.LFSRNumerator{<:StandardNormalSample})(prior::Normal)
     Z = location(target)
     _post = _normal_normal_posterior(prior, Z)
     ratio = ccdf(_post, 0)
     denom = MarginalDensityTarget(Z)(prior)
     ratio*denom
end

#function (target::PosteriorMeanNumerator{<:StandardNormalSample})(prior::Normal)
#    x = response(location(target)) #ok this notation is not nice...
#    pdf(marginalize(prior, StandardNormalSample(0.0)), x)
#end

#function cf(target::MarginalDensityTarget{<:StandardNormalSample}, t)
#    error_dbn = Normal(response(location(target))) #TODO...
#    cf(error_dbn, t)
#end

#function riesz_representer(target::MarginalDensityTarget{<:StandardNormalSample}, t)
#    error_dbn = Normal(response(location(target))) #TODO...
#    pdf(error_dbn, t)
#end

# Normal: LocalFalseSignRate

#σ = std(dist)
#μ = mean(dist)
#(isfinite(σ) && isfinite(μ)) || throw(ArgumentError("Parameters σ, μ must be finite."))
#gh = gausshermite(n)
#nodes = gh[1].*(sqrt(2)*(σ)) .+ μ
#weights = gh[2]./sqrt(pi)

# TODO:

 # Type MarginalDistribution()...


#=
Example 2: LQ Ramsey model with discrete exogenous process.

@author : Spencer Lyon <spencer.lyon@nyu.edu>

@date: 2014-08-21

References
----------

Simple port of the file examples/lqramsey_discrete.py

http://quant-econ.net/lqramsey.html

=#
include("lqramsey.jl")

# Parameters
bet = 1 / 1.05
P = [0.8 0.2 0.0
     0.0 0.5 0.5
     0.0 0.0 1.0]

# Possible states of the world
# Each column is a state of the world. The rows are [g d b s 1]
x_vals = [0.5 0.5 0.25
          0.0 0.0 0.0
          2.2 2.2 2.2
          0.0 0.0 0.0
          1.0 1.0 1.0]
Sg = [1.0 0.0 0.0 0.0 0.0]
Sd = [0.0 1.0 0.0 0.0 0.0]
Sb = [0.0 0.0 1.0 0.0 0.0]
Ss = [0.0 0.0 0.0 1.0 0.0]
discrete = true
proc = DiscreteStochProcess(P, x_vals)

econ = Economy(bet, Sg, Sd, Sb, Ss, discrete, proc)
T = 15

path = compute_paths(econ, T)

gen_fig_1(path)


"""
Li, D., Wang, Z., Cao, C. and Liu, Y., 2018.
Information entropy based sample reduction for support vector data description.
Applied Soft Computing, 71, pp.1153-1160.

Information Entropy based Sample Reduction for Support Vector Data Description (IESRSVDD)Parameter guideline from the paper, given a data set of size N:
eps (on benchmark data eps ∈ [0.5, 0.6, 0.7])

Simplification of equations:

Dis_ij = K(x_i * x_i) + K(x_j * x_j) - 2 K(x_i * x_j) (Eq. 10)
       = 2 (1 - K(x_i * x_j)) (with RBF Kernel identity == 1)

p_ij = dis_ij / sum(dis_ik for k in 1:n) (Eq. 11)
     = 2 (1 - K(x_i * x_j)) / sum(2 (1 - K(x_i * x_k)) for k in 1:n)
     = (1 - K(x_i * x_j)) / sum(1 - K(x_i * x_k) for k in 1:n)

p_ii = 1

H_i = - sum(p_ik log_2(p_ik) for k in 1:n) (Eq. 12)
    = - sum(p_ik log_2(p_ik) for k in 1:n if k != i) (p_ii == 1 and log_2(1) = 0)
"""
mutable struct IESRSVDD <: Sampler
    kde_cache::Union{KDECache, Nothing}
    gamma::Union{Symbol, Real}
    eps::Real
    function IESRSVDD(kde_cache::Union{KDECache, Nothing}, gamma::Union{Symbol, Real}, eps::Real=0.5)
        (eps >= 0 && eps <= 1) || throw(ArgumentError("Invalid threshold $(eps)."))
        return new(kde_cache, gamma, eps)
    end
end

function IESRSVDD(kde_cache::KDECache, eps::Real=0.5)
    return IESRSVDD(kde_cache, :precomputed, eps)
end

function IESRSVDD(gamma::Union{Symbol, Real}, eps::Real=0.5)
    return IESRSVDD(nothing, gamma, eps)
end

function sample(sampler::IESRSVDD, data::DataSet, labels::Vector{Symbol})::BitArray{1}
    if sampler.kde_cache === nothing || sampler.kde_cache.data != data
        sampler.kde_cache = KDECache(data, sampler.gamma)
    end
    dis_ij = 1 .- sampler.kde_cache.K
    p_ij = dis_ij ./ sum(dis_ij, dims=1)
    p_ij[diagind(p_ij)] .= 1
    h_i = [-sum(p_ij[i, k] * log(2, p_ij[i, k]) for k in 1:size(p_ij, 2)) for i in 1:size(p_ij, 2)]
    h_i_min, h_i_max = extrema(h_i)
    θ = h_i_max - sampler.eps * (h_i_max - h_i_min)
    sample_mask = h_i .> θ
    return sample_mask
end


module MapTransform

    mutable struct Relation

        lat      :: AbstractArray{Float64, 2}
        area     :: AbstractArray{Float64, 2}
        mask     :: AbstractArray{Float64, 2}

        lat_bnd  :: AbstractArray{Float64, 1}
        ∂a       :: AbstractArray{Float64, 1}
        A        :: Float64
        indices  :: Any

        function Relation(;
            lat     :: AbstractArray{Float64, 2},
            area    :: AbstractArray{Float64, 2},
            mask    :: AbstractArray{Float64, 2},
            lat_bnd :: AbstractArray{Float64, 1},
        )
            Δlat = lat_bnd[2:end] - lat_bnd[1:end-1]

            if any(Δlat .<= 0)
                throw(ErrorException("lat_bnd not monotonically increasing."))
            end

            indices = []
            mask_is_one = mask .== 1.0
            push!(indices, (lat_bnd[1] .<= lat .<= lat_bnd[2]) .& mask_is_one)
            for i = 2:length(lat_bnd)-1
                push!(indices, (lat_bnd[i] .< lat .<= lat_bnd[i+1]) .& mask_is_one )
            end

            ∂a = zeros(Float64, length(lat_bnd)-1)
            for i = 1:length(∂a)
                ∂a[i] = sum(area[indices[i]])
            end

            area_chk = sum(area[mask.==1.0])
            if abs((sum(∂a) - area_chk) / area_chk) > 1e-10
                print(abs((sum(∂a))), "; area_chk=", area_chk)
                throw(ErrorException("Sum area not equal."))
            end

            return new(
                lat,
                area,
                mask,
                lat_bnd,
                ∂a,
                sum(∂a),
                indices,
            )

        end

    end

    function f∂a(
        r :: Relation,
        f :: AbstractArray{Float64, 2},
    )
        f∂a = zeros(Float64, length(r.lat_bnd)-1)
        for i = 1:length(r.∂a)
            index = r.indices[i]
            f∂a[i] = sum(r.area[index] .* f[index])
        end

        return f∂a
    end

    function ∫∂a(
        r :: Relation,
        f :: AbstractArray{Float64, 2},
    )
        _∫f∂a = zeros(Float64, length(r.lat_bnd))
        _f∂a = f∂a(r, f)

        for i = 1:length(_f∂a)
            _∫f∂a[i+1] = _∫f∂a[i] + _f∂a[i]
        end

        return _∫f∂a

    end

    function transform(
        r :: Relation,
        f :: AbstractArray{Float64, 2},
    )
        _f∂a = f∂a(r, f)

        return _f∂a ./ r.∂a

    end

    function mean(
        r :: Relation,
        f :: AbstractArray{Float64, 2},
    )
        return ∫∂a(r, f)[end] / r.A
    end
end


function kron_δ(i,j)
    if i == j
        return 1
    else
        return 0
    end
end

function nbm_matrix(g::SimpleGraph)
    edgeidmap = Dict{Edge, Int}()
    aristas = ne(g)
    m = 0
    for e in edges(g)
        m += 1
        edgeidmap[e] = m
	edgeidmap[reverse(e)] = m + aristas
    end

    B = spzeros(Int64, 2*aristas, 2*aristas)

    for (e,u) in edgeidmap
        i, j = src(e), dst(e)
	eles = neighbors(g,j)
	k = j
        for l in eles
          B[edgeidmap[k=>l],u] = (kron_δ(j,k)*(1-kron_δ(i,l)))
        end
    end
    return B, edgeidmap
end


using Primes
using LinearAlgebra

#returns a random orthogonal matrix of size n
function random_orthogonal(n::Int)
   A = rand(n,n)
   Q,R = qr(A)
   return Q
end

#returns the array A of length L such that A[i] = 1 if i ∉ t and A[i] = 2 else
function tuple_to_index(t::Array{Int},L)
   A = ones(Integer,L)
   for x=t
      A[x]=A[x]+1
   end
   return A
end

#random Slater determinant ψ_1 ∧ ⋯ ∧ ψ_N where (ψ_1,⋯,ψ_N)^T = V (ϕ_1,⋯,ϕ_L)^T for V ∈ R^{N × L} orthogonal
function random_slater(N,L;V=random_orthogonal(L))
   C = zeros(Float64, 2*ones(Integer,L)...)
   tuple_collection = collect(combinations(1:L,N))
   for x=1:size(tuple_collection,1)
      C[tuple_to_index(tuple_collection[x],L)...] = det(V[1:N,tuple_collection[x]])
   end
   return C
end

#returns a sum of random Slater determinants
# C = ∑_{(i_1,…,i_N) ∈ CAS) a_i1…i_N ψ_i1 ∧ ⋯ ∧ ψ_iN
function random_static_correlation_tensor(N,L;V=random_orthogonal(L),n=2,a=randn(n),
   CAS=collect(combinations(1:L,N))[vcat(1,rand(2:binomial(L,N),n-1))])
   a = a/norm(a)
   C = zeros(Float64, 2*ones(Integer,L)...)
   tuple_collection = collect(combinations(1:L,N))
   for x=1:size(tuple_collection,1)
      for k=1:n
         C[tuple_to_index(tuple_collection[x],L)...] += a[k]*det(V[CAS[k],tuple_collection[x]])
      end
   end
   return C
end

#returns the state C[μ_1,⋯,μ_L] = √(prime) iff ∑ μ_i = N with different primes
function random_prime_tensor(N,L)
   C = zeros(Float64, 2*ones(Integer,L)...)
   tuple_collection = collect(combinations(1:L,N))
   prime_collection = shuffle!(primes(2^(N+L))) #works for N+L moderate
   for x=1:size(tuple_collection,1)
      C[tuple_to_index(tuple_collection[x],L)...] = sqrt(prime_collection[x])
   end
   return C./norm(C)
end


export Segment,
    Segments,
    intersect!,
    is_lower_end,
    is_upper_end,
    contains,
    find_leftmost,
    find_rightmost,
    contains,
    is_singular,
    min_x,
    min_y,
    max_x,
    max_y,
    trivial_miss

struct Segment{T<:Float64}
    p::Point{T}
    q::Point{T}
    slope::T
    function Segment(p::Point{T}, q::Point{T}) where {T<:Float64}
        if p > q
            p, q = q, p
        end
        slope = (q.x - p.x) / (q.y - p.y)
        return new{T}(p, q, slope)
    end
end

Base.isless(s::Segment, t::Segment) = (s.p.y > t.q.y) | ((s.p.y == t.p.y) && (s.p.x < t.p.x))
Base.:(==)(s::Segment, t::Segment) = (s.p == t.p) && (s.q == t.q)

get_x(segment::Segment, y) = segment.p.x + segment.slope * (y - segment.p.y)
get_y(segment::Segment, x) = segment.p.y + (x - segment.p.x) / segment.slope

min_x(segment::Segment) = min(segment.p.x, segment.q.x)
min_y(segment::Segment) = min(segment.p.y, segment.q.y)
max_x(segment::Segment) = max(segment.p.x, segment.q.x)
max_y(segment::Segment) = max(segment.p.y, segment.q.y)

Segment(px, py, qx, qy) = Segment(Point(px, py), Point(qx, qy))

"""
Checks if the segment s is purely horizontal or vertical.
"""
function is_singular(s::Segment)
    if (s.p.x == s.q.x) || (s.p.y == s.q.y)
        return true
    else
        return false
    end
end

function trivial_miss(s1::Segment, s2::Segment)
    if s1.slope ≈ s2.slope rtol=1e-10
        return true
    elseif max_x(s1) < min_x(s2)
        return true
    elseif max_x(s2) < min_x(s1)
        return true
    elseif max_y(s1) < min_y(s2)
        return true
    elseif max_y(s2) < min_y(s1)
        return true
    end
    return false
end

"""
Checks for the intersection of two segments s1, s2.
"""
function Base.intersect!(
    s1::Segment{T},
    s2::Segment{T},
    A::Matrix{T},
    b::Vector{T},
    tol=1e-9
) where {T<:AbstractFloat}
    if is_singular(s1) || is_singular(s2)
        return false, Point(0.0, 0.0)
    end
    if trivial_miss(s1, s2)
        return false, Point(0.0, 0.0)
    end
    A[1, 1] = s1.q.x - s1.p.x
    A[1, 2] = s2.p.x - s2.q.x
    A[2, 1] = s1.q.y - s1.p.y
    A[2, 2] = s2.p.y - s2.q.y
    b[1] = s2.p.x - s1.p.x
    b[2] = s2.p.y - s1.p.y
    sol = 0.0
    try
        sol = A \ b
    catch
        @warn "Singular matrix. Check for edge cases!"
        return false, Point(0.0, 0.0)
    end
    if (-tol < sol[1] < 1+tol) && (-tol < sol[2] < 1+tol)
        intersection =
            Point(s1.p.x + sol[1] * (s1.q.x - s1.p.x), s1.p.y + sol[1] * (s1.q.y - s1.p.y))
        return true, intersection
    else
        return false, Point(0.0, 0.0)
    end
end

function Base.intersect!(s1::Segment{T}, s2::Segment{T}) where {T<:AbstractFloat}
    A = zeros((2, 2))
    b = zeros(2)
    return intersect!(s1, s2, A, b)
end

is_lower_end(segment::Segment, Point::Point) = (segment.q == Point)
is_upper_end(segment::Segment, Point::Point) = (segment.p == Point)
function Base.contains(segment::Segment, point::Point, tol=1e-9)
    if is_lower_end(segment, point) | is_upper_end(segment, point)
        return false
    end
    y = get_y(segment, point.x)
    if y ≈ point.y
        atol = tol
        return true
    else
        return false
    end
end

function find_leftmost(segment_set, y, tol=1e-9)
    ret = nothing
    xmin = Inf
    for segment in segment_set
        x = get_x(segment, y - tol)
        if x < xmin
            xmin = x
            ret = segment
        end
    end
    return ret, xmin
end

function find_rightmost(segment_set, y, tol=1e-9)
    ret = nothing
    xmax = 0
    for segment in segment_set
        x = get_x(segment, y - tol)
        if x > xmax
            xmax = x
            ret = segment
        end
    end
    return ret, xmax
end


pop(α,i) = [α[1:i-1]...,α[i+1:end]...] # pop out ith entry
pad(α,i) = [α[1:i-1]...,zero(eltype(α)),α[i:end]...] # pad by 0

"
function recursive_nodes(α,node_family):
Note α is a barycentric coordinate here, not a multi-index.
"
function recursive_nodes(α,node_family)
    d = length(α)-1
    xn = node_family[sum(α)+1]
    if d==1
        b = [xn[α[1]+1],xn[α[2]+1]]
        return b
    else
        b = zeros(eltype(xn),d+1)
        weight = zero(eltype(xn))
        for i = 1:d+1
            br = pad(recursive_nodes(pop(α,i),node_family),i)
            x_i = xn[sum(pop(α,i))+1]
            @. b += x_i*br
            weight += x_i
        end
        return b./weight
    end
end

# recursive_nodes(d::Int,N::Int,node_family) =
#     recursive_nodes(MultiIndexSet(d,N),node_family)
#
# function recursive_nodes(set::MultiIndexSet,node_family)
#     X = ntuple(x->zeros(length(Ndset)),d)
#     for (index,α) in enumerate(Ndset)
#         b = recursive_nodes((α...,N-sum(α)),node_family)
#         for i = 1:d
#             X[i][index] = sum(b.*VXYZ[i])
#         end
#     end
#     return X
# end


@testset "Schottky genus 4" begin
    τ = [0.95870734+0.73587725im 0.22092477+0.76863646im 0.53877459+0.87577267im 0.68177023+0.867436im; 0.22092477+0.76863646im 0.98812562+1.79674905im 0.54859032+1.10626215im 0.63310305+1.30158981im; 0.53877459+0.87577267im 0.54859032+1.10626215im 0.50173043+1.27729044im 0.49163557+1.33147334im; 0.68177023+0.867436im 0.63310305+1.30158981im 0.49163557+1.33147334im 0.35312207+1.60745975im];
    @test schottky_genus_4(τ) ≈ 2.16230-0.94677im atol=1e-3
end


using LinearAlgebra: cholesky

@testset "Factorization" begin
    @testset "Cholesky" begin
        A = [1 0 0.5;
             0 2 0;
             0.5 0 3]

        B = rand(4)
        B *= B'
        B += I

        K = A ⊗ B

        @test isposdef(K)

        KC = cholesky(K)
        @test isposdef(K)
        @test size(K) == (12, 12)

        C = cholesky(collect(K))

        @test det(KC) ≈ det(C)
        @test logdet(KC) ≈ logdet(C)
        @test collect(inv(KC)) ≈ inv(C)
    end
end


using Plots
using FFTW

function norm(a)
    sum = 0
    for i = 1:length(a)
        sum += a[i]
    end

    a=a/sum
    return a
end

function plot_fft(res, timesteps, max_freq)
    for i = 1:timesteps
        time_name = "images/time_" * string(lpad(i-1, 4, "0")) * ".png"
        freq_name = "images/freq_" * string(lpad(i-1, 4, "0")) * ".png"
        comb_name = "images/comb_" * string(lpad(i-1, 4, "0")) * ".png"
        a = []
        if (i-1 <= div(timesteps,4))
            freq = (2+max_freq*(i-1)/(timesteps/4))
            a = [sin(freq*pi*Float64(j)/res) for j = 1:res]
        elseif (i-1 > div(timesteps,4) && i-1 <= div(timesteps,2))
            freq = (2+max_freq*(timesteps/2-(i-1))/(timesteps/4))
            a = [sin(freq*pi*Float64(j)/res) for j = 1:res]
        elseif (i-1 > div(timesteps,2) && i-1 < 3*div(timesteps,4))
            freq = (2 + max_freq*((i-1)-timesteps/2)/(timesteps/4))
            a = [sin(freq*pi*Float64(j)/res) for j = 1:res]
            b = [sin(2*freq*pi*Float64(j)/res) for j = 1:res]
            c = [sin(4*freq*pi*Float64(j)/res) for j = 1:res]
            a = a+b+c
        else
            freq = (2 + max_freq*(timesteps - (i-1))/(timesteps/4))
            a = [sin(freq*pi*Float64(j)/res) for j = 1:res]
            b = [sin(2*freq*pi*Float64(j)/res) for j = 1:res]
            c = [sin(4*freq*pi*Float64(j)/res) for j = 1:res]
            a = a+b+c
        end
        plot(a, label="Time Domain", legend = :top, linewidth = 2,
             xlabel = "Time", ylabel = "Amplitude" )
        savefig(time_name)

        b = abs2.(fft(a))
        b = norm(b)
        plot((b)[1:div(res,16)+2], label="Frequency Domain",
             legend = :top, linewidth = 2, xlabel = "Frequency",
             ylabel = "Normalized Amplitude")
        savefig(freq_name)

        run(`convert +append $time_name $freq_name $comb_name`)

    end
end

plot_fft(2000, 120, 60)


"""
    stratified1D(N::Integer, jitter::Bool)

Divides the `[0,1]` interval into `N` bins and either returns an array with the bin centers
(if `jitter == false`), or a uniformly random point inside each bin (if `jitter == true`).

# Examples
```julia-repl
julia> stratified1D(2, false)
2-element Array{Float64,1}:
 0.25
 0.75
```
"""
stratified1D(N::Integer, jitter::Bool) = [(i-1 + (jitter ? rand() : 0.5))/N for i = 1:N]

"""
    stratified2D(Nx::Integer, Ny::Integer, jitter::Bool)

Performs stratified sampling in two dimensions by dividing the unit square into `Nx` by
`Ny` bins.

Returns a two-column array with `Nx * Ny` rows. Each row contains the `x` and `y`
coordinate of a point in the unit square: the values at the bin centres if `jitter ==
false` and randomly jittered inside the bin if `jitter == true`.

"""
function stratified2D(Nx::Integer, Ny::Integer, jitter::Bool)
    out = zeros(Float64, (2,Nx*Ny))
    n = 1
    for i = 1:Nx
        for j = 1:Ny
            out[1,n] = (i-1 + (jitter ? rand() : 0.5))/Nx
            out[2,n] = (j-1 + (jitter ? rand() : 0.5))/Ny
            n += 1
        end
    end
    out
end

"""
    CameraSample

A wrapper for four floating point values, generally to represent two random points needed
for ray creation from a camera: one point on the film and the other on the lens.

"""
struct CameraSample <: Sample
    imgX::Float64
    imgY::Float64
    lensU::Float64
    lensV::Float64
end

import Statistics.normalize
normalize(S::CameraSample, a::Number, b::Number) = CameraSample(S.imgX/a, S.imgY/b, S.lensU, S.lensV)

"""
    StratifiedSampler <: Sampler

A StratifiedSampler will generate CameraSamples. It handles a certain window of pixels and
can be divided into subsamplers which are new copies of StratifiedSampler with the desired
subwindow of pixels.

It can generate multiple samples per pixel, either divided into clean bins, or jittered.

# Fields

- `xstart::Int64`: The start of the pixel range in the x direction.
- `xend::Int64`: The end of the pixel range in the x direction.
- `ystart::Int64`: The start of the pixel range in the y direction.
- `yend::Int64`: The end of the pixel range in the y direction.
- `xs::Int64`: The number of sample bins per pixel in the x direction.
- `ys::Int64`: The number of sample bins per pixel in the y direction.
- `jitter::Bool`: Whether to jitter samples or not.
- `xnorm::Int64`: The maximum value of the overlying sampler. For a subsampler, equal to
  the `xnorm` of the previous level. For the top level, equal to `xend`.
- `ynorm::Int64`: As above, for y.

"""
struct StratifiedSampler <: Sampler
    xstart::Int64
    xend::Int64
    ystart::Int64
    yend::Int64
    xs::Int64
    ys::Int64
    jitter::Bool
    xnorm::Int64
    ynorm::Int64
    max::Int64
end

function StratifiedSampler(rangeX::Int64, rangeY::Int64, res::Int64)
    Nmax = rangeX * rangeY
    return StratifiedSampler(1, rangeX, 1, rangeY, res, res, true, rangeX, rangeY, Nmax)
end

"""
    compute_subwindow(sampler::Sampler, n::Int64, count::Int64)

This computes the pixel range which a given subsampler samples, given the index `n` of the
subsampler and the total number of areas, `count`.

Returns four integers, given the pixel coordinates (x0, x1, y0, y1) of the subwindow
(inclusive).

Note that indexing starts from 1 unlike in book's C++ code.

"""
function compute_subwindow(sampler::Sampler, n::Int64, count::Int64)
    dx = sampler.xend - sampler.xstart + 1
    dy = sampler.yend - sampler.ystart + 1
    n -= 1
    nx = count
    ny = 1
    while nx%2==0 && 2*dx*ny < dy*nx
        nx = div(nx, 2)
        ny *= 2
    end

    cx = div(dx,nx)
    cy = div(dy,ny)

    x0 = n % nx
    y0 = div(n, ny)
    return x0*cx+1, (x0+1)*cx, y0*cy+1, (y0+1)*cy
end

"""
    get_subsampler(sampler::StratifiedSampler, n::Integer, count::Integer)

Given a `StratifiedSampler`, a total number of subwindows `count` and a subwindow index
`n`, returns a new `StratifiedSampler` which samples that subwindow.

"""
function get_subsampler(sampler::StratifiedSampler, n::Integer, count::Integer)
    x0, x1, y0, y1 = compute_subwindow(sampler, n, count)
    if x0 == x1 || y0 == y1
        return nothing
    else
        Nmax = (1 + x1 - x0) * (1 + y1 - y0)
        return StratifiedSampler(x0, x1, y0, y1, sampler.xs, sampler.ys, sampler.jitter,
                sampler.xnorm, sampler.ynorm, Nmax)
    end
end

"""
    finished(sampler::StratifiedSampler, state::Integer)

Returns `true` when `state` indicates that `sampler` is "finished", i.e. cannot return
any more samples.

"""
finished(sampler::StratifiedSampler, state::Integer) = state >= sampler.max

"""
    get_samples(sampler::StratifiedSampler, state::Integer)

Generate samples from the `StratifiedSampler`, given its `state`, Returns the array of
samples and the new state of the sampler.

"""
function get_samples(sampler::StratifiedSampler, state::Integer)
    out = Array{CameraSample,1}(undef, sampler.xs * sampler.ys)
    state = get_samples!(sampler, state, out)
    return out, state
end

"""
    get_samples!(sampler::StratifiedSampler, state::Integer, out::Array{CameraSample,1})

Generate samples for a single pixel from the `StratifiedSampler`, given its `state`, and
fill the array `out` with them. Returns the updated state of the sampler.

"""
function get_samples!(sampler::StratifiedSampler, state::Integer, out::Array{CameraSample,1})
    dx = sampler.xend - sampler.xstart + 1
    dy = sampler.yend - sampler.ystart + 1
    if state >= dx*dy
        return state
    end

    xpos = sampler.xstart + state % dx
    ypos = sampler.ystart + div(state, dy)
    img_samples = stratified2D(sampler.xs, sampler.ys, sampler.jitter)
    lens_samples = stratified2D(sampler.xs, sampler.ys, sampler.jitter)
    shuffle!(lens_samples[1,:])
    shuffle!(lens_samples[2,:])
    N = sampler.xs * sampler.ys
    for i = 1:N
        out[i] = CameraSample(
            img_samples[1,i] + xpos - 1,
            img_samples[2,i] + ypos - 1,
            lens_samples[1,i],
            lens_samples[2,i]
        )
    end
    return state+1
end


function merge(a::Array, b::Array)
  len = length(a) + length(b)
  B = Array(Int64, len)

  for k = 1:len

    if first(a) <= first(b)
      B[k] = shift!(a)
     else
       B[k] = shift!(b)
    end

    if isempty(a)
      return transfer_tail(B, b, k)
    elseif isempty(b)
      return transfer_tail(B, a, k)
    end

  end
end

function transfer_tail(arr::Array, tail::Array, count::Int64)
  for k = (count + 1):(length(tail) + count)
    arr[k] = shift!(tail)
  end
  return arr
end

function merge_sort(arr::Array)
    if length(arr) <= 1
       return arr
    end

    q = div(length(arr),2)

    a = merge_sort(arr[1 : q])
    b = merge_sort(arr[q + 1 : end])
    merge(a,b)
end

A = [3,6,5,8,99,2,1,5,9]
println(A)

A = merge_sort(A)
println(A)


@testset "Basic 2D Orientation Tests" begin
	o = (0f0,0f0); x = (1,0); y = (0,1); p = (2,2); q = (3,4); r = (1f0,1f0);
	@test ccw(o,x,p) == ccw(o,x,toPoint2H(p))
	@test ccw(o,x,p) == ccw(o,toPoint2H(x),(p))
	@test ccw(o,x,p) == ccw(toPoint2H(o),x,(p))
	@test ccw(o,x,p) == ccw(x,p,toPoint2H(o))
	@test ccw(o,x,p) == ccw(toPoint2H(o),toPoint2H(x),toPoint2H(p))
	@test ccw(o,x,p) == ccw(toPoint2H(o),toPoint2H(x),(p))
	@test ccw(o,x,p) == ccw(toPoint2H(o),(x),toPoint2H(p))
	@test ccw(o,x,p) == ccw((o),toPoint2H(x),toPoint2H(p))
	@test 0 == ccw(o,r,toPoint2H(p))
	@test 0 == ccw(r,p,toPoint2H(o))
	@test 0 == ccw((o),toPoint2H(r),toPoint2H(p))
	@test 0 == ccw(toPoint2H(o),toPoint2H(r),(p))
	@test 0 == ccw(toPoint2H(o),toPoint2H(r),toPoint2H(p))

	@test leftturn(o,x,p)
	@test !rightAhead(o,x,p)
	@test rightturn(o,y,q)
	@test !leftAhead(o,y,q)
	@test !leftturn(x,p,q)
	@test leftAhead(x,p,q) # colinear
	@test rightAhead(x,p,q) # colinear
	@test !leftAhead(o,p,r) # colinear
	@test leftOnAhead(y,o,q)
	@test leftOnAhead(o,p,r) # colinear
	@test rightOnAhead(o,p,r) # colinear
	@test rightOnAhead(x,p,q) # colinear
	@test !rightOnAhead(o,x,p) # colinear
	@test leftofseg(p,y,x)  # query point with segment
	@test rightofseg(p,x,y)
	@test leftward(p,r, o,y)
	@test !rightward(p,r, o,y)
	@test !leftward(p,r, o,r) # parallel
	@test !rightward(p,r, o,r) # parallel
	@test line(o,y) == (0,-1,0)
	@test line(y,r) == (-1,0,1)
	@test leftturn(line(o,x), line(x,p), line(p,o))
	@test rightof(p,line(r,q))
	@test leftof(r,line(p,q))
	@test !leftof(o, line(p,r))
	@test !rightof(o, line(p,r))
	@test rightof(toPoint2H(p),line(r,q))
	@test leftof(toPoint2H(r),line(p,q))
	@test !leftof(toPoint2H(o), line(p,r))
	@test !rightof(toPoint2H(o), line(p,r))
end;


using StatsFuns: normcdf, normccdf, normlogcdf, normlogccdf, normlogpdf, normpdf, logsumexp

@marginalrule Probit(:out_in) (m_out::PointMass, m_in::UnivariateNormalDistributionsFamily, meta::ProbitMeta) = begin

    # extract parameters
    mz, vz = mean_cov(m_in)
    p = mean(m_out)
    @assert p >= zero(p) && p <= one(p) "The Probit node only accepts messages on its output with values between 0 and 1."

    # calculate auxiliary variables
    γ = mz/sqrt(1+vz)

    # calculate moments of g
    if γ > 0 && p > 0.5
        log_mom0_pz = logsumexp((log(1-p), log(2*p-1) + normlogccdf(-γ)))
    elseif γ <= 0 && p > 0.5
        log_mom0_pz = logsumexp((log(1-p), log(2*p-1) + normlogcdf(γ)))
    elseif γ > 0 && p <= 0.5
        log_mom0_pz = logsumexp((log(1-p) + normlogcdf(-γ), log(p) + normlogcdf(γ)))
    else
        log_mom0_pz = logsumexp((log(1-p) + normlogccdf(γ), log(p) + normlogcdf(γ)))
    end
    tmp = log(vz) + normlogpdf(γ) - log(1+vz)/2 - log_mom0_pz
    mom1_pz = mz + (2*p-1)*exp(tmp)
    mom2_pz = vz + mz^2 + (2*p-1)*2*mz*exp(tmp) - (2p-1)*γ*exp(log(vz) - log(1 + vz)/2 + tmp)

    # calculate parameters of posterior
    mpz = mom1_pz
    vpz = clamp(mom2_pz - mom1_pz^2, tiny, vz) # ensure variance of marginal is not larger than the variance of the cavity distribution.

    return (out = m_out, in = NormalMeanVariance(mpz, vpz))

end


"""
    (dvec_ngwp, BS_ngwp) = ngwp_bestbasis(dmatrix::Array{Float64,3}, GP_star::GraphPart;
                                          cfspec::Any = 1.0, flatten::Any = 1.0,
                                          j_start::Int = 1, j_end::Int = size(dmatrix, 2),
                                          useParent::Bool = true)

Select the best basis from the matrix of NGWP expansion coefficients.

# Input Arguments
- `dmatrix::Array{Float64,3}`: the matrix of expansion coefficients
- `GP_star::GraphPart`: an input GraphPart object of the dual graph
- `cfspec::Any`: the specification of cost functional to be used (default = 1.0,
    i.e., 1-norm)
- `flatten::Any`: the method for flattening vector-valued data to scalar-valued
    data (default = 1.0, i.e, 1-norm)
- `useParent::Bool`: the flag to indicate if we update the selected best basis
    subspace to the parent when parent and child have the same cost (default = false)

#  Output Arguments
- `dvec_ngwp::Matrix{Float64}`: the vector of expansion coefficients corresponding
    to the NGWP best basis
- `BS_ngwp::BasisSpec`: a BasisSpec object which specifies the NGWP best basis
"""
function ngwp_bestbasis(dmatrix::Array{Float64,3}, GP_star::GraphPart;
                            cfspec::Any = 1.0, flatten::Any = 1.0,
                            j_start::Int = 1, j_end::Int = size(dmatrix, 2),
                            useParent::Bool = false)
    dvec_ngwp, BS_ngwp = ghwt_c2f_bestbasis(dmatrix, GP_star; cfspec = cfspec,
                            flatten = flatten, j_start = j_start, j_end = j_end,
                            useParent = useParent)
    BS_ngwp.description = "NGWP Best Basis"
    return dvec_ngwp, BS_ngwp
end


# Uses build_chromosome() to specify some example circuits given in of Principles Evol. Design I (Miller et al 2000)
# Defines functions to compute goals for 2-bit and 3-bit multiplier circuits.
# Note that the computed goal for the 2-bit multiplier circuit does not agree with the output of the two_bit_multipler() circuit of Fig. 15b.
# Then defines functions to evolve 2 and 3 bit
# Successfully checked with one minor bug fix on 4/25/21.
#include("build_circuit.jl")
using Statistics

funcs = [AND,OR,XOR]
# Function of Figure 2 of Principles Evol. Design (Miller et al 2000)
function one_bit_adder()
  c = build_chromosome( (1,2,3), ((AND,[1,2]),(XOR,[1,2]), (AND,[5,3]), (XOR,[5,3]), (OR,[4,6])), (8,7))
  context = construct_context(c.params.numinputs)
  print_chromosome(c)
  outputs = execute_chromosome(c,context)
  println("output: ",outputs)
  outputs
end

# Function of Fig. 15b of Principles Evol. Design (Miller et al 2000)
function two_bit_multiplier()
  c = build_chromosome(
     (1,2,3,4),
     ((AND,[1,3]),  # gate 5
      (AND,[2,4]),  # gate 6
      (AND,[2,3]),  # gate 7
      (AND,[1,4]),  # gate 8
      (AND,[5,6]),  # gate 9
      (XOR,[7,8]),  # gate 10
      (AND,[7,10])), # gate 11
     (11,9,10,6))   # outputs P3,P2,P1,P0

  context = construct_context(c.params.numinputs)
  print_chromosome(c)
  outputs = execute_chromosome(c,context)
  println("output: ",outputs)
  outputs
end

# result should agree with result of two_bit_multiplier()
function check_two_bit_multiplier()
  context = construct_context(4)
  g1 = context[1]
  g2 = context[2]
  g3 = context[3]
  g4 = context[4]
  g5 = g1 & g3
  g6 = g2 & g4
  g7 = g2 & g3
  g8 = g1 & g4
  g9 = g5 & g6
  g10 = xor(g7,g8)
  g11 = g10 & g7
  p3 = g11
  p2 = g9
  p1 = g10
  p0 = g6
  [p3,p2,p1,p0]
end

function goal_for_two_bit_multiplier()
  factor1 = [0x3, 0x3, 0x3, 0x3, 0x2, 0x2, 0x2, 0x2, 0x1, 0x1, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0]
  factor2 = [0x3, 0x2, 0x1, 0x0, 0x3, 0x2, 0x1, 0x0, 0x3, 0x2, 0x1, 0x0, 0x3, 0x2, 0x1, 0x0]
  prod = map(*,factor1,factor2)
  b4 = map(x->convert(Int64,(0x8&x)>>3),prod)
  b3 = map(x->convert(Int64,(0x4&x)>>2),prod)
  b2 = map(x->convert(Int64,(0x2&x)>>1),prod)
  b1 = map(x->convert(Int64,(0x1&x)),prod)
  #println("b4: ",b4)
  #println("b3: ",b3)
  #println("b2: ",b2)
  #println("b1: ",b1)
  return map(convert_binary_list_to_UInt,[b4,b3,b2,b1])
end

function convert_binary_list_to_UInt( list::Vector{Int64} )
  result = MyInt(0)
  shift = 0
  for i = length(list):-1:1
    result += convert(MyInt,list[i]) << shift
    shift += 1
  end
  result
end

function UInt_to_binary_list( x::UInt64 )
  size = 64  # If the type of x is UInt64, must be 64.  If the type of x is UInt32, must be 32.
  shift = size-1
  result = fill(0,size)
  for i = 1:size
    result[i] = (x >> shift) & convert(UInt64,1)
    shift -= 1
  end
  result
end

function goal_for_three_bit_multiplier()
  f1=[fill(UInt64(i),8) for i=collect(0x7:-1:0x0)]
  factor1 = vcat(f1...)
  #println("factor1: ",map(x->convert(Int64,x),factor1))
  f2 = [reverse(collect(UInt64(0x00):UInt64(0x07))) for i = 0:7]
  factor2 = vcat(f2...)
  #println("factor2: ",map(x->convert(Int64,x),factor2))
  prod = map(*,factor1,factor2)
  #println("prod: ",prod)
  bb = [ map(x->convert(Int64,((0x2^i) & x)>>i),prod) for i = 5:-1:0 ]
  for i = 1:6
    #println("bb[",i,"]: ",bb[i])
  end
  map(convert_binary_list_to_UInt,bb)
end

# Evolves a two_bit multipler circuit with the goal defined by goal_for_two_bit_multiplier() defined above
function evolve_two_bit_multiplier( numinteriors::Int64=10, max_steps::Int64=100000;
      iterations::Int64=1, num_mutations::Int64=1, outfile::String="")
  println("num_mutations: ",num_mutations)
  steps_list = zeros(Int64,iterations)
  p2b = Parameters(numinputs=4,numoutputs=4,numlevelsback=numinteriors,numinteriors=numinteriors)
  mfuncs = [Func(&, 2, "&"),Func(xor, 2, "xor") ]
  #mfuncs = [Func(&, 2, "&"),Func(|, 2, "or"), Func(xor, 2, "xor") ]
  current_chromosome = random_chromosome(p2b,mfuncs)  # not used but establishes rc as a top-level scope variable
  goal = goal_for_two_bit_multiplier()
  #goal = UInt16[0x40c0, 0x8000, 0x6ac0, 0xa0a0]
  #goal = UInt16[0x4c00, 0x8000, 0x6ac0, 0xa0a0]
  goallist = [goal]
  for i = 1:iterations
    rc = random_chromosome(p2b,mfuncs)
    result = mut_evolve(rc,goallist,mfuncs,max_steps,num_mutations=num_mutations)
    current_chromosome = result[1]
    num_active = number_active(result[1])
    steps_list[i] = result[2]
    println("i: ",i,"  steps: ",steps_list[i],"  num active: ",num_active)
  end
  println("max_steps: ",max_steps)
  println("numinteriors=",numinteriors)
  println("num_mutations: ",num_mutations)
  if iterations==1
    steps = steps_list[1]
  else
    steps = Statistics.mean(steps_list)
  end
  fname = length(outfile)==0 ? "../data/7_10/two_bit_mult.csv" : outfile
  num_active = number_active(current_chromosome)
  println("evolve, steps, num_active, num_muts, num_ints, max_steps")
  println("2bit_mul,", steps,",",num_active,",",num_mutations,",",numinteriors,",",max_steps)
  open(fname,"a") do f
    println(f, "evolve, steps, num_active, num_muts, num_ints, max_steps")
    println(f, "2bit_mul,", steps,",",number_active(current_chromosome),",",num_mutations,",",numinteriors,",",max_steps)
  end
  if steps < max_steps
    println("output: ",sort(execute_chromosome(current_chromosome, construct_context(p2b.numinputs))))
    println("goal:   ",sort(goal))
    @assert sort(execute_chromosome(current_chromosome, construct_context(p2b.numinputs))) == sort(goal)
  end
  current_chromosome
end

# Evolves a three_bit multipler circuit with the goal defined by goal_for_three_bit_multiplier() defined above
# Usually succeeds (based on a sample of 5) with numinteriors=35 and max_steps=50000000
# Note:  MyInt must be UInt64
function evolve_three_bit_multiplier( numinteriors::Int64, max_steps::Int64;
      iterations::Int64=1, num_mutations::Int64=1, outfile::String="")
  @assert MyInt == UInt64
  println("num_mutations: ",num_mutations)
  steps_list = zeros(Int64,iterations)
  p3b = Parameters(numinputs=6,numoutputs=6,numlevelsback=25,numinteriors=numinteriors)
  mfuncs = [Func(&, 2, "&"),Func(xor, 2, "xor") ]
  #mfuncs = [Func(&, 2, "&"),Func(|, 2, "or"), Func(xor, 2, "xor") ]
  current_chromosome = random_chromosome(p3b,mfuncs)  # not used but establishes rc as a top-level scope variable
  goal = goal_for_three_bit_multiplier()
  goallist = [goal]
  for i = 1:iterations
    rc = random_chromosome(p3b,mfuncs)
    result = mut_evolve(rc,goallist,mfuncs,max_steps)
    current_chromosome = result[1]
    num_active = number_active(result[1])
    steps_list[i] = result[2]
    println("i: ",i,"  steps: ",steps_list[i],"  num active: ",num_active)
  end
  println("max_steps: ",max_steps)
  println("numinteriors=",numinteriors)
  println("num_mutations: ",num_mutations)
  if iterations==1
    steps = steps_list[1]
  else
    steps = Statistics.mean(steps_list)
  end
  num_active = number_active(current_chromosome)
  fname = length(outfile)==0 ? "../data/4_25/three_bit_mult.csv" : outfile
  println("evolve, steps, num_active, num_muts, num_ints, max_steps")
  println("3bit_mul,", steps, ",",  num_active,",",num_mutations,",",numinteriors,",",max_steps)
  open(fname,"a") do f
    println(f, "evolve, steps, num_active, num_muts, num_ints, max_steps")
    println(f, "3bit_mul,", steps,",",num_active,",",num_mutations,",",numinteriors,",",max_steps)
  end
  if steps < max_steps
    println("output: ",sort(execute_chromosome(current_chromosome, construct_context(p3b.numinputs))))
    println("goal:   ",sort(goal))
    @assert sort(execute_chromosome(current_chromosome, construct_context(p3b.numinputs))) == sort(goal)
  end
  current_chromosome
end


function f!(fvec, x)
    fvec[1] = 1 - x[1]
    fvec[2] = 10(x[2]-x[1]^2)
end
function j!(fjac, x)
    fjac[1,1] = -1
    fjac[1,2] = 0
    fjac[2,1] = -20x[1]
    fjac[2,2] = 10
end

function nlsolvers_f(x, Fx, Jx)
	if Fx !== nothing
		f!(Fx, x)
	end
	if Jx !== nothing
		j!(Jx, x)
	end
	objective_return(Fx, Jx)
end
x0 = [-1.2, 1.0]

nlsolve!(OnceDiffed(nlsolvers_f), copy(x0), TrustRegion(NLSolvers.Newton()))
nlsolve!(OnceDiffed(nlsolvers_f), copy(x0), TrustRegion(NLSolvers.Newton(), Dogleg()))
nlsolve!(OnceDiffed(nlsolvers_f), copy(x0), TrustRegion(NLSolvers.Newton(), NTR()))
nlsolve!(OnceDiffed(nlsolvers_f), copy(x0), TrustRegion(NLSolvers.Newton(), NWI()))
# initial convergence
nlsolve!(OnceDiffed(nlsolvers_f), fill(1.0, 2), TrustRegion(NLSolvers.Newton()))
nlsolve!(OnceDiffed(nlsolvers_f), fill(1.0, 2), TrustRegion(NLSolvers.Newton(), Dogleg()))
nlsolve!(OnceDiffed(nlsolvers_f), fill(1.0, 2), TrustRegion(NLSolvers.Newton(), NTR()))
nlsolve!(OnceDiffed(nlsolvers_f), fill(1.0, 2), TrustRegion(NLSolvers.Newton(), NWI()))


"""
CSRTOSPARSE
-----------
    convert a matrix from compressed sparse row to a sparse matrix A.
    It returns the arrays that feed the sparse function in julia.

Functions
---------
- (nzi,nzj,nzv) = csr_to_sparse{T}(rp::Vector{Int64},ci::Vector{Int64},ai::Vector{T})
- (nzi,nzj,nzv) = csr_to_sparse{T}(rp::Vector{Int64},ci::Vector{Int64},ai::Vector{T},nrows::Int64)

Example
-------
~~~
i = [1;2;3]
j = [3;4;4]
v = [8;9;10]
(rp,ci,ai,m) = sparse_to_csr(i,j,v)
(nzi,nzj,nzv) = csr_to_sparse(rp,ci,ai)
A = sparse(nzi,nzj,nzv,length(rp)-1,maximum(ci))
B = csr_to_sparse_matrix(rp,ci,ai)
isequal(A,B)
~~~
"""

##################
#	Functions    #
##################
function csr_to_sparse{T}(rp::Vector{Int64},ci::Vector{Int64},ai::Vector{T})
    nrows = length(rp)-1
    ncols = length(ci)
    nzi = zeros(Int64,ncols)
    for i=1:nrows
        for j=rp[i]:rp[i+1]-1
            nzi[j] = i
        end
    end
    nzj = ci
    nzv = ai
    return (nzi,nzj,nzv)
end

function csr_to_sparse_matrix{T}(rp::Vector{Int64},ci::Vector{Int64},ai::Vector{T})
    (i,j,k) = csr_to_sparse(rp,ci,ai)
    A = sparse(i,j,k)
    return A
end

function csr_to_sparse_matrix{T}(rp::Vector{Int64},ci::Vector{Int64},
                                            ai::Vector{T},nrows::Int64,ncols::Int64)
    (i,j,k) = csr_to_sparse(rp,ci,ai);
    A = sparse(i,j,k,nrows,ncols)
    return A
end


import Pkg; Pkg.add(Pkg.PackageSpec(url="https://github.com/JuliaComputing/JuliaAcademyData.jl"))
using JuliaAcademyData; activate("Deep learning with Flux")

# # Multiple neural network layers with `Flux.jl`

#-

# In a previous notebook, we saw that one layer of neurons wasn't enough to distinguish between three types of fruit (apples, bananas *and* grapes), since the data is quite complex. To solve this problem, we need to use more layers, so heading into the territory of **deep learning**!
#
# By adding another layer between the inputs and the output neurons, a so-called "hidden layer", we will get our first serious **neural network**, looking something like this:

include(datapath("scripts/draw_neural_net.jl"))
draw_network([2, 4, 3])

# We will continue to use two input data and try to classify into three types, so we will have three output neurons. We have chosen to add a single "hidden layer" in between, and have arbitrarily chosen to put 4 neurons there.
#
# Much of the *art* of deep learning is choosing a suitable structure for the neural network that will allow the model to be sufficiently complex to model the data well, but sufficiently simple to allow the parameters to be learned in a reasonable length of time.

#-

# ## Read in and process data
#
# As before, let's load some pre-processed data using code we've seen in the previous notebook.

## using Pkg; Pkg.add("Flux")
using Flux
using Flux: onehot

#-

## using Pkg; Pkg.add("CSV")
using CSV, DataFrames

apples_1 = DataFrame(CSV.File(datapath("data/Apple_Golden_1.dat"), delim='\t', allowmissing=:none, normalizenames=true))
apples_2 = DataFrame(CSV.File(datapath("data/Apple_Golden_2.dat"), delim='\t', allowmissing=:none, normalizenames=true))
apples_3 = DataFrame(CSV.File(datapath("data/Apple_Golden_3.dat"), delim='\t', allowmissing=:none, normalizenames=true))
bananas = DataFrame(CSV.File(datapath("data/Banana.dat"), delim='\t', allowmissing=:none, normalizenames=true))
grapes_1 = DataFrame(CSV.File(datapath("data/Grape_White.dat"), delim='\t', allowmissing=:none, normalizenames=true))
grapes_2 = DataFrame(CSV.File(datapath("data/Grape_White_2.dat"), delim='\t', allowmissing=:none, normalizenames=true))

apples = vcat(apples_1, apples_2, apples_3)
grapes = vcat(grapes_1, grapes_2);

#-

col1 = :red
col2 = :blue

x_apples  = [ [apples_1[i, col1], apples_1[i, col2]] for i in 1:size(apples_1)[1] ]
append!(x_apples, [ [apples_2[i, col1], apples_2[i, col2]] for i in 1:size(apples_2)[1] ])
append!(x_apples, [ [apples_3[i, col1], apples_3[i, col2]] for i in 1:size(apples_3)[1] ])

x_bananas = [ [bananas[i, col1], bananas[i, col2]] for i in 1:size(bananas)[1] ]

x_grapes = [ [grapes_1[i, col1], grapes_1[i, col2]] for i in 1:size(grapes_1)[1] ]
append!(x_grapes, [ [grapes_2[i, col1], grapes_2[i, col2]] for i in 1:size(grapes_2)[1] ])

xs = vcat(x_apples, x_bananas, x_grapes);

# We now we wish to classify the three types of fruit, so we again use one-hot vectors to represent the desired outputs $y^{(i)}$:

labels = [ones(length(x_apples)); 2*ones(length(x_bananas)); 3*ones(length(x_grapes))];

ys = [onehot(label, 1:3) for label in labels];  # onehotbatch(labels, 1:3)

# The input data is in `xs` and the one-hot vectors are in `ys`.

#-

# ## Multiple layers in Flux

#-

# Let's tell Flux what structure we want the network to have. We first specify the number of neurons in each layer, and then construct each layer as a `Dense` layer:

inputs = 2
hidden = 4
outputs = 3

layer1 = Dense(inputs, hidden, σ)
layer2 = Dense(hidden, outputs, σ)

# To stitch together multiple layers to make a multi-layer network, we use Flux's `Chain` function:

const model = Chain(layer1, layer2)

# #### Exercise 1
#
# What is the internal structure and sub-structure of this `model` object?

#-

# #### Solution

model.layers

# So `model` understands that it consists of two layers, and:

model.layers[1].W

#-

model.layers[1].b

# So both the layers have their own `W` and `b`, just as you might expect -- `Flux` wraps all this up in a way that is structured but easy to use.
#
# In particular, using `params` returns *all* the parameters hidden inside the `model` object, that is the pairs $(W, b)$ from both layers:

params(model)

# ## Training the model

#-

# We have now set up a model and we have some training data.
# How do we train the model on the data?
#
# The amazing thing is that the rest of the code in `Flux` is **exactly the same as before**. This is possible thanks to the design of Julia itself, and of the `Flux` package.

#-

# #### Exercise 2
#
# Train the model as before, now using the popular `ADAM` optimizer. You may need to train the network for longer than before, since we have many more parameters.

#-

# #### Solution

loss(x, y) = Flux.mse(model(x), y)

#-

data = zip(xs, ys);

#-

opt = ADAM(params(model), 0.02)

#-

@time for i in 1:100
    Flux.train!(loss, data, opt)
end

# Let's check the final values of the parameters:

params(model)

# ## Visualizing the results

#-

# What does this neural network represent? It is simply a more complicated function with two inputs and three outputs, i.e. a function $f: \mathbb{R}^2 \to \mathbb{R}^3$.
# Before, with a single layer, each component of the function $f$ basically corresponded to a hyperplane; now it will instead be a **more complicated nonlinear function** of the input data!

#-

# #### Exercise 3
#
# Visualize each component of the output separately as a heatmap and/or contours superimposed on the data. Interpret the results.

#-

# #### Solution

coords = 0:0.01:0.8

heatmap(coords, coords, (x,y)->model([x,y]).data[1])
#contour!(coords, coords, (x,y)->model([x,y]).data[1], levels=[0.5, 0.501], lw=3)

scatter!(first.(x_apples), last.(x_apples), m=:cross, label="apples")
scatter!(first.(x_bananas), last.(x_bananas), m=:circle, label="bananas")
scatter!(first.(x_grapes), last.(x_grapes), m=:square, label="grapes")

xlims!(0.4, 0.8)
ylims!(0.1, 0.5)

# We see that the first component, which is supposed to separate apples from non-apples, has been able to learn a set that has a much more complicated shape than simply a hyperplane: the hyperplane has been bent round. Sometimes it's able to encapsulate all the apple data, while sometimes it isn't, depending on how well it learnt.

coords = 0:0.01:1

#contour(coords, coords, (x,y)->model([x,y]).data[2], levels=[0.5, 0.501], lw=3)
heatmap(coords, coords, (x,y)->model([x,y]).data[2])

scatter!(first.(x_apples), last.(x_apples), m=:cross, label="apples")
scatter!(first.(x_bananas), last.(x_bananas), m=:circle, label="bananas")
scatter!(first.(x_grapes), last.(x_grapes), m=:square, label="grapes")

xlims!(0.4, 0.8)
ylims!(0.1, 0.5)

# The second component is very successful at separating out the bananas. Since there are some apples mixed in there, it can't be expected to do too much better.

coords = 0:0.01:1

heatmap(coords, coords, (x,y)->model([x,y]).data[3])
contour!(coords, coords, (x,y)->model([x,y]).data[3], levels=[0.5, 0.501], lw=3)

scatter!(first.(x_apples), last.(x_apples), m=:cross, label="apples")
scatter!(first.(x_bananas), last.(x_bananas), m=:circle, label="bananas")
scatter!(first.(x_grapes), last.(x_grapes), m=:square, label="grapes")

xlims!(0.4, 0.8)
ylims!(0.1, 0.5)

# The third component separates grapes from the rest pretty successfully.

#-

# ## What we have learned

#-

# Adding an intermediate layer allows the network to start to deform the separating surfaces that it is learning into more complicated, nonlinear (curved) shapes. This allows it to separate data that were previously unable to be separated!
#
# However, using only two features means that data from different classes overlaps. To distinguish it we would need to use more features.

#-

# ### Exercise 4
#
# Use three features (red, green and blue) and build a network with one hidden layer. Does this help to distinguish the data better?

Courses/Deep learning with Flux/1700.Introduction-to-deep-learning-quiz.jl


#!/usr/bin/env julia
# -*- coding: utf-8 -*-
#=
Apply HANTS process on a multidimensional array
=#
cd(dirname(@__FILE__))
include("../HANTS.jl")
using .HANTS, Plots, DelimitedFiles

arr = readdlm("test_data.csv", ',', Float64, '\n')
arr = reshape(arr, 4, 5, 23) # 3d array, the 3rd dimension to apply HANTS

nbase = 365
nfreq = 3
tseries = 1:16:365
validrange = (-1, 1)
fet = 0.02
dod = 1
δ = 0.1

A, φ, arr_rec = hants(
    arr, fet, dod, δ; dims=3, nbase=nbase, nfreq=nfreq, validrange=validrange, tseries=tseries, outlier=nothing
)
arr_reconstruct = reconstruct(A, φ, nbase; dims=3)

A_Hi, φ_Hi, arr_rec_Hi = hants(
    arr, fet, dod, δ; dims=3, nbase=nbase, nfreq=nfreq, validrange=validrange, tseries=tseries, outlier=:Hi
)
arr_reconstruct_Hi = reconstruct(A_Hi, φ_Hi, nbase; dims=3)

A_Lo, φ_Lo, arr_rec_Lo = hants(
    arr, fet, dod, δ; dims=3, nbase=nbase, nfreq=nfreq, validrange=validrange, tseries=tseries, outlier=:Lo
)
arr_reconstruct_Lo = reconstruct(A_Lo, φ_Lo, nbase; dims=3)

fig1 = plot(tseries, arr[2, 4, :]; lc=:black, label="Original Data", shape=:circle, mc=:black)
plot!(tseries, arr_rec[2, 4, :]; lc=:blue, label="HANTS - None", shape=:circle, mc=:blue)
plot!(tseries, arr_rec_Hi[2, 4, :]; lc=:red, label="HANTS - Hi", shape=:circle, mc=:red)
plot!(tseries, arr_rec_Lo[2, 4, :]; lc=:green, label="HANTS - Lo", shape=:circle, mc=:green)
title!("Testing HANTS Algorithm")

fig2 = plot(tseries, arr[2, 4, :]; lc=:black, label="Original Data", shape=:circle, mc=:black)
plot!(1:365, arr_reconstruct[2, 4, :]; lc=:blue, label="HANTS - None")
plot!(1:365, arr_reconstruct_Hi[2, 4, :]; lc=:red, label="HANTS - Hi")
plot!(1:365, arr_reconstruct_Lo[2, 4, :]; lc=:green, label="HANTS - Lo")
title!("Testing HANTS Reconstruct")

plot(fig1, fig2; layout=(1, 2), size=(800,300))


#=##############################################################################
# DESCRIPTION
    Methods for geometric manipulation.
# AUTHORSHIP
  * Author    : Eduardo J Alvarez
  * Email     : Edo.AlvarezR@gmail.com
  * Created   : Nov 2017
  * License   : MIT License
=###############################################################################

################################################################################
# MESHING AND DISCRETIZATION
################################################################################
"""
  `discretize(f, xlow, xhigh, n::Int64, r::Float64; central::Bool=false)`

Discretizes the continuous function `f` between the range `xlow` and `xhigh`
into `n` intervals, with `r` the ratio between first and last interval if
`central=false` or between first and central interval if `central=true`.

  **Examples**

  The following lines show the discretization of a semi-circumference perimeter
  into 100 intervals of uniform length:

  ```julia
    julia> f(x) = (x, sqrt(1-round(x,digits=8)^2), 0)  # Semi-circunference of radius 1
    julia> discretize(f, -1, 1, 100, 1.0)
  ```
"""
function discretize(f, xlow, xhigh, n::Int64, r::Float64; central::Bool=false,
                      check::Bool=true)

  # ERROR CASES
  if n == 0
    return Any[]
  elseif n < 0
    error("Invalid number of intervals (n < 0)")
  elseif r <= 0
    error("Invalid expansion ratio (r <= 0)")
  end

  out = Any[f(xlow)]

  l = xhigh - xlow        # Length of domain to be discretized
  cumlen = 0              # Cumulative length already walked

  for i in 1:n

    # Case of uniform discretization
    if r==1.0
      len = l/n

    # Linear increment discretization (see notebook entry 20170519)
    # Case of no central expansion
    elseif !central
      p = l/( (n*(n-1)/2)*(r+1)/(r-1) )
      d1 = p*(n-1)/(r-1)
      len = d1 + p*(i-1)
      # println("i=$i\tp=$p\td1=$d1\tlen=$len")

    # Case of central expansion
    else
      _central = 0.5
      # Left of the center
      if i<=floor(n*_central)
        _l = l*_central
        _n = floor(n*_central)
        _r = r
        _i = i
      # Right of the center
      else
        _l = l*(1-_central)
        _n = n-floor(n*_central)
        _r = 1/r
        _i = i-floor(n*_central)
      end
      p = _l/( (_n*(_n-1)/2)*(_r+1)/(_r-1) )
      d1 = p*(_n-1)/(_r-1)
      len = d1 + p*(_i-1)
      # println("$i\t$_r\t$cumlen\t_n=$_n\t_i=$_i")
    end

    cumlen += len
    this_x = xlow + (cumlen/l)*(xhigh-xlow)
    # println("i=$i\tx=$this_x\tlen=$len")
    this_f = f(this_x)
    push!(out, this_f)
  end

  # Verifies correct discretization
  if check && abs((cumlen-l)/l)>0.0001
    error("Critical logic error! cumlen!=l ($cumlen!=$l)")
  end

  return out
end

"""
  `multidiscretize(f, xlow, xhigh, sections)`

Discretizes the continuous function `f` between the range `xlow` and `xhigh`
into multiple sections of refinement as specified in `sections`.

  ** Arguments **
  * `f`         : Continuous function of the form `f(x)` to be discretized
                  between `xlow` and `xhigh`,
  * `xlow`      : Lower bound.
  * `xhigh`     : Upper bound.
  * `sections`  : Array `[sec1, sec2, ...]`specifying the
                  sections of discretization in the format
                  `sec = (c::Float64, n::Int64, r::Float64, central::Bool)`,
                  with `c` the normalized length of this section (the sum of all
                  c must equal one), `n` the number of intervals in this section
                  , `r` the increment ratio between first and last interval if
                  `central=false` or between first and central interval if
                  `central=true`.

  **Examples**

  The following lines show the discretization of a semi-circumference perimeter
  into 90 intervals done in three sections of discretization:

  ```julia
    julia> f(theta) = (cos(theta), sin(theta), 0)
    julia> sec = (1/3, 30, 1/8, true)
    julia> points = multidiscretize(f, 0, pi, [sec, sec, sec])
  ```
"""
function multidiscretize(f, xlow, xhigh, sections::multidisctype;
                                                              check::Bool=true)
  out = Any[f(xlow)]
  ctot = sum([sec[1] for sec in sections]) # Sum of all `c`s

  # Iterates over sections
  prev_xhigh = xlow
  for (c,n,r,central) in sections

    this_c = c/ctot # Makes sure that the sum of `c`s will add 1
    this_xlow = prev_xhigh
    this_xhigh = prev_xhigh + this_c*(xhigh-xlow)

    this_out = discretize(f, this_xlow, this_xhigh, n, r; central=central,
                                  check=check)
    out = vcat(out, this_out[2:end])

    prev_xhigh = this_xhigh
  end

  return out
end

"""
  Receives a closed 2D contour and splits it up in upper and lower surfaces as
  divided by lowest and highest x values. It returns `(upper, lower)` with
  `upper=(x,y)` the points of the upper surface, ditto for `lower`. Both `upper`
  and `lower` are given in increasing order in x. This function is useful for
  splitting up a closed contour into two sections that are injective in x, and
  that can be received by `parameterize()`.
"""
function splitcontour(x,y)

  # Flag indicating whether the contour start at the trailing or leading edge
  start_TE = x[1]==maximum(x)

  # Find the opposite end of the contour
  end_i = -1
  for (i, xi) in enumerate(x)
    if i==1
      nothing
    # Case of starting from the trailing edge
    elseif start_TE && xi > x[i-1]
      end_i = i-1
      break
    # Case of leading edge
    elseif !start_TE  && xi < x[i-1]
      end_i = i-1
      break
    end
  end

  # ERROR CASE
  if end_i==-1
    error("Logic error! End of contour not found!")
  end

  # Splits them up
  x_sec1, y_sec1 = x[1:end_i], y[1:end_i]
  x_sec2, y_sec2 = x[end_i:end], y[end_i:end]

  # Sorts them from LE to TE
  if x_sec1[1] > minimum(x); reverse!(x_sec1); reverse!(y_sec1); end;
  if x_sec2[1] > minimum(x); reverse!(x_sec2); reverse!(y_sec2); end;

  # Determines upper and lower surfaces
  if Statistics.mean(y_sec1) > Statistics.mean(y_sec2)
    upper = [x_sec1, y_sec1]
    lower = [x_sec2, y_sec2]
  else
    upper = [x_sec2, y_sec2]
    lower = [x_sec1, y_sec1]
  end

  return upper, lower
end

##### END OF MESHING ###########################################################

################################################################################
# PARAMETRIZATION
################################################################################
"""
  `parameterize(x, y, z; inj_var::Int64=1, s=0.0001, debug=false)`

Receives a contour (line) and returns a parametrization function f(s) of the
contour. The parametrization is done on the path of the splined
contour such that f(0.5) returns the point (x,y,z) where half of the entire
contour has been walked, and f(1.0) returns the last point in the contour. To
perform this parametrization, the contour must be injective in least one of the
variables (x, y, or z) (i.e., all values of the variable are unique), and that
variable must be given in increasing order.

  **Arguments**
  * `x::Array{Float64,1}`    : x-coordinates of the contour.
  * `y::Array{Float64,1}`    : y-coordinates of the contour.
  * `z::Array{Float64,1}`    : z-coordinates of the contour.

  **Optional Arguments**
  * `inj_var::Int64`         : Indicates the variable that is injective, with
                                  1=x, 2=y, and 3=z.
  * `s::Float64`             : Spline smoothness.

See Parametrization section in documentation for more details.
"""
function parameterize(x, y, z; inj_var::Int64=1, s=0.0001, debug=false, atol=0,
                                            kspl="automatic", bc="extrapolate")
  # ERROR CASES
  if size(x)!=size(y)!=size(z)  # Invalid contour
    error("Invalid contour. "*
              "$(size(x))!=$(size(y))!=$(size(z)) (size(x)!=size(y)!=size(z))")
  elseif !(inj_var in [1,2,3])  # Invalid input
    error("Invalid `inj_var`=$inj_var (!(inj_var in [1,2,3]))")
  end

  # Identifies injective and dependant variables
  inj = nothing   # Injective variable
  dep = []        # Dependant variables
  for (i,var) in enumerate([x, y, z])
    if i==inj_var
      inj = var
    else
      push!(dep, var)
    end
  end

  # Splines
  # bc="extrapolate"           # Out of boundary case
  # s=0.0001                 # Spline smoothness
  k = kspl=="automatic" ? min(size(x)[1]-1, 5) : kspl  # Spline order
  spl = []                   # Spline of each variable respect the injective

  for var in dep
    this_spl = Dierckx.Spline1D(inj, var; k=k, bc=bc, s=s)
    push!(spl, this_spl)
  end

  # Defines the path function
  dfdx1(x) = Dierckx.derivative(spl[1], x)    # Derivative of f respect x1
  dfdx2(x) = Dierckx.derivative(spl[2], x)    # Derivative of f respect x2
  fun(x) = sqrt.(1 .+ (dfdx1(x)).^2 .+ (dfdx2(x)).^2)   # Integrand
                                                  # Integral between xmin and x
  fun_s(this_x) = QuadGK.quadgk(fun, inj[1], this_x; atol=atol)[1]

  # Defines the normalized path function
  stot = fun_s(inj[end])      # Total length of the path
  norm_s(x) = fun_s(x)/stot         # Normalized path function

  # Defines the inverse normalized function
  function inv_norm_s(sstar; debug=false)
      # ERROR CASES
      if sstar<-0.001 || sstar>1.001 # Outside of domain
          error("Invalid normalized path length $(sstar) (sstar<0 || sstar>1)")
      end

      # Finds the x that matches the target (sstar)
      this_fun(x) = sstar - norm_s(x)   # Zeroed function
      bracket = [ inj[1]*(1-0.01*sign(inj[1])),# Bracket of the zero
                  inj[end]*(1+0.01*sign(inj[end]))]

      if debug
          println("sstar=$sstar\tbracket=$bracket")
          println("flow=$(this_fun(bracket[1]))")
          println("fup=$(this_fun(bracket[2]))")
      end

      # Zero
      this_x = Roots.find_zero(this_fun, bracket, Roots.Bisection())

      return this_x
  end

  # Redefine the original function in terms of the new parametrization s*
  function fstar(sstar)
      x = inv_norm_s(sstar; debug=debug)       # Finds the x that matches the path
      out = []                    # Output
      dep_i = 1                   # Next spline to evaluate
      for i in 1:3
        if i==inj_var
          push!(out, x)           # Injective variable
        else
          push!(out, spl[dep_i](x))   # Dependant variables
          dep_i += 1
        end
      end
      return out
  end

  return fstar
end

##### END OF PARAMETRIZATION ###################################################

################################################################################
# LINEAR ALGEBRA TRANSFORMATIONS
################################################################################
"""
  `transform(V::Array{Float64,1}, M::Array{Float64,2}, T::Array{Float64,1})`

Rotates and translates the vector V. Receives the i', j', k' unit vectors of an
euclidean system with origin T, and returns ``V'=M(V-T)`` (In this version, the
unit vectors have been organized as a matrix `M=[i'; j'; k']`).
"""
function transform(V::Array{T1,1}, M::Array{T2,2}, T::Array{T3,1}
                                        ) where{T1<:Real, T2<:Real, T3<:Real}
    return [
                M[1,1]*(V[1]-T[1]) + M[1,2]*(V[2]-T[2]) + M[1,3]*(V[3]-T[3]),
                M[2,1]*(V[1]-T[1]) + M[2,2]*(V[2]-T[2]) + M[2,3]*(V[3]-T[3]),
                M[3,1]*(V[1]-T[1]) + M[3,2]*(V[2]-T[2]) + M[3,3]*(V[3]-T[3])
            ]
end

function transform!(out::Array{T1, 1}, V::Array{T2,1},
                            M::Array{T3,2}, T::Array{T4,1}
                           ) where{T1<:Real, T2<:Real, T3<:Real, T4<:Real}
    out[1] = M[1,1]*(V[1]-T[1]) + M[1,2]*(V[2]-T[2]) + M[1,3]*(V[3]-T[3])
    out[2] = M[2,1]*(V[1]-T[1]) + M[2,2]*(V[2]-T[2]) + M[2,3]*(V[3]-T[3])
    out[3] = M[3,1]*(V[1]-T[1]) + M[3,2]*(V[2]-T[2]) + M[3,3]*(V[3]-T[3])
end

function transform(Vs::AbstractArray{Array{P,1},1}, M::AbstractArray{P,2}, T::AbstractArray{P,1}
                                                                ) where{P<:Real}
  out = Array{Float64,1}[]
  for V in Vs
    push!(out, transform(V, M, T))
  end
  return out
end

"""
  `countertransform(Vp::Array{Float64,1}, invM::Array{Float64,2},
T::Array{Float64,1})`

Rotates and translates back a vector V' that had been rotated and translated
into the system (i', j', k') with origin T, and returns the original
``V=M^{-1}V' + T``. To ease repetitive computation, instead of giving the unit
vectors, give the inverse of their matrix.
"""
function countertransform(Vp::Array{T1,1}, invM::Array{T2,2}, T::Array{T3,1}
                                        ) where{T1<:Real, T2<:Real, T3<:Real}
    return [
                invM[1,1]*Vp[1] + invM[1,2]*Vp[2] + invM[1,3]*Vp[3] + T[1],
                invM[2,1]*Vp[1] + invM[2,2]*Vp[2] + invM[2,3]*Vp[3] + T[2],
                invM[3,1]*Vp[1] + invM[3,2]*Vp[2] + invM[3,3]*Vp[3] + T[3]
            ]
end

function countertransform!(out::Array{T1, 1}, Vp::Array{T2,1},
                            invM::Array{T3,2}, T::Array{T4,1}
                           ) where{T1<:Real, T2<:Real, T3<:Real, T4<:Real}
   out[1] = invM[1,1]*Vp[1] + invM[1,2]*Vp[2] + invM[1,3]*Vp[3] + T[1]
   out[2] = invM[2,1]*Vp[1] + invM[2,2]*Vp[2] + invM[2,3]*Vp[3] + T[2]
   out[3] = invM[3,1]*Vp[1] + invM[3,2]*Vp[2] + invM[3,3]*Vp[3] + T[3]
end

function countertransform(Vps::AbstractArray{Array{P,1},1}, invM::AbstractArray{P,2},
                                                  T::AbstractArray{P,1}) where{P<:Real}
  out = Array{Float64,1}[]
  for Vp in Vps
    push!(out, countertransform(Vp, invM, T))
  end
  return out
end

"""
  `check_coord_sys(M::Array{Float64,2}; raise_error::Bool=true)`

Checks that the unit vectors given as the matrix `M=[i;j;k]` define a coordinate
system
"""
function check_coord_sys(M::Array{T,2}; raise_error::Bool=true) where{T<:Real}
  # Checks normalization
  for i in 1:size(M)[1]
    if abs(norm(M[i,:])-1) > 0.00000001
      println(M)
      if raise_error
        error("Not unitary axis: $(M[i,:])")
      else
        return false
      end
    end
  end

  # Checks ortogonality
  for i in size(M)[1]
    xi = M[i, :]
    xip1 = M[(i%size(M)[1])+1, :]
    proj = abs(dot(xi, xip1))
    if proj>0.00000001
      if raise_error
        error("Non-ortogonal system $M")
      else
        return false
      end
    end
  end
  return true
end

function check_coord_sys(M::Array{Array{T,1},1}; raise_error::Bool=true
                                                                ) where{T<:Real}
  dims = 3
  newM = Array{Float64}(undef, dims, dims)
  for i in 1:dims
    newM[i, :] = M[i]
  end
  return check_coord_sys(newM; raise_error=raise_error)
end

"""
  `axis_rotation(r::Array{Float64, 1}, angle_deg::Float64)`

Returns the transformation matrix of rotation around an arbitrary axis of unit
vector `r`
"""
function axis_rotation(r::Array{T, 1}, angle_deg::Real) where{T<:Real}
  ux, uy, uz = r
  C = cos(angle_deg*pi/180)
  S = sin(angle_deg*pi/180)
  t = 1 - C
  M = [t*ux^2+C t*ux*uy-S*uz t*ux*uz+S*uy;
        t*ux*uy+S*uz t*uy^2+C t*uy*uz-S*ux;
        t*ux*uz-S*uy t*uy*uz+S*ux t*uz^2+C]
  return M
end

"""
  `rotation_matrix(yaw::Real, pitch::Real, roll::Real)`

Receives yaw, pitch, and roll angles (in degrees) and returns the rotation
matrix corresponding to this rotation.
(see http://planning.cs.uiuc.edu/node102.html)

NOTE: Naming follows aircraft convention, with
* yaw:    rotation about z-axis.
* pitch:  rotation about y-axis.
* roll:   rotation about x-axis.
"""
function rotation_matrix(yaw::Real, pitch::Real, roll::Real)
  a, b, g = yaw*pi/180, pitch*pi/180, roll*pi/180
  Rz = [cos(a) -sin(a) 0; sin(a) cos(a) 0; 0 0 1]
  Ry = [cos(b) 0 sin(b); 0 1 0; -sin(b) 0 cos(b)]
  Rx = [1 0 0; 0 cos(g) -sin(g); 0 sin(g) cos(g)]
  return Rz*Ry*Rx
end

"""
  `rotation_matrix2(roll::Real, pitch::Real, yaw::Real)`

Receives yaw, pitch, and roll angles (in degrees) and returns the rotation
matrix corresponding to this rotation.
(see http://planning.cs.uiuc.edu/node102.html)

NOTE: Naming follows aircraft convention, with
* roll:   rotation about x-axis.
* pitch:  rotation about y-axis.
* yaw:    rotation about z-axis.

**Examples**
```jldoctest
julia> M = gt.rotation_matrix(0, 0, 0)
3×3 Array{Float64,2}:
  1.0  0.0  0.0
  0.0  1.0  0.0
  0.0  0.0  1.0

julia> M = gt.rotation_matrix(90, 0, 0)
3×3 Array{Float64,2}:
  1.0  0.0   0.0
  0.0  0.0  -1.0
  0.0  1.0   0.0

julia> X = [0, 1, 0];
julia> Xp = M*X
3-element Array{Float64,1}:
  0.0
  0.0
  1.0

julia> M = gt.rotation_matrix(0, 90, 0)
3×3 Array{Float64,2}:
  0.0  0.0  1.0
  0.0  1.0  0.0
 -1.0  0.0  0.0

julia> X = [1, 0, 0];
julia> Xp = M*X
3-element Array{Float64,1}:
  0.0
  0.0
 -1.0

julia> M = gt.rotation_matrix(0, 0, 90)
3×3 Array{Float64,2}:
  0.0  -1.0  0.0
  1.0   0.0  0.0
  0.0   0.0  1.0

julia> X = [1, 0, 0];
julia> Xp = M*X
3-element Array{Float64,1}:
  0.0
  1.0
  0.0

julia> M = gt.rotation_matrix(0, 45, 0)
3×3 Array{Float64,2}:
  0.707  0.0  0.707
  0.0    1.0  0.0
 -0.707  0.0  0.707

julia> X = [1, 0, 0];
julia> Xp = M*X
3-element Array{Float64,1}:
  0.707
  0.0
 -0.707

julia> M = gt.rotation_matrix(0, 45, 45)
3×3 Array{Float64,2}:
  0.5    -0.707  0.5
  0.5     0.707  0.5
 -0.707   0.0    0.707

julia> X = [1, 0, 0];
julia> Xp = M*X
3-element Array{Float64,1}:
  0.5
  0.5
 -0.707
```
"""
function rotation_matrix2(roll::Real, pitch::Real, yaw::Real)
  return rotation_matrix(yaw, pitch, roll)
end
##### END OF ALGEBRA ###########################################################

################################################################################
# ORTHOGONAL SPACE TRANSFORMATIONS
################################################################################

# ORTHOGONAL SYSTEMS (See https://en.wikipedia.org/wiki/Orthogonal_coordinates)
function cylindrical3D(X)
    r, theta, z = X
    return [ r*cos(theta), r*sin(theta), z ]
end
function cylindrical2D(X)
    r, theta = X
    return [ r*cos(theta),  r*sin(theta)]
end
function spherical3D(X)
    r, theta, phi = X
    return [  r*sin(theta)*cos(phi),
              r*sin(theta)*sin(phi),
              r*cos(theta) ]
end
function parabolic3D(X)
    u,v,z = X
    return [ 1/2*(u^2-v^2), u*v, z ]
end
function paraboloidal3D(X)
    u,v,phi = X
    return [u*v*cos(phi), u*v*sin(phi), 1/2*(u^2-v^2)]
end
function elliptic3D(X; a=1)
    u,v,z = X
    return [a*cosh(u)*cos(v), a*sinh(u)*sin(v), z]
end
function prolate3D(X; a=1)
    xi,etha,phi = X
    return [a*sinh(xi)*sin(etha)*cos(phi),
            a*sinh(xi)*sin(etha)*sin(phi),
            a*cosh(xi)*cos(etha)]
end
function oblate3D(X; a=1)
    xi,etha,phi = X
    return [a*cosh(xi)*cos(etha)*cos(phi),
            a*cosh(xi)*cos(etha)*sin(phi),
            a*sinh(xi)*sin(etha)]
end
function bipolar3D(X; a=1)
    u,v,z = X
    return [a*sinh(v)/(cosh(v)-cos(u)),
            a*sin(u)/(cosh(v)-cos(u)),
            z]
end
function toroidal3D(X; a=1)
    u,v,phi = X
    return [a*sinh(v)*cos(phi)/( cosh(v)-cos(u) ),
            a*sinh(v)*sin(phi)/( cosh(v)-cos(u) ),
            a*sin(u)/( cosh(v)-cos(u) )]
end
function conical3D(X; b=2, c=1)
    r,mu,nu = X
    return [r*mu*nu/(b*c),
            r/b*sqrt( (mu^2-b^2)*(nu^2-b^2) / (b^2-c^2) ),
            r/c*sqrt( (mu^2-c^2)*(nu^2-c^2) / (c^2-b^2) )]
end
##### END OF SPACE TRANSFORMATIONS #############################################


using Base
using Test
using LinearAlgebra
using Surrogates

#1D
lb = 0.0
ub = 4.0
x = [1.0,2.0,3.0]
y = [4.0,5.0,6.0]
linear = x -> norm(x)
cubic = x -> x^3
λ = 2.3
multiquadr = x -> sqrt(x^2+λ^2)
q = 1
my_rad = RadialBasis(x, y, lb, ub, rad = linearRadial)
est = my_rad(3.0)
@test est ≈ 6.0
add_point!(my_rad, 4.0, 10.0)
est = my_rad(3.0)
@test est ≈ 6.0
add_point!(my_rad,[3.2,3.3,3.4],[8.0,9.0,10.0])
est = my_rad(3.0)
@test est ≈ 6.0

my_rad = RadialBasis(x, y, lb, ub, rad = cubicRadial)
q = 2
my_rad = RadialBasis(x,y,lb,ub, rad = multiquadricRadial)

#ND
x = [(1.0, 2.0, 3.0), (4.0, 5.0, 6.0), (7.0, 8.0, 9.0)]
y = [4.0, 5.0, 6.0]
lb = [0.0,3.0,6.0]
ub = [4.0,7.0,10.0]
#bounds = [[0.0, 3.0, 6.0], [4.0, 7.0, 10.0]]
my_rad = RadialBasis(x, y, lb, ub)
est = my_rad((1.0,2.0,3.0))
@test est ≈ 4.0

#WITH ADD_POINT, adding singleton
x = [(1.0,2.0,3.0),(4.0,5.0,6.0),(7.0,8.0,9.0)]
y = [4.0,5.0,6.0]
lb = [0.0,3.0,6.0]
ub = [4.0,7.0,10.0]
#bounds = [[0.0,3.0,6.0],[4.0,7.0,10.0]]
my_rad = RadialBasis(x,y,lb,ub,rad = linearRadial, scale_factor = 1.0)
add_point!(my_rad,(9.0,10.0,11.0),10.0)
est = my_rad((1.0,2.0,3.0))
@test est ≈ 4.0

#WITH ADD_POINT, adding more
x = [(1.0,2.0,3.0),(4.0,5.0,6.0),(7.0,8.0,9.0)]
y = [4.0,5.0,6.0]
#bounds = [[0.0,3.0,6.0],[4.0,7.0,10.0]]
lb = [0.0,3.0,6.0]
ub = [4.0,7.0,10.0]
my_rad = RadialBasis(x,y,lb,ub)
add_point!(my_rad,[(9.0,10.0,11.0),(12.0,13.0,14.0)],[10.0,11.0])
est = my_rad((1.0,2.0,3.0))
@test est ≈ 4.0

lb = [0.0,0.0,0.0]
ub = [10.0,10.0,10.0]
#bounds = [lb,ub]
x = [(1.0,2.0,3.0),(4.0,5.0,6.0),(7.0,8.0,9.0)]
y = [4.0,5.0,6.0]

my_rad_ND = RadialBasis(x,y,lb,ub)
add_point!(my_rad_ND,(3.5,4.5,1.2),18.9)
add_point!(my_rad_ND,[(3.2,1.2,6.7),(3.4,9.5,7.4)],[25.72,239.0])
my_rad_ND = RadialBasis(x,y,lb,ub,rad = cubicRadial)
my_rad_ND = RadialBasis(x,y,lb,ub, rad = multiquadricRadial)
prediction = my_rad_ND((1.0,1.0,1.0))

# #100
f = x -> x[1]*x[2]
lb = [1.0, 2.0]
ub = [10.0, 8.5]
x = sample(500, lb, ub, SobolSample())
push!(x, (1.0, 2.0))
y = f.(x)
my_radial_basis = RadialBasis(x, y, lb, ub, rad = linearRadial)
@test my_radial_basis((1.0, 2.0)) ≈ 2
my_radial_basis = RadialBasis(x, y, lb, ub, rad =linearRadial)
@test my_radial_basis((1.0, 2.0)) ≈ 2

# #100
f = x -> x[1]*x[2]
lb = [1.0, 2.0]
ub = [10.0, 8.5]
x = sample(5, lb, ub, SobolSample())
push!(x, (1.0, 2.0))
y = f.(x)
my_radial_basis = RadialBasis(x, y, lb,ub, rad = linearRadial)
@test my_radial_basis((1.0, 2.0)) ≈ 2

# Multi-output #98
f  = x -> [x^2, x]
lb = 1.0
ub = 10.0
x  = sample(5, lb, ub, SobolSample())
push!(x, 2.0)
y  = f.(x)
my_radial_basis = RadialBasis(x, y, lb, ub, rad = linearRadial)
my_radial_basis(2.0)
@test my_radial_basis(2.0) ≈ [4, 2]

f  = x -> [x[1]*x[2], x[1]+x[2]^2]
lb = [1.0, 2.0]
ub = [10.0, 8.5]
x  = sample(5, lb, ub, SobolSample())
push!(x, (1.0, 2.0))
y  = f.(x)
my_radial_basis = RadialBasis(x, y, lb, ub, rad = linearRadial)
my_radial_basis((1.0, 2.0))
@test my_radial_basis((1.0, 2.0)) ≈ [2, 5]

x_new = (2.0, 2.0)
y_new = f(x_new)
add_point!(my_radial_basis, x_new, y_new)
@test my_radial_basis(x_new) ≈ y_new


# run this file using julia --project -t4 ch14_server.jl command

using Genie
using Statistics
using ThreadsX

function payoff_asian_sample(T, X0, K, r, s, m)::Float64
    X = X0
    sumX = X
    d = T / m
    for i in 1:m
        X *= exp((r - s^2 / 2) * d + s * sqrt(d) * randn())
        sumX += X
    end
    Y = sumX / (m + 1)
    return exp(-r * T) * max(Y - K, 0)
end

function asian_value(T, X0, K, r, s, m, max_time)
    result = Float64[]
    start_time = time()
    while time() - start_time < max_time
        append!(result,
                ThreadsX.map(_ -> payoff_asian_sample(T, X0, K, r, s, m),
                             1:10_000))
    end
    n = length(result)
    mv = mean(result)
    sdv = std(result)
    lo95 = mv - 1.96 * sdv / sqrt(n)
    hi95 = mv + 1.96 * sdv / sqrt(n)
    zero = mean(==(0), result)
    return (; n, mv, lo95, hi95, zero)
end

Genie.config.run_as_server = true

Genie.Router.route("/", method=POST) do
  message = Genie.Requests.jsonpayload()
  return try
      K = float(message["K"])
      max_time = float(message["max_time"])
      value = asian_value(1.0, 50.0, K, 0.05, 0.3, 200, max_time)
      Genie.Renderer.Json.json((status="OK", value=value))
  catch
      Genie.Renderer.Json.json((status="ERROR", value=""))
  end
end

Genie.startup()


################################################################################
# Intro Example
################################################################################
using Algames
using StaticArrays
using LinearAlgebra

T = Float64

# Define the dynamics of the system
p = 3 # Number of players
model = BicycleGame(p=p) # game with 3 players with unicycle dynamics
n = model.n
m = model.m

# Define the horizon of the problem
N = 20 # N time steps
dt = 0.1 # each step lasts 0.1 second
probsize = ProblemSize(N,model) # Structure holding the relevant sizes of the problem

# Define the objective of each player
# We use a LQR cost
Q = [Diagonal(10*ones(SVector{model.ni[i],T})) for i=1:p] # Quadratic state cost
R = [Diagonal(0.1*ones(SVector{model.mi[i],T})) for i=1:p] # Quadratic control cost
# Desrired state
xf = [SVector{model.ni[1],T}([2,+0.4,0,0]),
      SVector{model.ni[2],T}([2, 0.0,0,0]),
      SVector{model.ni[3],T}([3,-0.4,0,0]),
      ]
# Desired control
uf = [zeros(SVector{model.mi[i],T}) for i=1:p]
# Objectives of the game
game_obj = GameObjective(Q,R,xf,uf,N,model)
radius = 1.0*ones(p)
μ = 5.0*ones(p)
add_collision_cost!(game_obj, radius, μ)

# Define the constraints that each player must respect
game_con = GameConstraintValues(probsize)
# Add collision avoidance
radius = 0.08
add_collision_avoidance!(game_con, radius)
# Add control bounds
u_max =  5*ones(SVector{m,T})
u_min = -5*ones(SVector{m,T})
add_control_bound!(game_con, u_max, u_min)
# # Add state bounds for player 1
x_max =  5*ones(SVector{n,T})
x_min = -5*ones(SVector{n,T})
add_state_bound!(game_con, 1, x_max, x_min)
# Add wall constraint
walls = [Wall([0.0,-0.4], [1.0,-0.4], [0.,-1.])]
add_wall_constraint!(game_con, walls)
# Add circle constraint
xc = [1., 2., 3.]
yc = [1., 2., 3.]
radius = [0.1, 0.2, 0.3]
add_circle_constraint!(game_con, xc, yc, radius)

# Define the initial state of the system
x0 = SVector{model.n,T}([
    0.1, 0.0, 0.5,
   -0.4, 0.0, 0.7,
    0.0, 0.0, 0.0,
    0.0, 0.0, 0.0,
    ])

# Define the Options of the solver
opts = Options()
# Define the game problem
prob = GameProblem(N,dt,x0,model,opts,game_obj,game_con)

# Solve the problem
@time newton_solve!(prob)

# Visualize the Results
using Plots
plot(prob.model, prob.pdtraj.pr)
plot(prob.stats)


# https://sudonull.com/post/7909-Julia-Scripts-and-parsing-command-line-arguments

using DelimitedFiles, SharedArrays, Random, LinearAlgebra, Clustering, Base
using DataFrames, CSV
using Statistics, ArgParse
using ProgressMeter

nmats = 100 # iteraciones para samplear los eigenvalues de las correlaciones
n_kmedoids = 100 # iteraciones para calcular el minimo de kmedoids
n_kmedoids_out = 10 # Output de de las iteraciones del minimo de los kmedoids

############# FUNCTIONS
function max_eigv(gene)
    #returns largest eigenvalue of a randomized matrix generated from data.
    gran = zeros(size(gene))
    for i = 1:size(gene,2)
        gran[:,i] = gene[shuffle(1:end),i]
    end
 #    @time mr = corr_mat(gran)
    # println(mr[[end-1],end])
    mr = cor(transpose(gran))
    # println(mr[[end-1],end])
    return maximum(eigvals(mr))
end
#------------------------

############### MAIN
## JULIA_NUM_THREADS=$(nproc)
s = ArgParseSettings()
@add_arg_table! s begin
    "-c"
        help = "Flag to calculate number of clusters"
        action = :store_true
    "expfile"
        help = "Matriz de expresion (read a clean format exp matrix)"
        arg_type = String
        required = true
end
parsed_args = parse_args(s)
# parsed_args = Dict([("c", true), ("expfile", "Data/clean/Basal-clean.tsv")])
println(parsed_args)
fname = basename(parsed_args["expfile"])
subtype = split(fname,'-')[1]
println("Working on " * subtype)

println("Kamedoids clustering method on Cancer Expression Matrix")
flush(stdout)

Random.seed!(34568)

### INPUT FILES
#! gname s1 s2 ... gstart chromname
expr = readdlm(parsed_args["expfile"],skipstart=1)
#! cut cols: gname, gstart, chromname
gene = convert(Array{Float64,2},expr[:, 2:end-2])

###################
println("First correlation (m)")
@time  m = cor(transpose(gene))
gnames = expr[:,1]
gstarts = expr[:,end-1]
chroms= expr[:,end]
df = DataFrame(src = [], dst = [], src_gstart = Int64[],
    dst_gstart = Int64[], src_chrom = [], dst_chrom = [],
    pearson = Float64[])
@showprogress for i in 1:length(gnames)
    for j in i+1:length(gnames)
        # println(i," ",j," ",gnames[i]," ",gnames[j]," ",
        #     gstarts[i]," ",gstarts[j]," ",chroms[i]," ",
        #     chroms[j]," ", m[i,j])
        row = [gnames[i] gnames[j] gstarts[i] gstarts[j] chroms[i] chroms[j] m[i,j]]
        push!(df, row)
    end
end
odir = "Data/Pearson/"
mkpath(odir)
oname = odir * subtype * "-allchroms.csv"
CSV.write(oname,df)

###################
println("Automatic number of clusters detection for all chromosomes")
if parsed_args["c"] == true
    println("Get Eigenvalues from shuffles")
    l_evmax = SharedArray{Float64, 1}((nmats))
    for i in 1:nmats #doing the surrogate matrices and saving max eigv.
        @time l_evmax[i] = max_eigv(gene)
        println("Iteration: ",i,", max eigenvalue: ",l_evmax[i])
        flush(stdout)
    end
    #! get eigenvalue of gene
    m_ev = filter(x->x>0.0000001, eigvals(m))
    #! Todos los eigenvalues de la correlacion que sean mayores al mayor de las
    #! correlaciones muestreadas; el numero de estos eigenvalues es el numero de cluesters
    #! nc = 13, for Basal-All.txt with nmats=100
    nc = length(filter(x->x>maximum(l_evmax),m_ev))
    println("Number of clusters (nc): ",nc)
end

###################
println("Starting Kmedoids sampling for all chromosomes")
# make m in [0,1] insted [-1,1]
# correalacion solo en valores positivos
dmat = map(x-> 1-abs(x),m)

ens_c = []
for i = 1:n_kmedoids
    push!(ens_c, kmedoids(dmat,nc))
    if (i % n_kmedoids_out) == 0
    	println(i)
    end
end

#! De todas los clustereos cual es el que tiene el costo total minimo
inx_c = findmin(map(x -> x.totalcost, ens_c))[2]
c = ens_c[inx_c]
println("Cluster counts: ",c.counts)

###################
#! Saving
# writedlm("out.tsv", [expr[:,1] c.assignments c.acosts expr[:,end-1] expr[:,end]])
# header = ["gname" "clusterid" "assignmentcost" "gstart" "chromosome"]
# data = [expr[:,1] c.assignments c.costs expr[:,end-1] expr[:,end]]
# writedlm("out.tsv", [header ; data])
df = DataFrame([expr[:,1],c.assignments,c.costs,expr[:,end-1],expr[:,end]])
header = ["gname","clusterid","assignmentcost","gstart","chromosome"]
rename!(df, Symbol.(header)) # Force set header names
odir = "Data/Clustered/" * subtype * "/"
mkpath(odir)
oname = odir * subtype * "-all-clusters.csv"
CSV.write(oname,df)