#
#      Hn - The manifold of the n-dimensional sphere
#  Point is a Point on the n-dimensional sphere.
#
import LinearAlgebra: norm, dot
import Base: exp, log, show
export Hyperbolic, HnPoint, HnTVector, getValue
export distance, dot, exp, log, manifoldDimension, norm, parallelTransport
export typeofMPoint, typeofTVector, MinkowskiDot
export validateMPoint, validateTVector, zeroTVector
#
# Type definitions
#

@doc doc"""
    Hyperbolic <: Manifold
The manifold $\mathbb H^n$ is the set

```math
\mathbb H^n = \Bigl\{x\in\mathbb R^{n+1}
\ \Big|\ \langle x,x \rangle_{\mathrm{M}}= -x_{n+1}^2
+ \displaystyle\sum_{k=1}^n x_k^2 = -1, x_{n+1} > 0\Bigr\},
```

where $\langle\cdot,\cdot\rangle_{\mathrm{M}}$ denotes the [`MinkowskiDot`](@ref)
is Minkowski inner product, and this inner product in the embedded space yields
the Riemannian metric when restricted to the tangent bundle $T\mathbb H^n$.

This manifold is a matrix manifold (see [`IsMatrixM`](@ref)) and embedded (see
[`IsEmbeddedM`](@ref)).

# Abbreviation

`Hn`

# Constructor

    Hyperbolic(n)

generates the `n`-dimensional hyperbolic manifold embedded in $\mathbb R^{n+1}$.
"""
struct Hyperbolic <: Manifold
  name::String
  dimension::Int
  abbreviation::String
  Hyperbolic(dimension::Int) = new("$dimension-Hyperbolic Space",dimension,"Hn($(dimension-1))")
end
@doc doc"""
    HnPoint <: MPoint

A point $x$ on the manifold $\mathbb H^n$ represented by a vector
$x\in\mathbb R^{n+1}$ with Minkowski inner product

$\langle x,x\rangle_{\mathrm{M}} = -x_{n+1}^2 + \sum_{k=1}^n x_k^2 = -1$.
"""
struct HnPoint{T<:AbstractFloat} <: MPoint
  value::Vector{T}
  HnPoint{T}( value::Vector{T} ) where {T<:AbstractFloat} = new(value)
  HnPoint{T}( value::T) where {T<:AbstractFloat} = new([value])
end
HnPoint(value::Vector{T}) where {T<:AbstractFloat} = HnPoint{T}(value)
HnPoint(value::T) where {T <: AbstractFloat} = HnPoint{T}(value)

getValue(x::HnPoint) = length(x.value)==1 ? x.value[1] : x.value

@doc doc"""
    HnTVector <: TVector

A tangent vector $\xi \in T_x\mathbb H^n$ to a [`HnPoint`](@ref) $x$ on the
$n$-dimensional [`Hyperbolic`](@ref) space $\mathbb H^n$. To be precise
$\xi\in\mathbb R^{n+1}$ is hyperbocally orthogonal to $x\in\mathbb R^{n+1}$,
i.e. orthogonal with respect to the Minkowski inner product

$\langle \xi, x \rangle_{\mathrm{M}} = -\xi_{n+1}x_{n+1} + \sum_{k=1}^n \xi_k x_k = 0$
"""
struct HnTVector{T <: AbstractFloat}  <: TVector
    value::Vector{T}
    HnTVector{T}(value::Vector{T})  where {T <: AbstractFloat}  = new(value)
    HnTVector{T}(value::T) where {T <: AbstractFloat}  = new([value])
end
HnTVector(value::T) where {T <: AbstractFloat} = HnTVector{T}(value)
HnTVector(value::Vector{T})  where {T <: AbstractFloat}  = HnTVector{T}(value)

getValue(ξ::HnTVector) = length(ξ.value)==1 ? ξ.value[1] : ξ.value

# Traits
# ---
# (a) Hn is a MatrixManifold
@traitimpl IsMatrixM{Hyperbolic}
@traitimpl IsMatrixP{HnPoint}
@traitimpl IsMatrixTV{HnTVector}
# (b) Hn is a MatrixManifold
@traitimpl IsEmbeddedM{Hyperbolic}
@traitimpl IsEmbeddedP{HnPoint}
@traitimpl IsEmbeddedV{HnTVector}

# Functions
# ---
@doc doc"""
    distance(M,x,y)

compute the Riemannian distance on the [`Hyperbolic`](@ref) space $\mathbb H^n$
embedded in $\mathbb R^{n+1}$ can be computed as

```math
d_{\mathbb H^n}(x,y)
= \operatorname{acosh} \bigl(-\langle x,y\rangle_{\mathrm{M}}\bigr),
```

where $\langle x,y\rangle_{\mathrm{M}} = -x_{n+1}y_{n+1} +
\displaystyle\sum_{k=1}^n x_ky_k$ denotes the [`MinkowskiDot`](@ref) Minkowski
inner product on $\mathbb R^{n+1}$.
"""
distance(M::Hyperbolic,x::HnPoint{T},y::HnPoint{T}) where {T <: AbstractFloat} = acosh(  max(1.,-MinkowskiDot(getValue(x), getValue(y)))  )

@doc doc"""
    dot(M,x,ξ,ν)

compute the Riemannian inner product for two [`HnTVector`](@ref)s `ξ` and `ν`
from $T_x\mathcal M$ of the [`Hyperbolic`](@ref) space $\mathbb H^n$ given by
$\langle \xi, \nu \rangle_{\mathrm{M}}$ the [`MinkowskiDot`](@ref) Minkowski
inner product on $\mathbb R^{n+1}$.
"""
dot(M::Hyperbolic, x::HnPoint{T}, ξ::HnTVector{T}, ν::HnTVector{T}) where {T <: AbstractFloat} = MinkowskiDot( getValue(ξ), getValue(ν) )

@doc doc"""
    exp(M,x,ξ,[t=1.0])

computes the exponential map on the [`Hyperbolic`](@ref) space $\mathbb H^n$ with
respect to the [`HnPoint`](@ref) `x` and the [`HnTVector`](@ref) `ξ`, which can
be shortened with `t` to `tξ`. The formula reads

$\exp_x\xi = \cosh(\sqrt{\langle\xi,\xi\rangle_{\mathrm{M}}})x + \operatorname{sinh}(\sqrt{\langle\xi,\xi\rangle_{\mathrm{M}}})\frac{\xi}{\sqrt{\langle\xi,\xi\rangle_{\mathrm{M}}}}.$
"""
function exp(M::Hyperbolic,x::HnPoint{T},ξ::HnTVector{T},t::Float64=1.0)  where {T <: AbstractFloat}
  len = sqrt(  max( MinkowskiDot( getValue(ξ), getValue(ξ) ), 0.)  )
  if len < eps(Float64)
  	return x
  else
  	return HnPoint( cosh(t*len)*getValue(x) + sinh(t*len)/len * getValue(ξ) )
  end
end
@doc doc"""
    log(M,x,y)

computes the logarithmic map on the [`Hyperbolic`](@ref) space $\mathbb H^n$,
i.e., the [`HnTVector`](@ref) whose corresponding
[`geodesic`](@ref) starting from [`HnPoint`](@ref) `x` reaches the
[`HnPoint`](@ref) `y` after time 1 on the [`Hyperbolic`](@ref) space
$\mathbb H^n$.
The formula reads for $x\neq y$

```math
\log_x y = d_{\mathbb H^n}(x,y)\frac{y-\langle x,y\rangle_{\mathrm{M}} x}{\lVert y-\langle x,y\rangle_{\mathrm{M}} x \rVert_2}
```
and is zero otherwise.
"""
function log(M::Hyperbolic,x::HnPoint{T},y::HnPoint{T}) where {T <: AbstractFloat}
  scp = MinkowskiDot( getValue(x), getValue(y) )
  ξvalue = getValue(y) + scp*getValue(x)
  ξnorm = sqrt(max( scp.^2-1, 0.))
  if (ξnorm > eps(Float64))
    return HnTVector( acosh(max(1.,-scp))/ξnorm * ξvalue )
  else
    return zeroTVector(M,x)
  end
end
@doc doc"""
    manifoldDimension(x)

returns the dimension of the [`Hyperbolic`](@ref) space $\mathbb H^n$, the
[`HnPoint`](@ref) `x`, itself embedded in $\mathbb R^{n+1}$, belongs to.
"""
manifoldDimension(x::HnPoint)::Integer = length( getValue(x) )-1
@doc doc"""
    manifoldDimension(M)

returns the dimension of the [`Hyperbolic`](@ref) space $\mathbb H^n$.
"""
manifoldDimension(M::Hyperbolic)::Integer = M.dimension
@doc doc"""
    norm(M,x,ξ)
Computes the norm of the [`HnTVector`](@ref) `ξ` in the tangent space
$T_x\mathcal M$ at [`HnPoint`](@ref) `x` of the
[`Hyperbolic`](@ref) space $\mathbb H^n$.
"""
norm(M::Hyperbolic, x::HnPoint{T}, ξ::HnTVector{T})  where {T <: AbstractFloat}= sqrt(max( dot(M,x,ξ,ξ), 0.) )
@doc doc"""
    parallelTransport(M,x,y,ξ)

Compute the paralllel transport of the [`HnTVector`](@ref) `ξ` from
the tangent space $T_x\mathcal M$ at [`HnPoint`](@ref) `x` to
$T_y\mathcal M$ at [`HnPoint`](@ref) `y` on the
[`Hyperbolic`](@ref) space $\mathbb H^n$ along the unique [`geodesic`](@ref)
$g(\cdot;x,y)$.
The formula reads

$P_{x\to y}(\xi) = \xi - \frac{\langle \log_xy,\xi\rangle_x}
{d^2_{\mathbb H^n}(x,y)}\bigl(\log_xy + \log_yx \bigr).$
"""
function parallelTransport(M::Hyperbolic, x::HnPoint{T}, y::HnPoint{T}, ξ::HnTVector{T})  where {T <: AbstractFloat}
  ν = log(M,x,y)
  νL = norm(M,x,ν)
	return ξ - ( νL > 0 ? dot(M,x,ν,ξ)*(ν + log(M,y,x))/νL^2 : zeroTVector(M,x) )
end

typeofTVector(::Type{HnPoint{T}}) where T = HnTVector{T}
typeofMPoint(::Type{HnTVector{T}}) where T = HnPoint{T}

@doc doc"""
    typicalDistance(M)
returns the typical distance on the [`Hyperbolic`](@ref) space `M`: $\sqrt{n}$.
"""
typicalDistance(M::Hyperbolic) = sqrt(M.dimension);
@doc doc"""
    validateMPoint(M,x)

validate, that the [`HnPoint`](@ref) `x` is a valid point on the
[`Hyperbolic`](@ref) space `M`, i.e. that the dimension of $x\in\mathbb H^n$ is
correct and that its [`MinkowskiDot`](@ref) inner product is $\langle x,x\rangle_{\mathrm{M}} = -1$.
"""
function validateMPoint(M::Hyperbolic, x::HnPoint)
    if length(getValue(x)) ≠ M.dimension+1
    throw( ErrorException(
      "The Point $x is not on the $(M.name), since the vector dimension ($(length(getValue(x)))) is not $(M.dimension+1)."
    ))
  end
  if abs(MinkowskiDot(getValue(x),getValue(x))+1) >= 10^(-15)
    throw( ErrorException(
      "The Point $x is not on the $(M.name) since its minkowski inner product <x,x>_MN is $(MinkowskiDot(getValue(x),getValue(x))) and not -1"
    ))
  end
  return true
end
@doc doc"""
    validateTVector(M,x,ξ)

check that the [`HnTVector`](@ref) `ξ` is a valid tangent vector in the tangent
space of the [`HnPoint`](@ref) `x` on the [`Hyperbolic`](@ref) space `M`, i.e. `x`
is a valid point on `M`, the vectors within `ξ` and `x` agree in length and the
Minkowski inner product [`MinkowskiDot`](@ref)`(x,ξ) `is zero.
"""
function validateTVector(M::Hyperbolic, x::HnPoint, ξ::HnTVector)
    if !validateMPoint(M,x)
        return false
     end
    if length(getValue(x)) != length(getValue(ξ))
        throw( ErrorException(
            "The lengths of x ($(length(getValue(x)))) and ξ ($(length(getValue(ξ)))) do not agree, so ξ can not be a tangent vector of x."
        ))
    end
    if abs( MinkowskiDot(getValue(x),getValue(ξ)) ) > 10^(-15)
        throw( ErrorException(
            "The tangent vector should be (hyperbolically) orthogonal to its base, but the (Minkowski) inner product yields $(  MinkowskiDot(getValue(x),getValue(ξ)) )."
        ))
    end
    return true
end
@doc doc"""
    ξ = zeroTVector(M,x)

returns a zero vector in the tangent space $T_x\mathcal M$ of the
[`HnPoint`](@ref) $x\in\mathbb H^n$ on the [`Hyperbolic`](@ref) space `M`.
"""
zeroTVector(M::Hyperbolic, x::HnPoint{T}) where {T <: AbstractFloat} = HnTVector(  zero( getValue(x) )  );
# Display
# ---
show(io::IO, M::Hyperbolic) = print(io, "The $(M.name).")
show(io::IO, p::HnPoint) = print(io, "Hn($( getValue(p) ))")
show(io::IO, ξ::HnTVector) = print(io, "HnT($( getValue(ξ) ))")
# Helper
#
@doc doc"""
    MinkowskiDot(a,b)
computes the Minkowski inner product of two Vectors `a` and `b` of same length
`n+1`, i.e.

$\langle a,b\rangle_{\mathrm{M}} = -a_{n+1}b_{n+1} +
\displaystyle\sum_{k=1}^n a_kb_k.$
"""
MinkowskiDot(a::Vector,b::Vector) = -a[end]*b[end] + sum( a[1:end-1].*b[1:end-1] )


module Combinatorics

using ...Prerequisites.TypeTraits: corder,indtosub

export AbstractCombinatorics
export Combinations,DulCombinations,Permutations,DulPermutations

"""
    AbstractCombinatorics{M,C}

Abstract combinatoric algorithms.
"""
abstract type AbstractCombinatorics{M,C} end
Base.eltype(::Type{<:AbstractCombinatorics{M,C}}) where {M,C}=NTuple{M,C|>eltype}
@generated gettuple(contents,inds,::Val{M}) where M=Expr(:tuple,[:(contents[inds[$i]]) for i=1:M]...)

"""
    Combinations{M}(contents::C) where {M,C}

Combinations of M elements from contents. Duplicates are not allowed.
"""
struct Combinations{M,C} <: AbstractCombinatorics{M,C}
    contents::C
    N::Int
    Combinations{M}(contents::C) where {M,C}=new{M,C}(contents,contents|>length)
end
Base.length(c::Combinations{M}) where M=binomial(c.N,M)
Base.iterate(c::Combinations{M}) where M=M>c.N ? nothing : M==0 ? ((),[c.N+2]) : (gettuple(c.contents,1:M,Val(M)),nextmstate!(collect(1:M),c.N,M))
Base.iterate(c::Combinations{M},state) where M=state[1]>c.N-M+1 ? nothing : (gettuple(c.contents,state,Val(M)),nextmstate!(state,c.N,M))
function nextmstate!(state::Vector{Int},N::Int,M::Int)
    for i=M:-1:1
        state[i]+=1
        state[i]>(N-(M-i)) && continue
        for j=i+1:M
            state[j]=state[j-1]+1
        end
        break
    end
    state
end

"""
    DulCombinations{M}(contents::C) where {M,C}

Combinations of M elements from contents. Duplicates are allowed.
"""
struct DulCombinations{M,C} <: AbstractCombinatorics{M,C}
    contents::C
    N::Int
    DulCombinations{M}(contents::C) where {M,C}=new{M,C}(contents,contents|>length)
end
Base.length(c::DulCombinations{M}) where M=binomial(c.N+M-1,c.N-1)
Base.iterate(c::DulCombinations{M}) where M=M==0 ? ((),[c.N+1]) : (gettuple(c.contents,ntuple(i->1,M),Val(M)),nextdmstate!(collect(ntuple(i->1,M)),c.N,M))
Base.iterate(c::DulCombinations{M},state) where M=state[1]>c.N ? nothing : (gettuple(c.contents,state,Val(M)),nextdmstate!(state,c.N,M))
function nextdmstate!(state::Vector{Int},N::Int,M::Int)
    for i=M:-1:1
        state[i]+=1
        state[i]>N && continue
        for j=i+1:M
            state[j]=state[j-1]
        end
        break
    end
    state
end

"""
    Permutations{M}(contents::C) where {M,C}

Permutations of M elements from contents. Duplicates are allowed.
"""
struct Permutations{M,C} <: AbstractCombinatorics{M,C}
    contents::C
    N::Int
    Permutations{M}(contents::C) where {M,C}=new{M,C}(contents,contents|>length)
end
Base.length(p::Permutations{M}) where M=(0<=M<=p.N) ? prod((p.N-M+1):p.N) : 0
Base.iterate(p::Permutations{M}) where M=(p.N==0 && M>0 || 0<p.N<M) ? nothing : (state=collect(1:p.N);(gettuple(p.contents,state,Val(M)),nextpstate!(state,p.N,M)))
Base.iterate(p::Permutations{M},state) where M=(p.N==0 && M>0 || 0<p.N<max(state[1],M)) ? nothing : (gettuple(p.contents,state,Val(M)),nextpstate!(state,p.N,M))
function nextpstate!(state::Vector{Int},N::Int,M::Int)
    M<=0 && return [N+1]
    M<N && (j=M+1; while j<=N && state[M]>=state[j]; j+=1; end)
    if M<N && j<=N
        state[M],state[j]=state[j],state[M]
    else
        M<N && reverse!(state,M+1)
        i=M-1
        while i>=1 && state[i]>=state[i+1];i-=1; end
        if i>0
            j=N
            while j>i && state[i]>=state[j];j-=1; end
            state[i],state[j]=state[j],state[i]
            reverse!(state,i+1)
        else
            state[1]=N+1
        end
    end
    return state
end

"""
    DulPermutations{M}(contents::C) where {M,C}

Permutations of M elements from contents. Duplicates are not allowed.
"""
struct DulPermutations{M,C} <: AbstractCombinatorics{M,C}
    contents::C
    N::Int
    DulPermutations{M}(contents::C) where {M,C}=new{M,C}(contents,contents|>length)
end
Base.length(p::DulPermutations{M}) where M=p.N^M
Base.iterate(p::DulPermutations{M},state=1) where M=state>length(p) ? nothing : (indtosub(ntuple(i->p.N,M),state,corder),state+1)

end # module


## CLOSURE RELATIONS
#============================================#

# Momentum shape for density: H
H_Whitfield(H, M_e) = (H - 0.290M_e^2)/(1 + 0.113M_e^2)

## Laminar closure for walls - Falkner-Skan profiles
#============================================#

# Laminar kin-ergy shape: H*
lam_H_star(H_k) = 1.515 + ifelse(H_k < 4., 0.076, 0.040) * (H_k - 4)^2 / H_k

# Laminar skin-friction: Re_θ CF/2
lam_CF(H_k) = -0.067 + ifelse(H_k < 7.4, 0.01977(7.4 - H_k)^2 / (H_k - 1), 0.022(1. - 1.4 / (H_k - 6.))^2)

# Laminar dissipation: Re_θ 2CD / H_k*
lam_CD(H_k) = 0.207 + ifelse(H_k < 4., 0.00205(4 - H_k)^5.5, -0.003(H_k - 4.)^2 / (1. + 0.02(H_k - 4.)^2))

## Laminar closure for wakes
#============================================#

# Laminar wake kin-ergy shape: H*
lam_wake_H_star(H_k) = 1.50 + ifelse(H_k < 3.5, 0.025(3.5 - H_k)^3 + 0.001(3.5 - H_k)^5, 0.015(H_k - 3.5)^2 / H_k)

# Laminar wake dissipation: Re_θ 2CD / H*
lam_wake_CD(H_k) = 1.52(H_k - 1)^2 / (3 + H_k^3)

## Turbulent closure relations
#============================================#

# Skin-friction
turb_CF(H_k, Re_θ, M_e) = let F_c = √(1 + 0.2M_e^2); 0.3exp(-1.33H_k) * log(10, Re_θ / F_c)^(-1.74 - 0.31H_k) / F_c + 1.1e-4(tanh(4 - H_k/0.875) - 1) end

# Density thickness (negligible at low speeds): H**
turb_H_star_star(H_k, M_e) = (0.064 / (H_k - 0.8) + 0.251)M_e^2

# Y+
y_plus(u_t, μ_e, ρ_e, η) = ρ_e * u_t * η / μ_e

# Velocity profile
function turb_ue(Cf, θ, y_p, η, a, b)
    uτ_ue = √(Cf / 2)
    s     = Cf / abs(Cf)

    uτ_ue * s / 0.09 * atan(0.09y_p) + (1 - uτ_ue * s * π / 0.18) * √(tanh(a * (η, θ)^b))
end

# H*?
function turb_H_star(H_k, Re_θ)
    H_0 = ifelse(Re_θ < 400, 4, 3 + 400/Re_θ)

    1.505 + 4/Re_θ + ifelse(H_k < H_0, (0.165 - 1.6/√Re_θ) * (H_0 - H_k)^1.6 / H_k, (H_k - H_0)^2 * (0.04 / H_k + 0.007log(10, Re_θ)/(H_k - H_0 + 4/log(10, Re_θ))^2))
end

# CD
turb_CD(cf, u_s, c_τ) = cf * u_s / 2 + c_τ * (1 - u_s)

# U_s
wall_slip(H_star, H_k, H) = H_star / 2 * (1 - 4/3(H_k - 1) / H)

## Boundary layer thickness δ from Green's correlation
δ_thick(H_k, θ, δ_star) = θ * (3.15 + 1.72 / (H_k - 1)) + δ_star

# Equilibirum shear stress coefficient C_τ_EQ
c_τ_equilibrium(H, H_k, H_star, u_s) = 0.015H_star / (1 - u_s) * (H_k - 1)^3 / (H_k^2 * H)

## Transition
#============================================#

# Rate slope dñ/dReθ
dñdReθ(H_k) = 0.01 * √((2.4H_k - 3.7 + 2.5tanh(1.5H_k - 4.65))^2 + 0.25)

# Critical Reynolds number: log₁₀ Re_θ
log10Reθ(H_k) = (1.415/(H_k - 1) - 0.489)tanh(20/(H_k - 1) - 12.9) + 3.295/(H_k - 1) + 0.44

# G-β locus?
G(β) = 6.7 * √(1 + 0.75β)

# Weird amplification ratio DE
l(H_k) = (6.54H_k - 14.07) / H_k^2
m(H_k) = (0.058(H_k - 4)^2 / (H_k - 1) - 0.068) / l(H_k)
# New correlation: -0.05 + 2.7*H_k - 5.5H_k^2 + 3.0H_k^3

## Generic conversions
#============================================#

# Cf = 2lam_CF(H) / Re
cf(U_e, θ, H, ν = 1.5e-5) = 2 * lam_CF(H) / reynolds_number(U_e, θ, ν)

# CD = lam_H_star(H) * lam_CD(H) / 2Re
cD(U_e, θ, H, ν) = lam_H_star(H) * lam_CD(H) / (2 * reynolds_number(U_e, θ, ν) )

# Squire-Young formula for drag coefficient
cd_squire_young(θ_wake, u_e_wake, H_wake, V_inf) = 2θ_wake * (u_e_wake / V_inf)^((5 + H) / 2)


# frexp,ldexp,significand,exponent
@test frexp(12.8) == (0.8,4)
@test ldexp(0.8,4) == 12.8
@test significand(12.8) == 1.6
@test exponent(12.8) == 3

# degree-based trig functions
for T = (Float32,Float64)
    for x = -400:40:400
        @test_approx_eq_eps sind(convert(T,x))::T convert(T,sin(pi/180*x)) eps(deg2rad(convert(T,x)))
        @test_approx_eq_eps cosd(convert(T,x))::T convert(T,cos(pi/180*x)) eps(deg2rad(convert(T,x)))
    end
    for x = 0.0:180:720
        @test sind(convert(T,x)) === zero(T)
        @test sind(-convert(T,x)) === -zero(T)
    end

    for x = -3:0.3:3
        @test_approx_eq_eps sinpi(convert(T,x))::T convert(T,sin(pi*x)) eps(pi*convert(T,x))
        @test_approx_eq_eps cospi(convert(T,x))::T convert(T,cos(pi*x)) eps(pi*convert(T,x))
    end
    for x = 0.0:1.0:4.0
        @test sinpi(convert(T,x)) === zero(T)
        @test sinpi(-convert(T,x)) === -zero(T)
    end
end

# check type stability
for T = (Float32,Float64,BigFloat)
    for f = (sind,cosd,sinpi,cospi)
        @test Base.return_types(f,(T,)) == [T]
    end
end

# error functions
@test_approx_eq erf(1) 0.84270079294971486934
@test_approx_eq erfc(1) 0.15729920705028513066
@test_approx_eq erfcx(1) 0.42758357615580700442
@test_approx_eq erfi(1) 1.6504257587975428760
@test_approx_eq erfinv(0.84270079294971486934) 1
@test_approx_eq erfcinv(0.15729920705028513066) 1
@test_approx_eq dawson(1) 0.53807950691276841914
# TODO: complex versions only supported on 64-bit for now
@unix_only if WORD_SIZE==64
    @test_approx_eq erf(1+2im) -0.53664356577856503399-5.0491437034470346695im
    @test_approx_eq erfc(1+2im) 1.5366435657785650340+5.0491437034470346695im
    @test_approx_eq erfcx(1+2im) 0.14023958136627794370-0.22221344017989910261im
    @test_approx_eq erfi(1+2im) -0.011259006028815025076+1.0036063427256517509im
    @test_approx_eq dawson(1+2im) -13.388927316482919244-11.828715103889593303im
end
for x in logspace(-200, -0.01)
    @test_approx_eq_eps erf(erfinv(x)) x 1e-12*x
    @test_approx_eq_eps erf(erfinv(-x)) -x 1e-12*x
    @test_approx_eq_eps erfc(erfcinv(2*x)) 2*x 1e-12*x
    if x > 1e-20
        xf = float32(x)
        @test_approx_eq_eps erf(erfinv(xf)) xf 1e-5*xf
        @test_approx_eq_eps erf(erfinv(-xf)) -xf 1e-5*xf
        @test_approx_eq_eps erfc(erfcinv(2xf)) 2xf 1e-5*xf
    end
end

# airy
@test_approx_eq airy(1.8) 0.0470362168668458052247
@test_approx_eq airyprime(1.8) -0.0685247801186109345638
@test_approx_eq airybi(1.8) 2.595869356743906290060
@test_approx_eq airybiprime(1.8) 2.98554005084659907283
@test_throws Base.Math.AmosException airy(200im)
@test_throws Base.Math.AmosException airybi(200)

# besselh
true_h133 = 0.30906272225525164362 - 0.53854161610503161800im
@test_approx_eq besselh(3,1,3) true_h133
@test_approx_eq besselh(-3,1,3) -true_h133
@test_approx_eq besselh(3,2,3) conj(true_h133)
@test_approx_eq besselh(-3,2,3) -conj(true_h133)
@test_throws Base.Math.AmosException besselh(1,0)

# besseli
true_i33 = 0.95975362949600785698
@test_approx_eq besseli(3,3) true_i33
@test_approx_eq besseli(-3,3) true_i33
@test_approx_eq besseli(3,-3) -true_i33
@test_approx_eq besseli(-3,-3) -true_i33
@test_throws Base.Math.AmosException besseli(1,1000)

# besselj
@test besselj(0,0) == 1
for i = 1:5
    @test besselj(i,0) == 0
    @test besselj(-i,0) == 0
end

j33 = besselj(3,3.)
@test besselj(3,3) == j33
@test besselj(-3,-3) == j33
@test besselj(-3,3) == -j33
@test besselj(3,-3) == -j33

j43 = besselj(4,3.)
@test besselj(4,3) == j43
@test besselj(-4,-3) == j43
@test besselj(-4,3) == j43
@test besselj(4,-3) == j43

@test_approx_eq j33 0.30906272225525164362
@test_approx_eq j43 0.13203418392461221033
@test_throws DomainError    besselj(0.1, -0.4)
@test_approx_eq besselj(0.1, complex(-0.4)) 0.820421842809028916 + 0.266571215948350899im
@test_approx_eq besselj(3.2, 1.3+0.6im) 0.01135309305831220201 + 0.03927719044393515275im
@test_approx_eq besselj(1, 3im) 3.953370217402609396im
@test_throws Base.Math.AmosException besselj(20,1000im)

# besselk
true_k33 = 0.12217037575718356792
@test_approx_eq besselk(3,3) true_k33
@test_approx_eq besselk(-3,3) true_k33
true_k3m3 = -0.1221703757571835679 - 3.0151549516807985776im
@test_throws DomainError besselk(3,-3)
@test_approx_eq besselk(3,complex(-3)) true_k3m3
@test_approx_eq besselk(-3,complex(-3)) true_k3m3
@test_throws Base.Math.AmosException besselk(200,0.01)
# issue #6564
@test besselk(1.0,0.0) == Inf

# bessely
y33 = bessely(3,3.)
@test bessely(3,3) == y33
@test_approx_eq bessely(-3,3) -y33
@test_approx_eq y33 -0.53854161610503161800
@test_throws DomainError bessely(3,-3)
@test_approx_eq bessely(3,complex(-3)) 0.53854161610503161800 - 0.61812544451050328724im
@test_throws Base.Math.AmosException bessely(200.5,0.1)

# issue #6653
for f in (besselj,bessely,besseli,besselk,hankelh1,hankelh2)
    @test_approx_eq f(0,1) f(0,complex128(1))
    @test_approx_eq f(0,1) f(0,complex64(1))
end

# beta, lbeta
@test_approx_eq beta(3/2,7/2) 5π/128
@test_approx_eq beta(3,5) 1/105
@test_approx_eq lbeta(5,4) log(beta(5,4))
@test_approx_eq beta(5,4) beta(4,5)
@test_approx_eq beta(-1/2, 3) -16/3
@test_approx_eq lbeta(-1/2, 3) log(16/3)

# gamma, lgamma (complex argument)
@test gamma(1:25) == gamma(Float64[1:25])
@test_approx_eq gamma(1/2) sqrt(π)
@test_approx_eq gamma(-1/2) -2sqrt(π)
@test_approx_eq lgamma(-1/2) log(abs(gamma(-1/2)))
@test_approx_eq lgamma(1.4+3.7im) -3.7094025330996841898 + 2.4568090502768651184im
@test_approx_eq lgamma(1.4+3.7im) log(gamma(1.4+3.7im))

# digamma
for elty in (Float32, Float64)

    @test_approx_eq digamma(convert(elty, 9)) convert(elty, 2.140641477955609996536345)
    @test_approx_eq digamma(convert(elty, 2.5)) convert(elty, 0.7031566406452431872257)
    @test_approx_eq digamma(convert(elty, 0.1)) convert(elty, -10.42375494041107679516822)
    @test_approx_eq digamma(convert(elty, 7e-4)) convert(elty, -1429.147493371120205005198)
    @test_approx_eq digamma(convert(elty, 7e-5)) convert(elty, -14286.29138623969227538398)
    @test_approx_eq digamma(convert(elty, 7e-6)) convert(elty, -142857.7200612932791081972)
    @test_approx_eq digamma(convert(elty, 2e-6)) convert(elty, -500000.5772123750382073831)
    @test_approx_eq digamma(convert(elty, 1e-6)) convert(elty, -1000000.577214019968668068)
    @test_approx_eq digamma(convert(elty, 7e-7)) convert(elty, -1428572.005785942019703646)
    @test_approx_eq digamma(convert(elty, -0.5)) convert(elty, .03648997397857652055902367)
    @test_approx_eq digamma(convert(elty, -1.1)) convert(elty,  10.15416395914385769902271)

    @test_approx_eq digamma(convert(elty, 0.1)) convert(elty, -10.42375494041108)
    @test_approx_eq digamma(convert(elty, 1/2)) convert(elty, -γ - log(4))
    @test_approx_eq digamma(convert(elty, 1)) convert(elty, -γ)
    @test_approx_eq digamma(convert(elty, 2)) convert(elty, 1 - γ)
    @test_approx_eq digamma(convert(elty, 3)) convert(elty, 3/2 - γ)
    @test_approx_eq digamma(convert(elty, 4)) convert(elty, 11/6 - γ)
    @test_approx_eq digamma(convert(elty, 5)) convert(elty, 25/12 - γ)
    @test_approx_eq digamma(convert(elty, 10)) convert(elty, 7129/2520 - γ)
end

# trigamma
for elty in (Float32, Float64)
    @test_approx_eq trigamma(convert(elty, 0.1)) convert(elty, 101.433299150792758817)
    @test_approx_eq trigamma(convert(elty, 1/2)) convert(elty, π^2/2)
    @test_approx_eq trigamma(convert(elty, 1)) convert(elty, π^2/6)
    @test_approx_eq trigamma(convert(elty, 2)) convert(elty, π^2/6 - 1)
    @test_approx_eq trigamma(convert(elty, 3)) convert(elty, π^2/6 - 5/4)
    @test_approx_eq trigamma(convert(elty, 4)) convert(elty, π^2/6 - 49/36)
    @test_approx_eq trigamma(convert(elty, 5)) convert(elty, π^2/6 - 205/144)
    @test_approx_eq trigamma(convert(elty, 10)) convert(elty, π^2/6 - 9778141/6350400)
end

# invdigamma
for elty in (Float32, Float64)
    for val in [0.001, 0.01, 0.1, 1.0, 10.0]
        @test abs(invdigamma(digamma(convert(elty, val))) - convert(elty, val)) < 1e-8
    end
end

@test_approx_eq polygamma(20, 7.) -4.644616027240543262561198814998587152547

# eta, zeta
@test_approx_eq eta(1) log(2)
@test_approx_eq eta(2) pi^2/12
@test_approx_eq zeta(0) -0.5
@test_approx_eq zeta(2) pi^2/6
@test_approx_eq zeta(4) pi^4/90

# quadgk
@test_approx_eq quadgk(cos, 0,0.7,1)[1] sin(1)
@test_approx_eq quadgk(x -> exp(im*x), 0,0.7,1)[1] (exp(1im)-1)/im
@test_approx_eq quadgk(x -> exp(im*x), 0,1im)[1] -1im*expm1(-1)
@test_approx_eq_eps quadgk(cos, 0,BigFloat(1),order=40)[1] sin(BigFloat(1)) 1000*eps(BigFloat)
@test_approx_eq quadgk(x -> exp(-x), 0,0.7,Inf)[1] 1.0
@test_approx_eq quadgk(x -> exp(x), -Inf,0)[1] 1.0
@test_approx_eq quadgk(x -> exp(-x^2), -Inf,Inf)[1] sqrt(pi)
@test_approx_eq quadgk(x -> [exp(-x), exp(-2x)], 0, Inf)[1] [1,0.5]
@test_approx_eq quadgk(cos, 0,0.7,1, norm=abs)[1] sin(1)

# Ensure subnormal flags functions don't segfault
@test any(ccall("jl_zero_subnormals", Uint8, (Uint8,), 1) .== [0x00 0x01])
@test any(ccall("jl_zero_subnormals", Uint8, (Uint8,), 0) .== [0x00 0x01])

# isqrt (issue #4884)
@test isqrt(9223372030926249000) == 3037000498
@test isqrt(typemax(Int128)) == int128("13043817825332782212")
@test isqrt(int128(typemax(Int64))^2-1) == 9223372036854775806
@test isqrt(0) == 0
for i = 1:1000
    n = rand(Uint128)
    s = isqrt(n)
    @test s*s <= n
    @test (s+1)*(s+1) > n
    n = rand(Uint64)
    s = isqrt(n)
    @test s*s <= n
    @test (s+1)*(s+1) > n
end


import Pkg; Pkg.activate("../../")
using Turing

# Simulate data.
N = 1000
mu_true = [5.0 10.0 2.0]
sigma_true = 0.6
y = randn(N, length(mu_true)) * sigma_true .+ mu_true  # size(y) = (1000, 3)

# Model definition.
@model function demo_model(y)
    K = size(y, 2)
    mu ~ filldist(Normal(0, 10), K)
    sigma ~ LogNormal(0, 2)
    for k in 1:K
        y[:, k] .~ Normal(mu[k], sigma)
    end
end

# Sample from posterior via NUTS.
nadapt, target_accept_ratio = (1000, 0.8)
iterations = 1000 + nadapt
chain = sample(demo_model(y), NUTS(nadapt, target_accept_ratio), iterations)

# Get sigma
chain[:sigma]  # works for univariate parameters
# 2-dimensional AxisArray{Float64,2,...} with axes:
#     :iter, 1:1:1000
#     :chain, 1:1
# And data, a 1000×1 Array{Float64,2}:
#  0.5891087703064164
#  0.5982078191819813
#  0.6014525731322462
#  0.6049135394153814
#  0.6135096911950264
#  ⋮
#  0.6045248767595608
#  0.6043856761562517
#  0.6197436168542185
#  0.6036546321663407

# FIXME: Get mu
# chain[:mu]  # broken

# Workarouund for getting array parameters
# function extract_array_param(chain, sym)
#     rgx = Regex("$(sym)\\[\\d+(,\\d+)*\\]")
#     syms = filter(x -> x != nothing, match.(rgx, String.(chain.name_map[1])))
#     syms = map(x -> Symbol(x.match), syms)
#     return chain[syms]
# end
#
# extract_array_param(chain, :mu)

# This has been fixed.
group(chain, :mu)

# FIXME: Get log-likelihood.
loglike = logprob"y=y | model=demo_model, chain=chain"  # broken


@unionise begin

const tanh′ = x->1 - tanh(x)^2

"""
    Planar{Tu<:AbstractVector{<:Real}, Tw<:AbstractVector{<:Real}, Tb<:Real, Th, Th′} <: Invertible

A Planar flow as described in [1].

[1] - Rezende, D. J., & Mohamed, S. (2015). Variational inference with normalizing flows.
"""
struct Planar{Tu<:RealVec, Tû<:RealVec, Tw<:RealVec, Tb<:Real, Th, Th′} <: Invertible
    u::Tu
    û::Tû
    w::Tw
    b::Tb
    h::Th
    h′::Th′
    function Planar(u::Tu, w::Tw, b::Tb, h::Th, h′::Th′) where {Tu, Tw, Tb, Th, Th′}
        @assert length(u) == length(w)
        wu = w'u
        û = u + (log1p(exp(wu)) - 1 - wu) * w / (sum(abs2, w) + eps())
        return new{Tu, typeof(û), Tw, Tb, Th, Th′}(u, û, w, b, h, h′)
    end
end
Planar(u::Real, w::Real, b::Real, h, h′) = Planar([u], [w], b, h, h′)
Planar(u, w, b) = Planar(u, w, b, tanh, tanh′)
function Planar(θ::Vector, D::Int)
    @assert length(θ) == nparams(Planar, D)
    return Planar(θ[1:D], θ[D+1:2D], θ[2D+1])
end
dim(p::Planar) = length(p.u)
params(p::Planar) = [p.u, p.w, p.b]
nparams(::Type{<:Planar}, D::Int) = 2D + 1

"""
    apply(f::Planar, z)

Compute the planar transform of `z` specified by `f`.
"""
function apply(f::Planar, z::AbstractVecOrMat{<:Real})
    @assert dim(f) == size(z, 1)
    return z .+ f.û .* f.h.(f.w'z .+ f.b)
end

function logdetJ(f::Planar, z::AbstractVector{<:Real})
    @assert dim(f) == length(z)
    return log(abs(1 + f.h′(z'f.w + f.b[1]) * f.û'f.w) + eps())
end

function logdetJ(f::Planar, z::AbstractMatrix{<:Real})
    @assert dim(f) == size(z, 1)
    return log.(abs.(1 .+ f.h′.(z'f.w .+ f.b) .* f.û'f.w) .+ eps())
end

end # @unionise

function identity_init!(θ::AbstractVector{<:Real}, ::Type{<:Planar}, ::AbstractRNG, D::Int)
    @assert length(θ) == nparams(Planar, D)
    fill!(view(θ, 1:nparams(Planar, D)-1), zero(eltype(θ)))
    θ[end] = 2 * one(eltype(θ))
    return θ
end

# Invert the Planar transform using a root finding method.
function invert(f::Planar, y::AbstractVector{<:Real})
    @assert dim(f) == length(y)
    a, c = f.w'y, f.w'f.û
    try
        α = fzero(α->α + c * tanh(α + f.b) - a, 0.0; order=2)
        return y - f.û .* tanh(α + f.b)
    catch err
        @show err, f.b, a, c
        throw(err)
    end
end

# Batch-inversion method to conform with the defined interface.
function invert(f::Planar, Y::AbstractMatrix{<:Real})
    @assert dim(f) == size(Y, 1)
    Z = similar(Y)
    for n in 1:size(Y, 2)
        Z[:, n] = invert(f, view(Y, :, n))
    end
    return Z
end


@testset begin
  Qx, x = QQ["x"]
  K, a = NumberField(x^3 + 2, "a")
  Kt, t = K["t"]
  L, b = number_field([t^2 - 3, t^2 - t - 50], "b")

  r, s = @inferred signature(L)
  @test (r, s) == (4, 4)

  m = rand(L, -1:1)

  Labs, LabstoL = absolute_simple_field(L)

  c = conjugates(m, 64)
  cc = conjugates(LabstoL\m, 64)

  for _c in c
    @test count(_cc -> overlaps(_cc, _c), cc) == count(_cc -> overlaps(_cc, _c), c)
  end

  f = x^2+1
  K, a = number_field(f, cached = false, check = false);
  Kt, t = K["t"]
  f1 = t^3 + 249*t - 830
  f2 = t^7 - 1162*a*t^5 + (-29050*a + 29050)*t^4 - 542073*t^3 + (-14412286*a - 14412286)*t^2 - 80411562*a*t + 3869576532*a - 3869576532
  L, gL = number_field([f1, f2], cached = false)
  s, t = gL
  el = (1//978369084*a + 5//978369084)*t^6 + (-4//7411887*a + 89//978369084)*t^5 + (3181//489184542*a - 2753//489184542)*t^4 + (128197//978369084*a + 52165//81530757)*t^3 + (-6920957//489184542*a + 273422//22235661)*t^2 + (1243580//9058973*a - 2545783//18117946)*t + 67078685//27176919*a + 133062707//27176919
  c = t2(el)
  Labs, mLabs = absolute_simple_field(L)
  cc = t2(mLabs\el)
  @test overlaps(c, cc)
end


module Utils

    export @define, @big

    include("utils/macros.jl")

    export compensated_summation, compensated_summation!

    include("utils/summation.jl")

    export L2norm, l2norm

    include("utils/norms.jl")

    export relative_l2_error, relative_maximum_error

    include("utils/errors.jl")

end


# growth_witness()
# smoothen()

"""
    growth_witness(shift::AbstractArray, norm::AbstractArray, dt, [b]; skip=0) -> g
    growth_witness(df::DataFrame, [b]; skip=0) -> g
Compute the growth witness
```math
G^{(n)} = S^{(n)} - \\frac{\\vert\\mathbf{c}^{(n+1)}\\vert -
          \\vert\\mathbf{c}^{(n)}\\vert}{\\vert\\mathbf{c}^{(n)}\\vert d\\tau},
```
where `S` is the `shift` and \$\\vert\\mathbf{c}^{(n)}\\vert ==\$ `norm[n, 1]`.
Setting `b ≥ 1` a sliding average over `b` time steps is computed using
[`smoothen()`](@ref). The first `skip` time steps are skipped.
`mean(growth_witness)` is approximately the same as [`growth_estimator`](@ref) with `h=0`.

See also [`growth_estimator`](@ref).
"""
function growth_witness(shift::AbstractArray, norm::AbstractArray, dt; skip = 0)
    l = length(norm)
    @assert length(shift) == l "`shift` and `norm` arrays need to have the same length."
    l =  l-skip
    @assert l > 0 "`skip` must be larger than length of `shift` and `norm`"
    n = l - 1
    g = Vector{eltype(shift)}(undef, l)
    for i in 1:n
        g[i] = shift[skip+i] - (norm[skip+i+1] - norm[skip+i])/(dt*norm[skip+i])
    end
    # pad the vector g at the end
    g[n+1] = @views mean(g[1:n])
    return g
end
function growth_witness(shift::AbstractArray, norm::AbstractArray, dt, b; kwargs...)
    g_raw = growth_witness(shift, norm, dt; kwargs...)
    return smoothen(g_raw, b)
end
"""
    growth_witness(df::DataFrame, [b]; shift=:shift, norm=:norm, dτ=df.dτ[end], skip=0)
Calculate the growth witness directly from a `DataFrame` returned by
[`lomc!`](@ref). The keyword arguments `shift` and `norm`
can be used to change the names of the relevant columns.
"""
function growth_witness(
    df::DataFrame, b=Val(0);
    shift=:shift, norm=:norm, dτ=df.dτ[end], kwargs...
)
    shift_vec = getproperty(df, Symbol(shift))
    norm_vec = getproperty(df, Symbol(norm))
    return growth_witness(shift_vec, norm_vec, dτ, b; kwargs...)
end

"""
    smoothen(noisy::AbstractVector, b)
Smoothen the array `noisy` by averaging over a sliding window of length `b` and
wrapping `noisy` periodically. The [`mean(noisy)`](@ref) is preserved.
"""
function smoothen(noisy::AbstractVector, b::Integer)
    l = length(noisy)
    @assert 1 ≤ b ≤ l
    smooth = zeros(float(eltype(noisy)), l)
    offset = b÷2 + 1
    for i in 1:l
        for j = 1:b # average over `b` elements of `noisy`
            smooth[i] += noisy[mod1(i-offset+j, l)]/b
        end
    end
    return smooth
end
smoothen(noisy::AbstractVector, ::Val{0}) = noisy


using Mocha

############################################################
# Based on the paper "Auto-Encoding Variational Bayes"
# by Diederik P Kingma, Max Welling
# http://arxiv.org/abs/1312.6114
#
# We perform unsupervised learning of a generative model of
# binary MNIST data, using the "Variational Auto-Encoder" architecture.
#
# 50-dim Gaussian latent variable z, with a standard normal prior
#
# Encoder MLP:
# - one hidden layer with 500 hidden units and tanh()
# - two linear readout layers for mu(z) and logvar(z)
#
# (We'll just use elementwise layers and the RandomNormal layer to
# implement the "reparameterization trick".)
#
# Decoder MLP matches Encoder:
# - one hidden layer with 500 hidden units and tanh()
# - one linear readout layer for p(x)
#
############################################################

function make_vae(backend, N_LATENT=50, N_HIDDEN_DEC=500, N_HIDDEN_ENC=500, N_OUT=784)

  data_layer      = HDF5DataLayer(name="train-data", source="data/train.txt", batch_size=100)

  enc1_layer      = InnerProductLayer(name="enc1", output_dim=N_HIDDEN_ENC, neuron=Neurons.Tanh(),
                                      weight_init = GaussianInitializer(std=0.01),
                                      bottoms=[:data], tops=[:enc1])
  enc_split_layer = SplitLayer(name="enc-split", bottoms=[:enc1], tops=[:enc1_mu_in, :enc1_sigma_in])

  enc_mu_layer    = InnerProductLayer(name="enc1-mu", output_dim=N_LATENT, neuron=Neurons.Identity(),
                                      weight_init = GaussianInitializer(std=0.01),
                                      bottoms=[:enc1_mu_in], tops=[:z_mu])
  enc_sigma_layer = InnerProductLayer(name="enc1-sigma", output_dim=N_LATENT,
                                      # This layer outputs log(sigma^2) in the original paper
                                      neuron=Neurons.Exponential(), # still has large jumps in KL on CPU
                                      weight_init = GaussianInitializer(std=0.01),
                                      bottoms=[:enc1_sigma_in], tops=[:z_sigma])
  zm_split_layer = SplitLayer(name="zm-split", bottoms=[:z_mu],
                              tops=[:z_mu_1, :z_mu_2])
  zs_split_layer = SplitLayer(name="zs-split", bottoms=[:z_sigma],
                              tops=[:z_sigma_1, :z_sigma_2])

  eps_layer       = RandomNormalLayer(name="eps",
                                      batch_sizes=[data_layer.batch_size], output_dims=[N_LATENT],
                                      tops=[:eps])
  z_noise_layer   = ElementWiseLayer(name="z-noise", bottoms=[:z_sigma_1, :eps], tops=[:z_noise],
                                     operation=ElementWiseFunctors.Multiply())
  z_layer         = ElementWiseLayer(name="z", bottoms=[:z_noise, :z_mu_1], tops=[:z],
                                     operation=ElementWiseFunctors.Add())

  dec1_layer      = InnerProductLayer(name="dec1", output_dim=N_HIDDEN_DEC, neuron=Neurons.Tanh(),
                                      weight_init = GaussianInitializer(std=0.01),
                                      bottoms=[:z], tops=[:dec1])

  dec_out_layer   = InnerProductLayer(name="dec-out", output_dim=N_OUT, neuron=Neurons.Sigmoid(),
                                      weight_init = GaussianInitializer(std=0.01),
                                      bottoms=[:dec1], tops=[:out])

  bce_loss_layer  = BinaryCrossEntropyLossLayer(name="bce-loss", bottoms=[:out, :data])

  kl_loss_layer   = GaussianKLLossLayer(name="kl-loss", bottoms=[:z_mu_2, :z_sigma_2])

  enc_layers = [enc1_layer, enc_split_layer, enc_mu_layer, enc_sigma_layer, zm_split_layer, zs_split_layer]
  z_layers = [eps_layer, z_noise_layer, z_layer]
  dec_layers = [dec1_layer, dec_out_layer]
  common_layers = [enc_layers, z_layers, dec_layers]
  loss_layers = [bce_loss_layer, kl_loss_layer]
  # put training net together, note that the correct ordering will automatically be established by the constructor
  net = Net("MNIST-VAE-train", backend, [data_layer, common_layers..., loss_layers...])
  return net
end

# This function reproduces the feed-forward part of the model above, layers 12-13,
# as a simple function

asarray{T}(blob::CuTensorBlob{T}) = begin
  arr = zeros(T, blob.shape)
  copy!(arr, blob)
  return arr
end

asarray(blob::CPUBlob) = blob.data

function make_latent_to_output(net::Net, dec1_index, dec_out_index)
  sigmoid(x) = 1 ./ (1 + exp(-x))

  function latent_to_output(z)
    # pad z with zeros to match the expected latent size
    N_Z = size(net.states[dec1_index].parameters[1].blob)[1]
    if length(z) < N_Z
      z_ = zeros(eltype(z), N_Z)
      z_[1:length(z)] = z
      z = z_
    end

    w1 = asarray(net.states[dec1_index].parameters[1].blob)
    b1 = asarray(net.states[dec1_index].parameters[2].blob)

    w2 = asarray(net.states[dec_out_index].parameters[1].blob)
    b2 = asarray(net.states[dec_out_index].parameters[2].blob)

    dec1 = tanh(w1' * z + b1)
    dec_out = sigmoid(w2' * dec1 + b2)
    return reshape(dec_out, 28, 28)'
  end
  return latent_to_output
end


##############################
#      Moral Vector Type     #
##############################

"""
    type MoralVector{K, T <: Real}

Type representing a Moral Vector

### Fields
* `morals` (Vector{T}): normalized `K`-vector representing the moral dimension values (or, moral vector)
"""
struct MoralVector{K, T <: Real}
    morals::Vector{T}
end

"""
    MoralVector()

Construct a MoralVector with a random moral vector with default number of components `KMORAL`.
"""
MoralVector() = MoralVector{KMORAL, Float64}(randSphere(KMORAL))

"""
    MoralVector(k::Int)

Construct a MoralVector with a random moral vector with default number of components `k`.
"""
MoralVector(k::Int) = MoralVector{k, Float64}(randSphere(k))

"""
    MoralVector(m::Vector{T})

Construct a MoralVector with an input moral vector `m` to be normalized.
"""
MoralVector(m::Vector{T}) where T <: Real = MoralVector{length(m), T}(normalize(m))

###############################
# Moral Vector Redefinitions  #
###############################

length(mvector::MoralVector) = length(mvector.morals)
size(mvector::MoralVector)   = size(mvector.morals)

getindex(mvector::MoralVector, i::Int)   = mvector.morals[i]
getindex(mvector::MoralVector, c::Colon) = mvector.morals[:]

start(mvector::MoralVector)        = 1
done(mvector::MoralVector, s::Int) = s > length(mvector)
next(mvector::MoralVector, s::Int) = (mvector[s], s+1)

dot{K,T}(i::MoralVector{K,T}, j::MoralVector{K,T}) = (i[:] ⋅ j[:])

function show(io::IO, mvector::MoralVector)
    N = length(mvector)
    print(io, N, "-dimensional Moral Vector:")
    for i in mvector
        @printf io "\n %.5f" i
    end
end

+(mvector::MoralVector, nvector::MoralVector) = MoralVector(mvector[:] + nvector[:])
-(mvector::MoralVector, nvector::MoralVector) = MoralVector(mvector[:] - nvector[:])
-(mvector::MoralVector) = MoralVector(-mvector[:])

# need a better way to sum weigthed MoralVector's
*(p::T, mvector::MoralVector{K, T}) where {T <: Real, K} = p*mvector[:]

###############################
#  Moral Vector Definitions   #
###############################

"""
    consensus(ags::Vector{MoralVector}, i::Int, j::Int)

Evaluates the consensus of MoralVectors i and j from a Vector of MoralVectors
"""
function consensus(ags::Vector{MoralVector{K, T}}, i::Int, j::Int) where {T <: Real, K}
    N = length(ags)
    if 0 < i <= N && 0 < j <= N
        return ags[i] ⋅ ags[j]
    else
        error("The indices given are out of range")
    end
end

"""
    consensus(ags::::Vector{MoralVector})

Evaluates the consensus of Vector of MoralVectors, that is, the matrix of the correlations between the MoralVectors inner representations
"""
function consensus(ags::Vector{MoralVector{K, T}}; format = :none) where {T <: Real, K}
    N = length(ags)
    ψ = zeros(N, N)

    for i in 1:N for j in (i+1):N
        ψ[i, j] = consensus(ags, i, j)
    end end

    if format == :ordered
        return ψ[ψ .!= 0.]
    end
    return ψ
end


"""
IntersectingBoundaries()

Returns
-------
IntVert::AbstractArray{Float64, 2}            Contains all the intersecting points between all the boundaries of the simplices. Each row represents some intersecting point. Matrix of size D-by-N, where D = total number of intersecting points. Each row is an intersecting point.
ConvexExpIntVert::AbstractArray{Float64, 2}   Dimension D-by-(2N + 2), where D = total number of intersecting points. (2N + 2) corresponds to the number of vertices in each simplex * 2. The first n+1 columns correspond to the convex expansion coefficients of the intersecting points in terms of the vertices generating 's₁'. The remaining n+1 to (2N+2) columns correspond to the convex expansion coefficients of the intersecting points in terms of the vertices generating 's₂'. The faces of each simplex are numbered according to the column labels.
"""
function IntersectionOfBoundaries(s₁::AbstractArray{Float64, 2}, s₂::AbstractArray{Float64, 2},
                                    convexexp1in2::AbstractArray{Float64, 2},
                                    convexexp2in1::AbstractArray{Float64, 2},
                                    ordered_vertices1::AbstractVector{Int},
                                    ordered_vertices2::AbstractVector{Int},
                                    num1in2::Int,
                                    num2in1::Int,
                                    Ncomm::Int,
                                    tolerance::Float64)

    n::Int = size(s₁, 1)
    IntVert = Vector{Float64}(0)
    ConvexExpIntVert = Vector{Float64}(0)
    Z = zeros(Float64, 2*n+2, 1)

    if num1in2 <= num2in1
        # The last one is excluded because it corresponds to the whole simplex
        Labels = collect(2^num1in2:(2^(n+1)-2))

        #The one vertex boundaries correspond to the labels 2.^(0:n)
        #but since we discard all labels up to 2^(Nin(1))-1, then the one
        #vertex boundaries we still need to discard correspond to the labels
        #2.^(Nin(1):n), which correspond to the vertices with indices Nin(1)+1:n+1
        OneIndexLabels = transpose(2 .^ (num1in2:n)) -  2^num1in2 + 1
        Labels[OneIndexLabels] .= 0

        # Now all the boundaries are generated by 2 or more vertices
        # Notice that the first component of BoundaryBinaryLabels corresponds to the label 2^(Nin(1))+1
        BBL₁ = Binary(Labels[findall(Labels .> 0)], n)
        StartingIndexBound2 = 2^num2in1 - 2^num1in2 + 1 - (num2in1 - num1in2)
        BBL₂ = view(BBL₁, StartingIndexBound2:size(BBL₁, 1), :)
    else
        # The last one is excluded because it corresponds to the whole simplex
        Labels = transpose(collect(2^num2in1:(2^(n+1) - 2)))

        #The one vertex boundaries correspond to the labels 2.^(0:n)
        #but since we discard all labels up to 2^(Nin(1))-1, then the one
        #vertex boundaries we still need to discard correspond to the labels
        #2.^(Nin(1):n), which correspond to the vertices with indices Nin(1)+1:n+1
        OneIndexLabels = transpose(2 .^ (num2in1:n)) -  2^num2in1 + 1
        Labels[OneIndexLabels'] = zeros(size(OneIndexLabels))
        Labels = vec(Labels[findall(Labels .> 0)])

        # Now all the boundaries are generated by 2 or more vertices. The first component of
        # BoundaryBinaryLabels corresponds to the label 2^(Nin(1))+1
        BBL₂ = Binary(Labels, n)
        StartingIndexBound1 = 2^num1in2 - 2^num2in1 + 1 - (num1in2 - num2in1)
        BBL₁ = view(BBL₂, StartingIndexBound1:size(BBL₂, 1), :)
    end

    N = size(BBL₁, 1) * size(BBL₂, 1)
    N2 = size(BBL₂, 1)

    Indices = 1:n+1

    @inbounds for i = 1:N
        index1 = ceil(Int, i/N2)
        index2 = i - (index1 - 1) * N2
        b1 = view(BBL₁, index1, :) # column vector
        b2 = view(BBL₂, index2, :)# column vector

        num_vert = sum(b1 + b2) # total number of vertices in both boundaries (including repeatitions)
        no_common_vert = true

        #b1[1:Ncomm] + b2[1:Ncomm]   contains, 0,1 or 2
        #If there is any vertex shared by the boundaries, a 2 will appear somewhere
        if Ncomm > 0
            no_common_vert = maximum(b1[1:Ncomm] + b2[1:Ncomm]) < 2
        end

        if num_vert <= n+2 && no_common_vert
            dim1 = sum(b1)
            dim2 = sum(b2)
            if dim1 >= dim2
                r = dim1
                s = dim2
                TargetVertices = view(s₂,:,:)

                refboundary = view(ordered_vertices1, Indices[findall(b1 .> 0)]) # column vec
                targetboundary = view(ordered_vertices2, Indices[findall(b2 .> 0)]) # column vec
                β = view(convexexp2in1, :, :)
                Γ = view(convexexp2in1, setdiff(1:n+1, refboundary), targetboundary)
                Rank= rank(Γ)
                Rank0 = rank([Γ;ones(1, s)])
                no_vanishing_column = minimum(maximum(abs.(Γ), 1))
                switch = 0
            else
                s = dim1
                r = dim2
                TargetVertices = view(s₁, :, :)
                refboundary = view(ordered_vertices2, Indices[findall(b2 .> 0)]) # column vec
                targetboundary = view(ordered_vertices1, Indices[findall(b1 .> 0)]) # column vec
                β = view(convexexp1in2, :, :)
                Γ = view(convexexp1in2, setdiff(1:n+1, refboundary), targetboundary)
                Rank= rank(Γ)
                Rank0 = rank([Γ; ones(1, s)])
                no_vanishing_column = minimum(maximum(abs.(Γ), 1))
                switch = 1
            end

            if Rank0-Rank == 1 && Rank == s-1 && no_vanishing_column > 0
                λ = QR(Γ,tolerance)

                α = [1 - ones(Int, 1, r-1) * view(β, refboundary[2:r], targetboundary);
                        view(β, refboundary[2:r], targetboundary)] * λ
                α[abs.(α) .<= tolerance] = 0

                #if some of these coefficients are negative the boundaries simply do
                #not intersect.
                #if some are zero, then the boundaries are not minimal and the intersecting point has already been
                #computed or will be computed as the intersection of minimal
                #boundaries.
                #This also rules out duplication of points
                if min(minimum(α), minimum(λ)) > 0 && sum(findall(α .> 0)) > 1
                    # Filtering out non minimal boundaries
                    newpoint = view(TargetVertices, :, targetboundary)*λ
                    Z .= 0.0
                    if switch == 0
                        Z[refboundary] .= view(α, :)
                        Z[targetboundary .+ (n + 1)] .= view(λ, :)
                    else
                        Z[refboundary .+ (n + 1)] .= view(α, :)
                        Z[targetboundary] .= view(λ, :)
                    end
                    append!(IntVert,newpoint)
                    append!(ConvexExpIntVert, Z)
                end
            end
        end
    end

    return reshape(IntVert, n, transpose(div(length(IntVert), n))),
           reshape(ConvexExpIntVert, 2*n+2, transpose(div(length(ConvexExpIntVert), 2*n+2)))
end


function act(T::TensorMap{ComplexSpace, 2, 2}, psi::TensorMap{ComplexSpace, 2, 1})
    chi = get_chi(psi)
    d = dims(domain(T))[2]
    fuse_tensor = isomorphism(ℂ^(chi*d), ℂ^d*ℂ^chi)

    @tensor Tpsi[-1, -2; -3] := T[3, -2, 2, 5] * psi[1, 2, 4] * fuse_tensor[-1, 3, 1] * fuse_tensor'[5, 4, -3]
    return Tpsi
end

function rrule(::typeof(act), T::TensorMap{ComplexSpace, 2, 2}, psi::TensorMap{ComplexSpace, 2, 1})
    chi = get_chi(psi)
    d = dims(domain(T))[2]
    fuse_tensor = isomorphism(ℂ^(chi*d), ℂ^d*ℂ^chi)

    @tensor Tpsi[-1, -2; -3] := T[3, -2, 2, 5] * psi[1, 2, 4] * fuse_tensor[-1, 3, 1] * fuse_tensor'[5, 4, -3]

    fwd = Tpsi
    function act_pushback(f̄wd)
        @tensor psi_pushback[-1, -2; -3] := f̄wd[1,4,2] * conj(fuse_tensor[1,3,-1]) * conj(fuse_tensor'[5,-3,2]) * conj(T[3,4,-2,5])
        return NoTangent(), NoTangent(), psi_pushback
    end
    return fwd, act_pushback
end

function transf_mat(psi::TensorMap{ComplexSpace, 2, 1}, phi::TensorMap{ComplexSpace, 2, 1})
    function lop(v::TensorMap{ComplexSpace, 1, 1})
        @tensor result[-2; -1] := psi'[1, -1, 3] * phi[-2, 3, 2] * v[2, 1]
        return result
    end
    return lop
end

function transf_mat_T(psi::TensorMap{ComplexSpace, 2, 1}, phi::TensorMap{ComplexSpace, 2, 1})
    function lop_T(v::TensorMap{ComplexSpace, 1, 1})
        @tensor result[-2; -1] := psi[1, 3, -1] * phi'[-2, 2, 3] * v[2, 1]
        return result
    end
    return lop_T
end

function transf_mat(A::TensorMap{ComplexSpace, 2, 1}, O::TensorMap{ComplexSpace, 2, 2}, B::TensorMap{ComplexSpace, 2, 1})
    function lop(v::TensorMap{ComplexSpace, 2, 1})
        @tensor v1[-1, -2; -3] := A[-1, 5, 1] * O[-2, 4, 5, 3] * B'[2, -3, 4] * v[1, 3, 2]
        return v1
    end
    return lop
end

function transf_mat_T(A::TensorMap{ComplexSpace, 2, 1}, O::TensorMap{ComplexSpace, 2, 2}, B::TensorMap{ComplexSpace, 2, 1})
    function lop_T(v::TensorMap{ComplexSpace, 2, 1})
        @tensor v1[-1, -2; -3] := A'[-1, 1, 5] * O'[5, -2, 3, 4] * B[2, 4, -3] * v[1, 3, 2]
        return v1
    end
    return lop_T
end

function ovlp(psi::TensorMap{ComplexSpace, 2, 1}, phi::TensorMap{ComplexSpace, 2, 1})
    chi_psi, chi_phi = get_chi(psi), get_chi(phi)
    v0 = TensorMap(rand, ComplexF64, ℂ^chi_phi, ℂ^chi_psi)
    w, _ = eigsolve(transf_mat(psi, phi), v0, 1)
    return w[1]
end

function rrule(::typeof(ovlp), psi::TensorMap{ComplexSpace, 2, 1}, phi::TensorMap{ComplexSpace, 2, 1})
    chi_psi, chi_phi = get_chi(psi), get_chi(phi)
    v0 = TensorMap(rand, ComplexF64, ℂ^chi_phi, ℂ^chi_psi)
    wr, vr = eigsolve(transf_mat(psi, phi), v0, 1)
    _, vl = eigsolve(transf_mat_T(psi, phi), v0, 1)
    fwd = wr[1]
    vl, vr = vl[1], vr[1]
    vr = vr / dot(vl, vr)

    function ovlp_pushback(f̄wd)
        @tensor psi_pushback[-2, -3; -1] := vl'[-2, 1] * phi[1, -3, 2] * vr[2, -1] * conj(f̄wd)
        @tensor phi_pushback[-1, -2; -3] := conj(vl'[1, -1]) * conj(psi'[2, 1, -2]) * conj(vr[-3, 2]) * f̄wd
        return NoTangent(), psi_pushback, phi_pushback
    end
    return fwd, ovlp_pushback
end

function nonherm_cost_func(T::TensorMap{ComplexSpace, 2, 2}, psi::TensorMap{ComplexSpace, 2, 1})
    Tpsi = act(T, psi)

    up = ovlp(Tpsi, Tpsi) * ovlp(psi, psi)
    dn = ovlp(psi, Tpsi) * ovlp(Tpsi, psi)

    return log(norm(up / dn))
end

function lambda_gamma(psi::TensorMap{ComplexSpace, 2, 1})
    chi = get_chi(psi)

    lop = transf_mat(psi, psi)
    lopT = transf_mat_T(psi, psi)

    ρ0 = TensorMap(rand, ComplexF64, ℂ^chi, ℂ^chi)
    wr, ρr = eigsolve(lop, ρ0, 1)
    _, ρl = eigsolve(lopT, ρ0, 1)
    ρl, ρr = ρl[1], ρr[1]
    norm_psi = wr[1] |> norm |> sqrt
    psi = psi / norm_psi

    _, s, v = tsvd(ρr)
    Y = sqrt(s) * v
    Yinv = v' * inv(sqrt(s))

    _, s, v = tsvd(ρl')
    X = sqrt(s) * v
    Xinv = v' * inv(sqrt(s))

    u, Λ, v = tsvd(X * Y')
    @tensor Γ[-1, -2; -3] :=
        v[-1, 1] * Yinv'[1, 3] * psi[3, -2, 4] * Xinv[4, 2] * u[2, -3]
    return norm_psi, Γ, Λ
end

function left_canonical(psi::TensorMap{ComplexSpace, 2, 1})

    norm_psi, Γ, Λ = lambda_gamma(psi)
    @tensor psi_L[-1, -2; -3] := Λ[-1, 1] * Γ[1, -2, -3]
    return norm_psi, psi_L

end

function iTEBD_truncate(psi::TensorMap{ComplexSpace, 2, 1}, chi::Integer)
    norm_psi, Γ, Λ = lambda_gamma(psi)
    chi0 = get_chi(psi)

    P = isometry(ℂ^chi0, ℂ^chi)
    @tensor psi_truncated[-1, -2; -3] :=
        P'[-1, 1] * Λ[1, 2] * Γ[2, -2, 3] * P[3, -3]
    return norm_psi, psi_truncated
end

function ln_fidelity(psi::TensorMap{ComplexSpace, 2, 1}, phi::TensorMap{ComplexSpace, 2, 1})
    up = ovlp(psi, phi) * ovlp(phi, psi)
    dn = ovlp(psi, psi) * ovlp(phi, phi)
    return (up/dn) |> norm |> log
end

function free_energy(T::TensorMap{ComplexSpace, 2, 2}, psi::TensorMap{ComplexSpace, 2, 1})
    Tpsi = act(T, psi)
    up = ovlp(psi, Tpsi)
    dn = ovlp(psi, psi)
    return (up/dn) |> norm |> log
end

function mps_add(psi::TensorMap{ComplexSpace, 2, 1}, phi::TensorMap{ComplexSpace, 2, 1})
    # https://github.com/Jutho/TensorKit.jl/issues/54
    # actually doesn't work for infinite MPS
    chi_psi, chi_phi = get_chi(psi), get_chi(phi)
    chi = chi_psi + chi_phi

    embedder1 = isometry(ℂ^chi, ℂ^chi_psi)
    embedder2 = TensorMap(zeros, ComplexF64, ℂ^chi, ℂ^chi_phi)
    embedder2.data[chi_psi+1:chi, :] = Matrix{ComplexF64}(I, chi_phi, chi_phi)

    @tensor psi_plus_phi[-1, -2; -3] := embedder1[-1, 1] * psi[1, -2, 2] * embedder1'[2, -3] +
             embedder2[-1, 1] * phi[1, -2, 2] * embedder2'[2, -3]

    return psi_plus_phi
end

function right_canonical_QR(psi::TensorMap{ComplexSpace, 2, 1}, tol::Float64=1e-15)

    chi = get_chi(psi)
    L0 = id(ℂ^chi)

    L, Q = rightorth(permute(psi, (1, ), (2, 3)))
    psi_R = permute(Q, (1, 2), (3, ))
    L = L / norm(L)
    δ = norm(L - L0)
    L0 = L

    ix= 0
    while δ > tol && ix < 200
        if ix >= 20 && ix % 10 == 0
            lop = transf_mat(psi, psi_R)
            _, vr = eigsolve(lop, L0, 1; tol=max(tol, δ/10))
            L = vr[1]'
            #print("--> ")
        end

        L, Q = rightorth(permute(psi * L, (1, ), (2, 3)))
        psi_R = permute(Q, (1, 2), (3, ))
        L = L / norm(L)

        δ = norm(L-L0)
        L0 = L

        #println(δ)
        ix += 1
    end

    #println(ix, " iterations")
    δ > tol && @warn "right_canonical_QR failed to converge. δ: $δ , tol: $tol"

    return L0', psi_R
end

function left_canonical_QR(psi::TensorMap{ComplexSpace, 2, 1}, tol::Float64=1e-15)
    chi = get_chi(psi)
    R0 = id(ℂ^chi)

    psi_L, R = leftorth(psi)
    R = R / norm(R)
    δ = norm(R - R0)
    R0 = R

    ix = 0
    while δ > tol && ix < 200
        if ix >= 20 && ix % 10 == 0
            lop_T = transf_mat_T(psi, psi_L)
            _, vl = eigsolve(lop_T, R0, 1; tol=max(tol, δ/10))
            R = vl[1]
            #print("--> ")
        end

        @tensor psi_tmp[-1, -2; -3] := R[-1, 1] * psi[1, -2, -3]
        psi_L, R = leftorth(psi_tmp)
        R = R / norm(R)

        δ = norm(R - R0)
        R0 = R

        #println(δ)
        ix += 1
    end

    #println(ix, " iterations")
    δ > tol && @warn "left_canonical_QR failed to converge. δ: $δ , tol: $tol"

    return R0, psi_L
end

function entanglement_spectrum(psi::TensorMap{ComplexSpace, 2, 1})
    _, psi = left_canonical_QR(psi)
    Y, _ = right_canonical_QR(psi)
    _, s, _ = tsvd(Y')
    s = diag(s.data) .^ 2
    return s ./ sum(s)
end

function entanglement_entropy(psi::TensorMap{ComplexSpace, 2, 1})
    s = entanglement_spectrum(psi)
    return -sum(s .* log.(s))
end

function expand(psi::TensorMap{ComplexSpace, 2, 1}, chi::Integer, perturb::Float64=1e-3)
    chi0, d = get_chi(psi), get_d(psi)
    if chi <= chi0
        @warn "chi not larger than current bond D, not expanded "
        return psi
    end

    phi_arr = perturb*rand(ComplexF64, chi, d, chi)
    phi_arr[1:chi0, :, 1:chi0] += toarray(psi)
    phi = arr_to_TensorMap(phi_arr)

    return phi
end

function tangent_map_tn(O::TensorMap{ComplexSpace, 2, 2}, AL::TensorMap{ComplexSpace, 2, 1}, AR::TensorMap{ComplexSpace, 2, 1})
    lop_R = transf_mat(AR, O, AR)
    lop_L = transf_mat_T(AL, O, AL)
    chi_mps = get_chi(AL)
    chi_mpo = dims(domain(O))[2]

    ER = TensorMap(rand, ComplexF64, ℂ^chi_mps*ℂ^chi_mpo, ℂ^chi_mps)
    EL = TensorMap(rand, ComplexF64, ℂ^chi_mps*ℂ^chi_mpo, ℂ^chi_mps)
    _, ER = eigsolve(lop_R, ER, 1)
    _, EL = eigsolve(lop_L, EL, 1)
    EL, ER = EL[1], ER[1]

    @tensor norm = ER[1,2,3]*EL'[3,1,2]
    ER = ER / norm

    @tensor map_AC[-1, -2, -3; -4, -5, -6] := EL'[-1, -4, 1] * O[1, -2, -5, 2] * ER[-6, 2, -3]
    @tensor map_C[-1, -2; -3, -4] := EL'[-1, -3, 1] * ER[-4, 1, -2]

    return map_AC, map_C
end

function tangent_map(O::TensorMap{ComplexSpace, 2, 2}, AL::TensorMap{ComplexSpace, 2, 1}, AR::TensorMap{ComplexSpace, 2, 1})
    lop_R = transf_mat(AR, O, AR)
    lop_L = transf_mat_T(AL, O, AL)
    chi_mps = get_chi(AL)
    chi_mpo = dims(domain(O))[2]

    ER = TensorMap(rand, ComplexF64, ℂ^chi_mps*ℂ^chi_mpo, ℂ^chi_mps)
    EL = TensorMap(rand, ComplexF64, ℂ^chi_mps*ℂ^chi_mpo, ℂ^chi_mps)
    _, ER = eigsolve(lop_R, ER, 1)
    _, EL = eigsolve(lop_L, EL, 1)
    EL, ER = EL[1], ER[1]

    @tensor norm = ER[1,2,3]*EL'[3,1,2]
    ER = ER / norm

    function map_AC(AC::TensorMap{ComplexSpace, 2, 1})
        @tensor updated_AC[-1, -2; -3] := AC[1, 3, 2] * EL'[-1, 1, 4] * ER[2, 5, -3] * O[4, -2, 3, 5]
        return updated_AC
    end
    function map_C(C::TensorMap{ComplexSpace, 1, 1})
        @tensor updated_C[-1; -2] := C[1, 2] * EL'[-1, 1, 3] * ER[2, 3, -2]
        return updated_C
    end
    return map_AC, map_C
end

"""
    calculate_ALR(AC::TensorMap{ComplexSpace, 2, 1}, C::TensorMap{ComplexSpace, 1, 1}) -> AL::TensorMap{ComplexSpace, 2, 1}, AR::TensorMap{ComplexSpace, 2, 1}, ϵL::Float64, ϵR::Float64

    From `AC` and `C`, calculate the AL and AR tensors. Calculate the convergence measures `ϵL` and `ϵR` as well.
"""
function calculate_ALR(AC::TensorMap{ComplexSpace, 2, 1}, C::TensorMap{ComplexSpace, 1, 1})
    UAC_l, RAC = leftorth(AC; alg=Polar())
    UC_l, RC = leftorth(C; alg=Polar())

    LAC, UAC_r = rightorth(permute(AC, (1,), (2,3)); alg=Polar())
    LC, UC_r = rightorth(C; alg=Polar())

    AL = UAC_l * UC_l'
    AR = permute(UC_r' * UAC_r, (1, 2), (3,))

    ϵL = norm(RAC - RC)
    ϵR = norm(LAC - LC)
    #ϵL = norm(AC - AL * C)
    #ϵR = norm(permute(AC, (1,), (2, 3)) - C * permute(AR, (1,), (2, 3)))

    return AL, AR, ϵL, ϵR
end

"""
    biorth_right_canonical(AU::TensorMap{ComplexSpace, 2, 1}, AD::TensorMap{ComplexSpace, 2, 1}) -> AU_R::TensorMap{ComplexSpace, 2, 1}, AD_R::TensorMap{ComplexSpace, 2, 1}

    right biorthogonalization of MPSes `AU` and `AD` into `AU_R` and `AD_R`
"""
function biorth_right_canonical(AU::TensorMap{ComplexSpace, 2, 1}, AD::TensorMap{ComplexSpace, 2, 1})
    chiU, chiD = get_chi(AU), get_chi(AD)

    lop = transf_mat(AU, AD)
    wR, CR = eigsolve(lop, TensorMap(rand, ComplexF64, ℂ^chiD, ℂ^chiU), 1)
    wR, CR = wR[1], CR[1]

    #UR, S, VR = tsvd(CR)
    #S = sqrt(S)

    #wR_abs, wR_arg = abs(wR), angle(wR)
    #Sinv = pinv(S)
    #@tensor AU_R[-1, -2; -3] := Sinv[-1, 1] * VR[1, 2] * AU[2, -2, 3] * VR'[3, 4] * S[4, -3] / sqrt(wR_abs)
    #@tensor AD_R[-1, -2; -3] := Sinv[-1, 1] * UR'[1, 2] * AD[2, -2, 3] * UR[3, 4] * S[4, -3] / sqrt(wR_abs) * exp(-1im*wR_arg)

    #return AU_R, AD_R

    @tensor AD_R[-1, -2; -3] := pinv(CR)[-1, 1] * AD[1, -2, 2] * CR[2, -3] / wR
    return AU, AD_R
end

"""
    biorth_left_canonical(AU::TensorMap{ComplexSpace, 2, 1}, AD::TensorMap{ComplexSpace, 2, 1}) -> AU_L::TensorMap{ComplexSpace, 2, 1}, AD_L::TensorMap{ComplexSpace, 2, 1}

    left biorthogonalization of MPSes `AU` and `AD` into `AU_L` and `AD_L`
"""
function biorth_left_canonical(AU::TensorMap{ComplexSpace, 2, 1}, AD::TensorMap{ComplexSpace, 2, 1})
    chiU, chiD = get_chi(AU), get_chi(AD)

    lopT = transf_mat_T(AU, AD)
    wL, CL = eigsolve(lopT, TensorMap(rand, ComplexF64, ℂ^chiD, ℂ^chiU), 1)
    wL, CL = wL[1], CL[1]

    #UL, S, VL = tsvd(CL')
    #S = sqrt(S)

    #wL_abs, wL_arg = abs(wL), angle(wL)
    #Sinv = pinv(S)

    #@tensor AU_L[-1, -2; -3] := S[-1, 1] * UL'[1, 2] * AU[2, -2, 3] * UL[3, 4] * Sinv[4, -3] / sqrt(wL_abs)
    #@tensor AD_L[-1, -2; -3] := S[-1, 1] * VL[1, 2] * AD[2, -2, 3] * VL'[3, 4] * Sinv[4, -3] / sqrt(wL_abs) * exp(1im*wL_arg)

    @tensor AD_L[-1, -2; -3] := CL'[-1, 1] * AD[1, -2, 2] * pinv(CL')[2, -3] / wL'
    return AU, AD_L
end

# FIXME. correct this.
function tangent_map(O::TensorMap{ComplexSpace, 2, 2}, AL::TensorMap{ComplexSpace, 2, 1}, AR::TensorMap{ComplexSpace, 2, 1}, BL::TensorMap{ComplexSpace, 2, 1}, BR::TensorMap{ComplexSpace, 2, 1})
    lop_R = transf_mat(AR, O, BR)
    lop_L = transf_mat_T(AL, O, BL)
    chi_mpsA, chi_mpsB = get_chi(AL), get_chi(BL)
    chi_mpo = dims(domain(O))[2]

    ER = TensorMap(rand, ComplexF64, ℂ^chi_mpsA*ℂ^chi_mpo, ℂ^chi_mpsB)
    EL = TensorMap(rand, ComplexF64, ℂ^chi_mpsA*ℂ^chi_mpo, ℂ^chi_mpsB)
    _, ER = eigsolve(lop_R, ER, 1)
    _, EL = eigsolve(lop_L, EL, 1)
    EL, ER = EL[1], ER[1]

    @tensor norm = ER[1,2,3]*EL'[3,1,2]
    ER = ER / norm

    function map_AC(AC::TensorMap{ComplexSpace, 2, 1})
        @tensor updated_AC[-1, -2; -3] := AC[1, 3, 2] * EL'[-1, 1, 4] * ER[2, 5, -3] * O[4, -2, 3, 5]
        return updated_AC
    end
    function map_C(C::TensorMap{ComplexSpace, 1, 1})
        @tensor updated_C[-1; -2] := C[1, 2] * EL'[-1, 1, 3] * ER[2, 3, -2]
        return updated_C
    end
    return map_AC, map_C
end


#!/usr/local/bin/julia

using RDatasets: dataset
using DecisionTree

iris = dataset("datasets", "iris")

head(iris)

features = Array.(iris[:, 1:4])
labels = Array(iris[:, 5])

# Single Tree
model = build_tree(labels, features)
print_tree(model)

"""
Wrapper for producing a Random Forest model
"""
function RandomForest(x, y, ntrees=10)
  features = Array.(x)
  labels = Array(y)

  nfeatures = size(features)[2] |> sqrt |> floor |> Int
  model = build_forest(labels, features, nfeatures, ntrees)
  return(model)
end

model = RandomForest(iris[:, 1:4], iris[:, 5], 50)

# Predict on new data
newdata = [5.9,3.0,5.1,1.9]
apply_forest(model, newdata)

# Estimate predicted probabilities for each class
apply_forest_proba(model, newdata, ["setosa", "versicolor", "virginica"])

# Cross-validation for random forests
accuracy = nfoldCV_forest(labels, features, 2, 10, 3, 0.5)