for N in [Float64, Rational{Int}, Float32]
    # π/2 trigonometric rotation
    b = BallInf(N[1, 2], N(1))
    M = N[0 -1 ; 1 0]
    # Test Construction
    lm1 = LinearMap(M, b)
    @test lm1.M == M
    @test lm1.X == b
    # Test Dimension
    @test dim(lm1) == 2
    # Test Support Vector
    d = N[1, 1]
    @test σ(d, lm1) == N[-1, 2]
    d = N[-1, 1]
    @test σ(d, lm1) == N[-3, 2]
    d = N[-1, -1]
    @test σ(d, lm1) == N[-3, 0]
    d = N[1, -1]
    @test σ(d, lm1) == N[-1, 0]

    # 2D -> 1D Projection
    b = BallInf(N[1, 2], N(1))
    M = N[1 0]
    lm = M*b
    # Test Dimension
    @test dim(lm) == 1
    # Test Support Vector
    d = N[1]
    @test σ(d, lm) == N[2]
    d = N[-1]
    @test σ(d, lm) == N[0]

    # scalar multiplication
    b = BallInf(N[0, 0], N(1))
    lm = N(2) * b
    @test lm == b * N(2)
    # repeated scalar multiplication
    lm2 = N(2) * lm
    # Test Dimension
    @test dim(lm) == 2
    # Test Support Vector
    d = N[1, 1]
    @test σ(d, lm) == N[2, 2]
    d = N[-1, 1]
    @test σ(d, lm) == N[-2, 2]
    d = N[-1, -1]
    @test σ(d, lm) == N[-2, -2]
    d = N[1, -1]
    @test σ(d, lm) == N[2, -2]

    # Nested construction
    lm1_copy = LinearMap(Matrix{N}(I, 2, 2), lm1)
    @test lm1_copy.M == lm1.M
    @test lm1_copy.X == lm1.X

    # boundedness
    @test isbounded(ones(N, 2, 2) * Singleton(N[1, 2]))  # bounded set
    @test isbounded(zeros(N, 2, 2) * HalfSpace(N[1, 1], N(1)))  # zero map
    @test !isbounded(N[2 3; 1 2] * HalfSpace(N[1, 1], N(1)))  # invertible matrix
    @test !isbounded(N[2 3; 0 0] * HalfSpace(N[1, 1], N(1)))  # singular matrix (expensive check)

    # isempty
    @test !isempty(lm)

    # an_element function
    lm = N(2) * BallInf(N[0, 0], N(1))
    an_element(lm)
    @test an_element(lm) ∈ lm

    # check linear map between vector and set
    X = BallInf(N[1], N(1.10445))
    a = N[-1, 2]
    @test a * X isa LinearMap{N,BallInf{N,Array{N,1}},N,Array{N,2}}

    # absorbing elements
    for neutral in [ZeroSet{N}(2), EmptySet{N}(2)]
        @test N[0 -1 ; 1 0] * neutral == neutral
    end

    # vertices_list
    b = BallInf(N[0, 0], N(1))
    M = N[1 2; 3 4]
    vlist = vertices_list(LinearMap(M, b))
    @test ispermutation(vlist, [N[3, 7], N[1, 1], N[-1, -1], N[-3, -7]])
    M = zeros(N, 2, 2)
    vlist = vertices_list(LinearMap(M, b))
    @test vlist == [zeros(N, 2)]

    # test that redundant vertices are removed by default (see #1355)
    X = Interval(N(0), N(1)) × ZeroSet{N}(1)
    M = N[0 1; 0 2]
    @test vertices_list(M * X) == [N[0, 0]]

    # concrete linear map of a LinearMap
    b = BallInf(N[0, 0], N(1))
    M = N[2 3; 1 2]
    L = LinearMap(M, b)
    V = linear_map(M, LinearMap(M, b))

    # concretize
    @test concretize(L) == linear_map(M, b)
end

# tests that only work with Float64
for N in [Float64]
    b = BallInf(N[0, 0], N(1))

    if test_suite_polyhedra
        # concrete intersection with lazy linear map
        M = N[2 3; 1 2]
        L = M * b
        Lb = intersection(L, b)
        @test M * an_element(b) ∈ Lb

        # constraints_list
        b = BallInf(N[0, 0], N(1))
        M = N[2 3; 1 2]  # invertible
        lm1 = LinearMap(M, b)
        clist = constraints_list(lm1)
        p1 = HPolygon(clist)
        M = N[2 3; 0 0]  # not invertible
        lm2 = LinearMap(M, b)
        clist = constraints_list(lm2)
        p2 = HPolygon(clist)
        for d in BoxDiagDirections{N}(2)
            @test ρ(d, lm1) ≈ ρ(d, p1)
            @test ρ(d, lm2) ≈ ρ(d, p2)
        end
    end

    # concrete linear map of a LinearMap and membership
    b = BallInf(N[0, 0], N(1))
    M = N[2 3; 1 2]
    L = LinearMap(M, b)
    V = linear_map(M, LinearMap(M, b))
    @test M * M * N[1, 1] ∈ V
end


function is_primal_infeasible(δy, ws)

	settings = ws.settings
	# calculate unscaled norm of δy
	norm_δy = scaled_norm(ws.sm.E, δy, Inf)::eltype(δy)

	# make sure norm is unequal to zero before continuing
	if norm_δy > settings.eps_prim_inf
		# test condition A'δy = 0
		A_δy = ws.p.A' * δy
		A_δy = ws.sm.Dinv * A_δy
		if norm(A_δy, Inf) <= settings.eps_prim_inf*norm_δy
			minus_unit_δy = SplitVector(- δy / norm_δy, ws.p.C)
			sF = support_function(minus_unit_δy, ws.p.C, settings.eps_prim_inf) - dot(minus_unit_δy,ws.p.b)
			if sF <= settings.eps_prim_inf
				return true
			end
		end
	end
	return false
end

function is_dual_infeasible(δx, ws)
	settings = ws.settings
	# calculate unscaled norm of δx
	norm_δx = scaled_norm(ws.sm.D, δx, Inf)::eltype(δx)

	if norm_δx > settings.eps_dual_inf
		# test condition <q,δx> < 0
		if  dot(ws.p.q,δx) / (norm_δx * ws.sm.c[]) < -settings.eps_dual_inf
			# test condition Pδx == 0
			P_δx = ws.p.P * δx
			P_δx = ws.sm.Dinv * P_δx
			if norm(P_δx, Inf) / (norm_δx * ws.sm.c[]) <= settings.eps_dual_inf

				# test condition Ax in Ktilde_b∞
				A_δx = ws.p.A * δx
				A_δx = ws.sm.Einv * A_δx
				A_δx_split = SplitVector(A_δx / norm_δx, ws.p.C)
				if in_pol_recc(A_δx_split, ws.p.C, settings.eps_dual_inf)
					return true
				end
			end
		end
	end
	return false
end


using HypothesisTests, UncertainData, Distributions

# Single uncertain values
uval1 = UncertainValue(Normal, 0, 0.2)
uval2 = UncertainValue(Beta, 1, 2)

xs = [UncertainValue(Normal, 0, rand()) for i = 1:10]
ys = [UncertainValue(Gamma, rand(Uniform(2, 3)), rand(Uniform(4, 5))) for i = 1:10]

x = UncertainDataset(xs)
y = UncertainDataset(ys)

#######################
# On uncertain values
######################
@test pvalue(MannWhitneyUTest(uval1, uval2)) isa Float64
@test pvalue(MannWhitneyUTest(uval1, uval2, 10)) isa Float64

@test pvalue(OneSampleTTest(uval1)) isa Float64
@test pvalue(OneSampleTTest(uval1, μ0 = 0.0)) isa Float64
@test pvalue(OneSampleTTest(uval1, uval2)) isa Float64
@test pvalue(OneSampleTTest(uval1, uval2, μ0 = 0.0)) isa Float64

@test pvalue(OneSampleTTest(uval1, 10)) isa Float64
@test pvalue(OneSampleTTest(uval1, 10)) isa Float64
@test pvalue(OneSampleTTest(uval1, uval2, 10)) isa Float64
@test pvalue(OneSampleTTest(uval1, uval2, 10, μ0 = 0.0)) isa Float64

@test pvalue(EqualVarianceTTest(uval1, uval2)) isa Float64
@test pvalue(EqualVarianceTTest(uval1, uval2, 10)) isa Float64
@test pvalue(EqualVarianceTTest(uval1, uval2, μ0 = 0.0)) isa Float64
@test pvalue(EqualVarianceTTest(uval1, uval2, 10, μ0 = 0.0)) isa Float64

@test pvalue(UnequalVarianceTTest(uval1, uval2)) isa Float64
@test pvalue(UnequalVarianceTTest(uval1, uval2, 10)) isa Float64
@test pvalue(UnequalVarianceTTest(uval1, uval2, μ0 = 0.0)) isa Float64
@test pvalue(UnequalVarianceTTest(uval1, uval2, 10, μ0 = 0.0)) isa Float64

@test pvalue(ApproximateTwoSampleKSTest(uval1, uval2)) isa Float64
@test pvalue(ApproximateTwoSampleKSTest(uval1, uval2, 10)) isa Float64

@test pvalue(ExactOneSampleKSTest(uval1, Normal())) isa Float64
@test pvalue(OneSampleADTest(uval1, Normal())) isa Float64

#######################
# On uncertain datasets
######################
@test pvalue(MannWhitneyUTestPooled(x, y)) isa Float64
@test pvalue(MannWhitneyUTestPooled(x, y, 10)) isa Float64
@test pvalue.(MannWhitneyUTestElementWise(x, y)) isa Vector{Float64}
@test pvalue.(MannWhitneyUTestElementWise(x, y, 10)) isa Vector{Float64}

@test pvalue(OneSampleTTestPooled(x)) isa Float64
@test pvalue(OneSampleTTestPooled(x, y)) isa Float64
@test pvalue(OneSampleTTestPooled(x, y, 10)) isa Float64
@test pvalue.(OneSampleTTestElementWise(x, y)) isa Vector{Float64}
@test pvalue.(OneSampleTTestElementWise(x, y, 10)) isa Vector{Float64}
@test pvalue(OneSampleTTestPooled(x, μ0 = 0.0)) isa Float64
@test pvalue(OneSampleTTestPooled(x, y, μ0 = 0.0)) isa Float64
@test pvalue(OneSampleTTestPooled(x, y, 10, μ0 = 0.0)) isa Float64
@test pvalue.(OneSampleTTestElementWise(x, y, μ0 = 0.0)) isa Vector{Float64}
@test pvalue.(OneSampleTTestElementWise(x, y, 10, μ0 = 0.0)) isa Vector{Float64}

@test pvalue(EqualVarianceTTestPooled(x, y)) isa Float64
@test pvalue(EqualVarianceTTestPooled(x, y, 10)) isa Float64
@test pvalue.(EqualVarianceTTestElementWise(x, y)) isa Vector{Float64}
@test pvalue.(EqualVarianceTTestElementWise(x, y, 10)) isa Vector{Float64}

@test pvalue(UnequalVarianceTTestPooled(x, y)) isa Float64
@test pvalue(UnequalVarianceTTestPooled(x, y, 10)) isa Float64
@test pvalue.(UnequalVarianceTTestElementWise(x, y)) isa Vector{Float64}
@test pvalue.(UnequalVarianceTTestElementWise(x, y, 10)) isa Vector{Float64}

@test pvalue(ApproximateTwoSampleKSTestPooled(x, y)) isa Float64
@test pvalue.(ApproximateTwoSampleKSTestElementWise(x, y)) isa Vector{Float64}

@test pvalue(ExactOneSampleKSTestPooled(x, Normal())) isa Float64
@test pvalue.(ExactOneSampleKSTestElementWise(x, Normal())) isa Vector{Float64}

@test pvalue(OneSampleADTestPooled(x, Normal())) isa Float64
@test pvalue.(OneSampleADTestElementWise(x, Normal())) isa Vector{Float64}

@test pvalue(JarqueBeraTestPooled(x)) isa Float64
@test pvalue.(JarqueBeraTestElementWise(x)) isa Vector{Float64}


# ------------------------------------------------------------------------------
# hyperelasticity example
# ------------------------------------------------------------------------------

using LinearAlgebra

# setup
mesh = Mesh3D(1.0, 1.0, 1.0)
finep = 2
coarsep = 1
numbercomponents = 3
dimension = 3
finebasis = TensorH1LagrangeBasis(finep + 1, finep + 1, numbercomponents, dimension)
coarsebasis = TensorH1LagrangeBasis(coarsep + 1, finep + 1, numbercomponents, dimension)
ctofbasis = TensorH1LagrangeBasis(
    coarsep + 1,
    finep + 1,
    numbercomponents,
    dimension,
    lagrangequadrature = true,
)

# constants
E = 1E6                             # Young's modulus
ν = 0.3                             # Poisson's ratio
K = E / (3 * (1 - 2 * ν))           # bulk modulus
λ = E * ν / ((1 + ν) * (1 - 2 * ν)) # Lamé parameters
μ = E / (2 * (1 + ν))

# state
gradu = [1; 2; 3] * ones(1, 3);

function neohookeanweakform(deltadu::Array{Float64}, w::Array{Float64})
    # dP = dF S + F dS

    # deformation gradient
    F = gradu + I
    J = det(F)
    # Green-Lagrange strain tensor
    E = (gradu * gradu' + gradu' * gradu) / 2
    # right Cauchy-Green tensor
    C = 2 * E + I
    C_inv = C^-1
    # second Piola-Kirchhoff
    S = λ * log(J) * C_inv + 2 * μ * C_inv * E

    # delta du
    deltadu = deltadu'
    # dF
    dF = deltadu + I
    # deltaE
    deltaE = (deltadu * deltadu' + deltadu' * deltadu) / 2
    # dS
    dS = λ * sum(C_inv .* deltaE) * C_inv + 2 * (μ - λ * log(J)) * C_inv * deltaE * C_inv
    # dP
    dP = (dF * S + F * dS) * w[1]

    return [dP']
end

# linearized Neo-Hookean operators
function makeoperator(basis::TensorBasis)
    inputs = [
        OperatorField(basis, [EvaluationMode.gradient], "gradent of deformation"),
        OperatorField(basis, [EvaluationMode.quadratureweights], "quadrature weights"),
    ]
    outputs = [
        OperatorField(
            basis,
            [EvaluationMode.gradient],
            "test function gradient of deformation",
        ),
    ]
    return Operator(neohookeanweakform, mesh, inputs, outputs)
end
fineoperator = makeoperator(finebasis)
coarseoperator = makeoperator(coarsebasis)

# Chebyshev smoother
chebyshev = Chebyshev(fineoperator)

# p-multigrid preconditioner
multigrid = PMultigrid(fineoperator, coarseoperator, chebyshev, [ctofbasis])

# compute operator symbols
A = computesymbols(multigrid, [3], [1, 1], [π, π, π])
eigenvalues = real(eigvals(A))

# ------------------------------------------------------------------------------


include("ellpla.jl")
# -------------------------------------------
# Contact between an ellipsoid and a triangle
# ===========================================
# References
# -----------
# [1] http://gamedev.stackexchange.com/questions/23743/whats-the-most-efficient-way-to-find-barycentric-coordinates
# -----------------------------------------------------------------------------------------------------------------
# ellipsoid is defined by centre x, radii a, b, c, and orientation R;
# triangle is identified by points p, q, r;
# ----------------------------------------------
# returned (conpnt, depth, A, B, normal), where:
# conpnt - contact point (or [0.0, 0.0, 0.0] if no contact)
# normal - contact normal (or [0.0, 0.0, 0.0] if no contact)
# depth - positive penetration depth (or -1.0 if no contact)
# A, B - contact ellipse radii (or 0.0, 0.0 if no contact)
# ____________________________________________________________
function ellipsoid_triangle_contact(x, a, b, c, R, p, q, r)
  e0 = q-p
  e1 = r-p
  normal = cross(-e1, e0)
  normal = normal/norm(normal)
  (conpnt, depth, A, B) = ellipsoid_plane_contact(x, a, b, c, R, p, normal);
  if depth == -1.0
    normal = [0.0 0.0 0.0];
  else
    vv = conpnt-p;
    aa = dot(e0,e0)
    bb = dot(e0,e1)
    cc = dot(e1,e1)
    dd = dot(e0,vv)
    ee = dot(e1,vv)
    mm = aa*cc-bb*bb
    ss = (cc*dd-bb*ee)/mm
    tt = (aa*ee-bb*dd)/mm
    if !(ss >= 0.0 && tt >= 0.0 && ss + tt <= 1.0)
      conpnt = [0.0; 0.0; 0.0];
      normal = [0.0; 0.0; 0.0];
      depth = -1.0;
      A = 0.0;
      B = 0.0;
    end
  end
  return (conpnt, normal, depth, A, B)
end


module KTOptical
    using Parameters
# Write your package code here.

    # l,p: LG のモードパラメータ
    # x,y: 座標(使用する際はブロードキャストする)
    # ω,λ: モードフィールド径、波長

    # ラゲール倍多項式
    @with_kw struct struct_beamparam
        ω::Float64
        z::Float64
        λ::Float64
        k::Float64
        zr::Float64
        ωz::Float64
        Rz::Float64
    end

    struct power
        W
    end
    struct energy
        J
    end

    export LG_E
    export LG_I
    export HG_E
    export HG_I
    export gauss_mode
    export vortex_mode
    struct gauss_mode
        m::Int8
        n::Int8
    end
    struct vortex_mode
        # userはl,pで代入するが、
        # m,nのフィールドも持っておき、コードライティング中では、m,nでやらせる。
        l::Int8
        p::Int8
    end

    function setParam(ω_,z_,λ_)
        # 計算結果を使ってパラメータセットするときは、
        # 引数内の宣言内のスコープが引き継がれない。
        # よってω、ｚ、λを使った計算はk,zr,wz,rzに入れられない。
        # ω_、ｚ_、λ_を使う必要がある。
        k = 2π / λ_
        zr = k * ω_^2 / 2
        ωz = ω_ * √(1 + (z_^2 / zr^2))
        Rz = k * ω_^2 / 2

        global bp = struct_beamparam(ω_,z_,λ_,k,zr,ωz,Rz)
    end

    function __Llp__(l,p,u)
        function Llp_setinit(l,p,u,m)
            if m < 0 #再起関数の終了条件
                return 1
            else
                return Llp_setinit(l,p,u,m-1) + (-1)^m*(2p+abs(l)-m)* u^m / factorial(m)
            end
        end
        return Llp_setinit(l,p,u,p)
    end

    function LG_E(l,p,x,y)
        R = √(x^2 + y^2)
        Θ = atan(y/x)
        E = @. √(2 * factorial(p) / (π * factorial(p + abs(l)))) *
            (√2R / bp.ω)^abs(l) *
            __Llp__(l, p, 2 * R^2 / bp.ω^2) *
            (bp.ω / bp.ωz) *
            exp(-R^2 / bp.ω^2) *
            exp(1im * l * Θ) *
            exp(-1im * (1 + 2p + abs(l)) * atan(bp.z / bp.zr)) *
            exp(-1im * bp.k * R^2 / (2bp.Rz))
        return E
    end

    function LG_E(l,p,x)
        R = √(x^2)
        Θ = atan(0)
        E = @. √(2 * factorial(p) / (π * factorial(p + abs(l)))) *
            (√2R / bp.ω)^abs(l) *
            __Llp__(l, p, 2 * R^2 / bp.ω^2) *
            (bp.ω / bp.ωz) *
            exp(-R^2 / bp.ω^2) *
            exp(1im * l * Θ) *
            exp(-1im * (1 + 2p + abs(l)) * atan(bp.z / bp.zr)) *
            exp(-1im * bp.k * R^2 / (2bp.Rz))
        return E
    end

    function LG_I(l,p,x,y)
        return abs(LG_E(l,p,x,y))^2
    end

    function LG_I(l,p,x)
        return abs(HG_E(l,p,x))^2
    end

# m,n: HG のモードパラメータ
# x,y: 座標(使用する際はブロードキャストする)

# エルミート倍多項式
    function __Hn__(n,u)
        function Hn_setinit(n,u,m)
            if m > floor(n/2) #再起関数の終了条件
                return 0
            else
                return Hn_setinit(n, u, m + 1) + (-1)^m/(factorial(m)*factorial(n-2m))*(2u)^(n-2m)
            end
        end
        return factorial(n) * Hn_setinit(n,u,0)
    end

    function HG_E(m,n,x,y)
        E = __Hn__(n, √2 * y / bp.ω) *
        __Hn__(m, √2 * x / bp.ω) *
        exp(-(x^2 + y^2) / bp.ω^2) *
        exp(-1im * (1 + m + n) * atan(bp.z / bp.zr)) *
        exp(-1im * bp.k * (x^2 + y^2) / (2bp.Rz))
        return E
    end

    function HG_E(m,n,x)
        E = __Hn__(m, √2 * x / bp.ω) *
        exp(-(x^2) / bp.ω^2) *
        exp(-1im * (1 + m + n) * atan(bp.z / bp.zr)) *
        exp(-1im * bp.k * (x^2) / (2bp.Rz))
        return E
    end

    function HG_I(m,n,x,y)
        return abs(HG_E(m,n,x,y))^2
    end

    function HG_I(m,n,x)
        return abs(HG_E(m,n,x))^2
    end

    function E(mode::vortex_mode ,x,y)
        l = mode.l
        p = mode.p
        R = √(x^2 + y^2)
        Θ = atan(y/x)
        E = @. √(2 * factorial(p) / (π * factorial(p + abs(l)))) *
            (√2R / bp.ω)^abs(l) *
            __Llp__(l, p, 2 * R^2 / bp.ω^2) *
            (bp.ω / bp.ωz) *
            exp(-R^2 / bp.ω^2) *
            exp(1im * l * Θ) *
            exp(-1im * (1 + 2p + abs(l)) * atan(bp.z / bp.zr)) *
            exp(-1im * bp.k * R^2 / (2bp.Rz))
        return E
    end

    function E(mode::vortex_mode ,x)
        l = mode.l
        p = mode.p
        R = √(x^2)
        Θ = atan(0)
        E = @. √(2 * factorial(p) / (π * factorial(p + abs(l)))) *
            (√2R / bp.ω)^abs(l) *
            __Llp__(l, p, 2 * R^2 / bp.ω^2) *
            (bp.ω / bp.ωz) *
            exp(-R^2 / bp.ω^2) *
            exp(1im * l * Θ) *
            exp(-1im * (1 + 2p + abs(l)) * atan(bp.z / bp.zr)) *
            exp(-1im * bp.k * R^2 / (2bp.Rz))
        return E
    end

    function E(mode::gauss_mode,x,y)
        m = mode.m
        n = mode.n
        E = __Hn__(n, √2 * y / bp.ω) *
        __Hn__(m, √2 * x / bp.ω) *
        exp(-(x^2 + y^2) / bp.ω^2) *
        exp(-1im * (1 + m + n) * atan(bp.z / bp.zr)) *
        exp(-1im * bp.k * (x^2 + y^2) / (2bp.Rz))
        return E
    end

    function E(mode::gauss_mode,x)
        m = mode.m
        n = mode.n
        E = __Hn__(m, √2 * x / bp.ω) *
        exp(-(x^2) / bp.ω^2) *
        exp(-1im * (1 + m + n) * atan(bp.z / bp.zr)) *
        exp(-1im * bp.k * (x^2) / (2bp.Rz))
        return E
    end

    function I(mode::vortex_mode,x)
        l = mode.l
        p = mode.p
        return abs(HG_E(l,p,x))^2
    end

    function I(mode::vortex_mode,x,y)
        l = mode.l
        p = mode.p
        return abs(LG_E(l,p,x,y))^2
    end

    function I(mode::gauss_mode,x)
        m = mode.m
        n = mode.n
        return abs(HG_E(m,n,x))^2
    end

    function I(mode::gauss_mode,x,y)
        m = mode.m
        n = mode.n
        return abs(HG_E(m,n,x,y))^2
    end

    function set_I(I,I0,step_t)
        total_input_I = sum(I)
        ratio = (I0*step_t)/total_input_I
        return ratio .* I , ratio
    end

end


function _helperEkmanTransport(
    τ_x :: Float64,
    τ_y :: Float64,
    ϵ   :: Float64,
    f   :: Float64,
)
    s2 = ϵ^2 + f^2
    return (ϵ * τ_x + f * τ_y) / s, (ϵ * τ_y - f * τ_x) / s
end

function calEkmanFlow!(
    τ_x :: AbstractArray{Float64, 2},
    τ_y :: AbstractArray{Float64, 2},
    v_x :: AbstractArray{Float64, 2},
    v_y :: AbstractArray{Float64, 2},
    f   :: AbstractArray{Float64, 2},
    ϵ   :: AbstractArray{Float64, 2},
    h   :: AbstractFloat,
)
    for i = 1:occ.Nx, j = 1:occ.Ny
        M_x[i, j], M_y[i, j] = _helperEkmanTransport(
            τ_x[i, j],
            τ_y[i, j],
            ϵ[i, j],
            f[i, j],
        )
    end
end


using Roots

"""
Return y'(x) using forward numerical differentiation.
"""
function num_differentiate(xs,ys)
    ders=zeros(length(xs)-1)
    for i in 1:length(xs)-1
        ders[i]=(ys[i+1]-ys[i])/(xs[i+1]-xs[i])
    end
    return ders
end

"""
Function where root x is relative extension of
worm-like chain in l>P regime with
    F = force
    P = persistence length

Bouchiat C, Wang MD, Allemand J, Strick T, Block SM, Croquette V.
Estimating the persistence length of a worm-like chain molecule from force-extension measurements.
Biophys J. 1999;76:409-413.
doi:10.1016/s0006-3495(99)77207-3
"""
function g_long(x, F, P, kT)
    a_two=-0.5164228
    a_three=-2.737418
    a_four=16.07497
    a_five=-38.87607
    a_six=39.49944
    a_seven=-14.17718

    return F*P/kT -(1-x)^(-2)/4. + 1. /4. -x - a_two*x^2-a_three*x^3-a_four*x^4-a_five*x^5-a_six*x^6-a_seven*x^7
end

"""
Function where root x is relative extension of
worm-like chain in other regime with l<P
    F = force
    P = persistence length
    kT = k_B T

Broedersz C.P., MacKintosh F.C.
Modeling semiflexible polymer networks
Rev. Mod. Phys. 2014;86:995-1036
doi:10.1103/RevModPhys.86.995
"""
function g_short(x, F, P, l, kT)
    ϕ=F*l^2/(pi^2*P*kT)
    return ϕ-9/pi^2*((1-x)^(-2)-1-x/3)
end

"""
Finds the relative extension of a chain with
    F = force
    P = persistence length
    kT = k_B T
    Marko_Siggia : if true, use Marko-Siggia, else
"""
function wlc_ext(L,P,F,kT=1, Marko_Siggia=true)
    if Marko_Siggia
        return L*find_zero(x->g_long(x,F,P,kT),(0,1),Bisection())
    else
        return L*find_zero(x->g_short(x,F,P,L,kT),(0,1),Bisection())
    end

end


export heisenberg, transverse_ising

"""
    heisenberg(nbit::Int; periodic::Bool=true)

1D Heisenberg hamiltonian defined as ``\\sum_{i=1}^{n} X_{i}X_{i+1} + Y_{i}Y_{i+1} + Z_{i}Z_{i+1}``, where ``n`` is specified by `nbit`.
`periodic` means the boundary condition is periodic.

References
----------------------
* de Oliveira, Mário J. "Ground-state properties of the spin-1/2 antiferromagnetic Heisenberg chain obtained by use of a Monte Carlo method." Physical Review B 48.9 (1993): 6141-6143.
"""
function heisenberg(nbit::Int; periodic::Bool=true)
    map(1:(periodic ? nbit : nbit-1)) do i
        j=i%nbit+1
        repeat(nbit,X,(i,j)) + repeat(nbit, Y, (i,j)) + repeat(nbit, Z, (i,j))
    end |> sum
end

"""
    transverse_ising(nbit::Int, h::Number; periodic::Bool=true)

1D transverse Ising hamiltonian defined as ``\\sum_{i=1}^{n} hX_{i} + Z_{i}Z_{i+1}``, where ``n`` is specified by `nbit`.
`periodic` means the boundary condition is periodic.
"""
function transverse_ising(nbit::Int, h::Number; periodic::Bool=true)
    ising_term = map(1:(periodic ? nbit : nbit-1)) do i
        repeat(nbit,Z,(i,i%nbit+1))
    end |> sum
    ising_term + h*sum(map(i->put(nbit,i=>X), 1:nbit))
end


#=
Defining various quadrature routines.

Based on the quadrature routines found in the CompEcon toolbox by
Miranda and Fackler.

@author: Spencer Lyon

@date: 2014-08-15

References
----------
Miranda, Mario J, and Paul L Fackler. Applied Computational Economics
and Finance, MIT Press, 2002.
=#

const qnw_func_notes = """
##### Notes

If any of the parameters to this function are scalars while others are
`Vector`s of length `n`, the the scalar parameter is repeated `n` times.
"""

const qnw_returns = """
##### Returns

- `nodes::Array{Float64}` : An array of quadrature nodes
- `weights::Array{Float64}` : An array of corresponding quadrature weights
"""

const qnw_refs = """
##### References

Miranda, Mario J, and Paul L Fackler. Applied Computational Economics and
Finance, MIT Press, 2002.
"""

"""
Computes multivariate Guass-Legendre  quadrature nodes and weights.

##### Arguments

- `n::Union{Int, Vector{Int}}` : Number of desired nodes along each dimension
- `a::Union{Real, Vector{Real}}` : Lower endpoint along each dimension
- `b::Union{Real, Vector{Real}}` : Upper endpoint along each dimension

$(qnw_returns)

$(qnw_func_notes)

$(qnw_refs)
"""
:qnwlege

## 1d versions
function qnwlege(n::Int, a::Real, b::Real)
    maxit = 10000
    m = fix((n + 1) / 2.0)
    xm = 0.5 * (b+a)
    xl = 0.5 * (b-a)
    nodes = zeros(n)

    weights = copy(nodes)
    i = 1:m

    z = cos(pi * (i - 0.25) ./ (n + 0.5))

    # allocate memory for loop arrays
    p3 = similar(z)
    pp = similar(z)

    its = 0
    for its=1:maxit
        p1 = ones(z)
        p2 = zeros(z)
        for j=1:n
            p3 = p2
            p2 = p1
            p1 = ((2*j-1)*z.*p2-(j-1)*p3)./j
        end

        pp = n*(z.*p1-p2)./(z.*z-1)
        z1 = z
        z = z1 - p1./pp

        err = Base.maxabs(z - z1)
        if err < 1e-14
            break
        end
    end

    if its == maxit
        error("Maximum iterations in _qnwlege1")
    end

    nodes[i] = xm - xl * z
    nodes[n+1-i] = xm + xl * z

    weights[i] = 2*xl./((1-z.*z).*pp.*pp)
    weights[n+1-i] = weights[i]

    return nodes, weights
end

"""
Computes multivariate Guass-Checbychev quadrature nodes and weights.

##### Arguments

- `n::Union{Int, Vector{Int}}` : Number of desired nodes along each dimension
- `a::Union{Real, Vector{Real}}` : Lower endpoint along each dimension
- `b::Union{Real, Vector{Real}}` : Upper endpoint along each dimension

$(qnw_returns)

$(qnw_func_notes)

$(qnw_refs)
"""
:qnwcheb

function qnwcheb(n::Int, a::Real, b::Real)
    nodes = (b+a)/2 - (b-a)/2 .* cos(pi/n .* (0.5:(n-0.5)))
    weights = ((b-a)/n) .* (cos(pi/n .* (collect(1:n)-0.5)*collect(0:2:n-1)')
                            *vcat(1, -2./(collect(1:2:n-2).*collect(3:2:n))))
    return nodes, weights
end

"""
Computes nodes and weights for multivariate normal distribution

##### Arguments

- `n::Union{Int, Vector{Int}}` : Number of desired nodes along each dimension
- `mu::Union{Real, Vector{Real}}` : Mean along each dimension
- `sig2::Union{Real, Vector{Real}, Matrix{Real}}(eye(length(n)))` : Covariance
structure

$(qnw_returns)

##### Notes

This function has many methods. I try to describe them here.

`n` or `mu` can be a vector or a scalar. If just one is a scalar the other is
repeated to match the length of the other. If both are scalars, then the number
of repeats is inferred from `sig2`.

`sig2` can be a matrix, vector or scalar. If it is a matrix, it is treated as
the covariance matrix. If it is a vector, it is considered the diagonal of a
diagonal covariance matrix. If it is a scalar it is repeated along the diagonal
as many times as necessary, where the number of repeats is determined by the
length of either n and/or mu (which ever is a vector).

If all 3 are scalars, then 1d nodes are computed. `mu` and `sig2` are treated as
the mean and variance of a 1d normal distribution

$(qnw_refs)
"""
:qnwnorm

function qnwnorm(n::Int)
    maxit = 100
    pim4 = 1 / pi^(0.25)
    m = floor(Int, (n + 1) / 2)
    nodes = zeros(n)
    weights = zeros(n)

    z = sqrt(2n+1) - 1.85575 * ((2n+1).^(-1/6))

    for i=1:m
        # Reasonable starting values for root finding
        if i == 1
            z = sqrt(2n+1) - 1.85575 * ((2n+1).^(-1/6))
        elseif i == 2
            z = z - 1.14 * (n.^0.426)./z
        elseif i == 3
            z = 1.86z + 0.86nodes[1]
        elseif i == 4
            z = 1.91z + 0.91nodes[2]
        else
            z = 2z + nodes[i-2]
        end

        # root finding iterations
        it = 0
        pp = 0.0  # initialize pp so it is available outside while
        while it < maxit
            it += 1
            p1 = pim4
            p2 = 0

            for j=1:n
                p3 = p2
                p2 = p1
                p1 = z .* sqrt(2/j) .*p2 - sqrt((j-1)/j).*p3
            end

            pp = sqrt(2n).*p2
            z1 = z
            z = z1 - p1./pp

            if abs(z - z1) < 1e-14
                break
            end
        end

        if it >= maxit
            error("Failed to converge in qnwnorm")
        end

        nodes[n+1-i] = z
        nodes[i] = -z
        weights[i] = 2 ./ (pp.*pp)
        weights[n+1-i] = weights[i]
    end

    weights ./= sqrt(pi)
    nodes *= sqrt(2)

    nodes = ndims(nodes) == 2 && size(nodes, 2) == 1 ? squeeze(nodes, 2): nodes

    return nodes, weights
end

"""
Computes multivariate Simpson quadrature nodes and weights.

##### Arguments

- `n::Union{Int, Vector{Int}}` : Number of desired nodes along each dimension
- `a::Union{Real, Vector{Real}}` : Lower endpoint along each dimension
- `b::Union{Real, Vector{Real}}` : Upper endpoint along each dimension

$(qnw_returns)

$(qnw_func_notes)

$(qnw_refs)
"""
:qnwsimp

function qnwsimp(n::Int, a::Real, b::Real)
    if n<=1
        error("In qnwsimp: n must be integer greater than one.")
    end

    if n % 2 ==0
        warn("In qnwsimp: n must be odd integer - increasing by 1.")
        n += 1
    end

    dx = (b - a) / (n - 1)
    nodes = collect(a:dx:b)
    weights = repeat([2.0, 4.0], outer=Int[(n + 1.0) / 2.0])
    weights = weights[1:n]
    weights[1] = 1
    weights[end] = 1
    weights = (dx / 3) * weights
    return nodes, weights
end

"""
Computes multivariate trapezoid quadrature nodes and weights.

##### Arguments

- `n::Union{Int, Vector{Int}}` : Number of desired nodes along each dimension
- `a::Union{Real, Vector{Real}}` : Lower endpoint along each dimension
- `b::Union{Real, Vector{Real}}` : Upper endpoint along each dimension

$(qnw_returns)

$(qnw_func_notes)

$(qnw_refs)
"""
:qnwtrap

function qnwtrap(n::Int, a::Real, b::Real)
    if n < 1
        error("n must be at least 1")
    end

    dx = (b - a) / (n - 1)
    nodes = collect(a:dx:b)
    weights = fill(dx, n)
    weights[[1, n]] .*= 0.5
    return nodes, weights
end

"""
Computes nodes and weights for beta distribution

##### Arguments

- `n::Union{Int, Vector{Int}}` : Number of desired nodes along each dimension
- `a::Union{Real, Vector{Real}}` : First parameter of the beta distribution,
along each dimension
- `b::Union{Real, Vector{Real}}` : Second parameter of the beta distribution,
along each dimension

$(qnw_returns)

$(qnw_func_notes)

$(qnw_refs)
"""
:qnwbeta

function qnwbeta(n::Int, a::Real, b::Real)
    a -= 1
    b -= 1
    ab = a + b

    maxit = 25

    x = zeros(n)
    w = zeros(n)

    z::Float64 = 0.0

    for i=1:n
        if i == 1
            an = a / n
            bn = b / n
            r1 = (1 + a) * (2.78 / (4 + n * n) + 0.768an / n)
            r2 = 1 + 1.48 * an + 0.96bn + 0.452an*an + 0.83an*bn
            z = 1 - r1 / r2

        elseif i == 2
            r1 = (4.1 + a) / ((1 + a) * (1 + 0.156a))
            r2 = 1 + 0.06 * (n - 8) * (1 + 0.12a) / n
            r3 = 1 + 0.012b * (1 + 0.25 * abs(a)) / n
            z = z - (1 - z) * r1 * r2 * r3

        elseif i == 3
            r1 = (1.67 + 0.28a) / (1 + 0.37a)
            r2 = 1 + 0.22 * (n - 8) / n
            r3 = 1 + 8 * b / ((6.28 + b) * n * n)
            z = z - (x[1] - z) * r1 * r2 * r3

        elseif i == n - 1
            r1 = (1 + 0.235b) / (0.766 + 0.119b)
            r2 = 1 / (1 + 0.639 * (n - 4) / (1 + 0.71 * (n - 4)))
            r3 = 1 / (1 + 20a / ((7.5+ a ) * n * n))
            z = z + (z - x[n-3]) * r1 * r2 * r3

        elseif i == n
            r1 = (1 + 0.37b) / (1.67 + 0.28b)
            r2 = 1 / (1 + 0.22 * (n - 8) / n)
            r3 = 1 / (1 + 8 * a / ((6.28+ a ) * n * n))
            z = z + (z - x[n-2]) * r1 * r2 * r3

        else
            z = 3 * x[i-1] - 3 * x[i-2] + x[i-3]
        end

        its = 1
        temp = 0.0
        pp, p2 = 0.0, 0.0
        for its = 1:maxit
            temp = 2 + ab
            p1 = (a - b + temp * z) / 2
            p2 = 1
            for j=2:n
              p3 = p2
              p2 = p1
              temp = 2 * j + ab
              aa = 2 * j * (j + ab) * (temp - 2)
              bb = (temp - 1) * (a * a - b * b + temp * (temp - 2) * z)
              c = 2 * (j - 1 + a) * (j - 1 + b) * temp
              p1 = (bb * p2 - c * p3) / aa
            end
            pp = (n * (a - b - temp * z) * p1 +
                  2 * (n + a) * (n + b) * p2) / (temp * (1 - z * z))
            z1 = z
            z = z1 - p1 ./ pp
            if abs(z - z1) < 3e-14 break end
        end

        if its >= maxit
            error("Failure to converge in qnwbeta")
        end

        x[i] = z
        w[i] = temp / (pp * p2)
    end

    x = (1 - x) ./ 2
    w = w * exp(lgamma(a + n) +
                lgamma(b + n) -
                lgamma(n + 1) -
                lgamma(n + ab + 1))
    w = w / (2 * exp(lgamma(a + 1) +
                     lgamma(b + 1) -
                     lgamma(ab + 2)))

    return x, w
end

"""
Computes nodes and weights for beta distribution

##### Arguments

- `n::Union{Int, Vector{Int}}` : Number of desired nodes along each dimension
- `a::Union{Real, Vector{Real}}` : First parameter of the gamma distribution,
along each dimension
- `b::Union{Real, Vector{Real}}` : Second parameter of the gamma distribution,
along each dimension

$(qnw_returns)

$(qnw_func_notes)

$(qnw_refs)
"""
:qnwgamma

function qnwgamma(n::Int, a::Real=1.0, b::Real=1.0)
    a < 0 && error("shape parameter must be positive")
    b < 0 && error("scale parameter must be positive")

    a -= 1
    maxit = 10
    fact = -exp(lgamma(a+n)-lgamma(n)-lgamma(a+1))
    nodes = zeros(n)
    weights = zeros(n)

    z = (1+a)*(3 + 0.92a)/(1 + 2.4n + 1.8a)

    for i=1:n
        # get starting values
        if i==1
            z = (1+a)*(3 + 0.92a)/(1 + 2.4n + 1.8a)
        elseif i==2
            z += (15 + 6.25a)./(1 + 0.9a + 2.5n)
        else
            j = i-2
            z += ((1+2.55j)./(1.9j) + 1.26j*a./(1+3.5j)) * (z-nodes[j])./(1+0.3a)
        end

        # rootfinding iterations
        it = 0
        pp = 0.0
        p2 = 0.0
        for it = 1:maxit
            p1 = 1.0
            p2 = 0.0
            for j=1:n
                p3 = p2
                p2 = p1
                p1 = ((2j - 1 + a - z) * p2 - (j - 1 + a) * p3) ./ j
            end
            pp = (n*p1-(n+a)*p2)./z
            z1 = z
            z = z1-p1 ./ pp
            if abs(z - z1) < 3e-14
                break
            end
        end

        if it >= maxit
            error("failure to converge.")
        end

        nodes[i] = z
        weights[i] = fact / (pp * n * p2)
    end

    return nodes .* b, weights
end

## Multidim versions
for f in [:qnwlege, :qnwcheb, :qnwsimp, :qnwtrap, :qnwbeta, :qnwgamma]
    @eval begin
        function ($f)(n::Vector{Int}, a::Real, b::Real)
            n_n = length(n)
            ($f)(n, fill(a, n_n), fill(b, n_n))
        end

        function ($f)(n::Int, a::Vector, b::Real)
            n_a = length(a)
            ($f)(fill(n, n_a), a, fill(b, n_a))
        end

        function ($f)(n::Int, a::Real, b::Vector)
            n_b = length(b)
            ($f)(fill(n, n_b), fill(a, n_b), b)
        end

        function ($f)(n::Vector{Int}, a::Vector, b::Real)
            n_n, n_a = length(n), length(a)
            if n_n != n_a
                msg = "Cannot construct nodes/weights. n and a have different"
                msg *= " lengths"
                error(msg)
            end
            ($f)(n, a, fill(b, n_a))
        end

        function ($f)(n::Vector{Int}, a::Real, b::Vector)
            n_n, n_b = length(n), length(b)
            if n_n != n_b
                msg = "Cannot construct nodes/weights. n and b have different"
                msg *= " lengths"
                error(msg)
            end
            ($f)(n, fill(a, n_b), b)
        end

        function ($f)(n::Real, a::Vector, b::Vector)
            n_a, n_b = length(a), length(b)
            if n_a != n_b
                msg = "Cannot construct nodes/weights. a and b have different"
                msg *= " lengths"
                error(msg)
            end
            ($f)(fill(n, n_a), a, b)
        end

        function ($f)(n::Vector{Int}, a::Vector, b::Vector)
            n_n, n_a, n_b = length(n), length(a), length(b)

            if !(n_n == n_a == n_b)
                error("n, a, and b must have same number of elements")
            end

            nodes = Vector{Float64}[]
            weights = Vector{Float64}[]

            for i=1:n_n
                _1d = $f(n[i], a[i], b[i])
                push!(nodes, _1d[1])
                push!(weights, _1d[2])
            end
            weights = ckron(weights[end:-1:1]...)
            nodes = gridmake(nodes...)
            return nodes, weights
        end
    end  # @eval
end

## Multidim version for qnworm
function qnwnorm(n::Vector{Int}, mu::Vector, sig2::Matrix=eye(length(n)))
    n_n, n_mu = length(n), length(mu)

    if !(n_n == n_mu)
        error("n and mu must have same number of elements")
    end

    nodes = Vector{Float64}[]
    weights = Vector{Float64}[]

    for i=1:n_n
        _1d = qnwnorm(n[i])
        push!(nodes, _1d[1])
        push!(weights, _1d[2])
    end

    weights = ckron(weights[end:-1:1]...)
    nodes = gridmake(nodes...)

    new_sig2 = chol(sig2)

    nodes = nodes * new_sig2 .+ mu'

    nodes = size(nodes, 2) == 1 ? squeeze(nodes, 2) : nodes

    return nodes, weights
end

# other types of args
qnwnorm(n::Vector{Int}, mu::Vector, sig2::Real) =
    qnwnorm(n, mu, diagm(fill(convert(Float64, sig2), length(n))))

qnwnorm(n::Vector{Int}, mu::Real, sig2::Matrix=eye(length(n))) =
    qnwnorm(n, fill(mu, length(n)), sig2)

qnwnorm(n::Vector{Int}, mu::Real, sig2::Real) =
    qnwnorm(n, fill(mu, length(n)), diagm(fill(convert(Float64, sig2), length(n))))

qnwnorm(n::Int, mu::Vector, sig2::Matrix=eye(length(mu))) =
    qnwnorm(fill(n, length(mu)), mu, sig2)

qnwnorm(n::Int, mu::Vector, sig2::Real) =
    qnwnorm(fill(n, length(mu)), mu, diagm(fill(convert(Float64, sig2), length(mu))))

qnwnorm(n::Int, mu::Real, sig2::Matrix=eye(length(mu))) =
    qnwnorm(fill(n, size(sig2, 1)), fill(mu, size(sig2, 1)), sig2)

qnwnorm(n::Int, mu::Real, sig2::Real) =
    qnwnorm([n], [mu], fill(convert(Float64, sig2), 1, 1))

qnwnorm(n::Vector{Int}, mu::Vector, sig2::Vector) =
    qnwnorm(n, mu, diagm(convert(Array{Float64}, sig2)))

qnwnorm(n::Vector{Int}, mu::Real, sig2::Vector) =
    qnwnorm(n, fill(mu, length(n)), diagm(convert(Array{Float64}, sig2)))

qnwnorm(n::Int, mu::Vector, sig2::Vector) =
    qnwnorm(fill(n, length(mu)), mu, diagm(convert(Array{Float64}, sig2)))

qnwnorm(n::Int, mu::Real, sig2::Vector) =
    qnwnorm(fill(n, length(sig2)), fill(mu, length(sig2)), diagm(convert(Array{Float64}, sig2)))

"""
Computes quadrature nodes and weights for multivariate uniform distribution

##### Arguments

- `n::Union{Int, Vector{Int}}` : Number of desired nodes along each dimension
- `a::Union{Real, Vector{Real}}` : Lower endpoint along each dimension
- `b::Union{Real, Vector{Real}}` : Upper endpoint along each dimension

$(qnw_returns)

$(qnw_func_notes)

$(qnw_refs)
"""
function qnwunif(n, a, b)
    nodes, weights = qnwlege(n, a, b)
    weights ./= prod(b - a)
    return nodes, weights
end

"""
Computes quadrature nodes and weights for multivariate uniform distribution

##### Arguments

- `n::Union{Int, Vector{Int}}` : Number of desired nodes along each dimension
- `mu::Union{Real, Vector{Real}}` : Mean along each dimension
- `sig2::Union{Real, Vector{Real}, Matrix{Real}}(eye(length(n)))` : Covariance
structure

$(qnw_returns)

##### Notes

See also the documentation for `qnwnorm`

$(qnw_refs)
"""
function qnwlogn(n, mu, sig2)
    nodes, weights = qnwnorm(n, mu, sig2)
    return exp(nodes), weights
end

## qnwequi
const equidist_pp = sqrt(primes(7920))  # good for d <= 1000

"""
Generates equidistributed sequences with property that averages
value of integrable function evaluated over the sequence converges
to the integral as n goes to infinity.

##### Arguments

- `n::Union{Int, Vector{Int}}` : Number of desired nodes along each dimension
- `a::Union{Real, Vector{Real}}` : Lower endpoint along each dimension
- `b::Union{Real, Vector{Real}}` : Upper endpoint along each dimension
- `kind::AbstractString("N")`: One of the following:
    - N - Neiderreiter (default)
    - W - Weyl
    - H - Haber
    - R - pseudo Random

$(qnw_returns)

$(qnw_func_notes)

$(qnw_refs)

"""
:qnwequi

function qnwequi(n::Int, a::Vector, b::Vector, kind::AbstractString="N")
    # error checking
    n_a, n_b = length(a), length(b)
    if !(n_a == n_b)
        error("a and b must have same number of elements")
    end

    d = n_a
    i = reshape(1:n, n, 1)
    if kind == "N"
        j = 2.^((1:d)/(d+1))
        nodes = i*j'
        nodes -= fix(nodes)

    elseif kind == "W"
        j = equidist_pp[1:d]
        nodes = i*j'
        nodes -= fix(nodes)

    elseif kind == "H"
        j = equidist_pp[1:d]
        nodes = (i.*(i+1)./2)*j'
        nodes -= fix(nodes)

    elseif kind == "R"
        nodes = rand(n, d)

    else
        error("Unknown `kind` specified. Valid choices are N, W, H, R")
    end

    r = b - a
    nodes = a' .+ nodes .* r'  # use broadcasting here.
    weights = fill((prod(r) / n), n)

    nodes = size(nodes, 2) == 1 ? squeeze(nodes, 2) : nodes

    return nodes, weights
end

# Other argument types
qnwequi(n::Vector{Int}, a::Vector, b::Vector, kind::AbstractString="N") =
    qnwequi(prod(n), a, b, kind)

qnwequi(n::Vector{Int}, a::Real, b::Vector, kind::AbstractString="N") =
    qnwequi(prod(n), fill(a, length(b)), b, kind)

qnwequi(n::Vector{Int}, a::Vector, b::Real, kind::AbstractString="N") =
    qnwequi(prod(n), a, fill(b, length(a)), kind)

qnwequi(n::Vector{Int}, a::Real, b::Real, kind::AbstractString="N") =
    qnwequi(prod(n), fill(a, length(n)), fill(b, length(n)), kind)

qnwequi(n::Int, a::Real, b::Vector, kind::AbstractString="N") =
    qnwequi(n, fill(a, length(b)), b, kind)

qnwequi(n::Int, a::Vector, b::Real, kind::AbstractString="N") =
    qnwequi(n, a, fill(b, length(a)), kind)

qnwequi(n::Int, a::Real, b::Real, kind::AbstractString="N") =
    qnwequi(n, [a], [b], kind)

## Doing the quadrature
"""
Approximate the integral of `f`, given quadrature `nodes` and `weights`

##### Arguments

- `f::Function`: A callable function that is to be approximated over the domain
spanned by `nodes`.
- `nodes::Array`: Quadrature nodes
- `weights::Array`: Quadrature nodes
- `args...(Void)`: additional positional arguments to pass to `f`
- `;kwargs...(Void)`: additional keyword arguments to pass to `f`

##### Returns

- `out::Float64` : The scalar that approximates integral of `f` on the hypercube
formed by `[a, b]`

"""
:do_quad

function do_quad(f::Function, nodes::Array, weights::Vector, args...;
                 kwargs...)
    return dot(f(nodes, args...; kwargs...), weights)
end
do_quad(f::Function, nodes::Array, weights::Vector) = dot(f(nodes), weights)

"""
Integrate the d-dimensional function f on a rectangle with lower and upper bound
for dimension i defined by a[i] and b[i], respectively; using n[i] points.

##### Arguments

- `f::Function` The function to integrate over. This should be a function that
accepts as its first argument a matrix representing points along each dimension
(each dimension is a column). Other arguments that need to be passed to the
function are caught by `args...` and `kwargs...``
- `n::Union{Int, Vector{Int}}` : Number of desired nodes along each dimension
- `a::Union{Real, Vector{Real}}` : Lower endpoint along each dimension
- `b::Union{Real, Vector{Real}}` : Upper endpoint along each dimension
- `kind::AbstractString("lege")` Specifies which type of integration to perform. Valid
values are:
    - `"lege"` : Gauss-Legendre
    - `"cheb"` : Gauss-Chebyshev
    - `"trap"` : trapezoid rule
    - `"simp"` : Simpson rule
    - `"N"` : Neiderreiter equidistributed sequence
    - `"W"` : Weyl equidistributed sequence
    - `"H"` : Haber  equidistributed sequence
    - `"R"` : Monte Carlo
    - `args...(Void)`: additional positional arguments to pass to `f`
    - `;kwargs...(Void)`: additional keyword arguments to pass to `f`

##### Returns

- `out::Float64` : The scalar that approximates integral of `f` on the hypercube
formed by `[a, b]`

$(qnw_refs)

"""
:quadrect

function quadrect(f::Function, n, a, b, kind="lege", args...; kwargs...)
    if lowercase(kind)[1] == 'l'
        nodes, weights = qnwlege(n, a, b)
    elseif lowercase(kind)[1] == 'c'
        nodes, weights = qnwcheb(n, a, b)
    elseif lowercase(kind)[1] == 't'
        nodes, weights = qnwtrap(n, a, b)
    elseif lowercase(kind)[1] == 's'
        nodes, weights = qnwsimp(n, a, b)
    else
        nodes, weights = qnwequi(n, a, b, kind)
    end

    return do_quad(f, nodes, weights, args...; kwargs...)
end

function quadrect(f::Function, n, a, b, kind="lege")
    if lowercase(kind)[1] == 'l'
        nodes, weights = qnwlege(n, a, b)
    elseif lowercase(kind)[1] == 'c'
        nodes, weights = qnwcheb(n, a, b)
    elseif lowercase(kind)[1] == 't'
        nodes, weights = qnwtrap(n, a, b)
    elseif lowercase(kind)[1] == 's'
        nodes, weights = qnwsimp(n, a, b)
    else
        nodes, weights = qnwequi(n, a, b, kind)
    end

    return do_quad(f, nodes, weights)
end


using Test;
using LinearAlgebra;
using SparseArrays;
using ParamLevelSet;
using Random;
n = 10;
B = getSpMatBuilder(Int64,Float64,n, n, 30);

for k=1:10
	setNext!(B,k,k,1.0);
end
II = getSparseMatrix(B);
@test norm(II - SparseMatrixCSC(1.0I,n,n)) < 1e-14

B.V.*=100.0;
reset!(B);

for i=1:10
	JJ = randperm(n)[1:5];
	for j = 1:5
		setNext!(B,i,JJ[j],1.0);
	end
end
@test sum(B.V[:]) == 50;
II = getSparseMatrix(B);
IIT = getSparseMatrixTransposed(B);
@test norm(II - IIT') < 1e-14


#=

  The BUGS Book
  Example 2.7.1 Surgery (continued): prediction

  Cf ~/jags/bugs_book_2_7_1.R
  """
  model {
    theta ~ dbeta(3, 27) # prior distribution
    Y ~ dbin(theta, 20) # sampling distribution
    P6 <- step(Y - 5.5) # =1 if y >= 6, 0 otherwise
  }

Summary Statistics
  parameters      mean       std   naive_se      mcse         ess      rhat
      Symbol   Float64   Float64    Float64   Float64     Float64   Float64

       theta    0.0987    0.0524     0.0003    0.0011   1799.2552    1.0049
           y    1.9620    1.6057     0.0080    0.0367   1271.8025    1.0081

Quantiles
  parameters      2.5%     25.0%     50.0%     75.0%     97.5%
      Symbol   Float64   Float64   Float64   Float64   Float64

       theta    0.0193    0.0588    0.0900    0.1299    0.2219
           y    0.0000    1.0000    2.0000    3.0000    6.0000

Distributions of variable (num:40)
0.00000 =>   38931  (0.973275)
1.00000 =>    1069  (0.026725)
  4.176741 seconds (17.21 M allocations: 923.018 MiB, 28.68% gc time)

=#

using Turing, StatsPlots, DataFrames
include("jl_utils.jl")

@model function bugs_book_2_7_1()

    theta ~ Beta(3,27) # prior distribution
    y ~ Binomial(20, theta) # sampling distribution
    p6 ~ Dirac(y >= 6)

end

model = bugs_book_2_7_1()

num_chns = 4

# chns = sample(model, Prior(), MCMCThreads(), 10_000, num_chns)
# chns = sample(model, MH(), 100_000)
# chns = sample(model, MH(
#                        # :alpha => Normal(2,sqrt(2)),
#                        # :beta => Normal(2,sqrt(2)),
#                        # :sigma => Gamma(2,2)
#                        ), MCMCThreads(), 40_000, num_chns)

# chns = sample(model, PG(15), MCMCThreads(), 1_000, num_chns)
# chns = sample(model, SMC(10_000), MCMCThreads(), 20_000, num_chns)
chns = sample(model, IS(), 10_000)

# Both HMC and NUTS give the following error:
# ERROR: LoadError: TaskFailedException:
# TypeError: in typeassert, expected Float64, got a value of type ForwardDiff.Dual{Nothing,Float64,3}
# chns = sample(model, HMC(0.1,5), MCMCThreads(), 10_000, num_chns) # Error
# chns = sample(model, NUTS(0.65), MCMCThreads(), 10_000, num_chns) # Error

display(chns)
# display(plot(chns))

show_var_dist_pct(chns,:p6 )
show_var_dist_pct(chns,:y )
# show_var_dist_pct(chns,:theta )


using TensorOperations
import Base: ==, ≈
using LinearAlgebra

export QuantumRegister, FullStateQuantumRegister, to_str
export apply_1qubit_full, apply_1qubit, apply_1qubit!
export apply_2qubit_full, apply_2qubit, apply_2qubit!
export decompose_2_qubit_gate
export swap_2qubits, get_conf, measure, get_counts, measure_probs
export binary_repr, execute!

"Abstract type and parent of all quantum register types"
abstract type QuantumRegister end

"Implementation type for full state quantum register"
mutable struct FullStateQuantumRegister{T} <: QuantumRegister
    N::Integer
    state::Array{T, 1}
    FullStateQuantumRegister{T}(N) where T <: Number = new(NTuple, zeros(T, 2^N))
    FullStateQuantumRegister{T}(N, state) where T <: Number = new(N, state)
    function FullStateQuantumRegister{T}(N, conf::String) where T <: Number
        @assert length(conf) == N
        new(N, configuration2vector(conf))
    end
end

FullStateQuantumRegister(N) = FullStateQuantumRegister{ComplexF64}(N)
FullStateQuantumRegister(N, state) = FullStateQuantumRegister{ComplexF64}(N, state)
FullStateQuantumRegister(N, conf::String) = FullStateQuantumRegister{ComplexF64}(N, conf)

"""
    configuration2vector(conf::String)

Convert a configuration given as a string to a hilbert space vector
# Examples
```jldoctest
julia> configuration2vector("01")
4-element Array{Float64,1}:
 0.0
 1.0
 0.0
 0.0
```
"""
function configuration2vector(conf::String)
    state = 1
    for i in 1:length(conf)
        state = kron(state, conf[i] == '0' ? [1., 0.] : [0., 1.])
    end
    state
end

"""
    binary_repr(num, N)

Get the binary string representation
of a decimal integer.
# Examples
```jldoctest
julia> println(binary_repr(5, 6))
"000101"
```
"""
function binary_repr(num, N)
    if num >= 1 << N
        num %= 1 << N
    end
    binary_str = ""
    while N > 0
        if num >= (1 << (N - 1))
            binary_str *= "1"
            num -= 1 << (N - 1)
        else
            binary_str *= "0"
        end
        N -= 1
    end
    return binary_str
end

"""
    to_str(qreg::FullStateQuantumRegister)

Function to convert express fullstate quantum register as a string

# Examples
```jldoctest
julia> ψ = FullStateQuantumRegister{Int}(3, "000")
julia> ψ.state[end] = 1
julia> to_str(ψ)
"(1|000>) + (1|111>)"
```
"""
function to_str(qreg::FullStateQuantumRegister)
    N = qreg.N
    str_rep = ""
    for i in 1:length(qreg.state)
        if abs(qreg.state[i]) > 1e-15
            if str_rep != ""
                str_rep *= " + "
            end
            str_rep *= "($(qreg.state[i])|$(binary_repr(i-1, N))>)"
        end
    end
    str_rep
end

"""
    ≈(a::FullStateQuantumRegister{T},
      b::FullStateQuantumRegister{T}) where T <: Complex

Function which does an approximate comparision of quantum registers with complex elements
"""
function ≈(a::FullStateQuantumRegister{T},
           b::FullStateQuantumRegister{T}) where T <: Complex
    maximum(abs.(a.state - b.state)) < eps(T.types[1])*log2(length(a.state))*2
end

"""
    ≈(a::FullStateQuantumRegister{T},
      b::FullStateQuantumRegister{T}) where T <: Complex

Function which does an approximate comparision of quantum registers with float elements
"""
function ≈(a::FullStateQuantumRegister{T},
           b::FullStateQuantumRegister{T}) where T <: AbstractFloat
    maximum(abs.(a.state - b.state)) < eps(T)*log2(length(a.state))*2
end

"""
    ==(a::FullStateQuantumRegister{T},
       b::FullStateQuantumRegister{T})

Function which does an exact comparision of elements of quantum registers
"""
function ==(a::FullStateQuantumRegister, b::FullStateQuantumRegister)
    a.state == b.state
end

"""
    apply_1qubit_full(qreg, gate, i)

Apply the 1 qubit gate to qubit i by expanding operator fully
"""
function apply_1qubit_full(qreg::QuantumRegister, gate, i)
    op = 1
    for j = 1:qreg.N
        op = kron(op, j == i ? gate : Gates.Id)
    end
    op * qreg.state
end

"""
    apply_1qubit(qreg, gate, i)

Apply the 1 qubit gate to qubit i
"""
function apply_1qubit(qreg::QuantumRegister, gate, i)
    # reshape to separate link to apply tensor to
    state = reshape(qreg.state, (2^(qreg.N-i), 2, 2^(i-1)))
    @tensor state[a, d, c] := gate[d, b] * state[a, b, c]
    reshape(state, (2^qreg.N))
end

"""
    apply_1qubit!(qreg, gate, i)

Apply the 1 qubit gate to qubit i inplace
"""
function apply_1qubit!(qreg::QuantumRegister, gate, i)
    # reshape to separate link to apply tensor to
    # state = reshape(qreg.state, (2^(qreg.N-i), 2, 2^(i-1)))
    #@tensor state[a, d, c] = gate[d, b] * state[a, b, c]
    # state = reshape(state, (2^qreg.N))
    qreg.state = apply_1qubit(qreg, gate, i)
end

"""
    decompose_2_qubit_gate(gate; threshold=1e-15)

Decompose a 2 qubit gate into a sum of 1 qubit gates on each qubit
"""
function decompose_2_qubit_gate(gate; threshold=1e-15)
    gate = reshape(gate, (2, 2, 2, 2))
    gate = permutedims(gate, (1 ,3, 2, 4))
    gate = reshape(gate, (4, 4))
    F = svd(gate)
    s = [x for x in F.S if x > threshold]
    ops = Array{Any, 1}()
    for i in 1:length(s)
        push!(ops, (reshape(F.U[:, i:i] * s[i], (2, 2)), reshape(F.Vt[i:i, :], (2, 2))))
    end
    ops
end

"""
    apply_2qubit_full(qreg, gate, i, j)

Apply the provided 2 qubit gate to qubits i and j by expanding full operator matrix
"""
function apply_2qubit_full(qreg::QuantumRegister, gate, i, j)
    gate_ops = decompose_2_qubit_gate(gate)
    total_op = nothing
    for gate_op in gate_ops
        op = 1
        for k = 1:qreg.N
            op = kron(op, k == j ? gate_op[1] : (k == i ? gate_op[2] : Gates.Id))
        end
        if total_op == nothing
            total_op = op
        else
            total_op += op
        end
    end
    total_op * qreg.state
end

"""
    apply_2qubit_full!(qreg, gate, i, j)

Apply the provided 2 qubit gate to qubits i and j by expanding full operator matrix
"""
function apply_2qubit_full!(qreg::QuantumRegister, gate, i, j)
    qreg.state = apply_2qubit_full(qreg, gate, i, j)
end

"""
    swap_2qubits(X)

Given a two qubit operation, switch the order of the qubits
"""
function swap_2qubits(X)
    X = reshape(X, (2, 2, 2, 2))
    X = permutedims(X, (2, 1, 4, 3))
    reshape(X, (4, 4))
end

"""
    apply_2qubit(qreg, gate, i, j)

Apply the provided 2 qubit gate to qubits i and j
"""
function apply_2qubit(qreg::QuantumRegister, gate, i, j)
    if i > j
        return apply_2qubit(qreg, swap_2qubits(gate), j, i)
    else
        # reshape to separate i and j indices
        state = reshape(qreg.state, (2^(qreg.N-j), 2, 2^(j-i-1), 2, 2^(i-1)))
        gate = reshape(gate, (2, 2, 2, 2))
        @tensor state[a, b, c, d, e] := gate[b, d, x, y] * state[a, x, c, y, e]
        reshape(state, (2^qreg.N))
    end
end

"""
    apply_2qubit!(qreg, gate, i, j)

Apply the provided 2 qubit gate to qubits i and j inplace
"""
function apply_2qubit!(qreg::QuantumRegister, gate, i, j)
    qreg.state = apply_2qubit(qreg, gate, i, j)
end

"""
    measure_probs(qreg)

Get for each bitstring
"""
function measure_probs(qreg::QuantumRegister)
    real(qreg.state .* conj(qreg.state))
end

"""
    get_conf(cprobs, num, N)

Given an array of cumulative probabilities of
configurations and a random number from [0, 1],
return the bitstring it corresponds to
"""
function get_conf(cprobs, num, N)
    binary_repr(searchsortedfirst(cprobs, num)-1, N)
end

"""
    measure(qreg, shots)

Given a quantum register, return a typical set of
measurements expected
"""
function measure(qreg::QuantumRegister; shots=100)
    cprobs = cumsum(measure_probs(qreg))
    rands = rand(shots)
    return [get_conf(cprobs, x, qreg.N) for x in rands]
end

"""
    get_counts(results)

Given a set of measurement results in the form of bitstrings,
return the counts of each
"""
function get_counts(results)
    counts = Dict{String, Integer}()
    for conf in results
        if haskey(counts, conf)
            counts[conf] += 1
        else
            counts[conf] = 1
        end
    end
    counts
end

"""
    execute!(qc::QuantumCircuit, state::FullStateQuantumRegister)

Apply the given quantum circuit to the given quantum register in place
"""
function execute!(qc::Gates.QuantumCircuit, state::FullStateQuantumRegister)
    for op in qc.ops
        if op.n == 1
            apply_1qubit!(state, op.gate, op.qubits[1])
        elseif op.n == 2
            apply_2qubit!(state, op.gate, op.qubits[1], op.qubits[2])
        end
    end
end


using ModelingToolkit

# Bounds
@variables u [bounds = (-1, 1)]
@test getbounds(u) == (-1, 1)
@test hasbounds(u)

@variables y
@test !hasbounds(y)

# Disturbance
@variables u [disturbance = true]
@test isdisturbance(u)

@variables y
@test !isdisturbance(y)

# Tunable
@parameters u [tunable = true]
@test istunable(u)

@parameters y
@test !istunable(y)

# Distributions
struct FakeNormal end
d = FakeNormal()
@parameters u [dist = d]
@test hasdist(u)
@test getdist(u) == d

@parameters y
@test !hasdist(y)

## System interface
@parameters t
Dₜ = Differential(t)
@variables x(t)=0 u(t)=0 [input = true] y(t)=0 [output = true]
@parameters T [tunable = true, bounds = (0, Inf)]
@parameters k [tunable = true, bounds = (0, Inf)]
@parameters k2
eqs = [Dₜ(x) ~ (-k2 * x + k * u) / T
       y ~ x]
sys = ODESystem(eqs, t, name = :tunable_first_order)

p = tunable_parameters(sys)
sp = Set(p)
@test k ∈ sp
@test T ∈ sp
@test k2 ∉ sp
@test length(p) == 2

lb, ub = getbounds(p)
@test lb == [0, 0]
@test ub == [Inf, Inf]

b = getbounds(sys)
@test b[T] == (0, Inf)

p = tunable_parameters(sys, default = true)
sp = Set(p)
@test k ∈ sp
@test T ∈ sp
@test k2 ∈ sp
@test length(p) == 3


typeof(2) #> Int64
typeof(Int64) #> DataType
typeof(Complex{Int64}) #> DataType
typeof(Any) #> DataType
typeof(DataType) #> DataType

supertype(Int64) #> Signed
supertype(Signed) #> Integer
supertype(Integer) #> Real
supertype(Real) #> Number
supertype(Number) #> Any
supertype(Any) #> Any

using InteractiveUtils
subtypes(Integer) #> 3-element Array{Any,1}:
# Bool
# Signed
# Unsigned
subtypes(Signed) #> 6-element Array{Any,1}:
# BigInt
# Int128
# Int16
# Int32
# Int64
# Int8
subtypes(Int64) #> 0-element Array{Any,1}
typeof(Nothing) #> DataType

Bool <: Integer #> true
Bool <: Any #> true
Bool <: Char #> false
Bool <: Integer #> true
Float64 <: Integer #> false

typeof(5) #> Int64
isa(5, Number) #> true


using PtFEM

data = Dict(
# Plane(ndim, nst, nxe, nye, nip, direction=(:x|:y), finite_element(nod, nodof), axisymmetric=false)
# Plane(ndim, nst, nxe, nye, nip, direction=:z, finite_element(nod, nodof), axisymmetric=true)
# Plane(ndim, nst, nre, nze, nip, direction=:r, finite_element(nod, nodof), axisymmetric=false)
  :struc_el => Plane(2, 6, 1, 5, 4, :r, Quadrilateral(8, 3), false),
  :properties => [1.0e5 0.3;],
  :r_coords => [0.0, 0.5],
  :z_coords => [10.0, 8.0, 6.0, 4.0, 2.0, 0.0],
  :support => [
    ( 1, [1 0 1]),
    ( 4, [1 0 1]),
    ( 6, [1 0 1]),
    ( 9, [1 0 1]),
    (11, [1 0 1]),
    (14, [1 0 1]),
    (16, [1 0 1]),
    (19, [1 0 1]),
    (21, [1 0 1]),
    (24, [1 0 1]),
    (26, [0 0 0]),
    (27, [0 0 0]),
    (28, [0 0 0])
    ],
  :loaded_nodes => [
    (3, [1/pi 0.0 0.0])
    ],
  :lth => 1,
  :iflag => 1,
  :chi => 0.0
)

data |> display
println()

@time m, dis_df, fm_df = p52(data)
println()

dis_df |> display
println()

println("\nThe integration point (nip = $(data[:struc_el].nip)) stresses are:")
fm_df |> display
println()


"""
    wasserstein(μ, ν; metric=Euclidean(), p=Val(1), kwargs...)

Compute the `p`-Wasserstein distance with respect to the `metric` between measures `μ` and
`ν`.

Order `p` can be provided as a scalar of type `Real` or as a parameter of a value type
`Val(p)`. For certain combinations of `metric` and `p`, such as `metric=Euclidean()` and
`p=Val(2)`, the computations are more efficient if `p` is specified as a value type. The
remaining keyword arguments are forwarded to [`ot_cost`](@ref).

See also: [`squared2wasserstein`](@ref), [`ot_cost`](@ref)
"""
function wasserstein(μ, ν; metric=Euclidean(), p::Union{Real,Val}=Val(1), kwargs...)
    cost = ot_cost(p2distance(metric, p), μ, ν; kwargs...)
    return prt(cost, p)
end

# compute the cost function corresponding to a metric and exponent `p`
p2distance(metric, ::Val{1}) = metric
p2distance(metric, ::Val{P}) where {P} = (x, y) -> metric(x, y)^P
p2distance(d::Euclidean, ::Val{2}) = SqEuclidean(d.thresh)
p2distance(metric, p) = (x, y) -> metric(x, y)^p

# compute the `p` root
prt(x, ::Val{1}) = x
prt(x, ::Val{2}) = sqrt(x)
prt(x, ::Val{3}) = cbrt(x)
prt(x, ::Val{P}) where {P} = x^(1 / P)
prt(x, p) = x^(1 / p)

"""
    squared2wasserstein(μ, ν; metric=Euclidean(), kwargs...)

Compute the squared 2-Wasserstein distance with respect to the `metric` between measures `μ`
and `ν`.

The remaining keyword arguments are forwarded to [`ot_cost`](@ref).

See also: [`wasserstein`](@ref), [`ot_cost`](@ref)
"""
function squared2wasserstein(μ, ν; metric=Euclidean(), kwargs...)
    return ot_cost(p2distance(metric, Val(2)), μ, ν; kwargs...)
end


export SymbolicPermutation

"""
A probability estimator based on permutations.
"""
abstract type PermutationProbabilityEstimator <: SymbolicProbabilityEstimator end

"""
    SymbolicPermutation(; τ = 1, m = 3, lt = Entropies.isless_rand) <: ProbabilityEstimator
    SymbolicWeightedPermutation(; τ = 1, m = 3, lt = Entropies.isless_rand) <: ProbabilityEstimator
    SymbolicAmplitudeAwarePermutation(; τ = 1, m = 3, A = 0.5, lt = Entropies.isless_rand) <: ProbabilityEstimator

Symbolic, permutation-based probabilities/entropy estimators.
`m` is the permutation order (or the symbol size or the embedding dimension) and
`τ` is the delay time (or lag).

## Repeated values during symbolization

In the original implementation of permutation entropy [^BandtPompe2002], equal values are
ordered after their order of appearance, but this can lead to erroneous temporal
correlations, especially for data with low-amplitude resolution [^Zunino2017]. Here, we
resolve this issue by letting the user provide a custom "less-than" function. The keyword
`lt` accepts a function that decides which of two state vector elements are smaller. If two
elements are equal, the default behaviour is to randomly assign one of them as the largest
(`lt = Entropies.isless_rand`). For data with low amplitude resolution, computing
probabilities multiple times using the random approach may reduce these erroneous
effects.

To get the behaviour described in Bandt and Pompe (2002), use `lt = Base.isless`).

## Properties of original signal preserved

- **`SymbolicPermutation`**: Preserves ordinal patterns of state vectors (sorting information). This
    implementation is based on Bandt & Pompe et al. (2002)[^BandtPompe2002] and
    Berger et al. (2019) [^Berger2019].
- **`SymbolicWeightedPermutation`**: Like `SymbolicPermutation`, but also encodes amplitude
    information by tracking the variance of the state vectors. This implementation is based
    on Fadlallah et al. (2013)[^Fadlallah2013].
- **`SymbolicAmplitudeAwarePermutation`**: Like `SymbolicPermutation`, but also encodes
    amplitude information by considering a weighted combination of *absolute amplitudes*
    of state vectors, and *relative differences between elements* of state vectors. See
    description below for explanation of the weighting parameter `A`. This implementation
    is based on Azami & Escudero (2016) [^Azami2016].

## Probability estimation

### Univariate time series

To estimate probabilities or entropies from univariate time series, use the following methods:

- `probabilities(x::AbstractVector, est::SymbolicProbabilityEstimator)`. Constructs state vectors
    from `x` using embedding lag `τ` and embedding dimension `m`, symbolizes state vectors,
    and computes probabilities as (weighted) relative frequency of symbols.
- `genentropy(x::AbstractVector, est::SymbolicProbabilityEstimator; α=1, base = 2)` computes
    probabilities by calling `probabilities(x::AbstractVector, est)`,
    then computer the order-`α` generalized entropy to the given base.

#### Speeding up repeated computations

A pre-allocated integer symbol array `s` can be provided to save some memory
allocations if the probabilities are to be computed for multiple data sets.

*Note: it is not the array that will hold the final probabilities that is pre-allocated,
but the temporary integer array containing the symbolized data points. Thus, if
provided, it is required that `length(x) == length(s)` if `x` is a Dataset, or
`length(s) == length(x) - (m-1)τ` if `x` is a univariate signal that is to be embedded
first*.

Use the following signatures (only works for `SymbolicPermutation`).

```julia
probabilities!(s::Vector{Int}, x::AbstractVector, est::SymbolicPermutation) → ps::Probabilities
probabilities!(s::Vector{Int}, x::AbstractDataset, est::SymbolicPermutation) → ps::Probabilities
```

### Multivariate datasets

Although not dealt with in the original paper describing the estimators, numerically speaking,
permutation entropies can also be computed for multivariate datasets with dimension ≥ 2
(but see caveat below). Such datasets may be, for example, preembedded time series. Then,
just skip the delay reconstruction step, compute and symbols directly from the ``L``
existing state vectors ``\\{\\mathbf{x}_1, \\mathbf{x}_2, \\ldots, \\mathbf{x_L}\\}``.

- `probabilities(x::AbstractDataset, est::SymbolicProbabilityEstimator)`. Compute ordinal patterns of the
    state vectors of `x` directly (without doing any embedding), symbolize those patterns,
    and compute probabilities as (weighted) relative frequencies of symbols.
- `genentropy(x::AbstractDataset, est::SymbolicProbabilityEstimator)`. Computes probabilities from
    symbol frequencies using `probabilities(x::AbstractDataset, est::SymbolicProbabilityEstimator)`,
    then computes the order-`α` generalized (permutation) entropy to the given base.

*Caveat: A dynamical interpretation of the permutation entropy does not necessarily
hold if computing it on generic multivariate datasets. Method signatures for `Dataset`s are
provided for convenience, and should only be applied if you understand the relation
between your input data, the numerical value for the permutation entropy, and
its interpretation.*

## Description

All symbolic estimators use the same underlying approach to estimating probabilities.

### Embedding, ordinal patterns and symbolization

Consider the ``n``-element univariate time series ``\\{x(t) = x_1, x_2, \\ldots, x_n\\}``.
Let ``\\mathbf{x_i}^{m, \\tau} = \\{x_j, x_{j+\\tau}, \\ldots, x_{j+(m-1)\\tau}\\}``
for ``j = 1, 2, \\ldots n - (m-1)\\tau`` be the ``i``-th state vector in a delay
reconstruction with embedding dimension ``m`` and reconstruction lag ``\\tau``.
There are then ``N = n - (m-1)\\tau`` state vectors.

For an ``m``-dimensional vector, there are ``m!`` possible ways of sorting it in
ascending order of magnitude. Each such possible sorting ordering is called a
*motif*. Let ``\\pi_i^{m, \\tau}`` denote the motif associated with the
``m``-dimensional state vector ``\\mathbf{x_i}^{m, \\tau}``, and let ``R``
be the number of distinct motifs that can be constructed from the ``N`` state
vectors. Then there are at most ``R`` motifs; ``R = N`` precisely when all motifs
are unique, and ``R = 1`` when all motifs are the same.

Each unique motif ``\\pi_i^{m, \\tau}`` can be mapped to a unique integer
symbol ``0 \\leq s_i \\leq M!-1``. Let ``S(\\pi) : \\mathbb{R}^m \\to \\mathbb{N}_0`` be
the function that maps the motif ``\\pi`` to its symbol ``s``, and let ``\\Pi``
denote the set of symbols ``\\Pi = \\{ s_i \\}_{i\\in \\{ 1, \\ldots, R\\}}``.

### Probability computation

#### `SymbolicPermutation`

The probability of a given motif is its frequency of occurrence, normalized by the total
number of motifs (with notation from [^Fadlallah2013]),

```math
p(\\pi_i^{m, \\tau}) = \\dfrac{\\sum_{k=1}^N \\mathbf{1}_{u:S(u) = s_i} \\left(\\mathbf{x}_k^{m, \\tau} \\right) }{\\sum_{k=1}^N \\mathbf{1}_{u:S(u) \\in \\Pi} \\left(\\mathbf{x}_k^{m, \\tau} \\right)} = \\dfrac{\\sum_{k=1}^N \\mathbf{1}_{u:S(u) = s_i} \\left(\\mathbf{x}_k^{m, \\tau} \\right) }{N},
```

where the function ``\\mathbf{1}_A(u)`` is the indicator function of a set ``A``. That
    is, ``\\mathbf{1}_A(u) = 1`` if ``u \\in A``, and ``\\mathbf{1}_A(u) = 0`` otherwise.

#### `SymbolicAmplitudeAwarePermutation`

Amplitude-aware permutation entropy is computed analogously to regular permutation entropy
but probabilities are weighted by amplitude information as follows.

```math
p(\\pi_i^{m, \\tau}) = \\dfrac{\\sum_{k=1}^N \\mathbf{1}_{u:S(u) = s_i} \\left( \\mathbf{x}_k^{m, \\tau} \\right) \\, a_k}{\\sum_{k=1}^N \\mathbf{1}_{u:S(u) \\in \\Pi} \\left( \\mathbf{x}_k^{m, \\tau} \\right) \\,a_k} = \\dfrac{\\sum_{k=1}^N \\mathbf{1}_{u:S(u) = s_i} \\left( \\mathbf{x}_k^{m, \\tau} \\right) \\, a_k}{\\sum_{k=1}^N a_k}.
```

The weights encoding amplitude information about state vector ``\\mathbf{x}_i = (x_1^i, x_2^i, \\ldots, x_m^i)`` are

```math
a_i = \\dfrac{A}{m} \\sum_{k=1}^m |x_k^i | + \\dfrac{1-A}{d-1} \\sum_{k=2}^d |x_{k}^i - x_{k-1}^i|,
```

with ``0 \\leq A \\leq 1``. When ``A=0`` , only internal differences between the elements of
``\\mathbf{x}_i`` are weighted. Only mean amplitude of the state vector
elements are weighted when ``A=1``. With, ``0<A<1``, a combined weighting is used.

#### `SymbolicWeightedPermutation`

Weighted permutation entropy is also computed analogously to regular permutation entropy, but
adds weights that encode amplitude information too:

```math
p(\\pi_i^{m, \\tau}) = \\dfrac{\\sum_{k=1}^N \\mathbf{1}_{u:S(u) = s_i}
\\left( \\mathbf{x}_k^{m, \\tau} \\right)
\\, w_k}{\\sum_{k=1}^N \\mathbf{1}_{u:S(u) \\in \\Pi}
\\left( \\mathbf{x}_k^{m, \\tau} \\right) \\,w_k} = \\dfrac{\\sum_{k=1}^N
\\mathbf{1}_{u:S(u) = s_i}
\\left( \\mathbf{x}_k^{m, \\tau} \\right) \\, w_k}{\\sum_{k=1}^N w_k}.
```

The weighted permutation entropy is equivalent to regular permutation entropy when weights
are positive and identical (``w_j = \\beta \\,\\,\\, \\forall \\,\\,\\, j \\leq N`` and
``\\beta > 0)``. Weights are dictated by the variance of the state vectors.

Let the aritmetic mean of state vector ``\\mathbf{x}_i`` be denoted
by

```math
\\mathbf{\\hat{x}}_j^{m, \\tau} = \\frac{1}{m} \\sum_{k=1}^m x_{j + (k+1)\\tau}.
```

Weights are then computed as

```math
w_j = \\dfrac{1}{m}\\sum_{k=1}^m (x_{j+(k+1)\\tau} - \\mathbf{\\hat{x}}_j^{m, \\tau})^2.
```

*Note: in equation 7, section III, of the original paper, the authors write*

```math
w_j = \\dfrac{1}{m}\\sum_{k=1}^m (x_{j-(k-1)\\tau} - \\mathbf{\\hat{x}}_j^{m, \\tau})^2.
```
*But given the formula they give for the arithmetic mean, this is **not** the variance
of ``\\mathbf{x}_i``, because the indices are mixed: ``x_{j+(k-1)\\tau}`` in the weights
formula, vs. ``x_{j+(k+1)\\tau}`` in the arithmetic mean formula. This seems to imply
that amplitude information about previous delay vectors
are mixed with mean amplitude information about current vectors. The authors also mix the
terms "vector" and "neighboring vector" (but uses the same notation for both), making it
hard to interpret whether the sign switch is a typo or intended. Here, we use the notation
above, which actually computes the variance for ``\\mathbf{x}_i``*.

[^BandtPompe2002]: Bandt, Christoph, and Bernd Pompe. "Permutation entropy: a natural
    complexity measure for time series." Physical review letters 88.17 (2002): 174102.
[^Berger2019]: Berger, Sebastian, et al. "Teaching Ordinal Patterns to a Computer: Efficient Encoding Algorithms Based on the Lehmer Code." Entropy 21.10 (2019): 1023.
[^Fadlallah2013]: Fadlallah, Bilal, et al. "Weighted-permutation entropy: A complexity
    measure for time series incorporating amplitude information." Physical
    Review E 87.2 (2013): 022911.
[^Rényi1960]: A. Rényi, *Proceedings of the fourth Berkeley Symposium on Mathematics, Statistics and Probability*, pp 547 (1960)
[^Azami2016]: Azami, H., & Escudero, J. (2016). Amplitude-aware permutation entropy: Illustration in spike detection and signal segmentation. Computer methods and programs in biomedicine, 128, 40-51.
[^Fadlallah2013]: Fadlallah, Bilal, et al. "Weighted-permutation entropy: A complexity measure for time series incorporating amplitude information." Physical Review E 87.2 (2013): 022911.
[^Zunino2017]: Zunino, L., Olivares, F., Scholkmann, F., & Rosso, O. A. (2017). Permutation entropy based time series analysis: Equalities in the input signal can lead to false conclusions. Physics Letters A, 381(22), 1883-1892.
"""
struct SymbolicPermutation{F} <: PermutationProbabilityEstimator
    τ::Int
    m::Int
    lt::F
end
function SymbolicPermutation(; τ::Int = 1, m::Int = 3, lt::F=isless_rand) where {F <: Function}
    m >= 2 || error("Need m ≥ 2, otherwise no dynamical information is encoded in the symbols.")
    SymbolicPermutation{F}(τ, m, lt)
end

"""
    symbolize(x::AbstractVector{T}, est::SymbolicPermutation) where {T} → Vector{Int}
    symbolize!(s, x::AbstractVector{T}, est::SymbolicPermutation) where {T} → Vector{Int}

If `x` is a univariate time series, first `x` create a delay reconstruction of `x`
using embedding lag `est.τ` and embedding dimension `est.m`, then symbolizing the resulting
state vectors with [`encode_motif`](@ref).

Optionally, the in-place `symbolize!` can be used to put symbols in a pre-allocated
integer vector `s`, where `length(s) == length(x)-(est.m-1)*est.τ`.

    symbolize(x::AbstractDataset{m, T}, est::SymbolicPermutation) where {m, T} → Vector{Int}
    symbolize!(s, x::AbstractDataset{m, T}, est::SymbolicPermutation) where {m, T} → Vector{Int}

If `x` is an `m`-dimensional dataset, then motif lengths are determined by the dimension of
the input data, and `x` is symbolized by converting each `m`-dimensional
state vector as a unique integer in the range ``1, 2, \\ldots, m-1``, using
[`encode_motif`](@ref).

Optionally, the in-place `symbolize!` can be used to put symbols in a pre-allocated
integer vector `s`, where `length(s) == length(s)`.

## Examples

Symbolize a 7-dimensional dataset. Motif lengths (or order of the permutations) are
inferred to be 7.

```julia
using DelayEmbeddings, Entropies
D = Dataset([rand(7) for i = 1:1000])
s = symbolize(D, SymbolicPermutation())
```

Symbolize a univariate time series by first embedding it in dimension 5 with embedding lag 2.
Motif lengths (or order of the permutations) are therefore 5.

```julia
using DelayEmbeddings, Entropies
n = 5000
x = rand(n)
s = symbolize(x, SymbolicPermutation(m = 5, τ = 2))
```

The integer vector `s` now has length `n-(m-1)*τ = 4992`, and each `s[i]` contains
the integer symbol for the ordinal pattern of state vector `x[i]`.

[^Berger2019]: Berger, Sebastian, et al. "Teaching Ordinal Patterns to a Computer: Efficient Encoding Algorithms Based on the Lehmer Code." Entropy 21.10 (2019): 1023.
"""
function symbolize(x::AbstractDataset{m, T}, est::PermutationProbabilityEstimator) where {m, T}
    m >= 2 || error("Data must be at least 2-dimensional to symbolize. If data is a univariate time series, embed it using `genembed` first.")
    s = zeros(Int, length(x))
    symbolize!(s, x, est)
    return s
end

function symbolize(x::AbstractVector{T}, est::PermutationProbabilityEstimator) where {T}
    τs = tuple([est.τ*i for i = 0:est.m-1]...)
    x_emb = genembed(x, τs)

    s = zeros(Int, length(x_emb))
    symbolize!(s, x_emb, est)
    return s
end

function fill_symbolvector!(s, x, sp, m::Int; lt::Function = isless_rand)
    @inbounds for i = 1:length(x)
        sortperm!(sp, x[i], lt = lt)
        s[i] = encode_motif(sp, m)
    end
end

function symbolize!(s::AbstractVector{Int}, x::AbstractDataset{m, T}, est::SymbolicPermutation) where {m, T}
    @assert length(s) == length(x)
    #=
    Loop over embedding vectors `E[i]`, find the indices `p_i` that sort each `E[i]`,
    then get the corresponding integers `k_i` that generated the
    permutations `p_i`. Those integers are the symbols for the embedding vectors
    `E[i]`.
    =#
    sp = zeros(Int, m) # pre-allocate a single symbol vector that can be overwritten.
    fill_symbolvector!(s, x, sp, m, lt = est.lt)

    return s
end

function symbolize!(s::AbstractVector{Int}, x::AbstractVector{T}, est::SymbolicPermutation) where T
    τs = tuple([est.τ*i for i = 0:est.m-1]...)
    x_emb = genembed(x, τs)
    symbolize!(s, x_emb, est)
end

function probabilities!(s::AbstractVector{Int}, x::AbstractDataset{m, T}, est::SymbolicPermutation) where {m, T}
    length(s) == length(x) || throw(ArgumentError("Need length(s) == length(x), got `length(s)=$(length(s))` and `length(x)==$(length(x))`."))
    m >= 2 || error("Data must be at least 2-dimensional to compute the permutation entropy. If data is a univariate time series embed it using `genembed` first.")

    @inbounds for i = 1:length(x)
        s[i] = encode_motif(x[i], m)
    end
    probabilities(s)
end

function probabilities!(s::AbstractVector{Int}, x::AbstractVector{T}, est::SymbolicPermutation) where {T<:Real}
    L = length(x)
    N = L - (est.m-1)*est.τ
    length(s) == N || error("Pre-allocated symbol vector `s`needs to have length `length(x) - (m-1)*τ` to match the number of state vectors after `x` has been embedded. Got length(s)=$(length(s)) and length(x)=$(L).")

    τs = tuple([est.τ*i for i = 0:est.m-1]...)
    x_emb = genembed(x, τs)

    probabilities!(s, x_emb, est)
end

function probabilities(x::AbstractDataset{m, T}, est::SymbolicPermutation) where {m, T}
    s = zeros(Int, length(x))
    probabilities!(s, x, est)
end

function probabilities(x::AbstractVector{T}, est::SymbolicPermutation) where {T<:Real}
    τs = tuple([est.τ*i for i = 0:est.m-1]...)
    x_emb = genembed(x, τs)

    s = zeros(Int, length(x_emb))
    probabilities!(s, x_emb, est)
end

function genentropy!(
    s::AbstractVector{Int}, x::AbstractDataset{m, T}, est::SymbolicPermutation;
    q = 1.0, α = nothing, base::Real = MathConstants.e
    ) where {m, T}

    if α ≠ nothing
        @warn "Keyword `α` is deprecated in favor of `q`."
        q = α
    end
    length(s) == length(x) || error("Pre-allocated symbol vector s need the same number of elements as x. Got length(s)=$(length(s)) and length(x)=$(L).")
    ps = probabilities!(s, x, est)

    genentropy(ps, α = α, base = base)
end

function genentropy!(
        s::AbstractVector{Int}, x::AbstractVector{T}, est::SymbolicPermutation;
        q::Real = 1.0, α = nothing, base::Real = MathConstants.e
    ) where {T<:Real}
    if α ≠ nothing
        @warn "Keyword `α` is deprecated in favor of `q`."
        q = α
    end
    L = length(x)
    N = L - (est.m-1)*est.τ
    length(s) == N || error("Pre-allocated symbol vector `s` needs to have length `length(x) - (m-1)*τ` to match the number of state vectors after `x` has been embedded. Got length(s)=$(length(s)) and length(x)=$(L).")

    ps = probabilities!(s, x, est)
    genentropy(ps, α = α, base = base)
end


using Test
using JuMP
using MIPVerify
using MIPVerify: check_size, increment!
@isdefined(TestHelpers) || include("../../TestHelpers.jl")

function test_convolution_layer(
    p::MIPVerify.Conv2d,
    input::AbstractArray{T,4},
    expected_output::AbstractArray{T,4},
) where {T<:Real}
    """
    Tests that passing `input` into the convolution layer `p` produces
    `expected_output`. We test three combinations:

      1) Passing numerical input into a numerical Conv2d layer, verifying
         that we recover the value of `expected_output`.

      2) Setting up an optimization problem with variables corresponding
         to the convolution layer and the output (`p_v` and `output_v`).

         `output_v` is constrained to be the result of applying `p_v` to
         `input`, and is also constrained to be equal to `expected_output`.

         We verify that, when the optimization problem is solved, applying
         `p_v` to `input` recovers the value of `expected_output`.

         Note that since the optimization problem is under-determined, we
         cannot assert that `p_v` is equal to `p`.

      3) Setting up an optimization problem with variables corresponding
         to the input and output (`input_v` and `output_v`).

         `output_v` is constrained to be the result of applying `p` to
         `input_v`, and is also constrained to be equal to `expected_output`.

         We verify that, when the optimization problem is solved, applying
         `p` to `input_v` recovers the value of `expected_output`.

         As in case 2), we cannot assert that `input_v` is equal to input.
    """
    input_size = size(input)
    filter_size = size(p.filter)
    bias_size = size(p.bias)
    @testset "Numerical Input, Numerical Layer Parameters" begin
        evaluated_output = MIPVerify.conv2d(input, p)
        @test evaluated_output == expected_output
    end
    @testset "Numerical Input, Variable Layer Parameters" begin
        m = TestHelpers.get_new_model()
        filter_v = map(_ -> @variable(m), CartesianIndices(filter_size))
        bias_v = map(_ -> @variable(m), CartesianIndices(bias_size))
        p_v = MIPVerify.Conv2d(filter_v, bias_v, p.stride, p.padding)
        output_v = MIPVerify.conv2d(input, p_v)
        @constraint(m, output_v .== expected_output)
        solve(m)

        p_solve = MIPVerify.Conv2d(getvalue(filter_v), getvalue(bias_v), p.stride, p.padding)
        solve_output = MIPVerify.conv2d(input, p_solve)
        @test solve_output ≈ expected_output
    end
    @testset "Variable Input, Numerical Layer Parameters" begin
        m = TestHelpers.get_new_model()
        input_v = map(_ -> @variable(m), CartesianIndices(input_size))
        output_v = MIPVerify.conv2d(input_v, p)
        @constraint(m, output_v .== expected_output)
        solve(m)

        solve_output = MIPVerify.conv2d(getvalue(input_v), p)
        @test solve_output ≈ expected_output
    end
end

function test_convolution_layer_with_default_values(
    input_size::NTuple{4,Int},
    filter_size::NTuple{4,Int},
    expected_output_2d::AbstractArray{T,2},
    stride::Int,
    padding::Padding,
) where {T<:Real}
    """
    Generates test input of dimension `input_size`, and a Conv2d layer with
    a filter of dimension `filter_size` and specified `stride` and `padding`,
    running `test_convolution_layer` to ensure that passing the generated
    `input` into the generated Conv2d layer produces `expected_output_2d`.

      + The input generated consists of natural numbers in increasing order from
        left to right and then top to bottom.
      + The filter generated is all 1s, and the convolution layer has bias 0.
      + For convenience, the expected output only needs to be specified with the
        non-singleton dimensions.
    """
    input = reshape([1:prod(input_size);], input_size)
    filter = ones(filter_size...)
    bias = [0]
    expected_output = reshape(expected_output_2d, (1, size(expected_output_2d)..., 1))
    p = Conv2d(filter, bias, stride, padding)
    test_convolution_layer(p, input, expected_output)
end

@testset "conv2d.jl" begin
    @testset "Conv2d" begin
        @testset "Base.show" begin
            filter = ones(3, 3, 2, 5)
            p = Conv2d(filter)
            io = IOBuffer()
            Base.show(io, p)
            @test String(take!(io)) ==
                  "Conv2d(2, 5, kernel_size=(3, 3), stride=(1, 1), padding=same)"
        end
        @testset "With Bias" begin
            @testset "Matched Size" begin
                out_channels = 5
                filter = ones(3, 3, 2, out_channels)
                bias = ones(out_channels)
                p = Conv2d(filter, bias)
                @test p.filter == filter
                @test p.bias == bias
            end
            @testset "Unmatched Size" begin
                filter_out_channels = 4
                bias_out_channels = 5
                filter = ones(3, 3, 2, filter_out_channels)
                bias = ones(bias_out_channels)
                @test_throws AssertionError Conv2d(filter, bias)
            end
        end
        @testset "No Bias" begin
            filter = ones(3, 3, 2, 5)
            p = Conv2d(filter)
            @test p.filter == filter
        end
        @testset "JuMP Variables" begin
            m = Model()
            filter_size = (3, 3, 2, 5)
            filter = map(_ -> @variable(m), CartesianIndices(filter_size))
            p = Conv2d(filter)
            @test p.filter == filter
        end
        @testset "check_size" begin
            filter = ones(3, 3, 2, 5)
            p = Conv2d(filter)
            @test check_size(p, (3, 3, 2, 5)) === nothing
            @test_throws AssertionError check_size(p, (3, 3, 2, 4))
        end
    end

    @testset "increment!" begin
        @testset "Real * Real" begin
            @test 7 == increment!(1, 2, 3)
        end
        @testset "JuMP.AffExpr * Real" begin
            m = TestHelpers.get_new_model()
            x = @variable(m, start = 100)
            y = @variable(m, start = 1)
            s = 5 * x + 3 * y
            t = 3 * x + 2 * y
            increment!(s, 2, t)
            @test getvalue(s) == 1107
            increment!(s, t, -1)
            @test getvalue(s) == 805
            increment!(s, x, 3)
            @test getvalue(s) == 1105
            increment!(s, y, 2)
            @test getvalue(s) == 1107
        end
    end

    @testset "conv2d" begin
        input_size = (1, 4, 4, 2)
        input = reshape(collect(1:prod(input_size)), input_size) .- 16
        filter_size = (3, 3, 2, 1)
        filter = reshape(collect(1:prod(filter_size)), filter_size) .- 9
        bias = [1]
        expected_output_2d = [
            225 381 405 285
            502 787 796 532
            550 823 832 532
            301 429 417 249
        ]
        expected_output = reshape(transpose(expected_output_2d), (1, 4, 4, 1))
        p = Conv2d(filter, bias)
        test_convolution_layer(p, input, expected_output)
    end

    @testset "conv2d with non-unit stride" begin
        input_size = (1, 6, 6, 2)
        input = reshape(collect(1:prod(input_size)), input_size) .- 36
        filter_size = (3, 3, 2, 1)
        filter = reshape(collect(1:prod(filter_size)), filter_size) .- 9
        bias = [1]
        stride = 2
        expected_output_2d = [
            1597 1615 1120
            1705 1723 1120
            903 879 513
        ]
        expected_output = reshape(transpose(expected_output_2d), (1, 3, 3, 1))
        p = Conv2d(filter, bias, stride)
        test_convolution_layer(p, input, expected_output)
    end

    @testset "conv2d with stride 2, odd input shape with even filter shape" begin
        input_size = (1, 5, 5, 2)
        input = reshape(collect(1:prod(input_size)), input_size) .- 25
        filter_size = (4, 4, 2, 1)
        filter = reshape(collect(1:prod(filter_size)), filter_size) .- 16
        bias = [1]
        stride = 2
        expected_output_2d = [
            1756 2511 1310
            3065 4097 1969
            1017 1225 501
        ]
        expected_output = reshape(transpose(expected_output_2d), (1, 3, 3, 1))
        p = Conv2d(filter, bias, stride)
        test_convolution_layer(p, input, expected_output)
    end

    @testset "conv2d with 'valid' padding" begin
        @testset "conv2d with 'valid' padding, odd input and filter size, stride = 1" begin
            expected_output_2d = [
                63 72 81
                108 117 126
                153 162 171
            ]
            test_convolution_layer_with_default_values(
                (1, 5, 5, 1),
                (3, 3, 1, 1),
                transpose(expected_output_2d),
                1,
                ValidPadding(),
            )
        end

        @testset "conv2d with 'valid' padding, odd input and filter size, stride = 1, channels != 1" begin
            expected_output_2d = [
                351 369 387
                441 459 477
                531 549 567
            ]
            test_convolution_layer_with_default_values(
                (1, 5, 5, 2),
                (3, 3, 2, 1),
                transpose(expected_output_2d),
                1,
                ValidPadding(),
            )
        end

        @testset "conv2d with 'valid' padding, stride = 1, input width != input height" begin
            expected_output_2d = [
                63 72 81
                108 117 126
                153 162 171
                198 207 216
            ]
            test_convolution_layer_with_default_values(
                (1, 5, 6, 1),
                (3, 3, 1, 1),
                transpose(expected_output_2d),
                1,
                ValidPadding(),
            )
        end

        @testset "conv2d with 'valid' padding, stride = 1, filter width != filter height" begin
            expected_output_2d = [
                39 45 51 57
                69 75 81 87
                99 105 111 117
            ]
            test_convolution_layer_with_default_values(
                (1, 5, 5, 1),
                (2, 3, 1, 1),
                transpose(expected_output_2d),
                1,
                ValidPadding(),
            )
        end

        @testset "conv2d with 'valid' padding, odd input and filter size, stride != 1" begin
            expected_output_2d = [
                63 81
                153 171
            ]
            test_convolution_layer_with_default_values(
                (1, 5, 5, 1),
                (3, 3, 1, 1),
                transpose(expected_output_2d),
                2,
                ValidPadding(),
            )
        end

        @testset "conv2d with 'valid' padding, odd input size, even filter size, stride = 1" begin
            expected_output_2d = [
                16 20 24 28
                36 40 44 48
                56 60 64 68
                76 80 84 88
            ]
            test_convolution_layer_with_default_values(
                (1, 5, 5, 1),
                (2, 2, 1, 1),
                transpose(expected_output_2d),
                1,
                ValidPadding(),
            )
        end

        @testset "conv2d with 'valid' padding, odd input size, even filter size, stride != 1" begin
            expected_output_2d = [
                16 24
                56 64
            ]
            test_convolution_layer_with_default_values(
                (1, 5, 5, 1),
                (2, 2, 1, 1),
                transpose(expected_output_2d),
                2,
                ValidPadding(),
            )
        end

        @testset "conv2d with 'valid' padding, even input size, odd filter size, stride = 1" begin
            expected_output_2d = [
                72 81 90 99
                126 135 144 153
                180 189 198 207
                234 243 252 261
            ]
            test_convolution_layer_with_default_values(
                (1, 6, 6, 1),
                (3, 3, 1, 1),
                transpose(expected_output_2d),
                1,
                ValidPadding(),
            )
        end

        @testset "conv2d with 'valid' padding, even input size, odd filter size, stride != 1" begin
            expected_output_2d = [
                72 90
                180 198
            ]
            test_convolution_layer_with_default_values(
                (1, 6, 6, 1),
                (3, 3, 1, 1),
                transpose(expected_output_2d),
                2,
                ValidPadding(),
            )
        end

        @testset "conv2d with 'valid' padding, even input and filter size, stride = 1" begin
            expected_output_2d = [
                18 22 26 30 34
                42 46 50 54 58
                66 70 74 78 82
                90 94 98 102 106
                114 118 122 126 130
            ]
            test_convolution_layer_with_default_values(
                (1, 6, 6, 1),
                (2, 2, 1, 1),
                transpose(expected_output_2d),
                1,
                ValidPadding(),
            )
        end

        @testset "conv2d with 'valid' padding, even input and filter size, stride != 1" begin
            expected_output_2d = [
                18 30
                90 102
            ]
            test_convolution_layer_with_default_values(
                (1, 6, 6, 1),
                (2, 2, 1, 1),
                transpose(expected_output_2d),
                3,
                ValidPadding(),
            )
        end
    end

    @testset "conv2d wit fixed padding" begin
        @testset "conv2d with (0, 0) padding, stride = 1" begin
            expected_output_2d = [
                63 72 81
                108 117 126
                153 162 171
            ]
            test_convolution_layer_with_default_values(
                (1, 5, 5, 1),
                (3, 3, 1, 1),
                transpose(expected_output_2d),
                1,
                (0, 0),
            )
        end

        @testset "conv2d with (0, 0) padding, stride != 1" begin
            expected_output_2d = [
                63 81
                153 171
            ]
            test_convolution_layer_with_default_values(
                (1, 5, 5, 1),
                (3, 3, 1, 1),
                transpose(expected_output_2d),
                2,
                (0, 0),
            )
        end

        @testset "conv2d with (1, 1) padding, stride = 1" begin
            expected_output_2d = [
                16 27 33 39 28
                39 63 72 81 57
                69 108 117 126 87
                99 153 162 171 117
                76 117 123 129 88
            ]
            test_convolution_layer_with_default_values(
                (1, 5, 5, 1),
                (3, 3, 1, 1),
                transpose(expected_output_2d),
                1,
                (1, 1),
            )
        end

        @testset "conv2d with (1, 1) padding, stride != 1" begin
            expected_output_2d = [
                16 33 28
                69 117 87
                76 123 88
            ]
            test_convolution_layer_with_default_values(
                (1, 5, 5, 1),
                (3, 3, 1, 1),
                transpose(expected_output_2d),
                2,
                (1, 1),
            )
        end

        @testset "conv2d with 1 padding, stride = 1" begin
            expected_output_2d = [
                16 27 33 39 28
                39 63 72 81 57
                69 108 117 126 87
                99 153 162 171 117
                76 117 123 129 88
            ]
            test_convolution_layer_with_default_values(
                (1, 5, 5, 1),
                (3, 3, 1, 1),
                transpose(expected_output_2d),
                1,
                1,
            )
        end

        @testset "conv2d with (1, 1) padding, input width != input_height, stride = 1" begin
            expected_output_2d = [
                18 30 36 42 48 34
                45 72 81 90 99 69
                81 126 135 144 153 105
                117 180 189 198 207 141
                90 138 144 150 156 106
            ]
            test_convolution_layer_with_default_values(
                (1, 6, 5, 1),
                (3, 3, 1, 1),
                transpose(expected_output_2d),
                1,
                (1, 1),
            )
        end

        @testset "conv2d with (1, 1) padding, input width != input_height, stride != 1" begin
            expected_output_2d = [
                18 36 48
                81 135 153
                90 144 156
            ]
            test_convolution_layer_with_default_values(
                (1, 6, 5, 1),
                (3, 3, 1, 1),
                transpose(expected_output_2d),
                2,
                (1, 1),
            )
        end

        @testset "conv2d with (1, 2) padding, stride = 1" begin
            expected_output_2d = [
                3 6 9 12 9
                16 27 33 39 28
                39 63 72 81 57
                69 108 117 126 87
                99 153 162 171 117
                76 117 123 129 88
                43 66 69 72 49
            ]
            test_convolution_layer_with_default_values(
                (1, 5, 5, 1),
                (3, 3, 1, 1),
                transpose(expected_output_2d),
                1,
                (1, 2),
            )
        end

        @testset "conv2d with (1, 2) padding, stride != 1" begin
            expected_output_2d = [
                3 9 9
                39 72 57
                99 162 117
                43 69 49
            ]
            test_convolution_layer_with_default_values(
                (1, 5, 5, 1),
                (3, 3, 1, 1),
                transpose(expected_output_2d),
                2,
                (1, 2),
            )
        end

        @testset "conv2d with (1, 1) padding, channels != 1, stride = 1" begin
            expected_output_2d = [
                132 204 216 228 156
                228 351 369 387 264
                288 441 459 477 324
                348 531 549 567 384
                252 384 396 408 276
            ]
            test_convolution_layer_with_default_values(
                (1, 5, 5, 2),
                (3, 3, 2, 1),
                transpose(expected_output_2d),
                1,
                (1, 1),
            )
        end

        @testset "conv2d with (1, 1) padding, channels != 1, stride != 1" begin
            expected_output_2d = [
                132 216 156
                288 459 324
                252 396 276
            ]
            test_convolution_layer_with_default_values(
                (1, 5, 5, 2),
                (3, 3, 2, 1),
                transpose(expected_output_2d),
                2,
                (1, 1),
            )
        end

        @testset "conv2d with (1, 1, 1, 1) padding, stride = 1" begin
            expected_output_2d = [
                16 27 33 39 28
                39 63 72 81 57
                69 108 117 126 87
                99 153 162 171 117
                76 117 123 129 88
            ]
            test_convolution_layer_with_default_values(
                (1, 5, 5, 1),
                (3, 3, 1, 1),
                transpose(expected_output_2d),
                1,
                (1, 1, 1, 1),
            )
        end

        @testset "conv2d with (1, 2, 3, 4) padding, stride = 1" begin
            expected_output_2d = [
                0 0 0 0 0 0
                3 6 9 12 9 5
                16 27 33 39 28 15
                39 63 72 81 57 30
                69 108 117 126 87 45
                99 153 162 171 117 60
                76 117 123 129 88 45
                43 66 69 72 49 25
                0 0 0 0 0 0
                0 0 0 0 0 0
            ]
            test_convolution_layer_with_default_values(
                (1, 5, 5, 1),
                (3, 3, 1, 1),
                transpose(expected_output_2d),
                1,
                (1, 2, 3, 4),
            )
        end
    end
end