"""
    absorb_fromleft(A::Array{T, 2}, B::Array{T, 3}) where T<:Number

Absorb the matrix A in B where A is at the left of B.

 -- A -- B -- = -- C --
         |         |

"""
function absorb_fromleft(A::Array{T, 2}, B::Array{T, 3}) where T<:Number
    a1, a2 = size(A)
    b1, b2, b3 = size(B)
    B = reshape(B, b1, b2*b3)
    return reshape(A*B, a1, b2, b3)
end

"""
    absorb_fromright(A::Array{T, 3}, B::Array{T, 2}) where T<:Number

Absorb the matrix B in A where B is at the right of A.

 -- A -- B -- = -- C --
    |              |

"""
function absorb_fromright(A::Array{T, 3}, B::Array{T, 2}) where T<:Number
    a1, a2, a3 = size(A)
    b1, b2 = size(B)
    A = reshape(A, a1*a2, a3)
    return reshape(A*B, a1, a2, b2)
end

"""
    prop_right2(L::Array{T, 2}, A::Array{T, 3}, B::Array{T, 3}) where T<:Number

Propagate the tensor L through the tensors A, conj(B).

 ---- A ---     ---
 |    |         |
 L    |     =  L2
 |    |         |
 ---- B ---     ---

"""
function prop_right2(L::Array{T, 2}, A::Array{T, 3},
                     B::Array{T, 3}) where T<:Number
    Ar = reshape(A, size(A, 1), size(A, 2)*size(A, 3))
    T1 = BLAS.gemm('T', 'N', L, Ar)
    T1 = reshape(T1, size(T1, 1)*size(B, 2), size(A, 3))
    Br = reshape(B, size(B, 1)*size(B, 2), size(B, 3))
    L2 = BLAS.gemm('T', 'N', T1, conj(Br))
    return L2
end

"""
    prop_right3(L::Array{T, 3}, A::Array{T, 3}, M::Array{T, 4},
                B::Array{T, 3}) where T<:Number

Propagate the tensor L through the tensors A, M, conj(B).

 ---- A ---         ---
 |    |             |
 L--- M ---  =  new_L
 |    |             |
 ---- B ---         ---

"""
function prop_right3(L::Array{T, 3}, A::Array{T, 3}, M::Array{T, 4},
                     B::Array{T, 3}) where T<:Number
    @tensoropt L1[l1, l2, s2, c] := L[l1, l2, l3]*conj(B[l3, s2, c])
    @tensoropt L2[l1, s1, b, c] := L1[l1, l2, s2, c]*M[l2, s1, s2, b]
    @tensoropt new_L[a, b, c] := L2[l1, s1, b, c]*A[l1, s1, a]
    return new_L
end

"""
    prop_right4(L::Array{T, 4}, A::Array{T, 3}, M1::Array{T, 4},
                M2::Array{T, 4}, B::Array{T, 3}) where T<:Number

Propagate the tensor L through the tensors A, M1, M2, conj(B).

 ---- A ---         ---
 |    |             |
 |--- M1---         |--
 L    |      =  new_L
 |----M2---         |--
 |    |             |
 ---- B ---         ---

"""
function prop_right4(L::Array{T, 4}, A::Array{T, 3}, M1::Array{T, 4},
                     M2::Array{T, 4}, B::Array{T, 3}) where T<:Number
    @tensoropt L1[l1, l2, l3, s3, d] := L[l1, l2, l3, l4]*conj(B[l4, s3, d])
    @tensoropt L2[l1, l2, s2, c, d] := L1[l1, l2, l3, s3, d]*M2[l3, s2, s3, c]
    @tensoropt L3[l1, s1, b, c, d] := L2[l1, l2, s2, c, d]*M1[l2, s1, s2, b]
    @tensoropt new_L[a, b, c, d] := L3[l1, s1, b, c, d]*A[l1, s1, a]
    return new_L
end

"""
    prop_left2(A::Array{T, 3}, B::Array{T, 3}, R::Array{T, 2}) where T<:Number

Propagate the tensor R through the tensors A, conj(B).

 --- A ----      ---
     |    |        |
     |    R   =   R2
     |    |        |
 --- B ----      ---

"""
function prop_left2(A::Array{T, 3}, B::Array{T, 3},
                    R::Array{T, 2}) where T<:Number
    Ar = reshape(A, size(A, 1)*size(A, 2), size(A, 3))
    T1 = BLAS.gemm('N', 'N', Ar, R)
    T1 = reshape(T1, size(A, 1), size(A, 2)*size(T1, 2))
    Br = reshape(B, size(B, 1), size(B, 2)*size(B, 3))
    L2 = BLAS.gemm('N', 'C', T1, Br)
    return L2
end

"""
    prop_left3(A::Array{T, 3}, M::Array{T, 4}, B::Array{T, 3},
               R::Array{T, 3}) where T<:Number

Propagate the tensor R through the tensors A, M, conj(B).

 --- A ----      ---
     |    |        |
 --- M -- R   =    new_R
     |    |        |
 --- B ----      ---

"""
function prop_left3(A::Array{T, 3}, M::Array{T, 4}, B::Array{T, 3},
                    R::Array{T, 3}) where T<:Number
    @tensoropt new_R[a, b, c] := (
        A[a, s1, r1]*M[b, s1, s2, r2]*conj(B[c, s2, r3])*R[r1, r2, r3]
    )
    return new_R
end

"""
    prop_left4(A::Array{T, 3}, M1::Array{T, 4}, M2::Array{T, 4},
               B::Array{T, 3}, R::Array{T, 4}) where T<:Number

Propagate the tensor R through the tensors A, M1, M2, conj(B).

 --- A -----     ----
     |     |        |
 --- M1 ---|     ---|
     |     R   =    new_R
 --- M2 ---|     ---|
     |     |        |
 --- B -----     ----

"""
function prop_left4(A::Array{T, 3}, M1::Array{T, 4}, M2::Array{T, 4},
                    B::Array{T, 3}, R::Array{T, 4}) where T<:Number
    @tensoropt R1[a, s1, r2, r3, r4] := A[a, s1, r1]*R[r1, r2, r3, r4]
    @tensoropt R2[a, b, s2, r3, r4] := R1[a, s1, r2, r3, r4]*M1[b, s1, s2, r2]
    @tensoropt R3[a, b, c, s3, r4] := R2[a, b, s2, r3, r4]*M2[c, s2, s3, r3]
    @tensoropt new_R[a, b, c, d] := R3[a, b, c, s3, r4]*conj(B[d, s3, r4])
    return new_R
end

"""
    prop_right_subexp(L::Array{T, 3}, W::Array{T, 4}, M::Array{T, 3}) where T<:Number

Propagate the tensor L through the tensors W, conj(M) for the DMRG3S algorithm.

 ----               ---
 |                  |
 |    |             |----
 L -- W ---     =   P
 |    |             |----
 |    |             |
 ---- M ---         ---

"""
function prop_right_subexp(L::Array{T, 3}, W::Array{T, 4},
                           M::Array{T, 3}) where T<:Number
    @tensoropt P[l1, s1, r2, r3] := (
        L[l1, l2, l3]*conj(M[l3, s2, r3])*W[l2, s1, s2, r2]
        )
    return P
end

"""
    prop_left_subexp(W::Array{T, 4}, M::Array{T, 3},
                     R::Array{T, 3}) where T<:Number

Propagate the tensor R through the tensors W, conj(M) for the DMRG3S algorithm.

        ----     ----
           |        |
           |     ---|
     |     |   =    P
 --- W --- R     ---|
     |     |        |
     |     |        |
 --- B -----     ----

"""
function prop_left_subexp(W::Array{T, 4}, M::Array{T, 3},
                          R::Array{T, 3}) where T<:Number
    @tensoropt R1[r1, r2, s2, l3] := R[r1, r2, r3]*conj(M[l3, s2, r3])
    @tensoropt P[s1, r1, l2, l3] := R1[r1, r2, s2, l3]*W[l2, s1, s2, r2]
    return P
end

"""
    absorb_Le(Le::Array{T, 3}, M::Array{T, 2}) where T<:Number

Absorb M and conj(M) into the left environment Le.

 ----- M ------     ----
 |                  |
Le ------------ =  Le --
 |                  |
 --- conj(M) --     ----

"""
function absorb_Le(Le::Array{T, 3}, M::Matrix{T}) where T<:Number
    if size(M, 1) == size(M, 2)
        @tensoropt Le[l1, l2, l3] = Le[a, l2, b]*M[a, l1]*conj(M[b, l3])
    else
        @tensoropt Le[l1, l2, l3] := Le[a, l2, b]*M[a, l1]*conj(M[b, l3])
    end
    return Le
end

"""
    absorb_Re(Re::Array{T, 3}, M::Array{T, 2}) where T<:Number

Absorb M and conj(M) into the right environment Re.

 ---- M ------     ----
             |        |
 ---------- Re  =  -- Re
             |        |
 - conj(M) ---     ----

"""
function absorb_Re(Re::Array{T, 3}, M::Matrix{T}) where T<:Number
    if size(M, 1) == size(M, 2)
        @tensoropt Re[r1, r2, r3] = M[r1, a]*conj(M[r3, b])*Re[a, r2, b]
    else
        @tensoropt Re[r1, r2, r3] := M[r1, a]*conj(M[r3, b])*Re[a, r2, b]
    end
    return Re
end

"""
    build_local_hamiltonian(Le::Array{T, 3}, W::Array{T, 4},
                            Re::Array{T, 3}) where T<:Number

Build the local Hamiltonian with the left and right environments.

 ---      ----
 |     |     |     |
Le --- W --- Re =  Hi
 |     |     |     |
 ---      ----

"""
function build_local_hamiltonian(Le::Array{T, 3}, W::Array{T, 4},
                                 Re::Array{T, 3})  where T<:Number
    @tensoropt Hi[l1, s1, r1, l3, s2, r3] := (
        Le[l1, l2, l3]*W[l2, s1, s2, r2]*Re[r1, r2, r3]
        )
    Hi = reshape(Hi, (size(Le, 1)*size(W, 2)*size(Re, 1),
                      size(Le, 3)*size(W, 3)*size(Re, 3)))
    return Hi
end

"""
    build_local_hamiltonian(Le::Array{T, 3}, W::Array{T, 4}, Re::Array{T, 3},
                            cache::Cache{T}) where T<:Number

Build the local Hamiltonian with the left and right environments. Pass a
preallocated cache that can be used to store the final tensor.
"""
function build_local_hamiltonian(
    Le::Array{T, 3}, W::Array{T, 4}, Re::Array{T, 3}, cache::Cache{T}
    )  where T<:Number
    # Check if any cache element of appropriate size can be used.
    needed_space = (size(Le, 1)*size(W, 2)*size(Re, 1)
                    *size(Le, 3)*size(W, 3)*size(Re, 3))
    loc_in_cache = is_in_cache(cache, needed_space)
    if loc_in_cache > 0
        Hi = reshape(cache.elts[loc_in_cache],
                     (size(Le, 1), size(W, 2), size(Re, 1),
                      size(Le, 3), size(W, 3), size(Re, 3)))
        @tensoropt Hi[l1, s1, r1, l3, s2, r3] = (
            Le[l1, l2, l3]*W[l2, s1, s2, r2]*Re[r1, r2, r3]
            )
    else
        @tensoropt Hi[l1, s1, r1, l3, s2, r3] := (
            Le[l1, l2, l3]*W[l2, s1, s2, r2]*Re[r1, r2, r3]
            )
    end
    Hi = reshape(Hi, (size(Le, 1)*size(W, 2)*size(Re, 1),
                      size(Le, 3)*size(W, 3)*size(Re, 3)))
    update_cache!(cache, Hi)
    return Hi
end

"""
    build_local_hamiltonian_2(Le::Array{T, 3}, W1::Array{T, 4},
                              W2::Array{T, 4}, Re::Array{T, 3},
                              cache::Cache{T}) where T<:Number

Build the local Hamiltonian with the left and right environments for DMRG2.

 ---            ---
 |    |     |     |    |
Le -- W1 -- W2 -- Re = Hi
 |    |     |     |    |
 ---            ---

"""
function build_local_hamiltonian_2(Le::Array{T, 3}, W1::Array{T, 4},
                                   W2::Array{T, 4}, Re::Array{T, 3},
                                   cache::Cache{T}) where T<:Number
    @tensoropt Hi[l1, s1, s3, r1, l3, s2, s4, r3] := (Le[l1, l2, l3]
                                                      *W1[l2, s1, s2, a]
                                                      *W2[a, s3, s4, r2]
                                                      *Re[r1, r2, r3])
    Hi = reshape(Hi, (size(Le, 1)*size(W1, 2)*size(W2, 2)*size(Re, 1),
                      size(Le, 3)*size(W1, 3)*size(W2, 3)*size(Re, 3)))
    # Check if any cache element of appropriate size can be used.
    needed_space = (size(Le, 1)*size(W1, 2)*size(W2, 2)*size(Re, 1)
                    *size(Le, 3)*size(W1, 3)*size(W2, 3)*size(Re, 3))
    loc_in_cache = is_in_cache(cache, needed_space)
    if loc_in_cache > 0
        Hi = reshape(cache.elts[loc_in_cache],
                     ((size(Le, 1), size(W1, 2), size(W2, 2), size(Re, 1),
                      size(Le, 3),size(W1, 3),size(W2, 3),size(Re, 3))))
        @tensoropt Hi[l1, s1, s3, r1, l3, s2, s4, r3] = (
            Le[l1, l2, l3]*W1[l2, s1, s2, a]*W2[a, s3, s4, r2]*Re[r1, r2, r3]
            )
    else
        @tensoropt Hi[l1, s1, s3, r1, l3, s2, s4, r3] := (
            Le[l1, l2, l3]*W1[l2, s1, s2, a]*W2[a, s3, s4, r2]*Re[r1, r2, r3]
            )
    end
    Hi = reshape(Hi, ((size(Le, 1)*size(W1, 2)*size(W2, 2)*size(Re, 1),
                       size(Le, 3)*size(W1, 3)*size(W2, 3)*size(Re, 3))))
    update_cache!(cache, Hi)
    return Hi
end


# mri_objects.jl

using MIRT: mri_objects

using MIRT: jim, image_geom, prompt, max_percent_diff
import MIRT: mri_objects_trap
using FFTW: fft, fftshift, ifftshift
using Plots
using Test: @test, @test_throws #, @inferred

# 2D test
#NOTE: Dirac will not plot very well.
#Just make sure it prints a gray image, and that's fine. FFT will also be screwed up.
function mri_objects_test2()
    ig = image_geom(nx = 2^7, ny = 2^7 + 2, dx = 4, offsets = :dsp)

    shift = [0.1, 0.2] .* ig.fovs
    sizes = [.15, .1] .* ig.fovs
    tests = ( # 3 separate tests:
                (:circ2, [shift' sizes[1] 2]),
                (:gauss2, [shift' sizes' 2]),
                (:rect2, [shift' sizes' 2]),
                (:dirac2,[shift' 2])
            )
    ntest = length(tests)
    pl = Matrix{Plots.Plot}(undef, ntest,3)

    for (i,test) = enumerate(tests)
        st = mri_objects(test)
        i2 = st.image(ig.xg, ig.yg, dx=ig.dx, dy=ig.dy)
        s2 = ifftshift(fft(fftshift(i2))) * abs(ig.dx * ig.dy)
        f2 = st.kspace(ig.fg...)
        isinteractive() && (@show max_percent_diff(f2,s2))
    #   @show round.(maximum.([i2, abs.(f2), abs.(s2)]), digits=2)
        pl[i,1] = jim(i2, title = "$(test[1])")
        pl[i,2] = jim(abs.(s2), title = "|fft|")
        pl[i,3] = jim(abs.(f2), title = "|kspace|")
    end
    plot(pl...)
    return plot(pl...)
end

function mri_objects_test_case1()
    ig = image_geom(nx = 2^7, ny = 2^7 + 2, dx = 4, offsets = :dsp)
    xt = mri_objects(:case1,unit = :cm).image(ig.xg, ig.yg)
    jim(xt, title = "case1cm")
end

function mri_objects_test_case4()
    ig = image_geom(nx = 2^5, ny = 2^5 + 2,nz = 2^5, dx = 1,dz = .5, offsets = :dsp)
    xt = mri_objects(:case4,unit = :cm).image(ig.xg, ig.yg, ig.zg)
    jim(xt, title = "case4cm")
end

# 3D test
# NOTE: Dirac will not plot very well.
# Just make sure it prints a gray image, and that's fine. FFT will also be screwed up.
function mri_objects_test3()
    ig = image_geom(nx = 2^7, nz = 2^5, offsets = :dsp, dx = 4, dz = 3)
    shift = [0.1, 0.2, 0.3] .* ig.fovs
    sizes = [0.15, 0.1, 0.2] .* ig.fovs
    tests = ((:cyl3, [shift' sizes[1] sizes[3] 2]),
             (:gauss3, [shift' sizes' 2]),
             (:rect3, [shift' sizes' 2]),
             (:dirac3,[shift' 2]))
    ntest = length(tests)
    pl = Matrix{Plots.Plot}(undef, ntest,3)
    for (i,test) in enumerate(tests)
        st = mri_objects(test)
        i3 = st.image(ig.xg,ig.yg,ig.zg, dx=ig.dx, dy=ig.dy, dz=ig.dz)
        s3 = ifftshift(fft(fftshift(i3)))
        s3 *= abs(ig.dx * ig.dy * ig.dz)
        fg = ig.fg
        f3 = st.kspace(fg...)
        isinteractive() && (@show max_percent_diff(f3,s3))
        pl[i,1] = jim(i3, title = "$(test[1])")
        pl[i,2] = jim(abs.(s3), title = "fft")
        pl[i,3] = jim(abs.(f3), title = "kspace")
        plot(pl[i,:]...)
        prompt()
    end
    return plot(pl...)
end

# plot the trapezoid
function mri_objects_trap_test()
    len = 6
    dz = 2
    z = LinRange(-8,8,201)
    trap = mri_objects_trap(z, dz, len)
    trap0 = mri_objects_trap(z, 0, len)
    plot(z, trap, label="trap")
    plot!(z, trap0, label="rect")
    plot!(xtick = [0 -len/2 len/2-dz/2 len/2 len/2+dz/2])
end

@test_throws String mri_objects(:bad)

@test mri_objects_trap_test() isa Plots.Plot
prompt()

@test mri_objects_test2() isa Plots.Plot
prompt()

@test mri_objects_test_case1() isa Plots.Plot
prompt()

@test mri_objects_test3() isa Plots.Plot

@test mri_objects_test_case4() isa Plots.Plot
prompt()


# %% Comparison with analytically resolved models
using DrWatson
@quickactivate :FractalDimension # uses DynamicalSystems, PyPlot
include(srcdir("style.jl"))

datas = Vector(undef, 6)
labels = Vector{String}(undef, 6)

N = Int(1e5)

datas[1] = :roessler_periodic
labels[1] = "periodic"

# datas[2] = :torus2
# labels[2] = "2-torus"
datas[2] = :henonheiles_quasi
labels[2] = "quasiperiodic"

datas[3] = :koch
labels[3] = "Koch snowflake"

datas[4] = :kaplanyorke_map
labels[4] = "Kaplan-Yorke map"

datas[5] = :uniform_sphere
labels[5] = "sphere"

datas[6] = :standardmap_chaotic
labels[6] = "SM (uniform)"

qH = 0
qC = 2
Cmethod = "standard" # bueno or standard. Decides εmax for correlation sum.

# Calculate values for H, C
eHs, eCs, Hs, Cs = [Vector{Float64}[] for i in 1:4]
for i in 1:length(datas)
    # These are the parameters that change between the different elements of the plot
    data = datas[i]

    # Here we simply pack all parameters into a dictionary
    # (other parameters are (probably) globals)
    params = @strdict N qH qC data
    if Cmethod ≠ "standard"
        params["Cmethod"] = Cmethod
    end
    if data == :koch
        params["maxk"] = 7
        delete!(params, "N")
    end

    # This is the main call that calculates everything
    output, s = produce_or_load(
        datadir("main"), params, make_C_H;
        prefix = string(data), suffix = "jld2", force = false,
        ignores = ["data"], storepatch = false,
    )
    @unpack eH, eC, H, C = output
    push!(eHs, eH); push!(Hs, H); push!(eCs, eC); push!(Cs, C)
end

# Do the actual plot
legendtitle = "analytically known \$\\Delta\$"

fig, axs = mainplot(
    Hs, Cs, eHs, eCs, labels, legendtitle;
    qH, qC, tol = 0.25,

    # For this plot we use standard regression fit because
    # the estimate is already so accurate, we don't want to
    # have the unecessary larger confidence intervals from the
    # logarithmic correction
    dimension_fit_C = FractalDimension.linear_regression_fit_glm,
)

wsave(plotsdir("paper", "analytic"), fig)


# This file was generated, do not modify it.

using MLJ, DataFrames, Random
@load RidgeRegressor pkg=MultivariateStats

Random.seed!(5) # for reproducibility
x1 = rand(300)
x2 = rand(300)
x3 = rand(300)
y = exp.(x1 - x2 -2x3 + 0.1*rand(300))
X = DataFrame(x1=x1, x2=x2, x3=x3)

test, train = partition(eachindex(y), 0.8);

Xs = source(X)
ys = source(y, kind=:target)

std_model = Standardizer()
stand = machine(std_model, Xs)
W = transform(stand, Xs)

box_model = UnivariateBoxCoxTransformer()
box = machine(box_model, ys)
z = transform(box, ys)

ridge_model = RidgeRegressor(lambda=0.1)
ridge = machine(ridge_model, W, z)
ẑ = predict(ridge, W)

ŷ = inverse_transform(box, ẑ)

@from_network CompositeModel(std=std_model, box=box_model,
                             ridge=ridge_model) <= ŷ;

cm = machine(CompositeModel(), X, y)
res = evaluate!(cm, resampling=Holdout(fraction_train=0.8),
                measure=rms)
round(res.measurement[1], sigdigits=3)

mutable struct CompositeModel2 <: DeterministicNetwork
    std_model::Standardizer
    box_model::UnivariateBoxCoxTransformer
    ridge_model::RidgeRegressor
end

function MLJ.fit(m::CompositeModel2, verbosity::Int, X, y)
    Xs = source(X)
    ys = source(y, kind=:target)
    W = transform(machine(m.std_model, Xs), Xs)
    box = machine(m.box_model, ys)
    z = transform(box, ys)
    ẑ = predict(machine(m.ridge_model, W, z), W)
    ŷ = inverse_transform(box, ẑ)
    fit!(ŷ, verbosity=0)
    return fitresults(ŷ)
end

mdl = CompositeModel2(Standardizer(), UnivariateBoxCoxTransformer(),
                      RidgeRegressor(lambda=0.1))
cm = machine(mdl, X, y)
res = evaluate!(cm, resampling=Holdout(fraction_train=0.8), measure=rms)
round(res.measurement[1], sigdigits=3)


using Soss

m = @model Prior,x,σ begin
    β ~ Prior
    yhat = β .* x
    y ~ For(eachindex(x)) do j
        Normal(yhat[j], σ)
    end
end

x = randn(3);
Prior = Normal()
truth = rand(m(Prior=Prior, x=x, σ=2.0));

pairs(truth)

post = dynamicHMC(m(Prior=Prior, x=x, σ=2.0), (y=truth.y,));
particles(post)

pred = predictive(m, :β)

predictive(m,:yhat)


# Wrapper function for log likelihood

"""
    ll(y, model, θ; initiate = "first")
Likelihood function for Count Data models.

* `y`: Time series
* `model`: Model Specification
* `θ`: Parameters (Vector or parameter)
* `initiate`: How is the time series initiated?

The time series can be initiated by "first", the first observed value, "intercept",
the intercept parameter (possibly exponentiated if log-link), or "marginal", the marginal
mean of the process.

# Example
```julia-repl
ll(y, model, pars)
```
"""
function ll(y::Array{T, 1} where T<:Integer,
    model::INGARCHModel,
    θ::parameter;
    initiate = "first")::Tuple{Float64, Vector{Float64}}

    T = length(y)
    p = length(model.pastObs)
    q = length(model.pastMean)
    if p == 0
        P = 0
    else
        P = maximum(model.pastObs)
    end
    if q == 0
        Q = 0
    else
        Q = maximum(model.pastMean)
    end

    nb = model.distr == "NegativeBinomial"
    zi = model.zi

    M = convert(Int64, maximum([P, Q]))

    if !parametercheck(θ, model)
        return -Inf, fill(-Inf, T)
    end

    ϕ = θ.ϕ
    ω = θ.ω

    indrel = (M+1):T

    λ = LinPred(y, model, θ, initiate)
    lls = fill(0.0, T)

    if any(λ .< 0)
        return (-Inf, fill(-Inf, T))
    end

    if !nb
        d = Poisson.(λ[indrel])
        if ω > 0
            lls[indrel] = log.(pdf.(d, y[indrel]).*(1 - ω) .+ (y[indrel] .== 0).*ω)
        else
            lls[indrel] = logpdf.(d, y[indrel])
        end
        LL = sum(lls)
    else
        ptemp = @. ϕ[1]/(ϕ[1] + λ[indrel])
        d = NegativeBinomial.(ϕ[1], ptemp)
        if ω > 0
            lls[indrel] = log.(pdf.(d, y[indrel]).*(1 - ω) .+ (y[indrel].== 0).*ω)
        else
            lls[indrel] = logpdf.(d, y[indrel])
        end

        LL = sum(lls)
    end

    return LL, lls
end

function ll(y::Array{T, 1} where T<:Integer,
    model::INARCHModel,
    θ::parameter;
    initiate = "first")::Tuple{Float64, Vector{Float64}}

    T = length(y)
    X = model.X
    r = length(model.external)

    p = length(model.pastObs)
    if p == 0
        P = 0
    else
        P = maximum(model.pastObs)
    end

    M = P

    logl = (model.link == "Log")
    lin = !logl

    nb = model.distr == "NegativeBinomial"
    zi = model.zi

    M = P
    nObs = T - M
    nPar = 1 + p + r + nb + zi

    if !parametercheck(θ, model)
        return -Inf, fill(-Inf, T)
    end

    β0 = θ.β0
    αRaw = θ.α
    η = θ.η
    ϕ = θ.ϕ
    ω = θ.ω

    indrel = (M+1):T

    λ = LinPred(y, model, θ, initiate)
    if any(λ .< 0)
        return (-Inf, fill(-Inf, T))
    end

    lls = zeros(T)
    if !nb
        d = Poisson.(λ[indrel])
        if length(ω) == 1
            lls[indrel] = log.(pdf.(d, y[indrel]).*(1 - ω) .+ (y[indrel].== 0).*ω)
        elseif length(ω) == 0
            lls[indrel] = logpdf.(d, y[indrel])
        end
        LL = sum(lls)
    else
        ptemp = ϕ[1]./(ϕ[1] .+ λ[indrel])
        d = NegativeBinomial.(ϕ[1], ptemp)
        if length(ω) == 1
            lls[indrel] = log.(pdf.(d, y[indrel]).*(1 - ω) .+ (y[indrel].== 0).*ω)
        elseif length(ω) == 0
            lls[indrel] = logpdf.(d, y[indrel])
        end

        LL = sum(lls)
    end

    return LL, lls
end

function ll(y::Array{T, 1} where T<:Integer,
    model::IIDModel,
    θ::parameter;
    initiate = "first")::Tuple{Float64, Vector{Float64}}

    T = length(y)
    X = model.X

    r = length(model.external)

    logl = (model.link == "Log")
    lin = !logl

    nb = model.distr == "NegativeBinomial"
    zi = model.zi

    nObs = T
    nPar = 1 + r + nb + zi

    if !parametercheck(θ, model)
        return -Inf, fill(-Inf, T)
    end

    β0 = θ.β0
    η = θ.η
    ϕ = θ.ϕ
    ω = θ.ω

    λ = LinPred(y, model, θ, initiate)
    if any(λ .< 0)
        return (-Inf, fill(-Inf, T))
    end

    if !nb
        d = Poisson.(λ)
        if length(ω) == 1
            lls = log.(pdf.(d, y).*(1 - ω) .+ (y.== 0).*ω)
        elseif length(ω) == 0
            lls = logpdf.(d, y)
        end
        LL = sum(lls)
    else
        ptemp = ϕ[1]./(ϕ[1] .+ λ)
        d = NegativeBinomial.(ϕ[1], ptemp)
        if length(ω) == 1
            lls = log.(pdf.(d, y).*(1 - ω) .+ (y.== 0).*ω)
        elseif length(ω) == 0
            lls = logpdf.(d, y)
        end

        LL = sum(lls)
    end

    return LL, lls
end

function ll(y::Array{T, 1} where T<:Integer,
    model::INMAModel,
    θ::parameter)::Tuple{Float64, Vector{Float64}}

    T = length(y)
    ymax = maximum(y)
    X = model.X
    r = length(model.external)

    logl1 = model.link[1] == "Log"
    lin1 = !logl1

    logl2 = model.link[2] == "Log"
    lin2 = !logl2

    nb1 = model.distr[1] == "NegativeBinomial"
    nb2 = model.distr[2] == "NegativeBinomial"
    if r == 0
        nb2 = false
    else
        if sum(model.external) == 0
            nb2 = false
        end
    end

    zi = model.zi

    q = length(model.pastMean)
    if q == 0
        Q = 0
    else
        Q = maximum(model.pastMean)
    end
    M = Q

    if !parametercheck(θ, model)
        return -Inf, fill(-Inf, T)
    end

    β0 = θ.β0
    βRaw = θ.β
    η = θ.η
    ϕ = θ.ϕ
    ω = θ.ω

    if q > 0
        if q < Q
            β = zeros(Q)
            β[model.pastMean] = βRaw
        else
            β = βRaw
        end
    end

    rE = sum(model.external)
    rI = sum(.!model.external)

    if r > 0
        iE = findall(model.external)
        iI = findall(.!model.external)

        ηI = η[iI]
        ηE = η[iE]
    end

    μ = zeros(T)
    λ = fill(β0, T)

    if rI > 0
        for i = 1:rI
            λ .+= η[iI[i]].*X[iI[i], :]
        end
    end

    if logl1
        λ = exp.(λ)
    end

    if rE > 0
        for i = 1:rE
            μ .+= η[iE[i]].*X[iE[i], :]
        end
        if logl2
            μ = exp.(μ)
        end
    end

    PR = zeros(ymax + 1, T)
    PZ = zeros(ymax + 1, T)
    for t = 1:T
        if nb1
            PR[:, t] = pdf.(NegativeBinomial(ϕ[1], ϕ[1]/(ϕ[1] + λ[t])), 0:ymax)
        else
            PR[:, t] = pdf.(Poisson(λ[t]), 0:ymax)
        end

        if zi
            PR[2:ymax + 1, t] = PR[2:ymax + 1, t].*(1 .- ω./(1 .- PR[1, t]))
            PR[1, t] += ω
        end

        if rE > 0
            if nb2
                PZ[:, t] = pdf.(NegativeBinomial(ϕ[2], ϕ[2]/(ϕ[2] + μ[t])), 0:ymax)
            else
                PZ[:, t] = pdf.(Poisson(μ[t]), 0:ymax)
            end
        else
            PZ[:, t] = [1; fill(0, ymax)]
        end
    end

    lls = zeros(T)

    if q == 0
        for t = M+1:T
            lls[t] = log(dot(PR[1:y[t]+1], PZ[y[t]+1:-1:1]))
        end
        LL = sum(lls[M+1:end])

        return LL, lls
    end

    # Compute P(Σ α_i∘Y_{t-i} + Z_t = k) first --> pARZ
    PARZ = PZ

    if Q == 1
        QQ = zeros(ymax + 1, ymax + 1, T)
        a = zeros(ymax + 1, T)
        ϕmat = zeros(ymax + 1, T)
        u = zeros(ymax + 1, T)
        lls = zeros(T)
        B = zeros(ymax + 1, ymax + 1)

        for k = 0:ymax
            B[:, k + 1] = pdf.(Binomial(k, β[1]), 0:ymax)
        end

        a[1:y[M] + 1, M] = PR[1:y[M] + 1, M]./sum(PR[1:y[M] + 1, M])
        lls[M] = sum(a[:, M])
        ϕmat[:, M] = a[:, M]./lls[M]
        for t = M+1:T
            for k = 0:y[t]
                temp = PARZ[1:y[t] - k + 1, t]
                for l = 0:y[t-1]
                    QQ[k + 1, l+1, t] = dot(temp, B[y[t] - k + 1:-1:1, l+1])
                end
            end

            u[:, t] = diagm(PR[:, t])*QQ[:, :, t]*ϕmat[:, t-1]
            lls[t] = sum(u[:, t])
            ϕmat[:, t] = u[:, t]./lls[t]
        end

        lls[M:end] = log.(lls[M:end])
        LL = sum(lls[M+1:end])

        return LL, lls
    end

    if Q == 2
        b = zeros(T, ymax + 1, ymax + 1, ymax + 1)
        w = zeros(T)

        B1 = zeros(ymax + 1, ymax + 1)
        B2 = zeros(ymax + 1, ymax + 1)

        for i = 0:ymax
            B1[:, i + 1] = pdf.(Binomial(i, β[1]), 0:ymax)
            B2[:, i + 1] = pdf.(Binomial(i, β[2]), 0:ymax)
        end
        B = zeros(ymax + 1, ymax + 1, ymax + 1)
        for i = 0:ymax, j = 0:ymax
            B[i + 1, j + 1, :] = convolution(B1[i + 1, :], B2[j + 1, :])
        end

        for r1 = 0:y[M]
            temp1 = convolution(B1[:, r1 + 1], PARZ[:, M+1])
            trunc1 = PR[r1 + 1, M]/sum(PR[1:y[M]+1, M])
            for r2 = 0:y[M-1]
                temp = convolution(temp1, B2[:, r2 + 1])
                trunc2 = PR[r2 + 1, M - 1]/sum(PR[1:y[M-1]+1, M - 1])
                b[M+1, 1:y[M+1]+1, r1 + 1, r2 + 1] = PR[1:y[M+1]+1, M+1] .* trunc1 .* trunc2 .*temp[y[M+1]+1:-1:1]
            end
        end

        w[M+1] = sum(b[M+1, :, :, :])
        b[M+1, :, :, :] = b[M+1, :, :, :]./w[M+1]

        for t = M+2:T
            @inbounds for r1 = 0:ymax
                temp1 = convolution(B1[:, r1 + 1], PARZ[:, t])

                for r2 = 0:ymax
                    temp = convolution(temp1, B2[:, r2 + 1], y[t])
                    b[t, 1:y[t] + 1, r1 + 1, r2 + 1] = PR[1:y[t]+1, t].*sum(b[t-1, r1 + 1, r2 + 1, 1:y[t-3] + 1]).*temp[y[t]+1:-1:1]
                end
            end
            w[t] = sum(b[t, :, :, :])
            b[t, :, :, :] = b[t, :, :, :]./w[t]
        end

        lls = [fill(0, M); log.(w[M+1:end])]
        LL = sum(lls)
        bFinal = b[T, :, :, :]
    end

    if Q >= 3
        error("Exact likelihood evaluation for INARMA(p, q) with q ≥ 3 not supported.")
    end

    return LL, lls
end

function ll(y::Array{T, 1} where T<:Integer,
    model::T where T<:INGARCH,
    θ::Array{T, 1} where T<: AbstractFloat;
    initiate = "first")::Tuple{Float64, Vector{Float64}}

    θNew = θ2par(θ, model)
    return ll(y, model, θNew, initiate = initiate)
end

function ll(y::Array{T, 1} where T<:Integer,
    model::T where T<:INARMA,
    θ::Array{T, 1} where T<: AbstractFloat)::Tuple{Float64, Vector{Float64}}

    θNew = θ2par(θ, model)
    return ll(y, model, θNew)
end

####################################
### Ab hier wird alles schneller ###
####################################

function Pmat(λ::Vector{Float64},
    μ::Vector{Float64},
    ymax::Int64,
    T::Int64,
    nb1::Bool,
    nb2::Bool,
    ϕ::Vector{Float64},
    zi::Bool,
    ω::Float64,
    rE::Int64)::Tuple{Matrix{Float64}, Matrix{Float64}}

    PR = zeros(ymax + 1, T)
    PZ = zeros(ymax + 1, T)

    if nb1 & !zi
        for t = 1:T
            PR[:, t] = pdf.(NegativeBinomial(ϕ[1], ϕ[1]/(ϕ[1] + λ[t])), 0:ymax)
        end
    end
    if !nb1 & !zi
        for t = 1:T
            PR[:, t] = pdf.(Poisson(λ[t]), 0:ymax)
        end
    end

    if nb1 & zi
        PRtemp = zeros(T)
        for t = 1:T
            PRtemp = pdf.(NegativeBinomial(ϕ[1], ϕ[1]/(ϕ[1] + λ[t])), 0:ymax)
            if ω + PRtemp[1] < 1
                PR[2:ymax + 1, t] = PRtemp[2:ymax + 1].*(1 .- ω./(1 .- PRtemp[1]))
                PR[1, t] = PRtemp[1] + ω
            else
                PR[2:ymax + 1, t] .= 0.0
                PR[1, t] = 1.0
            end

        end
    end

    if !nb1 & zi
        PRtemp = zeros(T)
        for t = 1:T
            PRtemp = pdf.(Poisson(λ[t]), 0:ymax)
            if ω + PRtemp[1] < 1
                PR[2:ymax + 1, t] = PRtemp[2:ymax + 1].*(1 .- ω./(1 .- PRtemp[1]))
                PR[1, t] = PRtemp[1] + ω
            else
                PR[2:ymax + 1, t] .= 0.0
                PR[1, t] = 1.0
            end
        end
    end

    if nb2 & (rE > 0)
        for t = 1:T
            PZ[:, t] = pdf.(NegativeBinomial(ϕ[2], ϕ[2]/(ϕ[2] + μ[t])), 0:ymax)
        end
    end

    if !nb2 & (rE > 0)
        for t = 1:T
            PZ[:, t] = pdf.(Poisson(μ[t]), 0:ymax)
        end
    end

    if rE == 0
        PZ[1, :] .= 1.0
    end

    return PR, PZ
end

function Q1funInner(y::Vector{Int64},
                    t::Int64,
                    ymax::Int64,
                    PARZt::Vector{Float64},
                    PRt::Vector{Float64},
                    B::Matrix{Float64},
                    ϕmatOld::Vector{Float64})::Vector{Float64}
    #
    QQ = zeros(ymax + 1, ymax + 1)
    @inbounds for k = 0:y[t]
        temp = PARZt[y[t] - k + 1:-1:1]
        QQ[k+1, 1:y[t-1]+1] = temp'B[1:y[t] - k + 1, 1:y[t-1]+1]
    end

    # return diagm(PRt)*QQ*ϕmatOld # u
    return PRt.*(QQ*ϕmatOld) # u
end

function Q1fun(y::Vector{Int64},
               ymax::Int64,
               T::Int64,
               β::Vector{Float64},
               PR::Matrix{Float64},
               M::Int64,
               PARZ::Matrix{Float64})::Tuple{Float64, Vector{Float64}}
    a = zeros(ymax + 1, T)
    lls = zeros(T)
    B = zeros(ymax + 1, ymax + 1)

    for k = 0:ymax
        B[:, k + 1] = pdf.(Binomial(k, β[1]), 0:ymax)
    end

    a[1:y[M] + 1, M] = PR[1:y[M] + 1, M]./sum(PR[1:y[M] + 1, M])
    lls[M] = sum(a[:, M])
    ϕmatOld = a[:, M]./lls[M]

    @inbounds for t = M+1:T
        u = Q1funInner(y, t, ymax, PARZ[:, t], PR[:, t], B, ϕmatOld)
        lls[t] = sum(u)
        ϕmatOld = u./lls[t]
    end

    lls[M:end] = log.(lls[M:end])
    LL = sum(lls[M+1:end])

    return LL, lls
end

function createB(ymax::Int64, β::Vector{Float64})::Tuple{Matrix{Float64}, Matrix{Float64}}
    B1 = zeros(ymax + 1, ymax + 1)
    B2 = zeros(ymax + 1, ymax + 1)

    for i = 0:ymax
        B1[:, i + 1] = pdf.(Binomial(i, β[1]), 0:ymax)
        B2[:, i + 1] = pdf.(Binomial(i, β[2]), 0:ymax)
    end
    B1, B2
end

function Q2funDeeper(t::Int64,
    y::Vector{Int64},
    ymax::Int64,
    PARZt::Vector{Float64},
    PRt::Vector{Float64},
    B1::Matrix{Float64},
    B2::Matrix{Float64},
    bOldSum::Matrix{Float64})::Tuple{Float64, Array{Float64, 2}}

    bNew = zeros(y[t-2] + 1, ymax + 1, y[t] + 1)

    bOldSumRel = bOldSum[1:y[t-2] + 1, :]
    B2rel = B2[:, 1:y[t-2]+1]
    @simd for r1 = 0:y[t-1]
        # Convolution of β_1 ∘ R_{t-1}, AR part and regressors Z_t
        temp1 = convolution(B1[:, r1 + 1], PARZt)
        tempMat = convolutionRevScale(temp1, B2rel, y[t], PRt)
        bNew[:, r1 + 1, :] = bOldSumRel[:, r1 + 1].*tempMat
    end

    wt = sum(bNew)

    return wt, sum(bNew ./ wt, dims = 1)[1, :, :]
end

function Q2funInner(y::Vector{Int64},
                    ymax::Int64,
                    M::Int64,
                    T::Int64,
                    B1::Matrix{Float64},
                    B2::Matrix{Float64},
                    PARZ::Matrix{Float64},
                    PR::Matrix{Float64})::Vector{Float64}
    bOld = zeros(ymax + 1, ymax + 1, ymax + 1)
    w = zeros(T)

    for r1 = 0:y[M]
        temp1 = convolution(B1[:, r1 + 1], PARZ[:, M+1])
        trunc1 = PR[r1 + 1, M]/sum(PR[1:y[M]+1, M])
        for r2 = 0:y[M-1]
            temp = convolution(temp1, B2[:, r2 + 1])
            trunc2 = PR[r2 + 1, M - 1]/sum(PR[1:y[M-1]+1, M - 1])
            bOld[r2 + 1, r1 + 1, 1:y[M+1]+1] = PR[1:y[M+1]+1, M+1] .* trunc1 .* trunc2 .*temp[y[M+1]+1:-1:1]
        end
    end

    w[M+1] = sum(bOld)
    bOld = bOld./w[M+1]
    bOldSum = sum(bOld[1:y[M-1] + 1, :, :], dims = 1)[1, :, :]

    @inbounds for t = M+2:T
        PARZt = PARZ[:, t]
        PRt = PR[:, t]
        w[t], bOldSum = Q2funDeeper(t, y, ymax, PARZt, PRt[1:y[t] + 1], B1, B2, bOldSum)
    end

    return w
end

function Q2fun(y::Vector{Int64},
               ymax::Int64,
               T::Int64,
               β::Vector{Float64},
               PARZ::Matrix{Float64},
               PR::Matrix{Float64},
               M::Int64)::Tuple{Float64, Vector{Float64}}

    B1, B2 = createB(ymax, β)

    w = Q2funInner(y, ymax, M, T, B1, B2, PARZ, PR)

    lls = [fill(0, M); log.(w[M+1:end])]
    LL = sum(lls)
    return LL, lls
end

function ll(y::Array{T, 1} where T<:Integer,
    model::INARMAModel,
    θ::parameter)::Tuple{Float64, Vector{Float64}}

    T = length(y)
    ymax = maximum(y)
    X = model.X
    r = Int64(length(model.external))

    logl1 = model.link[1] == "Log"
    lin1 = !logl1

    logl2 = model.link[2] == "Log"
    lin2 = !logl2

    nb1 = model.distr[1] == "NegativeBinomial"
    nb2 = model.distr[2] == "NegativeBinomial"
    if r == 0
        nb2 = false
    else
        if sum(model.external) == 0
            nb2 = false
        end
    end

    zi = model.zi

    p = length(model.pastObs)
    if p == 0
        P = 0
    else
        P = Int64(maximum(model.pastObs))
    end

    q = Int64(length(model.pastMean))
    if q == 0
        Q = 0
    else
        Q = Int64(maximum(model.pastMean))
    end
    M = maximum([P, Q])

    if !parametercheck(θ, model)
        return -Inf, fill(-Inf, T)
    end

    β0 = Float64(θ.β0)
    αRaw = Vector{Float64}(θ.α)
    βRaw = Vector{Float64}(θ.β)
    η = Vector{Float64}(θ.η)
    ϕ = Vector{Float64}(θ.ϕ)
    ω = Float64(θ.ω)

    if p > 0
        if p < P
            α = zeros(P)
            α[model.pastObs] = αRaw
        else
            α = αRaw
        end
    end

    if q > 0
        if q < Q
            β = zeros(Q)
            β[model.pastMean] = βRaw
        else
            β = βRaw
        end
    end

    rE = sum(model.external)
    rI = r - rE

    if r > 0
        iE = findall(model.external)
        iI = findall(.!model.external)

        ηI = η[iI]
        ηE = η[iE]

        λ, μ = createλμ(T, β0, rI, iI, η, X, rE, iE, logl1, logl2)
    else
        λ, μ = createλμ(T, β0, logl1)
    end

    PR, PZ = Pmat(λ, μ, ymax, T, nb1, nb2, ϕ, zi, ω, rE)

    lls = zeros(T)

    # Below only for INAR Process, rather irrelevant
    if q == 0
        PY = zeros(p, ymax + 1, ymax + 1)
        for i = 1:p
            for j = sort(unique(y))
                PY[i, j + 1, 1:j + 1] = pdf.(Binomial.(j, α[i]), 0:j)
            end
        end

        PAR = zeros(ymax + 1, P + 1)

        for t = M+1:T
            for i = 1:P
                PAR[:, i] = PY[i, y[t-i] + 1, :]
            end
            PAR[:, P + 1] = PZ[:, t]

            ptemp = GetProbFromP(PAR)
            lls[t] = log(dot(PR[1:y[t]+1], ptemp[y[t]+1:-1:1]))
        end
        LL = sum(lls[M+1:end])

        return LL, lls
    end

    # Compute P(Σ α_i∘Y_{t-i} + Z_t = k) first --> pARZ
    PAR = zeros(ymax + 1, P + 1)
    PARZ = zeros(ymax + 1, T)
    for t = M+1:T
        for i = 1:P
            PAR[:, i] = pdf.(Binomial(y[t - i], α[i]), 0:ymax)
        end
        PAR[:, end] = PZ[:, t]
        PARZ[:, t] = GetProbFromP(PAR)
    end

    if Q == 1
        return Q1fun(y, ymax, T, β, PR, M, PARZ)
    end

    if Q == 2
        return Q2fun(y, ymax, T, β, PARZ, PR, M)
    end

    if Q >= 3
        error("Exact likelihood evaluation for INARMA(p, q) with q ≥ 3 not supported.")
    end

    return LL, lls
end

function CoreINAR(y::Vector{Int64},
                  ymax::Int64,
                  P::Int64,
                  M::Int64,
                  T::Int64,
                  PY::Array{Float64, 3},
                  PZ::Matrix{Float64},
                  PR::Matrix{Float64})::Vector{Float64}
    #
    lls = zeros(T)
    PAR = zeros(ymax + 1, P + 1)
    @inbounds for t = M+1:T
        out = 0.0
        for i = 1:P
            PAR[:, i] = PY[i, y[t-i] + 1, :]
        end
        PAR[:, P + 1] = PZ[:, t]

        ptemp = GetProbFromP(PAR)
        for i = 0:y[t]
            out += PR[i+1]*ptemp[y[t] - i + 1]
        end
        lls[t] = log(out)
    end
    lls
end

function CreatePY(y::Vector{Int64},
    ymax::Int64,
    p::Int64,
    α::Vector{Float64})::Array{Float64, 3}

    PY = zeros(p, ymax + 1, ymax + 1)
    allVals = sort(unique(y))
    @inbounds for i = 1:p
        for j = allVals
            PY[i, j + 1, 1:j + 1] = pdf.(Binomial.(j, α[i]), 0:j)
        end
    end
    PY
end

function createλμ(T::Int64,
    β0::Float64,
    rI::Int64,
    iI::Vector{Int64},
    η::Vector{Float64},
    X::Matrix{Float64},
    rE::Int64,
    iE::Vector{Int64},
    logl1::Bool,
    logl2::Bool)::Tuple{Vector{Float64}, Vector{Float64}}

    μ = zeros(T)
    λ = fill(β0, T)

    if rI > 0
        λ .+= (η[iI]'X[iI, :])[1, :]
    end

    if logl1
        λ = exp.(λ)
    end

    if rE > 0
        μ .+= (η[iE]'X[iE, :])[1, :]

        if logl2
            μ = exp.(μ)
        end
    end
    λ, μ
end

function createλμ(T::Int64,
    β0::Float64,
    logl1::Bool)::Tuple{Vector{Float64}, Vector{Float64}}

    if logl1
        λ = fill(exp(β0), T)
    else
        λ = fill(β0, T)
    end

    λ, zeros(T)
end

function ll(y::Array{T, 1} where T<:Integer,
    model::INARModel,
    θ::parameter)::Tuple{Float64, Vector{Float64}}

    T = length(y)
    ymax = maximum(y)
    X = model.X

    r = length(model.external)

    logl1 = model.link[1] == "Log"
    lin1 = !logl1

    logl2 = model.link[2] == "Log"
    lin2 = !logl2

    nb1 = model.distr[1] == "NegativeBinomial"
    nb2 = model.distr[2] == "NegativeBinomial"
    if r == 0
        nb2 = false
    else
        if sum(model.external) == 0
            nb2 = false
        end
    end

    zi = model.zi

    p = length(model.pastObs)
    if p == 0
        P = 0
    else
        P = maximum(model.pastObs)
    end

    M = P

    if !parametercheck(θ, model)
        return -Inf, fill(-Inf, T)
    end

    β0 = θ.β0
    αRaw = θ.α
    η = θ.η
    ϕ = θ.ϕ
    ω = θ.ω

    if p > 0
        if p < P
            α = zeros(P)
            α[model.pastObs] = αRaw
        else
            α = αRaw
        end
    end

    rE = sum(model.external)
    rI = sum(.!model.external)

    if r > 0
        iE = findall(model.external)
        iI = findall(.!model.external)

        ηI = η[iI]
        ηE = η[iE]

        λ, μ = createλμ(T, β0, rI, iI, η, X, rE, iE, logl1, logl2)
    else
        λ, μ = createλμ(T, β0, logl1)
    end
    PR, PZ = Pmat(λ, μ, ymax, T, nb1, nb2, ϕ, zi, ω, rE)
    PY = CreatePY(y, ymax, p, α)
    lls = CoreINAR(y, ymax, P, M, T, PY, PZ, PR)

    LL = sum(lls[M+1:end])

    return LL, lls
end


include("../envs/RLEnvironments.jl")
using .RLEnvironments
using Plots

function act(Q, s, ϵ)
    rand() <= ϵ && return rand(1:size(Q, 1))
    a = @view Q[:, s]
    return rand(findall(a .== maximum(a)))
end

function run!(Q, env; epochs=20, ϵ=0.1, α=0.5, γ=0.99)
    rewards = zeros(epochs)
    steps = zeros(Int64, epochs)

    for i in 1:epochs
        done = false
        s1 = reset!(env)
        while !done
            a = act(Q, s1, ϵ)
            s2, r, done = step!(env, a)
            r2 = done ? 0.0 : maximum(view(Q, :, s2))
            Q[a, s1] += α * (r + γ * r2 - Q[a, s1])
            s1 = s2

            rewards[i] += r
            steps[i] += 1
        end
        ϵ = max(ϵ * 0.995, 0.01)
    end
    return rewards, steps
end

env = SimpleRooms{Int64, Int64, Float64}()
Q = zeros(length(env.actionspace), length(env.observationspace))

r, s = run!(Q, env)
plt = plot([r s], labels=["Reward" "Steps"], color=[:red :blue], layout=(2, 1))
display(plt)
#savefig(plt, "qlearning")


# This file is a part of AstroLib.jl. License is MIT "Expat".
# Copyright (C) 2016 Mosè Giordano.

function _geo2mag{T<:AbstractFloat}(lat::T, long::T, pole_lat::T, pole_long::T)
    r       = 1.0 # Distance from planet center.  Value unimportant -- just need
                  # a length for conversion to rectangular coordinates
    lat  = deg2rad(lat)
    long = deg2rad(long)
    alt  = lat*0.0 + r
    x = alt*cos(lat)*cos(long)
    y = alt*cos(lat)*sin(long)
    z = alt*sin(lat)

    # Compute first rotation matrix: rotation around plane of the equator, from
    # the Greenwich meridian to the meridian containing the magnetic dipole
    # pole.
    geolong2maglong = Array(T, 3, 3)
    geolong2maglong[:,1] = [cos(pole_long), -sin(pole_long), 0.0]
    geolong2maglong[:,2] = [sin(pole_long),  cos(pole_long), 0.0]
    geolong2maglong[:,3] = [           0.0,             0.0, 1.0]
    out = geolong2maglong * [x, y, z]

    # Second rotation: in the plane of the current meridian from geographic pole
    # to magnetic dipole pole.
    tomaglat = Array(T, 3, 3)
    tomaglat[:,1] = [ cos(pi/2 - pole_lat), 0.0, sin(pi/2 - pole_lat)]
    tomaglat[:,2] = [                  0.0, 1.0,                  0.0]
    tomaglat[:,3] = [-sin(pi/2 - pole_lat), 0.0, cos(pi/2 - pole_lat)]
    out = tomaglat * out

    maglat  = rad2deg(atan2(out[3], hypot(out[1], out[2])))
    maglong = rad2deg(atan2(out[2], out[1]))
    # I don't care about that one...just put it there for completeness' sake
    # magalt  = vecnorm(out) - r
    return maglat, maglong
end

"""
    geo2mag(latitude, longitude[, year]) -> geomagnetic_latitude, geomagnetic_longitude

### Purpose ###

Convert from geographic to geomagnetic coordinates.

### Explanation ###

Converts from geographic (latitude, longitude) to geomagnetic (latitude,
longitude).  Altitude is not involved in this function.

### Arguments ###

* `latitude`: geographic latitude (North), in degrees.
* `longitude`: geographic longitude (East), in degrees.
* `year` (optional numerical argument): the year in which to perform conversion.
  If omitted, defaults to current year.

The coordinates can be passed as arrays of the same length.

### Output ###

The 2-tuple of magnetic (latitude, longitude) coordinates, in degrees.

If geographical coordinates are given as arrays, a 2-tuple of arrays of the same
length is returned.

### Example ###

Kyoto has geographic coordinates 35° 00' 42'' N, 135° 46' 06'' E, find its
geomagnetic coordinates in 2016:

``` julia
geo2mag(ten(35,0,42), ten(135,46,6), 2016)
# => (36.86579228937769,-60.184060536651614)
```

### Notes ###

This function uses list of North Magnetic Pole positions provided by World
Magnetic Model (https://www.ngdc.noaa.gov/geomag/data/poles/NP.xy).

`mag2geo` converts geomagnetical coordinates to geographic coordinates.

Code of this function is based on IDL Astronomy User's Library.
"""
geo2mag(lat::Real, long::Real, year::Real=Dates.year(Dates.now())) =
    _geo2mag(promote(float(lat), float(long),
                     deg2rad(POLELATLONG[year][1]::AbstractFloat),
                     deg2rad(POLELATLONG[year][2]::AbstractFloat))...)

function geo2mag{LA<:Real, LO<:Real}(lat::AbstractArray{LA},
                                     long::AbstractArray{LO},
                                     year::Real=Dates.year(Dates.now()))
    @assert length(lat) == length(long)
    typela   = typeof(float(one(LA)))
    maglat   = similar(lat, typela)
    maglong  = similar(lat, typela)
    polelat  = deg2rad(POLELATLONG[year][1])
    polelong = deg2rad(POLELATLONG[year][2])
    for i in eachindex(lat)
        maglat[i], maglong[i] =
            _geo2mag(promote(float(lat[i]), float(long[i]),
                             polelat, polelong)...)
    end
    return maglat, maglong
end


using Test, PolyChaos

d, nunc = 5, 10
op = OrthoPoly("genlaguerre",d,Dict(:shape=>1.23))
opq = OrthoPolyQ(op)
mop = MultiOrthoPoly([op for i in 1:nunc],d)
mopq = MultiOrthoPoly([opq for i in 1:nunc],d)
@testset "dimenions" begin
    @test isequal(PolyChaos.dim(op), d + 1)
    @test isequal(PolyChaos.dim(opq), d + 1)
    @test isequal(PolyChaos.dim(mop), factorial(d + nunc) / (factorial(d) * factorial(nunc)))
end

@testset "degrees" begin
    @test isequal(PolyChaos.deg(op),d)
    @test isequal(PolyChaos.deg(opq),d)
    @test isequal(PolyChaos.deg(mop),d)
end

NW = [ opq.quad.nodes opq.quad.weights ]
@testset "nodes and weights" begin
    @test PolyChaos.nw(opq) == NW
    @test PolyChaos.nw([opq for i in 1:nunc])[1] == [NW[:,1] for i in 1:nunc]
    @test PolyChaos.nw([opq for i in 1:nunc])[2] == [NW[:,2] for i in 1:nunc]
    @test PolyChaos.nw(mopq)[1] == [NW[:,1] for i in 1:nunc]
    @test PolyChaos.nw(mopq)[2] == [NW[:,2] for i in 1:nunc]
end

c = [ op.α op.β ]
@testset "coefficients" begin
    @test PolyChaos.coeffs(op) == c
    @test PolyChaos.coeffs(opq) == c
    @test PolyChaos.coeffs([op for i in 1:nunc])[1] == [ c[:,1] for i in 1:nunc ]
    @test PolyChaos.coeffs([op for i in 1:nunc])[2] == [ c[:,2] for i in 1:nunc ]
    @test PolyChaos.coeffs([opq for i in 1:nunc])[1] == [ c[:,1] for i in 1:nunc ]
    @test PolyChaos.coeffs([opq for i in 1:nunc])[2] == [ c[:,2] for i in 1:nunc ]
    @test PolyChaos.coeffs(mop)[1] == [ c[:,1] for i in 1:nunc ]
    @test PolyChaos.coeffs(mop)[2] == [ c[:,2] for i in 1:nunc ]
end

@test issymmetric(op) == issymmetric(opq) == issymmetric(opq.quad)


# PDEs
abstract type AbstractPDEs end

mutable struct VarElliptic <: AbstractPDEs
    # domain is fixed [0,1]*[0,1]
    a::Function
    rhs::Function
    bdy_type::Function # 1 Diri, 2 Neum, 3 Robin
    bdy_Diri::Function
    bdy_Neum::Function
    bdy_Robin::Function
end


"""Perform the dataset `xs` separation by thresholding a dataset `bins`ed histogram based on the algorith `T`"""
function separation(::Type{T}, xs::Vector{S};
                    bins = 20, tol::Real = 1.0e-5) where {T <: Thresholding, S <: Real}
    minX, maxX = extrema(xs)

    # get histogram
    r = range(minX, stop=maxX, length=bins+1)
    H = fit(Histogram, xs, r, closed=:left)
    N = length(H.weights)
    @logmsg DEBUG_SEPARATION "Separation histogram" H

    return try
        thresh = fit(T, H; tol=tol)

        minIdx = minimum(thresh)
        s = stats(thresh)
        threshold = minX + (minIdx)*(maxX - minX)/N
        # println(s[minIdx,:])
        discriminability = (abs(s[minIdx,3]-s[minIdx,4]))/(sqrt(s[minIdx,5]+s[minIdx,6]))

        Separation(convert(S, depth(thresh)), convert(S, discriminability), threshold, minIdx, minX, maxX, bins)
    catch ex
        # rethrow(ex)
        @logmsg TRACE "Separation error" ex
        Separation(zero(S), zero(S), maxX, 0, minX, maxX, bins) # threshold is set to maximal element
    end
end

"""Kittler algorithm thresholding type"""
struct Kittler <: Thresholding
    statistics::Matrix{Float64}
    depth::Float64
    minindex::Int
end
depth(t::Kittler) = t.depth
stats(t::Kittler) = t.statistics
Base.minimum(t::Kittler) = t.minindex

"""Performs Kittler's minimal thresholding algorithm

J. Kittler & J. Illingworth: "Minimum Error Thresholding", Pattern Recognition, Vol 19, nr 1. 1986, pp. 41-47.
"""
function fit(::Type{Kittler}, H::Histogram; tol::Real=1.0e-5)
    N = length(H.weights)

    # calculate stats
    S = histstats(H.weights./sum(H.weights))

    # Compute criterion function
    J = fill(2*log(eps()), N-1)
    for t=1:(N-1)
        @logmsg TRACE "Statistics" P₁=S[t,1] P₂=S[t,2] σ₁=S[t,5] σ₂=S[t,6]
        if S[t,1]!=0 && S[t,2]!=0 && S[t,5]>0 && S[t,6]>0
            σ₁ = sqrt(max(0, S[t,5]))
            logσ₁ = log(σ₁ == 0 ? eps() : σ₁)
            σ₂ = sqrt(max(0, S[t,6]))
            logσ₂ = log(σ₂ == 0 ? eps() : σ₂)
            ses = S[t,1]*logσ₁ + S[t,2]*logσ₂
            se = S[t,1]*log(S[t,1]) + S[t,2]*log(S[t,2])
            J[t] = 1 + 2*ses - 2*se
        end
    end
    @logmsg DEBUG_SEPARATION "Kittler criterion function" J=J' extrema=extrema(J)

    # Global minimum parameters
    depth, global_min = find_global_min(J, tol)

    return Kittler(S, depth, round(Int, global_min))
end

"""Otsu algorith thresholding type"""
struct Otsu <: Thresholding
    statistics::Matrix{Float64}
    depth::Float64
    minindex::Int
end
depth(t::Otsu) = t.depth
stats(t::Otsu) = t.statistics
Base.minimum(t::Otsu) = t.minindex

"""Performs Otsu thresholding algorithm

N. Otsu: "A threshold selection method from gray-level histograms", Automatica, 1975, 11, 285-296.
"""
function fit(::Type{Otsu}, H::Histogram; tol=1.0e-5)
    N = length(H.weights)

    # calculate stats
    S = histstats(H.weights./sum(H.weights))

    # Compute criterion function
    J = zeros(N-1)
    for t in 1:N-1
        σb = S[t,1]*S[t,2]*(S[t,4]-S[t,3])^2
        σw = S[t,1]*S[t,5]^2 - S[t,2]*S[t,6]^2
        J[t] = σb/σw
    end
    @logmsg DEBUG_SEPARATION "Otsu criterion function" J=J'

    # Global minimum parameters
    depth, global_min = find_global_min(J, tol)

    return Otsu(S, depth, round(Int, global_min))
end


using RandomMatrix, LinearAlgebra
using Test

@test randDiagonal(2) !== nothing
@test randTriangular(1:3,3,upper=false,Diag=false) !== nothing
@test !isreal(randMatrix(10)|>eigvals)

@test isreal(randHermitian(3)|>eigvals)
@test randHermitian(ComplexNormal(im,2),3,diag=Elliptic(0.1,c=im,R=9)) !==nothing

@test pdf(MarchenkoPastur(rand()),0) == 0


function get_operator(process::AbstractBlackScholesProcess, grid::Vector{Float64}, residualTime::Float64, ::Bool)
  # if timeDependent
  #   return BSMTermOperator(grid, process, residualTime)
  # else
  return BSMOperator(grid, process, residualTime)
  # end
end

function BSMOperator(grid::Vector{Float64}, process::AbstractBlackScholesProcess, residualTime::Float64)
  tridiag = TridiagonalOperator(length(grid))
  logGrid = LogGrid(grid)
  cc = PdeConstantCoeff(process, residualTime, state_variable(process).value)
  generate_operator!(cc, residualTime, logGrid, tridiag)

  return tridiag
end


α = U1(rand() * 2pi)
β = U1(rand() * 2pi)
a = SU2(α, β)
b = SU2(U1(rand() * 2pi), U1(rand() * 2pi))
M = Matrix(a)
N = [α -conj(β); β conj(α)]

@test a == SU2(M)
@test size(Matrix(a)) == (2, 2)
@test a + b isa Array{Complex,2}
@test a + M isa Array{Complex,2}
@test M + a isa Array{Complex,2}
@test a + N isa Array{Complex,2}
@test N + a isa Array{Complex,2}
@test a * b isa SU2
@test a * M isa SU2
@test M * a isa SU2
@test a * N isa Array{Complex,2}
@test N * a isa Array{Complex,2}
@test a .* α isa SU2
@test α .* a isa SU2


import ForwardDiff
using ForwardDiff: derivative
using Zygote: gradient

function ∇_x(f)
  function ∇_xf(x, y)
    return first(gradient(x -> f(x,y), x))
  end
  return ∇_xf
end

function ∇_y(f)
  function ∇_yf(x, y)
    return first(gradient(y -> f(x,y), y))
  end
  return ∇_yf
end

function D_yx(f)
  ∇_yf = ∇_y(f)
  function D_yxf(x, y, v)
    # return derivative(h ->  ∇_yf(x + h .* v, y), zero(eltype(v)))
    return ForwardDiff.partials.(∇_yf(ForwardDiff.Dual{Nothing}.(x, v), ForwardDiff.Dual{Nothing}.(y)), 1)
  end
  return D_yxf
end

function D_xy(f)
  ∇_xf = ∇_x(f)
  function D_xyf(x, y, v)
    # return derivative(h ->  ∇_xf(x, y + h .* v), zero(eltype(v)))
    return ForwardDiff.partials.(∇_xf(ForwardDiff.Dual{Nothing}.(x), ForwardDiff.Dual{Nothing}.(y, v)), 1)
  end
  return D_xyf
end


let
    order = 2

    # Haverly1 standard pooling instance
    x14, x24, x35, x36, x45, x46, w4 = variables(["x_{14}", "x_{24}", "x_{35}", "x_{36}", "x_{45}", "x_{46}", "w_4"])

    c14, c24, c35, c36, c45, c46 = [6, 16, 1, -9, -5, -15]
    q1, q2, q3, q5, q6 = [ 3, 1, 2, 5//2, 3//2 ]
    ub1, ub2, ub3, ub4, ub5, ub6 = [ 300, 300, 300, 300, 100, 200 ] // 300

    flow_eq    = [ x14+x24-(x45+x46) ]
    blend_eq   = [ w4*(x45+x46)-(q1*x14+q2*x24) ]
    qual_bnd   = [ q5*(x45+x36)-(w4*x45+q3*x35), q6*(x46+x36)-(w4*x46+q3*x36) ]
    cap_bnd    = [ ub1-x14, ub2-x24, ub3-(x35+x36), ub4-(x45+x46), ub5-(x35+x45), ub6-(x36+x46)]
    flow_bnd   = [ x14,x24,x35,x36,x45,x46]

    obj = dot([c14, c24, c35, c36, c45, c46], [x14, x24, x35, x36, x45, x46])
    prob = momentprob(order, obj, [qual_bnd, cap_bnd, flow_bnd], [flow_eq,blend_eq])

    y, objval, solsta = solve_mosek(prob)
    @test norm(y[1:8]-[1,0,1/3,0,1/3,0,1/3,1]) < 1e-4
end


using MultipleViewGeometry, Test
using MultipleViewGeometry.ModuleTypes

ℳ = [Point2D(rand(1,2)) for i = 1:10000]

moments(FundamentalMatrix(), (ℳ,ℳ))

@benchmark moments(HomographyMatrix(), (ℳ,ℳ))

# TODO

FundamentalMatrix()
HomographyMatrix()


#=
# 3D Poisson, Dirichlet bc
# Uses aMat target
=#

### If the Finch package has already been added, use this line #########
using Finch # Note: to add the package, first do: ]add "https://github.com/paralab/Finch.git"

### If not, use these four lines (working from the examples directory) ###
# if !@isdefined(Finch)
#     include("../Finch.jl");
#     using .Finch
# end
##########################################################################

init_finch("p3damat");

useLog("p3damatlog", level=3)

generateFor("../targets/target_amat_cg.jl")

domain(3)
functionSpace(order=2)

mesh(HEXMESH, elsperdim=4)

u = variable("u")
testSymbol("v")

boundary(u, 1, DIRICHLET, "0")

# Write the weak form
coefficient("f", "-29*pi*pi*sin(2*pi*x)*sin(3*pi*y)*sin(4*pi*z)")
weakForm(u, "-dot(grad(u),grad(v)) - f*v")

solve(u);

finalize_finch()


using Test
using Elemental
using LinearAlgebra

m, n = 10, 10

for T in (Float32, Float64, Complex{Float32}, Complex{Float64})
    for mat in (Elemental.Matrix, Elemental.DistMatrix)
        A = mat(T)
        Elemental.gaussian!(A, m, n)

        @test opnorm(A, 1)   ≈ opnorm(T[A[i,j] for i = 1:m, j = 1:n], 1)
        @test opnorm(A)      ≈ opnorm(T[A[i,j] for i = 1:m, j = 1:n])
        @test opnorm(A, Inf) ≈ opnorm(T[A[i,j] for i = 1:m, j = 1:n], Inf)
    end
end


using Compat, DataFrames, StatsBase
using Compat.Test
using GLM
using Alistair

# PERFORMANCE COMPARISON OPTIONS
const TEST_SIZE_1 = 100000

Profile.clear()

beta = [0.7; 0.45]
X = hcat(ones(TEST_SIZE_1,1), collect(1.0:TEST_SIZE_1))
Y = X * beta + 0.1*randn(TEST_SIZE_1, 1)

@time data = DataFrame(X1=X[:,2], Y1=Y[:])
@time OLS_result = glm(@formula(Y1 ~ X1), data, Normal(), IdentityLink())

@time OLSRegression = OLS(intercept=true, robust=false)
@time result = linregress(X, Y, OLSRegression)


## References
# - [1] Zhan, Zhang, and Chung. Adaptive particle swarm optimization, IEEE Transactions on Systems, Man, and Cybernetics, Part B: CyberneticsVolume 39, Issue 6 (2009): 1362-1381
struct ParticleSwarm{Tn,T}
    n_particles::Tn
    limit_search_space::Bool
    elitist_learning::Bool
    c₁::T
    c₂::T
    σmin::T
    σmax::T
end
# defaults for c's are from the flowchart of page 1370, σ's are from the bottom of 1370
ParticleSwarm(;
    n_particles = nothing,
    limit_search_space = false,
    elitist_learning = true,
    c₁ = 2.0,
    c₂ = 2.0,
    σmin = 0.1,
    σmax = 1.0,
) = ParticleSwarm(n_particles, limit_search_space, elitist_learning, c₁, c₂, σmin, σmax)

summary(::ParticleSwarm) = "Adaptive Particle Swarm"
function solve(
    problem::OptimizationProblem,
    x0,
    method::ParticleSwarm,
    options::OptimizationOptions,
)
    if !(mstyle(problem) === InPlace())
        throw(
            ErrorException("solve() not defined for OutOfPlace() with $(summary(method))"),
        )
    end
    t0 = time()
    n_particles = method.n_particles isa Nothing ? max(length(x0), 5) : method.n_particles
    lower, upper = lowerbounds(problem), upperbounds(problem)

    T, n = eltype(x0), length(x0)

    c₁, c₂, σmin, σmax = T(method.c₁), T(method.c₂), T(method.σmin), T(method.σmax)
    ω = T(9) / 10 # see page 1368

    X, V = [copy(x0) for i = 1:n_particles], [x0 * T(0) for i = 1:n_particles]
    X_best = [x0 * T(0) for i = 1:n_particles]

    Fs, Fs_best = zeros(T, n_particles), zeros(T, n_particles)

    x = copy(x0)
    if method.elitist_learning
        x_learn = copy(x0)
    end
    current_state = 0

    # spread the initial population uniformly over the whole search space
    width = upper .- lower
    for i = 1:n_particles
        X[i] .= lower .+ width .* rand(T)
        X_best[i] .= X[i]
    end

    best_f = T(0)
    swarm_f = best_f
    f0 = swarm_f
    X_best[1] .= x0
    X[1] .= x0
    iter = 0
    while iter <= options.maxiter
        iter += 1
        limit_X!(X, lower, upper)
        Fs = batched_value(problem, Fs, X)

        if iter == 1
            copyto!(Fs_best, Fs)
            best_f = Base.minimum(Fs)
        end
        best_f = housekeeping!(Fs, Fs_best, X, X_best, x, best_f)
        if iter == 1
            f0 = best_f
        end
        if method.elitist_learning # try to avoid non-global minima
            x_learn .= x
            # Perturb a random dimension and replace the current worst
            # solution in X with x_learn if x_learn presents the new
            # best solution. Else, discard x_learn.
            worst_Fs, i_worst = findmax(Fs)
            random_index = rand(1:n)

            # Eqn (14) - See Section VI-C for a discussion of max and min
            # Could allow for an annealing scheme. Maybe a SAMIN style?
            σlearn = σmax - (σmax - σmin) * iter / options.maxiter

            x_learn[random_index] =
                x_learn[random_index] + width[random_index] * σlearn * randn()
            x_learn[random_index] =
                max(lower[random_index], min(upper[random_index], x_learn[random_index]))

            Fs_learn = value(problem, x_learn)
            if Fs_learn < best_f
                X_best[i_worst] .= x_learn
                X[i_worst] .= x_learn
                x .= x_learn

                Fs_best[i_worst] = Fs_learn
                Fs[i_worst] = Fs_learn
                best_f = Fs_learn
            end
        end
        # TODO find a better name for _f (look inthe paper, it might be called f there)
        current_state, swarm_f = get_swarm_state(X, Fs, x, current_state)
        ω, c₁, c₂ = update_swarm_params!(c₁, c₂, ω, current_state, swarm_f)
        update_swarm!(X, X_best, x, n, V, ω, c₁, c₂)
    end
    best_f, x
    ConvergenceInfo(
        method,
        (
            swarm_f = swarm_f,
            X = X,
            Fs = Fs,
            solution = x,
            minimum = best_f,
            f0 = f0,
            iter = iter,
            time = time() - t0,
        ),
        options,
    )
end

function update_swarm!(X, X_best, best_point, n, V, ω, c₁, c₂)

    Tx = eltype(first(X))
    # compute new positions for the swarm particles
    # FIXME vmax should be 20%
    for i in eachindex(X, X_best)
        for j = 1:n
            r1 = rand(Tx)
            r2 = rand(Tx)
            vx = X_best[i][j] - X[i][j]
            vg = best_point[j] - X[i][j]
            V[i][j] = V[i][j] * ω + c₁ * r1 * vx + c₂ * r2 * vg
            X[i][j] = X[i][j] + V[i][j]
        end
    end
end

function get_mu_1(f::Tx) where {Tx}
    if Tx(0) <= f <= Tx(4) / 10
        return Tx(0)
    elseif Tx(4) / 10 < f <= Tx(6) / 10
        return Tx(5) * f - Tx(2)
    elseif Tx(6) / 10 < f <= Tx(7) / 10
        return Tx(1)
    elseif Tx(7) / 10 < f <= Tx(8) / 10
        return -Tx(10) * f + Tx(8)
    else
        return Tx(0)
    end
end

function get_mu_2(f::Tx) where {Tx}
    if Tx(0) <= f <= Tx(2) / 10
        return Tx(0)
    elseif Tx(2) / 10 < f <= Tx(3) / 10
        return Tx(10) * f - Tx(2)
    elseif Tx(3) / 10 < f <= Tx(4) / 10
        return Tx(1)
    elseif Tx(4) / 10 < f <= Tx(6) / 10
        return -Tx(5) * f + Tx(3)
    else
        return Tx(0)
    end
end

function get_mu_3(f::Tx) where {Tx}
    if Tx(0) <= f <= Tx(1) / 10
        return Tx(1)
    elseif Tx(1) / 10 < f <= Tx(3) / 10
        return -Tx(5) * f + Tx(3) / 2
    else
        return Tx(0)
    end
end

function get_mu_4(f::Tx) where {Tx}
    if Tx(0) <= f <= Tx(7) / 10
        return Tx(0)
    elseif Tx(7) / 10 < f <= Tx(9) / 10
        return Tx(5) * f - Tx(7) / 2
    else
        return Tx(1)
    end
end

function get_swarm_state(X, Fs, best_point, previous_state)
    # swarm can be in 4 different states, depending on which
    # the weighing factors c₁ and c₂ are adapted.
    # New state is not only depending on the current swarm state,
    # but also from the previous
    n_particles = length(X)
    n = length(first(X))
    Tx = eltype(first(X))
    f_best, i_best = findmin(Fs)
    d = zeros(Tx, n_particles)
    for i = 1:n_particles
        dd = Tx(0)
        for k = 1:n_particles
            for dim = 1:n
                @inbounds ddd = (X[i][dim] - X[k][dim])
                dd += ddd * ddd
            end
        end
        d[i] = sqrt(dd)
    end
    dg = d[i_best]
    dmin = Base.minimum(d)
    dmax = Base.maximum(d)

    f = (dg - dmin) / max(dmax - dmin, sqrt(eps(Tx)))

    mu = zeros(Tx, 4)
    mu[1] = get_mu_1(f)
    mu[2] = get_mu_2(f)
    mu[3] = get_mu_3(f)
    mu[4] = get_mu_4(f)
    best_mu, i_best_mu = findmax(mu)
    current_state = 0

    if previous_state == 0
        current_state = i_best_mu
    elseif previous_state == 1
        if mu[1] > 0
            current_state = 1
        else
            if mu[2] > 0
                current_state = 2
            elseif mu[4] > 0
                current_state = 4
            else
                current_state = 3
            end
        end
    elseif previous_state == 2
        if mu[2] > 0
            current_state = 2
        else
            if mu[3] > 0
                current_state = 3
            elseif mu[1] > 0
                current_state = 1
            else
                current_state = 4
            end
        end
    elseif previous_state == 3
        if mu[3] > 0
            current_state = 3
        else
            if mu[4] > 0
                current_state = 4
            elseif mu[2] > 0
                current_state = 2
            else
                current_state = 1
            end
        end
    elseif previous_state == 4
        if mu[4] > 0
            current_state = 4
        else
            if mu[1] > 0
                current_state = 1
            elseif mu[2] > 0
                current_state = 2
            else
                current_state = 3
            end
        end
    end
    return current_state, f
end

function update_swarm_params!(c₁, c₂, ω, current_state, f::T) where {T}

    Δc₁ = T(5) / 100 + rand(T) / T(20)
    Δc₂ = T(5) / 100 + rand(T) / T(20)

    if current_state == 1
        c₁ += Δc₁
        c₂ -= Δc₂
    elseif current_state == 2
        c₁ += Δc₁ / 2
        c₂ -= Δc₂ / 2
    elseif current_state == 3
        c₁ += Δc₁ / 2
        c₂ += Δc₂ / 2
    elseif current_state == 4
        c₁ -= Δc₁
        c₂ -= Δc₂
    end

    if c₁ < T(3) / 2
        c₁ = T(3) / 2
    elseif c₁ > T(5) / 2
        c₁ = T(5) / 2
    end

    if c₂ < T(3) / 2
        c₂ = T(5) / 2
    elseif c₂ > T(5) / 2
        c₂ = T(5) / 2
    end

    if c₁ + c₂ > T(4)
        c_total = c₁ + c₂
        c₁ = c₁ / c_total * 4
        c₂ = c₂ / c_total * 4
    end

    ω = 1 / (1 + T(3) / 2 * exp(-T(26) / 10 * f))
    return ω, c₁, c₂
end

function housekeeping!(Fs, Fs_best, X, X_best, best_point, F)
    for i in eachindex(X, X_best)
        if Fs[i] <= Fs_best[i]
            Fs_best[i] = Fs[i]
            X_best[i] .= X[i]

            if Fs[i] <= F
                best_point .= X[i]
                F = Fs[i]
            end
        end
    end
    return F
end

function limit_X!(X, lower, upper)
    # limit X values to boundaries
    for x in X
        x .= min.(max.(x, lower), upper)
    end
    X
end


#################### Monte Carlo Standard Errors ####################

function mcse{T<:Real}(x::Vector{T}, method::Symbol=:imse; args...)
  method == :bm ? mcse_bm(x; args...) :
  method == :imse ? mcse_imse(x) :
  method == :ipse ? mcse_ipse(x) :
    throw(ArgumentError("unsupported mcse method $method"))
end

function mcse_bm{T<:Real}(x::Vector{T}; size::Integer=100)
  n = length(x)
  m = div(n, size)
  m >= 2 ||
    throw(ArgumentError(
      "iterations are < $(2 * size) and batch size is > $(div(n, 2))"
    ))
  mbar = [mean(x[i * size + (1:size)]) for i in 0:(m - 1)]
  sem(mbar)
end

function mcse_imse{T<:Real}(x::Vector{T})
  n = length(x)
  m = div(n - 2, 2)
  ghat = autocov(x, [0, 1])
  Ghat = sum(ghat)
  value = -ghat[1] + 2 * Ghat
  for i in 1:m
    Ghat = min(Ghat, sum(autocov(x, [2 * i, 2 * i + 1])))
    Ghat > 0 || break
    value += 2 * Ghat
  end
  sqrt(value / n)
end

function mcse_ipse{T<:Real}(x::Vector{T})
  n = length(x)
  m = div(n - 2, 2)
  ghat = autocov(x, [0, 1])
  value = ghat[1] + 2 * ghat[2]
  for i in 1:m
    Ghat = sum(autocov(x, [2 * i, 2 * i + 1]))
    Ghat > 0 || break
    value += 2 * Ghat
  end
  sqrt(value / n)
end


ϵ0 = SolidStateDetectors.ϵ0 * u"F / m"
e = SolidStateDetectors.elementary_charge * u"C"

@testset "Infinite Parallel Plate Capacitor" begin
    sim = Simulation{T}(SSD_examples[:InfiniteParallelPlateCapacitor])
    calculate_electric_potential!(sim, convergence_limit = 1e-6, refinement_limits = [0.2, 0.1], verbose = false)
    calculate_weighting_potential!(sim, 1, convergence_limit = 1e-6, refinement_limits = [0.2, 0.1], verbose = false)
    calculate_weighting_potential!(sim, 2, convergence_limit = 1e-6, refinement_limits = [0.2, 0.1], verbose = false)
    # calculate_electric_field!(sim)
    BV_true = (maximum(broadcast(c -> c.potential, sim.detector.contacts)) - minimum(broadcast(c -> c.potential, sim.detector.contacts)))* u"V"
    # Δd = (sim.detector.contacts[2].decomposed_surfaces[1].loc - sim.detector.contacts[1].decomposed_surfaces[1].loc) * u"m"
    Δd = (sim.detector.contacts[2].geometry.origin[1] - sim.detector.contacts[1].geometry.origin[1]) * u"m"
    Δx = (sim.world.intervals[2].right - sim.world.intervals[2].left) * u"m"
    A = Δx * Δx
    V = A * Δd
    E_true = BV_true / Δd
    ϵr = sim.detector.semiconductor.material.ϵ_r
    W_true = uconvert(u"J", (ϵr * ϵ0 * E_true^2 / 2) * A * Δd)
    C_true = uconvert(u"pF", 2 * W_true / (BV_true^2))
    C_Analytical = [C_true -C_true; -C_true C_true]
    C_ssd = calculate_capacitance_matrix(sim)
    @testset "Capacity" begin
        @test all(isapprox.(C_ssd, C_Analytical, rtol = 0.001))
    end
end

@testset "Two Spehres Capacitor" begin
    sim = Simulation{T}(SSD_examples[:TwoSpheresCapacitor])
    simulate!(sim, refinement_limits = [0.2, 0.1, 0.05, 0.02], convergence_limit = 1e-7)
    C_ssd = calculate_capacitance_matrix(sim)

    R_1 = sim.detector.contacts[1].geometry.r * u"m"
    R_2 = sim.detector.contacts[2].geometry.r * u"m"
    d = abs((sim.detector.contacts[1].geometry.origin - sim.detector.contacts[2].geometry.origin)[3]) * u"m"

    # Analytical solutions for the elements of the capacitance matrix:
    function c_ii(r1, r2, d; n = 10)
        r1 = ustrip(uconvert(u"m", r1))
        r2 = ustrip(uconvert(u"m", r2))
        d  = ustrip(uconvert(u"m", d))
        c = 0
        F = 4π*ϵ0 * 1u"m"
        u = acosh((d^2 - r1^2 - r2^2) / (2*r1*r2))
        for i in 0:n
            c += inv(r1*sinh(i*u) + r2*sinh((i + 1)*u))
        end
        uconvert(u"pF", F * r1 * r2 * sinh(u) * c)
    end
    function c_ij(r1, r2, d; n = 10)
        r1 = ustrip(uconvert(u"m", r1))
        r2 = ustrip(uconvert(u"m", r2))
        d  = ustrip(uconvert(u"m", d))
        c = 0
        F = 4π*ϵ0 * 1u"m"
        u = acosh((d^2 - r1^2 - r2^2) / (2*r1*r2))
        for i in 1:n
            c += inv(sinh(i*u))
        end
        uconvert(u"pF", -F * r1 * r2 * sinh(u) * c / d)
    end
    A_c11 = c_ii(R_1, R_2, d)
    A_c22 = c_ii(R_2, R_1, d)
    A_c12 = c_ij(R_1, R_2, d)
    C_Analytical = [A_c11 A_c12; A_c12 A_c22]
    @test all(isapprox.(C_Analytical, C_ssd, rtol = 0.03))
end

struct DummyImpurityDensity{T} <: SolidStateDetectors.AbstractImpurityDensity{T} end

@testset "InfiniteCoaxialCapacitor" begin
    sim_cyl = Simulation{T}(SSD_examples[:InfiniteCoaxialCapacitor])
    sim_car = Simulation{T}(SSD_examples[:InfiniteCoaxialCapacitorCartesianCoords])
    BV_true = (maximum(broadcast(c -> c.potential, sim_cyl.detector.contacts)) - minimum(broadcast(c -> c.potential, sim_cyl.detector.contacts)))* u"V"
    ϵr = sim_cyl.detector.semiconductor.material.ϵ_r
    R1 = sim_cyl.detector.contacts[1].geometry.r[1][1] * u"m"
    R2 = sim_cyl.detector.contacts[2].geometry.r[1][1] * u"m"
    V1 = sim_cyl.detector.contacts[1].potential * u"V"
    V2 = sim_cyl.detector.contacts[2].potential * u"V"
    L = (sim_cyl.world.intervals[3].right - sim_cyl.world.intervals[3].left) * u"m"
    V = π * R2^2 * L
    intV = (R2^2 - R1^2) * π * L

    calculate_electric_potential!(sim_cyl, grid = Grid(sim_cyl, max_tick_distance = (0.3u"cm", 15u"°", 1u"m")),
        convergence_limit = 1e-6, refinement_limits = [0.2, 0.1, 0.05, 0.01], use_nthreads = 1, verbose = false
    )
    calculate_electric_potential!(sim_car, grid = Grid(sim_car, max_tick_distance = (0.3u"cm", 0.3u"cm", 1u"m")),
        convergence_limit = 1e-6, refinement_limits = [0.2, 0.1, 0.05, 0.01], use_nthreads = 1, verbose = false
    )
    calculate_weighting_potential!(sim_cyl, 1, grid = Grid(sim_cyl, max_tick_distance = (0.3u"cm", 15u"°", 1u"m")),
        convergence_limit = 1e-6, refinement_limits = [0.2, 0.1, 0.05, 0.01], use_nthreads = 1, verbose = false
    )
    calculate_weighting_potential!(sim_car, 1, grid = Grid(sim_car, max_tick_distance = (0.3u"cm", 0.3u"cm", 1u"m")),
        convergence_limit = 1e-3, refinement_limits = [0.2], use_nthreads = 1, verbose = false
    )
    calculate_weighting_potential!(sim_cyl, 2, grid = Grid(sim_cyl, max_tick_distance = (0.3u"cm", 15u"°", 1u"m")),
        convergence_limit = 1e-6, refinement_limits = [0.2, 0.1, 0.05, 0.01], use_nthreads = 1, verbose = false
    )
    calculate_weighting_potential!(sim_car, 2, grid = Grid(sim_car, max_tick_distance = (0.3u"cm", 0.3u"cm", 1u"m")),
        convergence_limit = 1e-3, refinement_limits = [0.2], use_nthreads = 1, verbose = false
    )

    C_true = uconvert(u"pF", 2π * ϵr * ϵ0 / log(R2/R1) * L )
    C_Analytical = [C_true -C_true; -C_true C_true]

    C_cyl_ssd = calculate_capacitance_matrix(sim_cyl)
    C_car_ssd = calculate_capacitance_matrix(sim_car)

    @testset "Capacity" begin
        @test all(isapprox.(C_cyl_ssd, C_Analytical, rtol = 0.01))
        @test all(isapprox.(C_car_ssd, C_Analytical, rtol = 0.06))
    end

    # Add impurity density and compare resulting potential to analytic solution
    Q = uconvert(u"C", -1e7 * e * u"cm^-3" * intV)
    Δz = L #(sim_cyl.electric_potential.grid.axes[3].ticks[end] - sim_cyl.electric_potential.grid.axes[3].ticks[1])* u"m"
    # Δz = 0.04f0 * u"m"
    ρ1 = 4Q / ((R2^4 - R1^4)*Δz*2π)
    function ρ_r(r)
        return ρ1 * r^2
    end
    a = ρ1 / (ϵ0 * ϵr)
    c1 = uconvert(u"V", ((a / 16) * (R2^4 - R1^4) - (V2-V1)) / log(R2/R1))
    c2 = uconvert(u"V", V2 + c1 * log(ustrip(R2)) - a * R2^4 / 16)  # unit: Volt
    function potential_analytic(r)
        return uconvert(u"V", a * r^4 / 16 - c1 * log(ustrip(r)) + c2)
    end
    rs_analytic = range(R1, stop = R2, length = 500);
    pot_analytic = map(r -> potential_analytic(r), rs_analytic);

    function SolidStateDetectors.get_impurity_density(cdm::DummyImpurityDensity{T}, pt::CylindricalPoint{T})::T where {T}
        if ustrip(R1) <= pt[1] <= ustrip(R2)
            return -ustrip(ρ1 / e) * pt[1]^2
        else
            return 0
        end
    end

    function SolidStateDetectors.get_impurity_density(cdm::DummyImpurityDensity{T}, pt::CartesianPoint{T})::T where {T}
        SolidStateDetectors.get_impurity_density(cdm, CylindricalPoint(pt))
    end
    sim_cyl.detector = SolidStateDetector(sim_cyl.detector, DummyImpurityDensity{T}());
    sim_car.detector = SolidStateDetector(sim_car.detector, DummyImpurityDensity{T}());

    calculate_electric_potential!(sim_cyl, grid = Grid(sim_cyl, max_tick_distance = (1u"cm", 15u"°", 1u"m")),
        convergence_limit = 1e-7, refinement_limits = [0.2, 0.1, 0.05], use_nthreads = 1, verbose = false
    )
    calculate_electric_potential!(sim_car, grid = Grid(sim_car, max_tick_distance = (0.5u"cm", 0.5u"cm", 1u"m")),
        convergence_limit = 1e-7, refinement_limits = [0.2, 0.1, 0.05, 0.02], use_nthreads = 1, verbose = false
    )

    idxR1 = searchsortedfirst( sim_cyl.electric_potential.grid.axes[1], ustrip(R1));
    rs = sim_cyl.electric_potential.grid.axes[1].ticks[idxR1:end];
    potential_rms_cyl = sqrt(sum((ustrip.(map(r -> potential_analytic(r * u"m"), rs)) .- sim_cyl.electric_potential.data[idxR1:end, 1, 1]).^2)) / (ustrip(BV_true)*length(rs));
    idxR1 = searchsortedfirst( sim_car.electric_potential.grid.axes[1], ustrip(R1));
    rs = sim_car.electric_potential.grid.axes[1].ticks[idxR1:end];
    idy0 = searchsortedfirst( sim_car.electric_potential.grid.axes[2], 0);
    potential_rms_car = sqrt(sum((ustrip.(map(r -> potential_analytic(r * u"m"), rs)) .- sim_car.electric_potential.data[idxR1:end, idy0, 1]).^2))/ (ustrip(BV_true)*length(rs));

    @testset "Potential RMS" begin
        @test potential_rms_cyl < 1e-3
        @test potential_rms_car < 1e-3
    end
end


mutable struct KalmanFilter
    estimate_position::Float64
    estimate_velocity::Float64
    p_xx::Float64
    p_xv::Float64
    p_vv::Float64
    position_variance::Float64
    velocity_variance::Float64
    r::Float64
    has_velocity::Bool

    function KalmanFilter()
        return KalmanFilter(0.0, false)
    end

    function KalmanFilter(position::Float64, video::Bool)
        local estimate_position, estimate_velocity, has_velocity
        local p_xx, p_xv, p_vv
        local position_variance, velocity_variance, r

        estimate_position = position
        estimate_velocity = 0.0
        has_velocity = false

        if !video
            # normal settings used for tracking X-Y coordinates
            p_xx = 0.1
            p_xv = 0.1
            p_vv = 0.1
            position_variance = 0.01
            velocity_variance = 0.01
            r = 100000.0
        else
            # special setting used by video streaming
            p_xx = 0.1 * position
            p_xv = 1.0
            p_vv = 1.0
            position_variance = 0.1 * position
            velocity_variance = 0.01 * position / 1000.0
            r = 0.01 * position
        end

        return new(
            estimate_position,
            estimate_velocity,
            p_xx,
            p_xv,
            p_vv,
            position_variance,
            velocity_variance,
            r,
            has_velocity,
        )
    end
end

function reset(filter::KalmanFilter, position::Float64)
    filter.estimate_position = position
    filter.estimate_velocity = 0.0
    filter.has_velocity = false
end

function update(filter::KalmanFilter, position::Float64, deltat::Float64)
    if deltat <= 0.0
        return
    end

    if !filter.has_velocity
        filter.estimate_velocity = (position - filter.estimate_position) / deltat
        filter.estimate_position = position
        filter.has_velocity = true
    else
        # Temporal update (predictive)
        filter.estimate_position += filter.estimate_velocity * deltat

        # Update covariance
        filter.p_xx += deltat * (2.0 * filter.p_xv + deltat * filter.p_vv)
        filter.p_xv += deltat * filter.p_vv

        filter.p_xx += deltat * filter.position_variance
        filter.p_vv += deltat * filter.velocity_variance

        # Observational update (reactive)
        vi = 1.0 / (filter.p_xx + filter.r)
        kx = filter.p_xx * vi
        kv = filter.p_xv * vi

        filter.estimate_position += (position - filter.estimate_position) * kx
        filter.estimate_velocity += (position - filter.estimate_position) * kv

        filter.p_xx *= 1.0 - kx
        filter.p_xv *= 1.0 - kx
        filter.p_vv -= kv * filter.p_xv
    end
end

function predict(filter::KalmanFilter, position::Float64, deltat::Float64)::Float64
    return position + filter.estimate_velocity * deltat
end


using FastTransforms, Compat
using Compat.Test

import FastTransforms: normalizecolumns!, maxcolnorm

@testset "Fast plan" begin
    m, n = 362, 300

    A = sphrandn(Float64, m, n);
    normalizecolumns!(A);
    B = zero(A);
    C = zero(A);
    D = zero(A);
    E = zero(A);

    @time FP = FastSphericalHarmonicPlan(A; sketch = :none);
    @time SP = SlowSphericalHarmonicPlan(A);

    @time mul!(B, SP, A);
    @time mul!(C, FP, A);
    @time At_mul_B!(D, SP, B);
    @time At_mul_B!(E, FP, C);

    @test maxcolnorm(B-C) < 10eps()*max(m,n)
    @test maxcolnorm(A-D) < 10eps()*max(m,n)
    @test maxcolnorm(A-E) < 10eps()*max(m,n)
    @test maxcolnorm(D-E) < 10eps()*max(m,n)

    println("The forward difference between slow and fast plans: ", maxcolnorm(B-C))
    println("The backward difference between slow plan and original: ", maxcolnorm(A-D))
    println("The backward difference between fast plan and original: ", maxcolnorm(A-E))
    println("The backward difference between slow and fast plans: ", maxcolnorm(D-E))
end


module GXBeam

using ArnoldiMethod
using ForwardDiff
using LinearAlgebra
using LinearMaps
using NLsolve
using SparseArrays
using StaticArrays
using WriteVTK
using DiffEqBase

export curve_length, discretize_beam
export System, Assembly, PrescribedConditions, DistributedLoads
export combine_loads
export system_state, reset_state!, set_state!
export set_element_deflections!, set_element_rotations!
export set_element_forces!, set_element_moments!
export set_element_linear_momenta!, set_element_angular_momenta!
export set_point_deflections!, set_point_rotations!
export set_point_forces!, set_point_moments!
export static_analysis, static_analysis!
export steady_state_analysis, steady_state_analysis!
export eigenvalue_analysis, eigenvalue_analysis!
export initial_condition_analysis, initial_condition_analysis!
export time_domain_analysis, time_domain_analysis!
export AssemblyState
export PointState, extract_point_state, extract_point_states, extract_point_states!
export ElementState, extract_element_state, extract_element_states, extract_element_states!
export wiener_milenkovic
export rotate, rotate!
export translate, translate!
export cross_section_velocities, cross_section_velocities!
export deform_cross_section, deform_cross_section!

export write_vtk
export left_eigenvectors, correlate_eigenmodes

const GAUSS_NODES = SVector(-0.8611363115940526, -0.3399810435848563, 0.3399810435848563, 0.8611363115940526)
const GAUSS_WEIGHTS = SVector(0.34785484513745385, 0.6521451548625462, 0.6521451548625462, 0.34785484513745385)

include("math.jl")
include("element.jl")
include("point.jl")
include("loads.jl")
include("assembly.jl")
include("system.jl")
include("analyses.jl")
include("postprocess.jl")

include("interfaces/differentialequations.jl")
include("interfaces/forwarddiff.jl")

end


module WhittleLikelihoodInference

using FFTW, LinearAlgebra, ToeplitzMatrices, StaticArrays, RecipesBase
using Distributions, SpecialFunctions, LazyArrays
using ChainRulesCore
using Optim, DSP

import Base: ndims, show, size, getindex, @propagate_inbounds
import StaticArrays: triangularnumber

export
    TimeSeriesModel,
    UnknownAcvTimeSeriesModel,
    AdditiveTimeSeriesModel,
    npars,
    ndims,
    parameternames,
    lowerbounds,
    upperbounds,
    parameter,
    sdf,
    asdf,
    acv,
    EI,
    coherancy,
    coherance,
    groupdelay,
    simulate_gp,
    fit,
    ## non-parametrics
    Periodogram,
    BartlettPeriodogram,
    SpectralEstimate,
    CoherancyEstimate,
    ## models
    OU,
    CorrelatedOU,
    Matern,
    Matern1D,
    Matern2D,
    Matern3D,
    Matern4D,
    ## Whittle
    DebiasedWhittleLikelihood,
    WhittleLikelihood

    include("typestructure.jl")
    include("memoryallocation.jl")
    include("properties/spectraldensity.jl")
    include("properties/autocovariance.jl")
    include("properties/expectedperiodogram.jl")
    include("properties/coherancy.jl")

    include("simulation.jl")
    include("nonparametric.jl")
    include("plotting.jl")

    include("whittle/whittledata.jl")
    include("whittle/generalwhittle.jl")
    include("whittle/standardwhittle.jl")
    include("whittle/debiasedwhittle.jl")
    include("whittle/chainrules.jl")
    include("fit.jl")

    include("models/OU.jl")
    include("models/CorrelatedOU.jl")
    include("models/OUUnknown.jl")
    include("models/CorrelatedOUUnknown.jl")
    include("models/Matern.jl")
    include("models/MaternAcv.jl")

end


using Base.BLAS

include("units.jl")

immutable Layer{T<:FloatingPoint}
	ni::Int
	no::Int
	W::Matrix{T}
	B::Vector{T}
	GW::Matrix{T}
	GB::Vector{T}
	UW::Matrix{T}
	UB::Vector{T}
	NET::Vector{T}
	ACT::Vector{T}
	DE_DNET::Vector{T}
	E::Vector{T}
	DE_DYH::Vector{T}
	DELTAS::Vector{T}
	activation::ASCIIString
	activation_function!::Function
	activation_backward!::Function

	function Layer(ni::Int, no::Int, activation::ASCIIString)
		if ni > 0 && no > 0
			# Initialize weights and biases for layer.
			sigma = 1e-2
			W = zeros(T, (ni, no))
			for i = 1:length(W)
				W[i] = sigma * randn()
			end
			B = zeros(T, no)
			for i = 1:length(B)
				B[i] = sigma * randn()
			end
			# Initialize storage for gradient information.
			GW, GB = zeros(W), zeros(B)
			UW, UB = zeros(W), zeros(B)
			# Initialize storage for upper level units.
			NET = zeros(T, no)
			ACT = zeros(T, no)
			DE_DNET = zeros(T, no)
			E = zeros(T, no)
			DE_DYH = zeros(T, no)
			DELTAS = zeros(T, ni)
			# Set activation function for layer.
			activation, activation_function!, activation_backward! = activation_function_selector(activation)
			# Create and return the object.
			new(ni, no, W, B, GW, GB, UW, UB, NET, ACT, DE_DNET, E, DE_DYH, DELTAS, activation, activation_function!, activation_backward!)
		else
			error("Invalid number of units used to initialize Layer object (ni=$ni; no=$no) to create Layer object.")
		end
	end
end

# Propagate delta quantities coming from the layer above downward through the layer.
function backward!{T<:FloatingPoint}(L::Layer{T}, DELTAS_ABOVE::Vector{T})
	@inbounds begin
		# Set DE_DNET on the layer.
		L.activation_backward!(L.NET, L.ACT, DELTAS_ABOVE, L.DE_DNET)
		# Set DELTAS on the layer.
		fill!(L.DELTAS, 0.0)
		gemv!('N', 1.0, L.W, L.DE_DNET, 1.0, L.DELTAS)
	end
end

# Propagate an incoming pattern forward through a layer.
function forward!{T<:FloatingPoint}(L::Layer{T}, IN::AbstractVector{T})
	@inbounds begin
		# Calculate net values.
		blascopy!(L.no, L.B, 1, L.NET, 1)
		gemv!('T', 1.0, L.W, IN, 1.0, L.NET)
		# Set activations and gradient information related to activations.
		L.activation_function!(L.NET, L.ACT)
	end
end


using PkgName, Base.Test

function ortho(V::AbstractMatrix{T})where{T}
    n = size(V,2)
    t = true
    for i in 1:n-1
        for j in (i+1):n
            if(vecdot(view(V, :, i),view(V, :, j))>(10*eps(T)))
                t = false
                break
            end
        end
    end
    return t
end

@test ortho(hessenberg(sprand(100,100,0.3),30)[2]) == true


using JSON

## build_groups
f = jldopen("data/small.ChromNet.jld")
data = read(f, "data")
ids = read(f, "ids")
metadata = read(f, "metadata")
close(f)
C = cov([data' data'])
groups = build_groups(C, [ids; ids])
for i in 1:8:length(groups[end][2])
    @test groups[end][2][i] == groups[end][2][i+4] # make sure duplicates are grouped together
end

## build_groupgm
C = cov([data[1,:]' data[1,:]' data[2,:]'.+0.01*data[1,:]' data[3,:]'] .+ 0.0001*randn(size(data)[2], 4))
groups = build_groups(C, ids[1:4])
I = inv(C)
G,idsG = build_groupgm(I, ids[1:4], groups)
@test abs(I[1,3] + I[2,3] - G[5,3]) < 1e-7

## parse_config
metadataTmp = open(f->parse_config(f, "data"), "data/single_bed.config")
@test metadataTmp["EX001"]["name"] == "K562_tss"
@test metadataTmp["EX001"]["lab"] == "Sample lab"

## build_network
C = cor(data, vardim=2)
groups = build_groups(C, ids)
G,idsG = build_groupgm(inv(C), ids, groups)
network = build_network(G, groups, metadata; threshold=0.03, groupScoreThreshold=0.7, quiet=true)


mutable struct GLLayer{
    T<:Number,T1<:Number,Tm1<:AbstractMatrix{T1},
    T2<:Number,Tm2<:AbstractMatrix{T2},
    Talg<:OrdinaryDiffEq.OrdinaryDiffEqCompositeAlgorithm
    }
    input_size::Int

    Win::Tm1

    G::SimpleGraph{Int64}
    W::Tm2
    γ::T
    Γ::T
    g::T
    D::T

    alg::Talg
    tinput::Float64
    toutput::Float64
end

function GLLayer(tinput::Real, toutput::Real, input_size::Int,
    γ::Number, Γ::Number, g::Number, G::SimpleGraph{Int64},
    D::Number=0.0, W=missing, Win=missing;
    alg::OrdinaryDiffEq.OrdinaryDiffEqCompositeAlgorithm=AutoTsit5(Rosenbrock23())
    )
    N = nv(G)
    @assert input_size > 0
    T = any(map(x->typeof(x)<:Union{Float32, ComplexF32}, [γ, Γ, g, D])) ? Float32 : typeof(γ)
    T = any(map(x->typeof(x)<:Complex, [γ, Γ, g, D])) ? complex(T) : T;

    _W = if ismissing(W)
        rand_couplings(N, T)
    else
        @assert size(W,1) == size(W,2) == N
        W
    end

    _Win = if ismissing(Win)
        rand_input_matrix(N, input_size, T)
    else
        @assert size(Win, 1) == N
        @assert size(Win, 2) == input_size
        Win
    end

    return GLLayer{T,eltype(_Win),typeof(_Win),eltype(_W),typeof(_W), typeof(alg)}(
          input_size, _Win, G, _W, T(γ), T(Γ), T(g), T(D),
          alg, Float64(tinput), Float64(toutput)
          )
end

function rand_couplings(N::Int,T::DataType)
    W = randn(T, N, N) ./ 2T(sqrt(N))
    for i in 1:N
        W[i,i] = zero(T)
    end
    (W .+ W') ./ 2
end

function rand_input_matrix(nr, nc, T)
    Win = rand(T, nr, nc)
    x = rand(T, nc)
    x ./= norm(x)
    Win ./= norm(Win * x)
    Win
end

function (m::GLLayer)(u::InputSignal)
    tspan = (m.tinput, m.toutput)

    function dψ_det!(dψ, ψ, p, t)
        mul!(dψ, m.Win, u(t))
        @. @inbounds dψ += (m.γ - (m.Γ + im*m.g)*abs2(ψ)) * ψ

        for v in vertices(m.G)
            Nv = neighbors(m.G, v)
            @inbounds @views dψ[v] -= im .* 10dot(m.W[v,Nv],ψ[Nv])
        end

        return dψ
    end

    function dψ_stoch!(dψ, ψ, p, t)
        @. @inbounds dψ = m.D

        return dψ
    end

    T = first(typeof(m).parameters)
    ψ0 = zeros(complex(T), nv(m.G))

    pb = SDEProblem(dψ_det!, dψ_stoch!, ψ0, tspan)
    solve(pb, m.alg; dtmax=0.1u.Δt, save_start=false, save_on=false).u[]
end

function (m::GLLayer)(u::AbstractArray{I}, args...; kwargs...) where I<:InputSignal
    tspan = (m.tinput, m.toutput)

    function dψ_det!(dψ, ψ, i, t)
        mul!(dψ, m.Win, u[i][1](t))
        @. @inbounds dψ += (m.γ - (m.Γ + im*m.g)*abs2(ψ)) * ψ

        for v in vertices(m.G)
            Nv = neighbors(m.G, v)
            @inbounds @views dψ[v] -= im .* 10dot(m.W[v,Nv],ψ[Nv])
        end
    end

    function dψ_stoch!(dψ, ψ, i, t)
        @. @inbounds dψ = m.D

        return dψ
    end

    T = first(typeof(m).parameters)
    ψ0 = zeros(complex(T), nv(m.G))

    pb = SDEProblem(dψ_det!, dψ_stoch!, ψ0, tspan, [1])
    function prob_func(prob, i, repeat)
        prob.p[1] = i
        prob
    end
    Δt = minimum(map(x->x.Δt, u))
    sol = solve(EnsembleProblem(pb, prob_func=prob_func), m.alg, args...; kwargs..., trajectories=length(u), dtmax=0.1Δt, save_start=false, save_on=false)
    reshape([sol[i].u[:][] for i in 1:length(u)], size(u))
end


module Laplace

## Package imports
#==========================================================================================#

using StaticArrays
using LinearAlgebra

import ..MathTools: Point2D, Point3D, magnitude, angle

import ..AeroMDAO: velocity

abstract type AbstractLaplace end

## Legacy (to be removed?)
#==========================================================================================#

# Performs velocity and potential computations for an object on a grid
grid_data(object :: AbstractLaplace, xs) = velocity(object, xs), potential(object, xs)
velocity(object :: AbstractLaplace, xs) = map(x -> velocity(object, x...), xs)
potential(object :: AbstractLaplace, xs) = map(x -> potential(object, x...), xs)

# Performs velocity and potential calculations on a grid
function grid_data(objects :: Vector{<: AbstractLaplace}, xs)
    vels = foldl((v1, v2) -> [ u .+ v for (u, v) ∈ zip(v1, v2) ], velocity(object, xs) for object ∈ objects)
    pots = foldl((v1, v2) -> v1 + v2, potential(object, xs) for object ∈ objects)

    vels, pots
end

## 2D singularities
#============================================#

struct Singularity2D{T <: Real} <: AbstractLaplace
    strength :: T
    r        :: Point2D{T}
end

# Getters
strength(s :: Singularity2D) = s.strength
x(s :: Singularity2D)        = s.r.x
y(s :: Singularity2D)        = s.r.y

source_velocity(src :: Singularity2D, x, y) = SVector(strength(src) / (2π) * (x - y(src)) / ((x - y(src))^2 + (y - x(src))^2), str / (2π) * (y - x(src)) / ((x - y(src))^2 + (y - x(src))^2))
source_potential(src :: Singularity2D, x, y) = strength(src) / (4π) * log((x - y(src))^2 + (y - x(src))^2)
source_stream(src :: Singularity2D, x, y) = strength(src) / (2π) * atan(y - x(src), x - y(src))

doublet_velocity(dub :: Singularity2D, x, y) = SVector(strength(dub) / (2π) * ((x - y(dub))^2 - (y - x(dub))^2) / ((x - y(dub))^2 + (y - x(dub))^2)^2, - strength(dub) / (2π) * 2 * (x - y(dub)) * (y - x(dub)) / ((x - y(dub))^2 + (y - x(dub))^2)^2)
doublet_potential(dub :: Singularity2D, x, y) = -strength(dub) / (2π) * (y - x(dub)) / ((x - y(dub))^2 + (y - x(dub))^2)
doublet_stream(dub :: Singularity2D, x, y) = -strength(dub) / (2π) * (y - x(dub)) / ((x - y(dub))^2 + (y - x(dub))^2)

vortex_velocity(vor :: Singularity2D, x, y) = SVector(-strength(vor) / (2π) * (y - x(vor)) / ((x - y(vor))^2 + (y - x(vor))^2), str / (2π) * (x - y(vor)) / ((x - y(vor))^2 + (y - x(vor))^2))
vortex_potential(vor :: Singularity2D, x, y) = strength(vor) / (2π) * atan(y - x(vor), x - y(vor))
vortex_stream(vor :: Singularity2D, x, y) = -strength(vor) / (4π) * log((x - y(vor))^2 + (y - x(vor))^2)

struct Uniform2D{T <: Real} <: AbstractLaplace
    magnitude :: T
    angle     :: T
    Uniform2D{T}(mag, ang) where T <: Real = new(mag, deg2rad(ang))
end

magnitude(uni :: Uniform2D) = uni.magnitude
angle(uni :: Uniform2D)     = uni.angle

Uniform2D(mag :: T, ang :: T) where T <: Real = Uniform2D{T}(mag, ang)
Uniform2D(mag, ang) = Uniform2D(promote(mag, ang)...)
Uniform2D(velocity) = Uniform2D((sqrt ∘ sum)(velocity.^2), atan(velocity[2], velocity[1]))
Uniform2D(; angle)  = Uniform2D(1., angle)

velocity(uni :: Uniform2D) = let (sa, ca) = sincos(uni.angle); uni.magnitude * SVector(ca, sa) end
potential(uni :: Uniform2D, x, y) = uni.magnitude * (x * cos(uni.angle) + y * sin(uni.angle))
stream(uni :: Uniform2D, x, y)    = uni.magnitude * (y * cos(uni.angle) - x * sin(uni.angle))

## 3D singularities
#============================================#

# struct Singularity3D{T <: Real} <: AbstractLaplace
#       str :: T
#       r   :: Point3D{T}
# end

# source_velocity(src :: Source2D, x, y, z)
# source_potential(src :: Source2D, x, y, z)
# source_stream(src :: Source2D, x, y, z)

struct DoubletLine3D{T <: Real} <: AbstractLaplace
    strength :: T
    r1       :: SVector{3,T}
    r2       :: SVector{3,T}
    eta      :: SVector{3,T}
end

function doublet_influence(r, φ, η)
    r_φ = dot(r, φ)
    r_η = dot(r, η)
    den = (norm(r)^2 - dot(r, φ)^2 ) * r

    ((r_φ * η + r_η * φ) * den - (den * r / norm(r)^2 + 2 * (r - r_φ * η) * r) * r_φ * r_η) / den^2
end

function velocity(src :: DoubletLine3D)
    l = normalize(src.r2 - src.r1)

    f(x) = src.strength / 4π * (doublet_influence(x - src.r2, l, src.eta) - doublet_influence(x - src.r1, l, src.eta))
end

## Freestream
#============================================#
"""
    freestream_to_cartesian(r, θ, φ)

Convert freestream flow (spherical polar) coordinates to Cartesian coordinates.
"""
freestream_to_cartesian(r, θ, φ) = r * SVector(cos(θ) * cos(φ), -sin(φ), sin(θ) * cos(φ))

"""
    cartesian_to_freestream(U)

Convert Cartesian coordinates to freestream (spherical polar) flow coordinates.
"""
cartesian_to_freestream(U) = SVector(norm(U), -atand(U[3], U[1]), -atand(U[2], √(U[1]^2 + U[3]^2)))

# 2D versions
cartesian_to_freestream(u, w) = magnitude(u, w), angle(u, w)
freestream_to_cartesian(V, α) = V * cos(α), V * sin(α)

end


@testset "1d approximation" begin
  xs = collect(0:0.1:10)
  fs = [sin(x) for x in xs]

  m = mwls_kd(xs, fs, 1, (d, e) -> exp(-d^2))
  cll = mwls_cll(xs, fs, 1, (d, e) -> exp(-d^2))

  teststatus = true
  for p in 0:0.1:10
    teststatus &= isapprox(m(p), sin(p), atol=0.1)
  end
  @test teststatus == true

  teststatus = true
  for p in 0:0.1:10
    teststatus &= isapprox(cll(p), sin(p), atol=0.1)
  end
  @test teststatus == true

  # not enough data for approximation, zero returned by default
  @test isapprox(m(-0.9), 0, atol=0.1)
  @test isapprox(cll(-0.9), 0, atol=0.1)
  @test isapprox(m(10.9), 0, atol=0.1)
  @test isapprox(cll(10.9), 0, atol=0.1)
end


#! /usr/bin/julia

# Rosetta Code, Grayscale image

using Color, Images, FixedPointNumbers

const M_RGB_Y = reshape(Color.M_RGB_XYZ[2,:], 3)

function rgb2gray(img::Image)
    g = red(img)*M_RGB_Y[1] + green(img)*M_RGB_Y[2] + blue(img)*M_RGB_Y[3]
    g = clamp(g, 0.0, 1.0)
    return grayim(g)
end

function gray2rgb(img::Image)
    colorspace(img) == "Gray" || return img
    g = map((x)->RGB{Ufixed8}(x, x, x), img.data)
    return Image(g, spatialorder=spatialorder(img))
end

ima = imread("grayscale_image_color.png")
imb = rgb2gray(ima)
imc = gray2rgb(imb)
imwrite(imc, "grayscale_image_rc.png")

ima = imread("grayscale_image_color.png")
imb = convert(Image{Gray{Ufixed8}}, ima)
imwrite(imb, "grayscale_image_julia.png")


ts = collect(now():Minute(10):(99*Minute(10)+now()))
N = length(ts)

xf = randn(N)
xt = 2*xf + 1 + 0.1*randn(N)

qf = Stage(ts,xf)
qt = Temperature(ts,xt)

c = Calibration(qt,qf)

m1 = fit(PolynomialCalibrationModel,c,1)

xf2 = randn(N)
qf2 = Stage(ts,xf2)

qt2 = predict(m1,qf2)

@testset "Polynomial calibration models" begin

    @test quantity(qt2) == m1.β[1] + m1.β[2]*xf2
    @test typeof(qt2) == Temperature{Float64}

end


"""Given a regularization parameter `regulα`, a vector `x` and a direction `d`,
   return `σ` > 0 such that

    h(σ) ~ argmin_{α>0} h(α) = q(x + α d) +  ‖x + αd‖³/3regulα

in the Euclidean norm.
"""
function to_minimum(A :: LinearOperator, b :: Array{T,1},
                               x :: Vector{T},
                               p :: Vector{T}, Ap :: Vector{T}, pAp :: T,
                               α :: T,
                               regulα :: T) where T
    regulα > 0 || error("regulα must be positive")
    #println("regulα = ",regulα)

    Ax = copy(A * x)
    xAx = dot(x, Ax)
    xAp = dot(x, Ap)
    bx = dot(b, x)
    bp = dot(b, p)
    h = α -> 0.5 * (xAx + 2.0 * xAp * α + α^2 * pAp) - bx - α * bp + norm(x + α * p)^3/(3 * regulα)
    h0 = h(0)

    #dh = α -> xAp + α*pAp - bp + norm(x+α*p)*(x+α*p)'*p
    xx = dot(x, x)
    xp = dot(x, p)
    pp = dot(p, p)
    dh = α -> xAp + α * pAp - bp + sqrt(xx + 2 * α * xp + α^2 * pp) * (xp + α * pp) / regulα

#println("h0 = ",h(0),"  dh0 = ",dh(0))
    # search for a positive dh
    α = regulα
    while dh(α)<0
        α *= 5
    end
#println("h(α) = ",h(α),"  dh(α) = ",dh(α), "  α = ",α)

    a = T(0.0)
    σ = T(α/2)
    @assert ((dh(a) < 0) & (dh(α) > 0))
    while (((α - a) / max(α, a)) > 1e-8) & (abs(dh(σ)) > 1e-8)
        if dh(σ) < 0
            a = σ
        else
            α = σ
        end
        σ = (a + α) / 2
#println("  dh(σ) = ",dh(σ),"  dh(α) = ",dh(α),"  dh(a) = ",dh(a), "  σ = ",σ, "  α = ",α, "  a = ",a)
    end
#println("h(σ) = ",h(σ),"  dh(σ) = ",dh(σ), "  σ = ",σ)

    return σ
end