#  Tests for the projected NEPs

using NonlinearEigenproblemsTest
using NonlinearEigenproblems
using Test
using LinearAlgebra
using Random

#using Winston # For plotting

@testset "Projected problems" begin

    @bench @testset "Problem $nepstr" for nepstr in ("pep", "dep", "sqrtm")

        local nep::NEP
        if (nepstr == "pep")
            nep=nep_gallery("pep0_sparse");
        elseif (nepstr == "dep")

            n=5;
            Random.seed!(1)
            A0=(1:5)*(1:5)'
            A1=(1:5)*(3:7)'+I;
            t::Float64=3.0

            minusop= S-> -S
            oneop= S -> one(S)
            expmop= S -> exp(-t*S)
            fi=[minusop, oneop, expmop];

            nep = SPMF_NEP([Matrix(1.0I, n, n), A0, A1], fi)

        elseif (nepstr == "sqrtm")

            n=8;
            Random.seed!(1)
            A1=I+(1:n)*(1:n)'/n
            A0=(1:n)*(3:(n+2))'+2I;
            A2=(-1:(n-2))*(3:(n+2))'/8-I
            t=3.0

            minusop= S-> -S
            oneop= S -> one(S)  # one(S) replaced by I it would not return a matrix
            expmop= S -> sqrt(-t*S + 30*one(S))
            fi=[minusop, oneop, expmop];

            nep=SPMF_NEP([A0,A1,A2],fi)
        end

        #

        n=size(nep,1);
        @info "Running Newton Raphson ($nepstr)"
        tol=1e-12
        λ,x =newton(nep,maxit=30,λ=1+1im,tol=tol,
                    v=ones(n));
        # Check residual is small
        λ_exact=λ
        @test norm(compute_Mlincomb(nep,λ,x))<tol*1000;

        # Create a projected NEP
        pnep=create_proj_NEP(nep,4); # maxsize=4
        V=(1:n)*(1:2)'/n; V[1,1]=pi;
        Q,R=qr(hcat(V,x)) # Make the eigenspace a part of the projection subspace
        Q = Matrix(Q)
        set_projectmatrices!(pnep,Q,Q);
        @info "Running Newton on projected problem with very good start value ($nepstr)"
        λ1,z1=newton(pnep,λ=(λ_exact+0.00001),logger=0,v=ones(size(pnep,1)))

        x1=Q*z1; x1=x1/x1[1];

        # should be small since the eigenvector is in the subspace
        @test norm(x/x[1]-x1)<1e-8

        @info "Running IAR for projected problem ($nepstr)"
        λv,X=iar(pnep,σ=complex(round(λ_exact*10)/10),
                 neigs=3,maxit=100, v=ones(size(pnep,1)))

        mydiff = minimum(abs.(λv .- λ_exact))
        @info "Difference of solution from projected problem ($nepstr): $mydiff"
        @test mydiff < sqrt(eps())

        @info "Testing to expand and running Newton ($nepstr)"
        n=size(nep,1);
        Vnew=[Q ones(n)];
        Wnew=[Q ones(n)];
        expand_projectmatrices!(pnep,Wnew,Vnew);
        λ1,zz1=newton(pnep,λ=(λ_exact+0.0000001),logger=0,
                      v=Vnew'*x.+0.00001*ones(size(pnep,1)),maxit=30)

        xx1=Vnew*zz1; xx1=xx1/xx1[1]

        # should be small since the eigenvector is in the subspace
        @test norm(x/x[1]-x1)<1e-8

    end
end


@testset "Monodromy" begin
    function toric_ed(A)
        d, n = size(A)
        @polyvar t[1:d] y[1:n] u[1:n]

        φ = map(j -> prod(i -> t[i]^A[i, j], 1:d), 1:n)
        Dφ = [differentiate(φ[i], t[j]) for i = 1:n, j = 1:d]

        F = [φ + y - u; Dφ' * y]

        # We sample a random t, find a normal vector on this point and then assemble
        # an u where we know the solution to.
        t_rand = randn(Float64, size(A, 1))
        null = nullspace(map(fi -> fi(t => t_rand), Dφ'))
        y_rand = null * randn(Float64, size(null, 2))
        u₀ = map(fi -> fi(t => t_rand), φ) + y_rand

        x₀ = complex.([t_rand; y_rand])

        F, u, u₀, x₀
    end

    @testset "monodromy_solve" begin
        F, p, p₀, x₀ = toric_ed([3 2 1 0; 0 1 2 3])

        result = monodromy_solve(
            F,
            x₀,
            p₀,
            parameters = p,
            target_solutions_count = 21,
            max_loops_no_progress = 20,
            threading = false,
        )
        @test is_success(result)
        @test length(solutions(result)) == 21
        @test length(solutions(result, only_real = true)) >= 1
        @test result.statistics.ntrackedpaths ≥ 21
        @test result.statistics.nparametergenerations ≥ 1
        @test length(HC.UniquePoints(result.solutions).points) == 21
        @test isempty(sprint(show, result)) == false

        # test seed
        result2 = monodromy_solve(
            F,
            x₀,
            p₀,
            parameters = p,
            target_solutions_count = 21,
            max_loops_no_progress = 20,
            threading = false,
            seed = result.seed,
        )
        @test result2.statistics.ntrackedpaths == result.statistics.ntrackedpaths

        result = monodromy_solve(
            F,
            x₀,
            p₀,
            parameters = p,
            target_solutions_count = 21,
            max_loops_no_progress = 20,
            threading = true,
        )
        @test is_success(result)
        @test length(solutions(result)) == 21

        # test that timeout works
        result_timeout = monodromy_solve(
            F,
            x₀,
            p₀,
            parameters = p,
            target_solutions_count = 21,
            timeout = 1e-12,
        )
        @test length(result_timeout.solutions) < 21

        # test input of length > 1
        result = monodromy_solve(F, [x₀ for _ = 1:30], p₀, parameters = p)
        @test length(solutions(result)) == 21

        # test with false input
        result = monodromy_solve(F, [rand(6) for _ = 1:10], p₀, parameters = p)
        @test result.returncode == :invalid_startvalue
        result = monodromy_solve(F, [x₀, rand(6)], p₀, parameters = p)
        @test length(solutions(result)) == 21

        # different distance function
        result = monodromy_solve(F, x₀, p₀, parameters = p, distance = (x, y) -> 0.0)
        @test length(solutions(result)) == 1

        # Test stop heuristic using too high target_solutions_count
        result = monodromy_solve(F, x₀, p₀, parameters = p, target_solutions_count = 25)
        @test is_heuristic_stop(result)
        # Test stop heuristic with no target solutions count
        result = monodromy_solve(F, x₀, p₀, parameters = p)
        @test is_heuristic_stop(result)
        # Test stop heuristic with no target solutions count
        result = monodromy_solve(
            F,
            x₀,
            p₀,
            parameters = p,
            strategy = Triangle(useweights = true),
        )
        @test is_heuristic_stop(result)

        # By group_actions=nothing we force that complex conjugation is not used.
        result2 = monodromy_solve(
            F,
            x₀,
            p₀,
            parameters = p,
            target_solutions_count = 21,
            complex_conjugation = false,
            max_loops_no_progress = 100,
        )
        @test is_success(result2)

        result = monodromy_solve(
            F,
            x₀,
            p₀,
            parameters = p,
            target_solutions_count = 21,
            done_callback = ((_, _) -> true),
        )
        @test length(result.solutions) == 2

        roots_of_unity(s) = begin
            t = cis(π * 2 / 3)
            t² = t * t
            (vcat(t * s[1], t * s[2], s[3:end]), vcat(t² * s[1], t² * s[2], s[3:end]))
        end

        result = monodromy_solve(
            F,
            x₀,
            p₀,
            parameters = p,
            target_solutions_count = 21,
            max_loops_no_progress = 100,
            equivalence_classes = false,
            group_action = roots_of_unity,
        )
        @test length(result.solutions) == 21

        result = monodromy_solve(
            F,
            x₀,
            p₀,
            parameters = p,
            target_solutions_count = 21,
            max_loops_no_progress = 100,
            complex_conjugation = false, # disable complex conjugation to test it as a group action.
            equivalence_classes = false,
            group_actions = (roots_of_unity, s -> (conj.(s),)),
        )
        @test length(result.solutions) == 21
        @test length(solutions(result)) == 21
        @test length(real_solutions(result)) < 21
        test_treeviews(result)

        # group_actions as a vector
        result = monodromy_solve(
            F,
            x₀,
            p₀,
            parameters = p,
            target_solutions_count = 21,
            max_loops_no_progress = 100,
            complex_conjugation = false,
            equivalence_classes = false,
            group_actions = [roots_of_unity, s -> (conj.(s),)],
        )
        @test length(result.solutions) == 21

        # equivalence classes
        result = monodromy_solve(
            F,
            x₀,
            p₀,
            parameters = p,
            equivalence_classes = true,
            target_solutions_count = 7,
            max_loops_no_progress = 100,
            group_actions = roots_of_unity,
        )
        @test length(result.solutions) == 7
        # Test that equivalence classes are on by default if we supply a group action
        result = monodromy_solve(
            F,
            x₀,
            p₀,
            parameters = p,
            group_action = roots_of_unity,
            max_loops_no_progress = 20,
        )
        @test length(result.solutions) == 7

        # Test affine tracking
        result = monodromy_solve(
            F,
            x₀,
            p₀,
            parameters = p,
            affine_tracking = true,
            group_action = roots_of_unity,
            target_solutions_count = 7,
            max_loops_no_progress = 200,
        )
        @test length(result.solutions) == 7

        # AbstractSystem as input
        F_p = SPSystem(F; parameters = p)
        result = monodromy_solve(
            F_p,
            x₀,
            p₀,
            affine_tracking = true,
            group_action = roots_of_unity,
            target_solutions_count = 7,
            max_loops_no_progress = 200,
        )
        @test length(result.solutions) == 7

        # test reuse strategies
        result = monodromy_solve(
            F_p,
            x₀,
            p₀,
            group_action = roots_of_unity,
            target_solutions_count = 7,
            reuse_loops = :all,
            max_loops_no_progress = 200,
        )
        @test length(result.solutions) == 7
        result = monodromy_solve(
            F_p,
            x₀,
            p₀,
            group_action = roots_of_unity,
            target_solutions_count = 7,
            reuse_loops = :random,
            max_loops_no_progress = 200,
        )
        @test length(result.solutions) == 7
        result = monodromy_solve(
            F_p,
            x₀,
            p₀,
            group_action = roots_of_unity,
            target_solutions_count = 7,
            reuse_loops = :none,
            max_loops_no_progress = 200,
        )
        @test length(result.solutions) == 7
    end

    @testset "Method of Moments" begin
        Random.seed!(130793)
        @polyvar a[1:3] x[1:3] s[1:3]

        f0 = a[1] + a[2] + a[3]
        f1 = a[1] * x[1] + a[2] * x[2] + a[3] * x[3]
        f2 = a[1] * (x[1]^2 + s[1]) + a[2] * (x[2]^2 + s[2]) + a[3] * (x[3]^2 + s[3])
        f3 = a[1] * (x[1]^3 + 3 * s[1] * x[1]) +
            a[2] * (x[2]^3 + 3 * s[2] * x[2]) +
            a[3] * (x[3]^3 + 3 * s[3] * x[3])
        f4 = a[1] * (x[1]^4 + 6 * s[1] * x[1]^2 + 3 * s[1]^2) +
            a[2] * (x[2]^4 + 6 * s[2] * x[2]^2 + 3 * s[2]^2) +
            a[3] * (x[3]^4 + 6 * s[3] * x[3]^2 + 3 * s[3]^2)
        f5 = a[1] * (x[1]^5 + 10 * s[1] * x[1]^3 + 15 * x[1] * s[1]^2) +
            a[2] * (x[2]^5 + 10 * s[2] * x[2]^3 + 15 * x[2] * s[2]^2) +
            a[3] * (x[3]^5 + 10 * s[3] * x[3]^3 + 15 * x[3] * s[3]^2)
        f6 = a[1] * (x[1]^6 + 15 * s[1] * x[1]^4 + 45 * x[1]^2 * s[1]^2 + 15 * s[1]^3) +
            a[2] * (x[2]^6 + 15 * s[2] * x[2]^4 + 45 * x[2]^2 * s[2]^2 + 15 * s[2]^3) +
            a[3] * (x[3]^6 + 15 * s[3] * x[3]^4 + 45 * x[3]^2 * s[3]^2 + 15 * s[3]^3)
        f7 = a[1] * (
            x[1]^7 + 21 * s[1] * x[1]^5 + 105 * x[1]^3 * s[1]^2 + 105 * x[1] * s[1]^3
        ) +
            a[2] * (
            x[2]^7 + 21 * s[2] * x[2]^5 + 105 * x[2]^3 * s[2]^2 + 105 * x[2] * s[2]^3
        ) +
            a[3] * (
            x[3]^7 + 21 * s[3] * x[3]^5 + 105 * x[3]^3 * s[3]^2 + 105 * x[3] * s[3]^3
        )
        f8 = a[1] * (
            x[1]^8 +
                28 * s[1] * x[1]^6 +
                210 * x[1]^4 * s[1]^2 +
                420 * x[1]^2 * s[1]^3 +
                105 * s[1]^4
        ) +
            a[2] * (
            x[2]^8 +
                28 * s[2] * x[2]^6 +
                210 * x[2]^4 * s[2]^2 +
                420 * x[2]^2 * s[2]^3 +
                105 * s[2]^4
        ) +
            a[3] * (
            x[3]^8 +
                28 * s[3] * x[3]^6 +
                210 * x[3]^4 * s[3]^2 +
                420 * x[3]^2 * s[3]^3 +
                105 * s[3]^4
        )
        f = [f0, f1, f2, f3, f4, f5, f6, f7, f8]

        S₃ = SymmetricGroup(3)
        relabeling(v) = map(p -> (v[1:3][p]..., v[4:6][p]..., v[7:9][p]...), S₃)

        @polyvar p[1:9]

        function sample_moments(_ = nothing)
            a₀ = rand(3)
            a₀ ./= sum(a₀)
            x₀ = randn(3)
            s₀ = rand(3) .* 2

            y₀ = [a₀; x₀; s₀]
            p₀ = [fᵢ(a => a₀, x => x₀, s => s₀) for fᵢ in f]
            y₀, p₀
        end

        y₀, p₀ = sample_moments()

        R = monodromy_solve(
            f - p,
            y₀,
            p₀;
            parameters = p,
            group_action = relabeling,
            parameter_sampler = last ∘ sample_moments,
            show_progress = false,
            max_loops_no_progress = 5,
        )
        @test length(solutions(R)) ≤ 225

        R = monodromy_solve(
            f - p,
            y₀,
            p₀;
            parameters = p,
            group_action = relabeling,
            target_solutions_count = 225,
            affine_tracking = true,
            parameter_sampler = last ∘ sample_moments,
            show_progress = false,
            max_loops_no_progress = 20,
        )
        @test length(solutions(R)) == 225
    end

    @testset "Trace Test" begin

        @testset "Simple" begin
            @polyvar x y a b c
            f = x^2 + y^2 - 1
            l = a * x + b * y + c
            res = monodromy_solve(
                [f, l],
                [-0.6 - 0.8im, -1.2 + 0.4im],
                [1, 2, 3];
                parameters = [a, b, c],
            )
            @test verify_solution_completeness([f, l], res; parameters = [a, b, c])
        end

        @testset "ED Example" begin
            Random.seed!(1222)
            # define f
            @polyvar x y
            f = (x^4 + y^4 - 1) * (x^2 + y^2 - 2) + x^5 * y
            # define new variables u₁, u₂ and λ₁
            @polyvar u[1:2] λ[1:1]
            # define the jacobian of F
            J = differentiate([f], [x, y])
            # J' defines the transpose of J
            C = [[x, y] - u - J' * λ; f]

            # create a random linear space
            l = sum(randn(ComplexF64, 3) .* [x, y, 1])
            # compute intersection points of the V(f) and V(l) and take the first solution
            s₀ = first(solutions(solve([f, l])))
            # The normal space is now generated by
            J_s₀ = [p([x, y] => s₀) for p in J]
            # create normal vector
            λ₀ = cis(2π * rand()) / norm(J_s₀)
            # x₀ - u₀ = J_s₀' * λ₀ => u = x -  J_s₀' * λ₀
            u₀ = s₀ - transpose(J_s₀) * λ₀ |> vec

            # This is not as realiable as I would like it to be, so just make a couple runs...
            is_complete = false
            for i = 1:5
                res = monodromy_solve(C, [s₀; λ₀], u₀; parameters = u)
                if verify_solution_completeness(C, res; parameters = u)
                    is_complete = true
                    break
                end
            end
            @test is_complete
        end
    end

    @testset "monodromy - automatic start pair" begin
        @polyvar x[1:4]
        # declare the variables for the 3 points
        @polyvar p[1:3, 1:3]
        p₁, p₂, p₃ = p[:, 1], p[:, 2], p[:, 3]
        quartic_monomials = monomials(sum(x)^4)
        @polyvar c[1:35]
        f = sum(c .* quartic_monomials)
        ∇f = differentiate(f, x)
        @polyvar u v
        F = [
            subs(f, x => [p₁; 1])
            subs(f, x => [p₂; 1])
            subs(f, x => [p₃; 1])
            (subs(∇f, x => [p₁; 1]) - u * subs(∇f, x => [p₂; 1]))
            (subs(∇f, x => [p₁; 1]) - v * subs(∇f, x => [p₃; 1]))
        ]

        x₀, c₀ = HC.find_start_pair(F, c)
        @test norm([f([p₁; p₂; p₃; u; v] => x₀, c => c₀) for f in F]) < 1e-8

        F1 = [subs(f, c[1] => 0.232) for f in F]
        x₀, c₀ = HC.find_start_pair(F1, c[2:end])
        @test norm([f([p₁; p₂; p₃; u; v] => x₀, c[2:end] => c₀) for f in F1]) < 1e-8

        # only use a small target_solutions_count, this is just to test that we actually
        # compute something
        res = monodromy_solve(F; parameters = c, target_solutions_count = 50)
        @test is_success(res)

        # Example where we cannot compute a start pair naively.
        @polyvar x y
        f = (x^4 + y^4 - 1) * (x^2 + y^2 - 2) + x^5 * y
        # define new variables u₁, u₂ and λ₁
        @polyvar u[1:2] λ[1:1]
        # define the jacobian of F
        J = differentiate([f], [x, y])
        # J' defines the transpose of J
        C = [[x, y] - u - J' * λ; f]

        @test_throws ArgumentError monodromy_solve(C; parameters = u)
        @test_throws ArgumentError HC.find_start_pair(C, u)
    end

    @testset "projective" begin
        @polyvar x y z a b c
        f = x^2 + y^2 - z^2
        l = a * x + b * y + c * z

        v = PVector([-0.6 - 0.8im, -1.2 + 0.4im, 1])
        res = monodromy_solve(
            [f, l],
            v,
            [1, 2, 3];
            parameters = [a, b, c], max_loops_no_progress = 20,
        )
        @test is_heuristic_stop(res)
        @test nsolutions(res) == 2
        @test isempty(real_solutions(res))

        @polyvar x y u v a b
        f = [x * y - a * u * v, x^2 - b * u^2]
        res = monodromy_solve(
            f,
            PVector([√2, 1], [√2, 1]),
            [2, 2];
            variable_groups = [[x, u], [y, v]],
            parameters = [a, b],
            max_loops_no_progress = 20,
        )
        @test is_heuristic_stop(res)
        @test nsolutions(res) == 2
        @test nreal(res) == 2
        @test sum(fubini_study.(real_solutions(res), solutions(res))) ≈ 0.0 atol = 1e-6
    end

    @testset "monodromy group" begin
        @polyvar x[1:2]  a b c
        p₁ = x - [2;0]
        p₂ = x - [-2;0]
        c₁ = p₁[1]^2 + p₁[2]^2 - 1
        c₂ = p₂[1]^2 + p₂[2]^2 - 1

        F = [c₁ * c₂; a * x[1] + b * x[2] - c]
        S = monodromy_solve(F, [[1.0, 0.0]], [1, 1, 1], parameters = [a;b;c])
        A = permutations(S)
        B = permutations(S, reduced = false)

        @test size(A) == (2,2)
        @test A == [1 2; 2 1] || A == [2 1; 1 2]
        @test size(B, 1) == 2
        @test size(B, 2) > 2

        F₀ = [c₁ * c₂; x[1] + x[2] - 1]
        start_sols=solutions(solve(F₀))
        S = monodromy_solve(F, start_sols, [1, 1, 1], parameters = [a;b;c])
        C = permutations(S)

        @test size(C, 1) == 4
    end

    @testset "monodromy group Fano problem" begin
        # Problem from Thomas Yahl
        k = 1; #dim of linear spaces
        m = 6; #ambient dim
        n = [2,2,3]; #hypersurface degrees
        r = length(n);
        np = sum(binomial.(n.+m,m));

        #hypersurface equations
        @polyvar x[1:m+1] t[1:k+1] y[1:(m-k)*(k+1)] p[1:np];
        monoms = [reshape(monomials(x,[s]),(1,binomial(s+m,m))) for s in n];
        splits = vcat([0],[sum(length.(monoms[1:i])) for i=1:r]);
        G = [(monoms[i]*p[splits[i]+1:splits[i+1]])[1] for i=1:r];

        #linear space substitution
        ℓ = vcat([sum(y[(k+1)*(i-1)+1:(k+1)*i].*t) for i=1:m-k],t);
        F̂ = subs(G,x=>ℓ).+0*prod(p);

        # Compute all monomials of a given degree d
        function segre(x, d)
            if d == 1
                return x
            else
                return unique(kron(x,segre(x,d-1)))
            end
        end

        F = vcat(map(1:3) do i
        	d = n[i]
        	g = F̂[i]
        	[coefficient(g, v, t) for v in segre(t,d)]
        end...)
        F = [subs(f, t=>zeros(2)) for f in F] # annoying bug in MVP

        #base system
        startSol = randn(ComplexF64,(m-k)*(k+1));
        params = (nullspace(transpose(hcat(map(f->coefficient.(f,p),subs.(F,y=>startSol))...)))*randn(ComplexF64,np-(m-k)*(k+1),1))[:,1];
        #subs.(F,y=>startSol,p=>params) #testing base system and solution

        #monodromy_solve
        F̂ = subs.(F,p=>params);
        Ĝ = subs.(F,p=>randn(ComplexF64,np));
        @polyvar τ
        H = @. (1-τ)*F̂+τ*Ĝ;
        S = monodromy_solve(H,startSol,[0.0+0.0im];parameters = [τ], target_solutions_count = 720)

        #permutations
        T = permutations(S)
        @test size(T,1) == 720
    end
end


######## Stan diagnose example  ###########

using StanSample, Test

ProjDir = @__DIR__

gq = "
  data {
    int<lower=0> N;
    int<lower=0> y[N];
  }
  parameters {
    real<lower=0.00001> Theta;
  }
  model {
    Theta ~ gamma(4, 1000);
    for (n in 1:N)
      y[n] ~ exponential(Theta);
  }
  generated quantities{
    real y_pred;
    y_pred = exponential_rng(Theta);
  }
";

gq_data = Dict(
  "N" => 3,
  "y" => [100, 950, 450]
);

stanmodel = SampleModel("generate_quantities", gq);

rc = stan_sample(stanmodel; data=gq_data)

if success(rc)
  # Convert to an MCMCChains.Chains object
  samples = read_samples(stanmodel)

  # Show the same output in DataFrame format
  df = StanSample.read_summary(stanmodel)

  df1 = stan_generate_quantities(stanmodel, 1)
  @test sum(df1[!, :y_pred])/size(df1, 1) ≈ 412.0 rtol=0.3
end


"""
    LD(A::Diagonal)
    LD(A::HBlikDiag)
    LD(A::DenseMatrix)

Return the value of `log(det(triu(A)))` calculated in place.
"""
LD(d::Diagonal) = sum(log, d.diag)

function LD(d::HBlkDiag)
    aa = d.arr
    p, q, k = size(aa)
    pq = p * q
    dd = diagind(p, q)
    r = sum(i -> log(aa[i]), dd)
    for j in 1:(k - 1)
        r += sum(i -> log(aa[i]), dd + j * pq)
    end
    r
end

LD(d::DenseMatrix) = sum(i -> log(d[i]), diagind(d))

"""
    logdet(m::LinearMixedModel)

Return the value of `log(det(Λ'Z'ZΛ + I))` calculated in place.
"""
function Base.logdet{T}(m::LinearMixedModel{T})
    k, R = length(m.Λ), m.R
    res = zero(T)
    for i in 1 : length(m.Λ)
        res += T(LD(R[i, i]))
    end
    2 * res
end


@testset "Genus representatives" begin
  Qx, x = PolynomialRing(FlintQQ, "x", cached = false)
  f = x-1;
  K, a = number_field(f)
  D = matrix(K, 3, 3, [-717210130, 0, 0, 0, -55, 0, 0, 0, -1173298395869600]);
  gens = [[1, 0, 0], [1, 0, 0], [4//5, 2//5, 0], [4//5, 2//5, 0], [1493459//5517001, 0, 1//15778622860], [1493459//5517001, 0, 1//15778622860]]
  L = quadratic_lattice(K, generators = gens, gram_ambient_space = D)
  @test length(genus_representatives(L)) == 6

  Qx, x = PolynomialRing(FlintQQ, "x", cached = false)
  f = x-1;
  K, a = number_field(f)
  D = matrix(K, 3, 3, [-34, 0, 0, 0, -17, 0, 0, 0, -17192032]);
  gens = [[26, 0, 0], [234, 0, 0], [177//17, 2//17, 0], [177//17, 2//17, 0], [27//2, 0, 1//1768], [27//2, 0, 1//1768]]
  L = quadratic_lattice(K, generators = gens, gram_ambient_space = D)
  @test length(genus_representatives(L)) == 93

  Qx, x = PolynomialRing(FlintQQ, "x", cached = false)
  f = x-1;
  K, a = number_field(f)
  D = matrix(K, 3, 3, [-98, 0, 0, 0, -2, 0, 0, 0, -5829824]);
  gens = [[8//7, 0, 0], [8//7, 0, 0], [5//14, 13//2, 0], [45//14, 117//2, 0], [29//28, 3//4, 1//1456], [29//28, 3//4, 1//1456]]
  L = quadratic_lattice(K, generators = gens, gram_ambient_space = D)
  @test length(genus_representatives(L)) == 114

  Qx, x = PolynomialRing(FlintQQ, "x", cached = false)
  f = x^2-2;
  K, a = number_field(f)
  D = matrix(K, 3, 3, [2, 0, 0, 0, -1, 0, 0, 0, -7436]);
  gens = [[13, 0, 0], [156*a+143, 0, 0], [3//2*a+5, 1, 0], [3//2*a+5, 1, 0], [21//2*a, 0, 1//26], [21//2*a, 0, 1//26]]
  L = quadratic_lattice(K, generators = gens, gram_ambient_space = D)
  @test length(genus_representatives(L)) == 1

  Qx, x = PolynomialRing(FlintQQ, "x", cached = false)
  f = x - 1
  K, a = number_field(f)
  M = matrix(QQ, 2, 2, [47, 80, 80, 560])
  w = Int[]
  for i in 1:10
    M[2, 2] = 560 + i
    V = quadratic_space(K, M)
    L = lattice(V, identity_matrix(K, 2))
    g = length(Hecke._genus_representatives_binary_quadratic(L))
    push!(w, g)
  end
  @test w == Int[37, 16, 19, 11, 6, 3, 52, 7, 13, 21]

  Qx, x = PolynomialRing(FlintQQ, "x", cached = false)
  f = x^2 - 2;
  K, a = number_field(f)
  D = matrix(K, 2, 2, [15, 2, 2, 32]);
  gens = [[1, 0], [1, 0], [0, 1], [0, 1]]
  L = quadratic_lattice(K, generators = gens, gram_ambient_space = D)
  @test length(genus_representatives(L)) == 6
end


using GLMakie, DynamicalSystems, InteractiveDynamics
using OrdinaryDiffEq

ds = Systems.henonheiles()
diffeq = (alg = Vern9(),)
u0s = [
    [0.0, -0.25, 0.42081, 0.0],
    [0.0, 0.1, 0.5, 0.0],
    [0.0, -0.31596, 0.354461, 0.0591255]
]
trs = [trajectory(ds, 10000, u0; diffeq...)[:, SVector(1,2,3)] for u0 ∈ u0s]
for i in 2:length(u0s)
    append!(trs[1], trs[i])
end

# Inputs:
j = 2 # the dimension of the plane
tr = trs[1]

brainscan_poincaresos(tr, j)


module Geo3x3
  function encode(lat, lng, level)
    if level < 1
      return null
    end
    res = ""
    if lng >= 0
      res = "E"
    else
      res = "W"
      lng += 180.0
    end
    lat += 90.0 # 180:the North Pole,  0:the South Pole
    unit = 180.0
    for i = 1:level - 1
      unit /= 3
      x = Int(floor(lng / unit))
      y = Int(floor(lat / unit))
      res = string(res, x + y * 3 + 1)
      lng -= x * unit
      lat -= y * unit
    end
    return res
  end

  function decode(code)
    clen = length(code)
    if clen == 0
      return null
    end
    beg = 0
    flg = false
    c = code[1]
    if c == '-' || c == 'W'
      flg = true
      beg = 1
    elseif c == '+' || c == 'E'
      beg = 1
    end
    unit = 180.0
    lat = 0.0
    lng = 0.0
    level = 1
    for i = beg:clen - 1
      n = Int(code[i + 1]) - 48
      if n < 1 || n > 9
        break
      end
      unit /= 3.0
      n -= 1
      lng += n % 3 * unit
      lat += floor(n / 3) * unit
      level += 1
    end
    lat += unit / 2.0
    lng += unit / 2.0
    lat -= 90.0
    if flg
      lng -= 180.0
    end
    return (lat, lng, level, unit)
  end
end


"""
	disjp : example from Bazinga.jl

	Mathematical program with disjunctive constraints, from [Meh19].

	Original formulations:

	minimize      f(x)
	subject to    x ∈ X1 ∪ X2
    with          f(x) := (x1 - 1)^2 + (x2 - 2)^2 + (x3 + 2)^2
                  X1 := { (x1,x2,x3) | x1 ≥ 4, x1 + (x2 - 2)^2 + (x3 + 2)^2 ≥ 5 }
                  X2 := { (x1,x2,x3) | x1^2 + x2^2 ≤ x3, (x1 - 1)^2 + x2^2 + x3 ≥ 1, x2 ≤ 0 }

	Reformulation as a constrained structured problem in the form

	minimize     f(x)
	subject to   c(x) in S

	where
	    c(x) := [ x1 - 4                           ]
       	        [ x1 + (x2 - 2)^2 + (x3 + 2)^2 - 5 ]
   	            [ x3 - x1^2 - x2^2                 ]
       	        [ (x1 - 1)^2 + x2^2 + x3 - 1       ]
                [ - x2                             ]
	    S    := { (c12,c345) | c12 ≥ 0  or  c345 ≥ 0 }

    References:
    [Meh19]     Mehlitz, "A comparison of first-order methods for the numerical
                solution of or-constrained optimization problems" (2019).
                ....
                arXiv:1905.01893v1 [math.OC] 6 May 2019
"""

using Bazinga, OptiMo
using Random, LinearAlgebra
using Printf

###################################################################################
# problem definition
###################################################################################
mutable struct DISJP <: AbstractOptiModel
  meta::OptiModelMeta
end

function DISJP()
    name = "DisjProgExample"
    nvar = 3
    ncon = 5
    meta = OptiModelMeta( nvar, ncon, x0=ones(Float64,nvar), name=name)
    return DISJP( meta )
end

# necessary methods:
# obj, grad!: cons!, jprod!, jtprod!, proj!, prox!, objprox!
function OptiMo.obj( prob::DISJP, x::AbstractVector )
    OptiMo.@lencheck prob.meta.nvar x
    return (x[1] - 1)^2 + (x[2] - 2)^2 + (x[3] + 2)^2
end

function OptiMo.grad!( prob::DISJP, x::AbstractVector, dfx::AbstractVector )
    OptiMo.@lencheck prob.meta.nvar x
    dfx .= [2*(x[1] - 1); 2*(x[2] - 2); 2*(x[3] + 2)]
    return nothing
end

function OptiMo.cons!( prob::DISJP, x::AbstractVector, cx::AbstractVector )
    OptiMo.@lencheck prob.meta.nvar x
    OptiMo.@lencheck prob.meta.ncon cx
    cx .= [ x[1] - 4;
            x[1] + (x[2] - 2)^2 + (x[3] + 2)^2 - 5;
            x[3] - x[1]^2 - x[2]^2;
            (x[1] - 1)^2 + x[2]^2 + x[3] - 1;
            - x[2] ]
    return nothing
end

function OptiMo.jprod!( prob::DISJP, x::AbstractVector, v::AbstractVector, Jv::AbstractVector )
    OptiMo.@lencheck prob.meta.nvar x v
    OptiMo.@lencheck prob.meta.ncon Jv
    Jv .= [ v[1];
            v[1] + 2*(x[2]-2)*v[2] + 2*(x[3]+2)*v[3];
            -2*x[1]*v[1] - 2*x[2]*v[2] + v[3];
            2*(x[1]-1)*v[1] + 2*x[2]*v[2] + v[3];
            -v[2] ]
    return nothing
end

function OptiMo.jtprod!( prob::DISJP, x::AbstractVector, v::AbstractVector, Jtv::AbstractVector )
    OptiMo.@lencheck prob.meta.nvar x Jtv
    OptiMo.@lencheck prob.meta.ncon v
    Jtv .= [ v[1] + v[2] - 2*x[1]*v[3];
             2*(x[2]-2)*v[2] - 2*x[2]*v[3] + 2*x[2]*v[4] - v[5];
             2*(x[3]+2)*v[2] + v[3] + v[4] ]
    return nothing
end

function OptiMo.prox!( prob::DISJP, x::AbstractVector, a::Real, z::AbstractVector )
    OptiMo.@lencheck prob.meta.nvar x z
    z .= x
    return nothing
end

function OptiMo.objprox!( prob::DISJP, x::AbstractVector, a::Real, z::AbstractVector )
    OptiMo.@lencheck prob.meta.nvar x z
    z .= x
    return 0.0
end

function OptiMo.proj!( prob::DISJP, cx::AbstractVector, px::AbstractVector )
    OptiMo.@lencheck prob.meta.ncon cx px
    px .= cx
    if any(cx[1:2] .< 0) && any(cx[3:5] .< 0)
        px .= max.(cx, 0)
        if norm(px[1:2]-cx[1:2], 2) < norm(px[3:5]-cx[3:5], 2)
            px[3:5] .= cx[3:5]
        else
            px[1:2] .= cx[1:2]
        end
    end
    return nothing
end

###################################################################################
# problem build
###################################################################################
problem = DISJP()

###################################################################################
# solver build
###################################################################################
solver = Bazinga.ALPX( max_iter=50,
                       max_sub_iter=1000,
                       tol_optim=1e-6,
                       tol_cviol=1e-8,
                       verbose=true,
                       subsolver=:zerofpr,
                       subsolver_verbose=false )

out = solver( problem )
print( out )
x = out.x;

#=
R = eltype( problem.meta.x0 )
nvar = problem.meta.nvar

global c22 = 0
global c44 = 0
global cun = 0

xmin = -5.0
xmax = +5.0

ntests = 1e+3
atol = 1e-4
ndots = round(Int,cbrt(ntests^2))

for i=1:ntests

    x0 = xmin .+ (xmax - xmin) .* rand(R,nvar);

    out = solver( problem, x0=x0 )

    x = out.x;

    if norm(x - [2; -2], Inf) <= atol
        global c22 += 1
    elseif norm(x - [4; 4], Inf) <= atol
        global c44 += 1
    else
        global cun += 1
        @printf "(%6.4f,%6.4f) from (%6.4f,%6.4f)\n" x[1] x[2] x0[1] x0[2]
    end

    @printf "."
    if mod(i,ndots) == 0
        @printf "\n"
    end

end
=#


##############################################################################
##
## Least Squares via Cholesky Factorization
##
##############################################################################

type CholfactFixedEffectProblem <: FixedEffectProblem
    fes::Vector{FixedEffect}
    m::SparseMatrixCSC{Float64,Int}
    chol::Base.SparseArrays.CHOLMOD.Factor{Float64}
    x::Vector{Float64}
end

function FixedEffectProblem(fes::Vector{FixedEffect}, ::Type{Val{:cholesky}})
    m = sparse(fes)
    chol = cholfact(At_mul_B(m, m))
    total_len = reduce(+, map(fe -> sum(fe.scale .!= 0), fes))
    CholfactFixedEffectProblem(fes, m, chol, Array(Float64, total_len))
end

function solve!(fep::CholfactFixedEffectProblem, r::AbstractVector{Float64} ; kwargs...)
    fep.chol \ At_mul_B!(fep.x, fep.m, r)
end

##############################################################################
##
## Least Squares via QR Factorization
##
##############################################################################

type QRfactFixedEffectProblem <: FixedEffectProblem
    fes::Vector{FixedEffect}
    m::SparseMatrixCSC{Float64,Int}
    qr::Base.SparseArrays.SPQR.Factorization{Float64}
    b::Vector{Float64}
end

function FixedEffectProblem(fes::Vector{FixedEffect}, ::Type{Val{:qr}})
    m = sparse(fes)
    qr = qrfact(m)
    b = Array(Float64, length(fes[1].refs))
    QRfactFixedEffectProblem(fes, m, qr, b)
end

function solve!(fep::QRfactFixedEffectProblem, r::AbstractVector{Float64} ; kwargs...)
    # since \ needs a vector
    copy!(fep.b, r)
    fep.qr \ fep.b
end

##############################################################################
##
## Methods used by all matrix factorization
##
##############################################################################

# construct the sparse matrix of fixed effects A in  A'Ax = A'r
function Base.sparse(fes::Vector{FixedEffect})
    # construct model matrix A constituted by fixed effects
    nobs = length(fes[1].refs)
    N = length(fes) * nobs
    I = Array(Int, N)
    J = similar(I)
    V = Array(Float64, N)
    start = 0
    idx = 0
    for fe in fes
       for i in 1:length(fe.refs)
           idx += 1
           I[idx] = i
           J[idx] = start + fe.refs[i]
           V[idx] = fe.interaction[i] * fe.sqrtw[i]
       end
       start += sum(fe.scale .!= 0)
    end
    sparse(I, J, V)
end

# updates r as the residual of the projection of r on A
function solve_residuals!(fep::Union{CholfactFixedEffectProblem, QRfactFixedEffectProblem}, r::AbstractVector{Float64}; kwargs...)
    x = solve!(fep, r; kwargs...)
    A_mul_B!(-1.0, fep.m, x, 1.0, r)
    return r, 1, true
end

# solves A'Ax = A'r
# transform x from Vector{Float64} (stacked vector of coefficients)
# to Vector{Vector{Float64}} (vector of coefficients for each categorical variable)
function solve_coefficients!(fep::Union{CholfactFixedEffectProblem, QRfactFixedEffectProblem}, r::AbstractVector{Float64}; kwargs...)
    x = solve!(fep, r; kwargs...)
    out = Vector{Float64}[]
    iend = 0
    for fe in get_fes(fep)
        istart = iend + 1
        iend = istart + sum(fe.scale .!= 0) - 1
        push!(out, x[istart:iend])
    end
    return out, 1, true
end

get_fes(fep::Union{CholfactFixedEffectProblem, QRfactFixedEffectProblem}) = fep.fes


# This file is a part of JuliaFEM.
# License is MIT: see https://github.com/JuliaFEM/JuliaFEM.jl/blob/master/LICENSE.md

# Let's drop here all integration schemes and some defaults for different element types maybe parse from txt file ..?

### Gauss quadrature rules for one dimension

function get_integration_points(::Type{Val{1}})
    return [2.0], [0.0]
end

function get_integration_points(::Type{Val{2}})
    return [1.0, 1.0], sqrt(1.0/3.0)*[-1.0, 1.0]
end

function get_integration_points(::Type{Val{3}})
    return 1.0/9.0*[5.0, 8.0, 5.0], sqrt(3.0/5.0)*[-1.0, 0.0, 1.0]
end

function get_integration_points(::Type{Val{4}})
    weights = 1.0/36.0*[
        18.0-sqrt(30.0),
        18.0+sqrt(30.0),
        18.0+sqrt(30.0),
        18.0-sqrt(30.0)]
    points = [
        -sqrt(3.0/7.0 + 2.0/7.0*sqrt(6.0/5.0)),
        -sqrt(3.0/7.0 - 2.0/7.0*sqrt(6.0/5.0)),
         sqrt(3.0/7.0 - 2.0/7.0*sqrt(6.0/5.0)),
         sqrt(3.0/7.0 + 2.0/7.0*sqrt(6.0/5.0))]
    return weights, points
end

function get_integration_points(::Type{Val{5}})
    weights = [
        1.0/900.0*(322.0-13.0*sqrt(70.0)),
        1.0/900.0*(322.0+13.0*sqrt(70.0)),
        128.0/225.0,
        1.0/900.0*(322.0+13.0*sqrt(70.0)),
        1.0/900.0*(322.0-13.0*sqrt(70.0))]
    points = [
        -1.0/3.0*sqrt(5.0 + 2.0*sqrt(10.0/7.0)),
        -1.0/3.0*sqrt(5.0 - 2.0*sqrt(10.0/7.0)),
         0.0,
         1.0/3.0*sqrt(5.0 - 2.0*sqrt(10.0/7.0)),
         1.0/3.0*sqrt(5.0 + 2.0*sqrt(10.0/7.0))]
    return weights, points
end

function get_integration_points(order::Int64)
    if order <= 5
        return get_integration_points(Val{order})
    else
        points, weights = Base.QuadGK.gauss(Float64, order)
        return weights, points
    end
end

### "cartesian" elements, integration rules comes from tensor product

function get_integration_points(element::Poi1)
    [ (1.0, [] ) ]
end

typealias CartesianLineElement Union{Seg2, Seg3, NSeg}
typealias CartesianSurfaceElement Union{Quad4, Quad8, Quad9, NSurf}
typealias CartesianVolumeElement Union{Hex8, Hex20, Hex27, NSolid}

function get_integration_points(element::CartesianLineElement, order::Int64)
    w, xi = get_integration_points(order)
    [ (w[i], [xi[i]]) for i=1:order ]
end

function get_integration_points(element::CartesianSurfaceElement, order::Int64)
    w, xi = get_integration_points(order)
    [ (w[i]*w[j], [xi[i], xi[j]]) for i=1:order, j=1:order ]
end

function get_integration_points(element::CartesianVolumeElement, order::Int64)
    w, xi = get_integration_points(order)
    [ (w[i]*w[j]*w[k], [xi[i], xi[j], xi[k]]) for i=1:order, j=1:order, k=1:order ]
end

### triangular and tetrahedral elements

# http://math2.uncc.edu/~shaodeng/TEACHING/math5172/Lectures/Lect_15.PDF
# http://libmesh.github.io/doxygen/quadrature__gauss__2D_8C_source.html

typealias TriangularElement Union{Tri3, Tri6, Tri7}

function get_integration_points(element::TriangularElement, ::Type{Val{1}})
    weights = [0.5]
    points = Vector{Float64}[1.0/3.0*[1.0, 1.0]]
    return zip(weights, points)
end

function get_integration_points(element::TriangularElement, ::Type{Val{2}})
    weights = 1.0/6.0*[1.0, 1.0, 1.0]
    points = Vector{Float64}[
        [2.0/3.0, 1.0/6.0],
        [1.0/6.0, 2.0/3.0],
        [1.0/6.0, 1.0/6.0]]
    return zip(weights, points)
end

function get_integration_points(element::TriangularElement, ::Type{Val{3}})
    weights = 0.5*[
        -0.5625,
         0.5208333333333333,
         0.5208333333333333,
         0.5208333333333333]
    points = Vector{Float64}[
        [1.0/3.0, 1.0/3.0],
        [0.2, 0.2],
        [0.2, 0.6],
        [0.6, 0.2]]
    return zip(weights, points)
end

function get_integration_points(element::TriangularElement, ::Type{Val{4}})
    weights = 0.5*[
        0.22338158967801,
        0.22338158967801,
        0.22338158967801,
        0.10995174365532,
        0.10995174365532,
        0.10995174365532]
    points = Vector{Float64}[
        [0.44594849091597, 0.44594849091597],
        [0.44594849091597, 0.10810301816807],
        [0.10810301816807, 0.44594849091597],
        [0.09157621350977, 0.09157621350977],
        [0.09157621350977, 0.81684757298046],
        [0.81684757298046, 0.09157621350977]]
    return zip(weights, points)
end

function get_integration_points(element::TriangularElement, ::Type{Val{5}})
    weights = 0.5*[
        0.22500000000000,
        0.13239415278851,
        0.13239415278851,
        0.13239415278851,
        0.12593918054483,
        0.12593918054483,
        0.12593918054483]
    points = Vector{Float64}[
        [0.33333333333333, 0.33333333333333],
        [0.47014206410511, 0.47014206410511],
        [0.47014206410511, 0.05971587178977],
        [0.05971587178977, 0.47014206410511],
        [0.10128650732346, 0.10128650732346],
        [0.10128650732346, 0.79742698535309],
        [0.79742698535309, 0.10128650732346]]
    return zip(weights, points)
end

### 3d elements

typealias TetrahedralElement Union{Tet4, Tet10}

function get_integration_points(element::TetrahedralElement, ::Type{Val{1}})
    weights = 1.0/6.0*[1.0]
    points = Vector{Float64}[
        1.0/4.0*[1.0, 1.0, 1.0]]
    return zip(weights, points)
end

function get_integration_points(element::TetrahedralElement, ::Type{Val{2}})
    a = (5.0+3.0*sqrt(5.0))/20.0
    b = (5.0-sqrt(5.0))/20.0
    w = 1.0/24.0
    weights = [w, w, w, w]
    points = Vector{Float64}[
        [a, b, b],
        [b, a, b],
        [b, b, a],
        [b, b, b]]
    return zip(weights, points)
end

function get_integration_points(element::TetrahedralElement, ::Type{Val{3}})
    a = 1.0/4.0
    b = 1.0/6.0
    c = 1.0/2.0
    weights = [-2.0/15.0, 3.0/40.0, 3.0/40.0, 3.0/40.0, 3.0/40.0]
    points = Vector{Float64}[
        [a, a, a],
        [b, b, b],
        [b, b, c],
        [b, c, b],
        [c, b, b]]
    return zip(weights, points)
end

function get_integration_points(element::TetrahedralElement, ::Type{Val{4}})
    a = 0.25
    b1 = 1.0/34.0*(7.0 + sqrt(15.0))
    b2 = 1.0/34.0*(7.0 - sqrt(15.0))
    c1 = 1.0/34.0*(13.0 - 3.0*sqrt(15.0))
    c2 = 1.0/34.0*(13.0 + 3.0*sqrt(15.0))
    d = 1.0/20.0*(5.0 - sqrt(15.0))
    e = 1.0/20.0*(5.0 + sqrt(15.0))
    w1 = 8.0/405.0
    w2 = (2665.0 - 14.0*sqrt(15.0))/226800.0
    w3 = (2665.0 + 14.0*sqrt(15.0))/226800.0
    w4 = 5.0/567.0
    weights = [w1, w2, w2, w2, w2, w3, w3, w3, w3, w4, w4, w4, w4, w4]
    points = Vector{Float64}[
        [a, a, a],
        [b1, b1, b1],
        [b1, b1, c1],
        [b1, c1, b1],
        [c1, b1, b1],
        [b2, b2, b2],
        [b2, b2, c2],
        [b2, c2, b2],
        [c2, b2, b2],
        [d, d, e],
        [d, e, d],
        [e, d, d],
        [d, e, e],
        [e, d, e],
        [e, e, d]]
    return zip(weights, points)
end

typealias PrismaticElement Union{Wedge6}

function get_integration_points(element::PrismaticElement, ::Type{Val{2}})
    weights = 1/6*[1.0, 1.0, 1.0, 1.0, 1.0, 1.0]
    points = Vector{Float64}[
        [0.5, 0.0, -1.0/sqrt(3)],
        [0.0, 0.5, -1.0/sqrt(3)],
        [0.5, 0.5, -1.0/sqrt(3)],
        [0.5, 0.0,  1.0/sqrt(3)],
        [0.0, 0.5,  1.0/sqrt(3)],
        [0.5, 0.5,  1.0/sqrt(3)]]
    return zip(weights, points)
end

function get_integration_points(element::Union{TriangularElement,
            TetrahedralElement, PrismaticElement}, order::Int64)
    return get_integration_points(element, Val{order})
end

### default number of integration points for each element
### 2 for linear elements, 3 for quadratic

typealias LinearElement Union{Seg2, Tri3, Quad4, Tet4, Wedge6, Hex8}

typealias QuadraticElement Union{Seg3, Tri6, Tri7, Tet10, Quad8, Quad9, Hex20, Hex27}

function get_integration_order(element::LinearElement)
    return 2
end

function get_integration_order(element::QuadraticElement)
    return 3
end

function get_integration_points(element::LinearElement)
    order = get_integration_order(element)
    get_integration_points(element, order)
end

function get_integration_points(element::QuadraticElement)
    order = get_integration_order(element)
    get_integration_points(element, order)
end


using Jets, JetPackTransforms, Test, Wavelets
# warning! This only tests the default Haar wavelet
@testset "dwt" begin
    # 1D transform Haar
    println("1D Haar")
    A = JopDwt(Float64, 128)
    m = rand(domain(A))
    d = rand(range(A))
    lhs, rhs = dot_product_test(A, m, d)
    @test isapprox(lhs, rhs, rtol=1e-7)

    # 2D transform Haar
    println("2D Haar")
    A = JopDwt(JetSpace(Float64, 128, 128))
    m = rand(domain(A))
    d = rand(range(A))
    lhs, rhs = dot_product_test(A, m, d)
    @test isapprox(lhs, rhs, rtol=1e-7)

    # 3D transform HAar
    println("3D Haar")
    A = JopDwt(Float64, 128, 128, 128)
    m = rand(domain(A))
    d = rand(range(A))
    lhs, rhs = dot_product_test(A, m, d)
    @test isapprox(lhs, rhs, rtol=1e-7)

    # db1
    wt = wavelet(WT.db1)
    # 1D transform
    println("1D $(wt.name)")
    A = JopDwt(Float64, 128; wt=wt)
    m = rand(domain(A))
    d = rand(range(A))
    lhs, rhs = dot_product_test(A, m, d)
    @test isapprox(lhs, rhs, rtol=1e-7)

    # 2D transform
    println("2D $(wt.name)")
    A = JopDwt(Float64, 128, 128; wt=wt)
    m = rand(domain(A))
    d = rand(range(A))
    lhs, rhs = dot_product_test(A, m, d)
    @test isapprox(lhs, rhs, rtol=1e-7)

    # 3D transform
    println("3D $(wt.name)")
    A = JopDwt(Float64, 128, 128, 128; wt=wt)
    m = rand(domain(A))
    d = rand(range(A))
    lhs, rhs = dot_product_test(A, m, d)
    @test isapprox(lhs, rhs, rtol=1e-7)

    # db6
    wt = wavelet(WT.db6)
    # 1D transform
    println("1D $(wt.name)")
    A = JopDwt(Float64, 128; wt=wt)
    m = rand(domain(A))
    d = rand(range(A))
    lhs, rhs = dot_product_test(A, m, d)
    @test isapprox(lhs, rhs, rtol=1e-7)

    # 2D transform
    println("2D $(wt.name)")
    A = JopDwt(Float64, 128, 128; wt=wt)
    m = rand(domain(A))
    d = rand(range(A))
    lhs, rhs = dot_product_test(A, m, d)
    @test isapprox(lhs, rhs, rtol=1e-7)

    # 3D transform
    println("3D $(wt.name)")
    A = JopDwt(Float64, 128, 128, 128; wt=wt)
    m = rand(domain(A))
    d = rand(range(A))
    lhs, rhs = dot_product_test(A, m, d)
    @test isapprox(lhs, rhs, rtol=1e-7)

    # db10
    wt = wavelet(WT.db10)
    # 1D transform
    println("1D $(wt.name)")
    A = JopDwt(Float64, 128; wt=wt)
    m = rand(domain(A))
    d = rand(range(A))
    lhs, rhs = dot_product_test(A, m, d)
    @test isapprox(lhs, rhs, rtol=1e-7)

    # 2D transform
    println("2D $(wt.name)")
    A = JopDwt(Float64, 128, 128; wt=wt)
    m = rand(domain(A))
    d = rand(range(A))
    lhs, rhs = dot_product_test(A, m, d)
    @test isapprox(lhs, rhs, rtol=1e-7)

    # 3D transform
    println("3D $(wt.name)")
    A = JopDwt(Float64, 128, 128, 128; wt=wt)
    m = rand(domain(A))
    d = rand(range(A))
    lhs, rhs = dot_product_test(A, m, d)
    @test isapprox(lhs, rhs, rtol=1e-7)
end


"""
    generate_raw_moment_eqs(rn::Union{ReactionSystem, ReactionSystemMod}, m_order::Int; combinatoric_ratelaw = true)

Given a [`ReactionSystem`](https://catalyst.sciml.ai/stable/api/catalyst_api/#ModelingToolkit.ReactionSystem)
or [`ReactionSystemMod`](@ref), return the [`RawMomentEquations`](@ref) of the system generated up to `m_order`.

Notes:
- The expansion order ``q``, denoted by `q_order` throughout the docs, is automatically
  determined from the given polynomial form of the propensity functions, see the
  [tutorial](@ref main_tutorial) and the [theory section](@ref raw_moment_eqs) for
  more details on how `q_order` is obtained.
- `combinatoric_ratelaw=true` uses binomials in calculating the propensity functions
  of a `ReactionSystem`, see the notes for [`ModelingToolkit.jumpratelaw`]
  (https://mtk.sciml.ai/stable/systems/ReactionSystem/#ModelingToolkit.jumpratelaw).
  *Note* that this field is irrelevant using `ReactionSystemMod` as then the
  propensities are defined directly by the user.
"""
function generate_raw_moment_eqs(rn::Union{ReactionSystem,ReactionSystemMod},
                                 m_order::Int; combinatoric_ratelaw = true)

    N = numspecies(rn)
    S = get_S_mat(rn)
    a = propensities(rn, combinatoric_ratelaw = combinatoric_ratelaw)

    term_factors, term_powers, poly_order = polynomial_propensities(a, rn)

    q_order = poly_order + m_order - 1

    # iterator over all moments from lowest to highest moment order
    iter_all = construct_iter_all(N, q_order)
    # iterator over raw moments up to order m
    iter_m = filter(x -> 1 < sum(x) <= m_order, iter_all)
    # iterator over raw moments of order rgrater than m up to q_order
    iter_q = filter(x -> m_order < sum(x) <= q_order, iter_all)
    # iterator over the first order moments
    iter_1 = filter(x -> sum(x) == 1, iter_all)

    μ = define_μ(N, q_order)

    dμ = Dict()
    for i in vcat(iter_1, iter_m)
        dμ[i] = 0
        for r = 1:numreactions(rn)
            iter_j = filter(x -> all(x .<= i) && sum(x) <= sum(i) - 1, iter_all)
            for j in iter_j
                factor_j = 1.0
                for k = 1:N
                    factor_j *= expected_coeff(S[k, r], i[k] - j[k]) * binomial(i[k], j[k])
                end
                suma = 0.0
                for k = 1:length(term_factors[r])
                    suma += term_factors[r][k] * μ[j.+Tuple(term_powers[r][k])]
                end
                dμ[i] += factor_j * suma
            end
        end
        dμ[i] = expand(dμ[i])
    end

    D = Differential(rn.iv)
    eqs = Equation[]
    for i in vcat(iter_1, iter_m)
        push!(eqs, D(μ[i]) ~ dμ[i])
    end

    vars = extract_variables(eqs, N, q_order)
    odes = ODESystem(eqs, rn.iv, vars, rn.ps)

    RawMomentEquations(
        odes,
        μ,
        N,
        m_order,
        q_order,
        iter_all,
        iter_m,
        iter_q,
        iter_1,
    )

end


using GAIO

# the Lorenz system
const σ, ρ, β = 10.0, 28.0, 0.4
v((x,y,z)) = (σ*(y-x), ρ*x-y-x*z, x*y-β*z)
f(x) = rk4_flow_map(v, x)

center, radius = (0,0,25), (30,30,30)
P = BoxPartition(Box(center, radius), (128,128,128))
F = BoxMap(f, P)

x = (sqrt(β*(ρ-1)), sqrt(β*(ρ-1)), ρ-1)         # equilibrium
W = unstable_set!(F, P[x])

plot(W)


### Day 13 - Problem 1
## A Maze of Twisty Little Cubicles
## Author: Thanasis Georgiou

# Cell type
immutable Cell
  x::Int
  y::Int
  weight::Int
end

# Equality check for cells
function Base.isequal(a::Cell, b::Cell)::Bool
  return a.x == b.x && a.y == b.y
end

# Manhattan distance from cell to cell
function man_distance(a::Cell, b::Cell)::Int
  return abs(a.x - b.x) + abs(a.y - b.y)
end

# Manhattan distance from coords to cell
function man_distance(x::Int, y::Int, b::Cell)::Int
  return abs(x - b.x) + abs(y - b.y)
end

createHeuristicCell(x::Int, y::Int, target::Cell) = Cell(x, y, man_distance(x, y, target))

# -1 For wall, anything else is step count
floor = Array(Int, (50, 500))

# Create floor plan
for x = 0:49, y = 0:49
  magic = x * x + 3 * x + 2 * x * y + y + y * y
  magic += 1358
  bitCount = count(bit -> bit == '1', bits(magic))

 floor[x + 1, y + 1] = isodd(bitCount)? -1 : 0
end

# Find optimal path
target = Cell(31 + 1, 39 + 1, 0)
visited = Array(Cell, 0)
queue = Array(Cell, 0)
from = Dict{Cell, Cell}()
path = Array(Cell, 0)

# Add starting position to queue
push!(queue, createHeuristicCell(2, 2, target))

while length(queue) > 0
  # Sort queue
  sort!(queue, lt = (a, b) -> a.weight < b.weight)

  # Grab most promising cell
  cell = queue[1]
  push!(visited, cell)
  deleteat!(queue, 1)
  steps = floor[cell.x, cell.y]

  # Check if we are at the target
  if isequal(cell, target)
    println("Found target, counting steps...")
    # Count steps
    steps = 0
    while (cell = get(from, cell, -1)) != -1
      steps += 1
      push!(path, cell)
    end
    println("Steps: $steps")
    break
  end

  # Find neighbors
  # For each neighbor check it's a wall. If not, create a cell, push to queue
  # add to dictionary
  neighbors = [
    createHeuristicCell(cell.x + 1, cell.y, target),
    createHeuristicCell(cell.x - 1, cell.y, target),
    createHeuristicCell(cell.x, cell.y + 1, target),
    createHeuristicCell(cell.x, cell.y - 1, target),
  ]
  for neighbor in neighbors
    if floor[neighbor.x, neighbor.y] == false
      # Check if we have visited this before
      visitedIndex = findfirst(x -> isequal(x, neighbor), visited)
      if visitedIndex == 0
        push!(queue, neighbor)
        from[neighbor] = cell
        floor[neighbor.x, neighbor.y] = steps + 1
      else
        oldCell = visited[visitedIndex]
        if floor[oldCell.x, oldCell.y] > steps + 1
          deleteat!(visited, visitedIndex)
          from[neighbor] = cell
          floor[neighbor.x, neighbor.y] = steps + 1
        end
      end
    end
  end
end

println("Finished execution")
# Print map
for x = 1:40
  for y = 1:45
    if x == target.x && y == target.y
      print("x")
    elseif findfirst(cell -> cell.x == x && cell.y == y, path) == 0
      print(floor[x, y] == -1? "█" : " ")
    else
      print("o")
    end
  end
  println()
end


struct analysis_lm_Gauss_NIP_Optim
    formula::FormulaTerm
    modelClass::String
    LikelihoodMod::String
    PriorMod::String
    Link::String
    ComputeMethod::String
    res
    fit
    LogLike::Float64
    AIC::Float64
    BIC::Float64
    R_sqr::Float64
    Adjusted_R_sqr::Float64
    sigma::Float64
    fittedResponse
    residuals
    Cooks_distance
end

function linear_reg(formula::FormulaTerm,data::DataFrame)

      modelClass = "LinearReg";
      LikelihoodMod="Gauss";
      PriorMod="NIP";
      Link = "Identity";
      ComputeMethod="Optimization";
      formla = formula;
      formula = apply_schema(formula, schema(formula, data));
      y, X = modelcols(formula, data);
      fm_frame=ModelFrame(formula,data);
      X=modelmatrix(fm_frame);
      p = size(X, 2);
      n = size(X, 1);

      res = lm(formula,data);
      fit = coeftable(res)
      R_sqr = GLM.r2(res);
      Adjusted_R_sqr = GLM.adjr2(res);

      logLike = GLM.loglikelihood(res)
      npar =(p+1);
      AIC = 2*npar - 2*logLike
      BIC = log(n)*npar - 2*logLike

      sse = GLM.deviance(res)
      degrees_of_freedon = GLM.dof_residual(res)
      mse = sse/degrees_of_freedon
      sigma = sqrt(mse)

      fittedResponse = predict(res);
      resid = residuals(res);

      cooks_distance = GLM.StatsBase.cooksdistance(res);
      ans = analysis_lm_Gauss_NIP_Optim(formla,modelClass,LikelihoodMod,PriorMod
      ,Link,ComputeMethod,res,fit, logLike, AIC, BIC, R_sqr
      ,Adjusted_R_sqr,sigma, fittedResponse,resid,cooks_distance)

      return ans

end

function linear_reg_predicts(obj,newdata::DataFrame)

    formula = obj.formula;
    fm_frame=ModelFrame(formula,newdata);
    X=modelmatrix(fm_frame);
    beta = coef(obj.res)
    y_pred = X*beta;
    y_pred
end

function linear_reg(formula::FormulaTerm,data::DataFrame,PriorMod::Prior_Ridge,h::Float64,sim_size::Int64)

    formula = apply_schema(formula, schema(formula, data));
    y, X = modelcols(formula, data);

    @model LinReg(X, y) = begin
      p=size(X, 2);

      #priors
      a0=0.1
      b0=0.1

      v ~ InverseGamma(h,h)
      σ ~ InverseGamma(a0,b0)
      α ~ Normal(0,v*σ)
      β ~ filldist(Normal(0,v*σ), p)

      #likelihood
      y ~ MvNormal(α .+ X * β, σ);
    end;
    LinReg_model=LinReg(X,y);
    chain = sample(CRRao_rng, LinReg_model, NUTS(), sim_size);
    summaries, quantiles = describe(chain);

    ans = MCMC_chain(chain,summaries,quantiles)
    ans
end

function linear_reg(formula::FormulaTerm,data::DataFrame,PriorMod::Prior_Laplace,h::Float64,sim_size::Int64)

    formula = apply_schema(formula, schema(formula, data));
    y, X = modelcols(formula, data);

    @model LinReg(X, y) = begin
      p=size(X, 2);

      #priors
      a0=0.1
      b0=0.1
      v ~ InverseGamma(h,h)
      σ ~ InverseGamma(a0,b0)
      α ~ Laplace(0,σ*v)
      β ~ filldist(Laplace(0,σ*v), p)

      #likelihood
      y ~ MvNormal(α .+ X * β, σ);
    end;
    LinReg_model=LinReg(X,y);
    chain = sample(CRRao_rng, LinReg_model, NUTS(), sim_size);
    summaries, quantiles = describe(chain);

    ans = MCMC_chain(chain,summaries,quantiles)
    ans

end

function linear_reg(formula::FormulaTerm,data::DataFrame,PriorMod::Prior_Cauchy,sim_size::Int64)

    formula = apply_schema(formula, schema(formula, data));
    y, X = modelcols(formula, data);

    @model LinReg(X, y) = begin
      p=size(X, 2);

      #priors
      σ ~ Truncated(TDist(1),0,Inf)
      α ~ TDist(1)*σ
      β ~ filldist(TDist(1)*σ, p)

      #likelihood
      y ~ MvNormal(α .+ X * β, σ);
    end;
    LinReg_model=LinReg(X,y);
    chain = sample(CRRao_rng, LinReg_model, NUTS(), sim_size);
    summaries, quantiles = describe(chain);

    ans = MCMC_chain(chain,summaries,quantiles)
    ans

end

function linear_reg(formula::FormulaTerm,data::DataFrame,PriorMod::Prior_TDist,h::Float64,sim_size::Int64)

    formula = apply_schema(formula, schema(formula, data));
    y, X = modelcols(formula, data);

    @model LinReg(X, y) = begin
      p=size(X, 2);

      #priors
      a0=0.1
      b0=0.1
      ν ~ InverseGamma(h,h)
      σ ~ InverseGamma(a0,b0)
      α ~ TDist(ν)*σ
      β ~ filldist(TDist(ν)*σ, p)

      #likelihood
      y ~ MvNormal(α .+ X * β, σ);
    end;
    LinReg_model=LinReg(X,y);
    chain = sample(CRRao_rng, LinReg_model, NUTS(), sim_size);
    summaries, quantiles = describe(chain);

    ans = MCMC_chain(chain,summaries,quantiles)
    ans

end

function linear_reg(formula::FormulaTerm,data::DataFrame,PriorMod::Prior_Uniform,h::Float64,sim_size::Int64)

    formula = apply_schema(formula, schema(formula, data));
    y, X = modelcols(formula, data);

    @model LinReg(X, y) = begin
      p=size(X, 2);

      #priors
      v=1/h;
      σ ~ Uniform(0,v)
      α ~ Uniform(-v*σ,v*σ)
      β ~ filldist(Uniform(-v,v), predictors)

      #likelihood
      y ~ MvNormal(α .+ X * β, σ);
    end;
    LinReg_model=LinReg(X,y);
    chain = sample(CRRao_rng, LinReg_model, NUTS(), sim_size);
    summaries, quantiles = describe(chain);

    ans = MCMC_chain(chain,summaries,quantiles)
    ans

end


@testset "Solver: Random QP w/ equalities (QDLDL)" begin
    """
        minimize   x' P x + q' x
        subject to    Ax = b
    """

    #####
    n = 10
    m = 3

    nw = n + m
    nθ = 2n + m * n + m

    function obj(z, θ)
        x = z[1:n]

        P = Diagonal(θ[1:n])
        p = θ[n .+ (1:n)]
        return transpose(x) * P * x + transpose(p) * x
    end

    function constraints(z, θ)
        x = z[1:n]
        A = reshape(θ[2n .+ (1:(m * n))], m, n)
        b = θ[2n + m * n .+ (1:m)]

        A * x - b
    end

    function lagrangian(w, θ)
        z = w[1:n]
        y = w[n .+ (1:m)]

        L = 0.0
        L += obj(z, θ)
        L += dot(y, constraints(z, θ))

        return L
    end

    # residual
    function _r(w, θ, κ)
        L = lagrangian(w, θ)
        Lz = Symbolics.gradient(L, w[1:n])
        c = constraints(w[1:n], θ)

        [
            Lz;
            c;
        ]
    end

    @variables z_sym[1:nw]
    @variables θ_sym[1:nθ]
    @variables κ_sym[1:1]

    r_sym = _r(z_sym, θ_sym, κ_sym)
    rf! = eval(Symbolics.build_function(r_sym, z_sym, θ_sym, κ_sym)[2])
    rz_exp = Symbolics.jacobian(r_sym, z_sym)
    rθ_exp = Symbolics.jacobian(r_sym, θ_sym)
    rz_sp = similar(rz_exp, Float64)
    rθ_sp = similar(rθ_exp, Float64)
    rzf! = eval(Symbolics.build_function(rz_exp, z_sym, θ_sym)[2])
    rθf! = eval(Symbolics.build_function(rθ_exp, z_sym, θ_sym)[2])

    x0 = randn(n)
    A = rand(m, n)
    b = A * x0
    z = [randn(n); zeros(m)]
    θ = [ones(n); zeros(n); vec(A); b]
    κ = [1.0]

    r = zeros(nw)
    rz = zeros(nw, nw)
    rθ = zeros(nw, nθ)

    opts = CALIPSO.InteriorPointOptions(diff_sol=true,
        undercut=10.0,
        max_ls=25,
        max_iter=100,
        verbose=false,
        solver=:lu_solver)

    idx = CALIPSO.IndicesOptimization(
        nw, nw,
        [collect(1:0), collect(1:0)],
        [collect(1:0), collect(1:0)],
        Vector{Vector{Int}}[],
        Vector{Vector{Int}}[],
        collect(1:(n + m)),
        Vector{Int}(),
        Vector{Int}(),
        Vector{Int}[],
        Vector{Int}())

    # solver
    ip = CALIPSO.interior_point(z, θ,
        idx=idx,
        r! = rf!, rz! = rzf!, rθ! = rθf!,
        rz = rz,
        rθ = rθ,
        opts = opts)

    status = CALIPSO.interior_point_solve!(ip)
    # @benchmark CALIPSO.interior_point_solve!($ip)

    # test
    @test status
    @test norm(ip.r, Inf) < opts.r_tol
    @test norm(A * ip.z[1:n] - b, Inf) < 1.0e-6
    @test norm(ip.δz, 1) != 0.0
end


using RecipesBase

@recipe function f(c::CircularAperture, npoints = 1000)
    seriestype := :path
    aspect_ratio --> :equal
    seriescolor --> :red
    label --> ""
    t = range(0, 2pi, length = npoints)

    x = @. c.x + c.r * sin(t) + 0.5
    y = @. c.y + c.r * cos(t) + 0.5

    x, y
end

@recipe function f(c::CircularAnnulus, npoints = 1000)
    t = range(0, 2pi, length = npoints)
    seriestype := :path
    seriescolor --> :red
    aspect_ratio --> :equal
    label --> ""

    # outer ring
    @series begin
        x = @. c.x + c.r_out * cos(t) + 0.5
        y = @. c.y + c.r_out * sin(t) + 0.5
        x, y
    end

    # inner ring
    @series begin
        x = @. c.x + c.r_in * cos(t) + 0.5
        y = @. c.y + c.r_in * sin(t) + 0.5
        x, y
    end

end

@recipe function f(e::EllipticalAperture, npoints = 1000)
    seriestype := :path
    seriescolor --> :red
    aspect_ratio --> :equal
    label --> ""

    t = range(0, 2pi, length = npoints)
    x = @. e.x + e.a * cos(t) * cosd(e.theta) - e.b * sin(t) * sind(e.theta) + 0.5
    y = @. e.y + e.a * cos(t) * sind(e.theta) + e.b * sin(t) * cosd(e.theta) + 0.5

    x, y
end

@recipe function f(e::EllipticalAnnulus, npoints = 1000)
    t = range(0, 2pi, length = npoints)
    seriestype := :path
    seriescolor --> :red
    aspect_ratio --> :equal
    label --> ""

    # outer ring
    @series begin
        x = @. e.x + e.a_out * cos(t) * cosd(e.theta) - e.b_out * sin(t) * sind(e.theta) + 0.5
        y = @. e.y + e.a_out * cos(t) * sind(e.theta) + e.b_out * sin(t) * cosd(e.theta) + 0.5
        x, y
    end

    # inner ring
    @series begin
        x = @. e.x + e.a_in * cos(t) * cosd(e.theta) - e.b_in * sin(t) * sind(e.theta) + 0.5
        y = @. e.y + e.a_in * cos(t) * sind(e.theta) + e.b_in * sin(t) * cosd(e.theta) + 0.5
        x, y
    end
end

@recipe function f(ap::RectangularAperture, npoints = 1000)
    seriestype := :path
    seriescolor --> :red
    aspect_ratio --> :equal
    label --> ""

    t = range(0, 2π, length = npoints)
    w2 = ap.w / 2
    h2 = ap.h / 2

    x = @. w2 * (abs(cos(t)) * cos(t) + abs(sin(t)) * sin(t))
    y = @. h2 * (abs(cos(t)) * cos(t) - abs(sin(t)) * sin(t))

    sinth, costh = sincos(deg2rad(ap.theta))
    u = @. ap.x + x * costh - y * sinth + 0.5
    v = @. ap.y + x * sinth + y * costh + 0.5

    u, v
end

@recipe function f(ap::RectangularAnnulus, npoints = 1000)
    seriestype := :path
    seriescolor --> :red
    aspect_ratio --> :equal
    label --> ""

    t = range(0, 2π, length = npoints)

    # outer box
    @series begin
        w2 = ap.w_out / 2
        h2 = ap.h_out / 2

        x = @. w2 * (abs(cos(t)) * cos(t) + abs(sin(t)) * sin(t))
        y = @. h2 * (abs(cos(t)) * cos(t) - abs(sin(t)) * sin(t))

        sinth, costh = sincos(deg2rad(ap.theta))
        u = @. ap.x + x * costh - y * sinth + 0.5
        v = @. ap.y + x * sinth + y * costh + 0.5

        u, v
    end

    # inner box
    @series begin
        w2 = ap.w_in / 2
        h2 = ap.h_in / 2

        x = @. w2 * (abs(cos(t)) * cos(t) + abs(sin(t)) * sin(t))
        y = @. h2 * (abs(cos(t)) * cos(t) - abs(sin(t)) * sin(t))

        sinth, costh = sincos(deg2rad(ap.theta))
        u = @. ap.x + x * costh - y * sinth + 0.5
        v = @. ap.y + x * sinth + y * costh + 0.5

        u, v
    end
end


using JuMP, EAGO

                     m = Model()

                     EAGO.register_eago_operators!(m)

                     @variable(m, -1 <= x[i=1:2] <= 1)

                     @variable(m, -3.892086739821988 <= q <= 1.133112254487742)

                     add_NL_constraint(m, :(log(1 + exp(-0.9215225812660202 + -0.8659264200816277*log(1 + exp(0.9754315885766114 + 0.9182647481159649*$(x[1]) + 0.2963572993024828*$(x[2]))) + -0.7655479829179535*log(1 + exp(0.7516798050016731 + -0.4187259277447679*$(x[1]) + -0.991762240929563*$(x[2]))))) + log(1 + exp(0.9152212659843197 + -0.09622429015475253*log(1 + exp(0.9754315885766114 + 0.9182647481159649*$(x[1]) + 0.2963572993024828*$(x[2]))) + 0.18727818139193309*log(1 + exp(0.7516798050016731 + -0.4187259277447679*$(x[1]) + -0.991762240929563*$(x[2]))))) - $q <= 0.0))

                     @objective(m, Min, q)

                     return m


# Octahedron Stein Distribution
#
# Represents a uniform distribution over an octahedron on the integer lattice
# in three dimensions, i.e., a scaled l1 ball in R^3 intersected with Z^3.
# The vertices of the octahedron are at (0,0,ulim), (0,0,-ulim), (0,ulim,0), etc.

using StatsBase: wsample

type SteinOctahedron <: SteinDistribution
    ulim::Int64
    c1::Array{Float64}
    c2::Array{Float64}
    c3::Array{Float64}
end

SteinOctahedron(ulim::Int64) = SteinOctahedron(
    ulim,
    [1.0 for i=1:3],
    [1.0 for i=1:3],
    [1.0 for i=1:3]
);

# Draw n independent samples from distribution via rejection sampling
function rand(d::SteinOctahedron, n::Int64)
    x = zeros(n,3);
    i = 0;
    while i < n
        y = rand(-d.ulim:d.ulim,3);
        if sum(abs(y)) <= d.ulim
            i += 1;
            x[i,:] = y;
        end
    end
    x
end

# Joint distribution evaluated at a single point (unnormalized)
function jointdistribution(d::SteinOctahedron, y::Array{Float64})
    sum(abs(y)) <= d.ulim ? 1.0 : 0.0
end

# Conditional distribution of jth coordinate of y, given all other coordinates,
# represented as a probability vector
function conddistribution(d::SteinOctahedron, y::Array{Float64,1}, j::Int64)
    conddist = [jointdistribution(d, [y[1:(j-1)], yj, y[(j+1):end]])
                for yj in -d.ulim:d.ulim]
    conddist / sum(conddist)
end

function supportlowerbound(d::SteinOctahedron, j::Int64)
    -d.ulim
end

function supportupperbound(d::SteinOctahedron, j::Int64)
    d.ulim
end

function numdimensions(d::SteinOctahedron)
    3
end


@testset "PositivePoisson" begin
using GLM: linkinv, linkfun, mueta, inverselink

λ0=3.4
pp = PositivePoisson(λ0)
@test params(pp) == (λ0,)
@test rate(pp) == λ0
@test mean(pp) == λ0 / (1-exp(-λ0))

xs = 1:10
@test pdf.(pp,xs) ≈ exp.(logpdf.(pp,xs)) atol=1e-4

xs = 1:100:10000
lp1=[logpdf(Poisson(λ0),x)::Float64 for x=xs]
lp2=[HurdleDMR.logpdf_approx(Poisson(λ0),x)::Float64 for x=xs]
@test lp1 ≈ lp2

lp1=[logpdf(PositivePoisson(λ0),x)::Float64 for x=xs]
lp2=[logpdf_exact(PositivePoisson(λ0),x)::Float64 for x=xs]
@test lp1 ≈ lp2

le1 = HurdleDMR.logexpm1.(xs)
le1big = HurdleDMR.logexpm1.(big.(xs))
le1big2 = broadcast(x->log(exp(x)-one(x)),big.(xs))
@test le1 ≈ le1big
@test le1 ≈ le1big2

ηs=-10:1.0:10
μs=broadcast(η->linkinv(LogProductLogLink(),η),ηs)
ηscheck=broadcast(μ->linkfun(LogProductLogLink(),μ),μs)
@test ηs ≈ ηscheck

μs=broadcast(η->mueta(LogProductLogLink(),η),ηs)
μscheck=broadcast(η->inverselink(LogProductLogLink(),η)[2],ηs)
@test μs ≈ μscheck

loglik(y, μ) = GLM.loglik_obs(PositivePoisson(λ0), y, μ, one(y), 0)

μs=1.01:10.0:1000.0
ηs=broadcast(μ->linkfun(LogProductLogLink(),μ),μs)
μscheck=broadcast(η->linkinv(LogProductLogLink(),η),ηs)
@test μs ≈ μscheck
#verify works for large μ
ys = round.(Int,μs) .+ 1.0
devresids = devresid.(PositivePoisson(λ0), ys, μs)
@test all(isfinite,devresids)
logliks = loglik.(ys, μs)
@test all(isfinite,logliks)

@test isinf(devresid(PositivePoisson(λ0), 3.0, 0.1))
@test iszero(devresid(PositivePoisson(λ0), 0.3, 0.1))

ys = fill(1.0, size(μs))
devresids1 = devresid.(PositivePoisson(λ0), ys, μs)
@test all(isfinite,devresids1)
logliks1 = loglik.(ys, μs)
@test all(isfinite,logliks1)

μsbig = big.(μs)
ηs=broadcast(μ->linkfun(LogProductLogLink(),μ),μsbig)
μscheck=broadcast(η->linkinv(LogProductLogLink(),η),ηs)
@test μsbig ≈ μscheck
#verify works for large μ
ysbig = round.(BigInt,μsbig) .+ 1.0
devresidsbig = devresid.(PositivePoisson(λ0), ysbig, μsbig)
@test all(isfinite,devresidsbig)
@test devresids ≈ Float64.(devresidsbig)

logliksbig = loglik.(ysbig, μsbig)
@test all(isfinite,logliksbig)
@test logliks ≈ Float64.(logliksbig)

ysbig = fill(big"1.0", size(μsbig))
devresidsbig1 = devresid.(PositivePoisson(λ0), ysbig, μsbig)
@test all(isfinite,devresidsbig1)
@test devresids1 ≈ Float64.(devresidsbig1)
logliksbig1 = loglik.(ysbig, μsbig)
@test all(isfinite,logliksbig1)

# μs close to 1.0
μs=big.(collect(1.0+1e-10:1e-10:1.0+1000*1e-10))
ηs=broadcast(μ->linkfun(LogProductLogLink(),μ),μs)
μscheck=broadcast(η->linkinv(LogProductLogLink(),η),ηs)
@test μs ≈ μscheck
#verify works for large μ
ys = round.(BigInt,μs) .+ big"1.0"
devresidsbig = devresid.(PositivePoisson(λ0), ys, μs)
@test all(isfinite,devresidsbig)
logliksbig = loglik.(ys, μs)
@test all(isfinite,logliksbig)

ys = fill(big"1.0", size(μs))
devresidsbig = devresid.(PositivePoisson(λ0), ys, μs)
@test all(isfinite,devresidsbig)
logliks1 = loglik.(ys, μs)
@test all(isfinite,logliks1)

# R benchmark
seed=12
nn=1000
b0=1.0
b1=-2.0
coefs0=[b0,b1]

# # uncomment code to generate R benchmark and save it to csv files
# using RCall
# R"if(!require(VGAM)){install.packages(\"VGAM\");library(VGAM)}"
# R"library(VGAM)"
# R"set.seed($seed)"
# R"pdata <- data.frame(x2 = runif(nn <- $nn))"
# R"pdata <- transform(pdata, lambda = exp($b0 + $b1 * x2))"
# R"pdata <- transform(pdata, y1 = rpospois(nn, lambda))"
# R"with(pdata, table(y1))"
# R"fit <- vglm(y1 ~ x2, pospoisson, data = pdata, trace = TRUE, crit = \"coef\")"
# print(R"summary(fit)")
# pdata=rcopy("pdata")
# coefsR=vec(rcopy(R"coef(fit, matrix = TRUE)"))
# coefsRdf = DataFrame(intercept=[coefsR[1]],x2=[coefsR[2]])
# writetable(joinpath(testdir,"data","positive_poisson_pdata.csv"),pdata)
# writetable(joinpath(testdir,"data","positive_poisson_coefsR.csv"),coefsRdf)

# load saved R benchmark
import CSV
pdata=CSV.read(joinpath(testdir,"data","positive_poisson_pdata.csv"))
coefsR=vec(convert(Matrix{Float64},CSV.read(joinpath(testdir,"data","positive_poisson_coefsR.csv"))))

X=convert(Array{Float64,2}, pdata[!,[:x2]])
Xwconst=[ones(size(X,1)) X]
y=convert(Array{Float64,1},pdata.y1)

glmfit = fit(GeneralizedLinearModel,Xwconst,y,PositivePoisson(),LogProductLogLink())

# glmfit = fit(GeneralizedLinearModel,Xwconst,y,PositivePoisson(),LogProductLogLink();convTol=1e-2)
coefsGLM = coef(glmfit)
@test coefsGLM ≈ coefsR rtol=1e-7
@test coefsGLM ≈ coefs0 rtol=0.05

# GammaLassoPath without actualy regularization
glpfit = fit(GammaLassoPath,X,y,PositivePoisson(),LogProductLogLink(); λ=[0.0])
coefsGLP = vec(coef(glpfit))
@test coefsGLP ≈ coefsGLM

# GammaLassoPath doing Lasso
lassofit = fit(GammaLassoPath,X,y,PositivePoisson(),LogProductLogLink(); γ=0.0)
coefsLasso = vec(coef(lassofit;select=MinAICc()))
@test coefsLasso ≈ coefsGLM rtol=0.02
@test coefsLasso ≈ coefs0 rtol=0.05

# GammaLassoPath doing concave regularization
glpfit = fit(GammaLassoPath,X,y,PositivePoisson(),LogProductLogLink(); γ=10.0)
coefsGLP = vec(coef(glpfit;select=MinAICc()))
@test coefsGLP ≈ coefsLasso rtol=0.0002
@test coefsGLP ≈ coefs0 rtol=0.05

# # problematic case 1
# X = [6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 6.693264063357201 5.354611250685761 5.354611250685761 5.354611250685761 5.354611250685761 5.354611250685761 5.354611250685761 5.354611250685761 5.354611250685761 5.354611250685761 5.354611250685761 5.354611250685761 5.354611250685761 5.354611250685761 5.354611250685761 5.354611250685761 5.354611250685761 5.354611250685761 5.354611250685761 5.354611250685761 5.354611250685761 5.354611250685761 5.354611250685761 5.354611250685761 5.354611250685761 5.354611250685761 5.354611250685761 5.354611250685761 5.354611250685761 5.354611250685761 5.354611250685761 5.354611250685761 5.354611250685761 5.354611250685761 5.354611250685761 5.354611250685761 5.354611250685761 5.354611250685761 5.354611250685761 5.354611250685761 5.354611250685761 5.354611250685761 5.354611250685761 4.015958438014321 4.015958438014321 4.015958438014321 4.015958438014321 4.015958438014321 4.015958438014321 4.015958438014321 4.015958438014321 4.015958438014321 2.6773056253428806 2.6773056253428806 2.6773056253428806 2.6773056253428806 1.3386528126714403 1.3386528126714403 1.3386528126714403 1.3386528126714403]'
# y = [1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,2.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,2.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0]
# yalt = [3.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,2.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,2.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0]
# y = [1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,2.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,2.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0]
# y=ones(size(X,1))
# y[end]=1
# problem1fit = fit(LassoPath,X,y,Binomial();verbose=true,naivealgorithm=true)
# problem1fit = fit(GeneralizedLinearModel,X,y,Poisson();verbose=true)
#
# problem1fit = fit(GammaLassoPath,X,y,Poisson();γ=0.0,verbose=true,naivealgorithm=false)
# problem1fit.m.pp
# m=[]
# problem1fit=[]
# (obj,objold,path,f,newcoef,oldcoef,b0,oldb0,b0diff,coefdiff,scratchmu,cd,r,α,curλ)=(0,0,[],0,[],[],0,0,0,[],[],[],[],0,0)
# try
#   problem1fit = fit(GammaLassoPath,X,y,PositivePoisson();γ=0.0,verbose=true,naivealgorithm=false)
# catch e
#   if typeof(e) == Lasso.ConvergenceException
#     (obj,objold,path,f,newcoef,oldcoef,b0,oldb0,b0diff,coefdiff,scratchmu,cd,r,α,curλ) = e.debugvars
#     m = path.m
#     error(e.msg)
#     nothing
#   else
#     e
#   end
# end
# coef(problem1fit)
# newcoef==oldcoef
# b0diff
# coefdiff
# newcoef
# oldcoef
# newcoef-oldcoef
# scratchmu
# m.rr.devresid
# sum(m.rr.devresid)
# println(m.rr.var)
# cd
# m.rr.mu
# m.pp.oldy
# deviance(m)
# fieldnames(path.m.pp)
# curλ
# b0=-3.67186085318965
# newcoef.coef[1]=-0.08566741748972534
#
# rnew = deepcopy(m.rr)
# pnew = deepcopy(m.pp)
# newmu = Lasso.linpred!(scratchmu, pnew, newcoef, b0)
# updateμ!(rnew, newmu)
# devnew = deviance(rnew)
# curλ*Lasso.P(α, newcoef, pnew.ω)
# objnew = devnew/2 + curλ*Lasso.P(α, newcoef, pnew.ω)
#
# rold = deepcopy(m.rr)
# pold = deepcopy(m.pp)
# oldmu = Lasso.linpred!(scratchmu, pold, oldcoef, oldb0)
# updateμ!(rold, oldmu)
# devold = deviance(rold)
# objold = devold/2 + curλ*Lasso.P(α, oldcoef, pold.ω)
#
# devnew==devold
# curλ*Lasso.P(α, oldcoef, pold.ω) > curλ*Lasso.P(α, newcoef, pnew.ω)
# objnew > objold
# # step-halving failure zoomin
# function devf(b0,newcoef,m,cd,f)
#   T = eltype(y)
#   m=deepcopy(m)
#   newcoef=deepcopy(newcoef)
#   b0=deepcopy(b0)
#   cd=deepcopy(cd)
#   r=m.rr
#   p=m.pp
#   for icoef = 1:nnz(newcoef)
#       oldcoefval = icoef > nnz(oldcoef) ? zero(T) : oldcoef.coef[icoef]
#       newcoef.coef[icoef] = oldcoefval+f*(coefdiff.coef[icoef])
#   end
#   b0 = oldb0+f*b0diff
#   updateμ!(r, Lasso.linpred!(scratchmu, cd, newcoef, b0))
#   dev = deviance(r)
#   curλ*Lasso.P(α, newcoef, cd.ω)
#   obj = dev/2 + curλ*Lasso.P(α, newcoef, cd.ω)
# end
# using Gadfly
# Δf=f*1
# fs=-100f:Δf:100f
# devfs=broadcast(f->devf(b0,newcoef,m,cd,f),fs)
# plot(layer(x=fs,y=devfs,Geom.line),layer(x=[0],y=[objold],Geom.point))
# r.var
# #
#
# fieldnames(newcoef)
# size(newcoef)
#
# # problematic case 2
#
# using Lasso, StatsBase
# X=[3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 3.461994189597978 2.7695953516783822 2.7695953516783822 2.7695953516783822 2.7695953516783822 2.7695953516783822 2.7695953516783822 2.7695953516783822 2.7695953516783822 2.7695953516783822 2.7695953516783822 2.7695953516783822 2.7695953516783822 2.7695953516783822 2.7695953516783822 2.7695953516783822 2.7695953516783822 2.7695953516783822 2.7695953516783822 2.7695953516783822 2.7695953516783822 2.7695953516783822 2.7695953516783822 2.7695953516783822 2.7695953516783822 2.7695953516783822 2.7695953516783822 2.7695953516783822 2.7695953516783822 2.7695953516783822 2.7695953516783822 2.0771965137587864 2.0771965137587864 2.0771965137587864 2.0771965137587864 2.0771965137587864 2.0771965137587864 2.0771965137587864 1.3847976758391911 1.3847976758391911 1.3847976758391911 1.3847976758391911 1.3847976758391911 1.3847976758391911 1.3847976758391911 1.3847976758391911 1.3847976758391911 1.3847976758391911 1.3847976758391911 1.3847976758391911 1.3847976758391911 1.3847976758391911 1.3847976758391911 1.3847976758391911 1.3847976758391911 1.3847976758391911 0.6923988379195956 0.6923988379195956 0.6923988379195956 0.6923988379195956 0.6923988379195956 0.6923988379195956 0.6923988379195956 0.6923988379195956 0.6923988379195956 0.6923988379195956 0.6923988379195956 0.6923988379195956 0.6923988379195956 0.6923988379195956 0.6923988379195956 0.6923988379195956 0.6923988379195956 0.6923988379195956 0.6923988379195956 0.6923988379195956 0.6923988379195956 0.6923988379195956 0.6923988379195956 0.6923988379195956 0.6923988379195956]'
# Xwconst=[ones(X) X]
# y=[1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,2.0,2.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,2.0,1.0,2.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,2.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,2.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0]
# sum(y.!=1.0)
# summarystats(y)
# std(y)
# problem2fit = fit(GeneralizedLinearModel,Xwconst,y,Poisson();verbose=true)
# problem2fit = fit(GeneralizedLinearModel,ones(X),y,Poisson();verbose=true)
# problem2fit = fit(LassoPath,X,y,Poisson();verbose=true,λ=[0.0])
# problem2fit = fit(LassoPath,X,y,Poisson();verbose=true,λ=[0.0],dofit=false)
# problem2fit = fit(LassoPath,X,y,Poisson();verbose=true)
# intercept=true
# n=length(y)
# T=eltype(X)
# d=Poisson()
# l=canonicallink(d)
# wts=ones(T, length(y))
# wts .*= convert(T, 1/sum(wts))
# offset=similar(y, 0)
# irls_tol =1e-7
# nullmodel = fit(GeneralizedLinearModel, ones(T, n, ifelse(intercept, 1, 0)), y, d, l;
#                     wts=wts, offset=offset, convTol=irls_tol)
# problem2fit = fit(GeneralizedLinearModel,ones(X),y,Poisson();verbose=true,convTol=irls_tol)
# problem2fit = fit(GeneralizedLinearModel,ones(X),y,Poisson();verbose=true,convTol=irls_tol,dofit=false)
#
# problem2fit = fit(GammaLassoPath,X,y,Poisson();γ=0.0,verbose=true)
# problem2fit = fit(LassoPath,ones(X),y,Poisson();verbose=true,intercept=false)

end


function predict_mu(gp::GPE, X::AbstractMatrix, cK::AbstractMatrix)
    n = size(X, 2)
    mu = mean(gp.mean, X) + cK*gp.alpha        # Predictive mean
    return mu
end

""" Copy of a Gaussian process that shares X and cK but allows
    modification of the outcomes Y. This is useful for bootstrapping.
"""
function modifiable(gp::GPE)
    gp_copy = GPE(gp.x, copy(gp.y), gp.mean, gp.kernel,
                  gp.logNoise, gp.covstrat, gp.data, gp.cK)
    return gp_copy
end
function update_alpha!(gp::GPE)
    m = mean(gp.mean,gp.x)
    gp.alpha = gp.cK \ (gp.y - m)
end
function mll(gp::GPE)
    μ = mean(gp.mean,gp.x)
    return -dot((gp.y - μ),gp.alpha)/2.0 - logdet(gp.cK)/2.0 - gp.nobs*log(2π)/2.0 # Marginal log-likelihood
end

function addcov!(cK::AbstractMatrix{Float64}, kernel::Kernel, X::AbstractMatrix{Float64}, data::KernelData)
    dim, nobs = size(X)
    @assert size(cK, 1) == nobs
    @assert size(cK, 2) == nobs
    @inbounds for j in 1:nobs
        for i in 1:j
            cK[i,j] += cov_ij(kernel, X, X, data, i, j, dim)
            cK[j,i] = cK[i,j]
        end
    end
end
function add_diag!(mat::AbstractMatrix, d::Real)
    n = size(mat, 1)
    if n!=size(mat, 2)
        throw("Matrix is not square")
    end
    @inbounds for i in 1:n
        mat[i,i] += d
    end
    return mat
end
function update_chol!(pd::PDMats.PDMat)
    Σbuffer = mat(pd)
    Σbuffer, chol = make_posdef!(Σbuffer, cholfactors(pd))
    new_pd = wrap_cK(pd, Σbuffer, chol)
    return new_pd
end


# EMalgorithm() currently does not use Jacobians. If profiling shows that most of the time is spent in the M-step rather than teh E-step,
# this might something useful to do.

"""
Generic EM algorithm.

	em(model,data,thetai,L=1000)

Maximization of the likelihood by EM algorithm. The E-step is handled by `estep()`'s
forward-backward algorithm and the M-step is a generic numerical optimization step.
`thetai` is a starting value and `L` a maximum numer of iterations.

"""
function em(model::DynamicDiscreteModel,data,thetai=rand(dim(model)),L=1000,tol=1e-10)
	dx,dy=size(model.mu)
	w=zeros(dx,dy,dx,dy)
	llks=zeros(L)
	theta=copy(thetai)
	coef!(model,theta)

	nonzeroindicies=find(model.m.!=0.0)

	llks[1]=loglikelihood(model,data)
	l=1
	go=true
	while go
		l+=1
		w[:]=0
		estep(model,data,w)
		function ff(xtheta)
			coef!(model,xtheta)
			#w[] has zero at lest wherever model.m has zeros, and maybe more
			#we still must be careful of not calling log(0)
			#this is what nonzeroindicies is for
			res=0.0
			for i in nonzeroindicies
				res+=w[i]*log(model.m[i])
			end
			-res
		end
		ret = Optim.optimize(ff, theta,method=:cg,iterations=L)
		theta[:]=ret.minimum
		coef!(model,theta)
		llks[l]=loglikelihood(model,data)
		go=(abs(llks[l]-llks[l-1])>tol && l<L)
	end
	println()
	println(" estimating dynamic discrete model via EM algorithm...")
	println("  $l iterations, final log-likelihood: $(round(llks[l],4))")
	# plot(llks[1:l])
	theta
end


using StaticArrays
using UniqueVectors
using Bravais
using ExactDiag

using JLD
using Combinatorics

using Test

using SparseArrays
using LinearAlgebra
using Arpack

debug = false

@inferred ExactDiag.exp_2πiη(0//1)
@test ExactDiag.exp_2πiη(0//1) == 1
@test ExactDiag.exp_2πiη(1//2) == -1
@test ExactDiag.exp_2πiη(-1//2) == -1
@test ExactDiag.exp_2πiη(-1//1) == 1

@test ExactDiag.exp_2πiη(-1//1) ≈ exp(-2π * im)
@test ExactDiag.exp_2πiη(1//2) ≈ exp(π * im)
@test ExactDiag.exp_2πiη(1//4) ≈ exp(π/2 * im)
@test ExactDiag.exp_2πiη(1//3) ≈ exp(2π/3 * im)
@test ExactDiag.exp_2πiη(1//3) ≈ exp(2π/3 * im)

@testset "Spin half         " begin include("spin_half.jl") end
@testset "Hubbard           " begin include("hubbard.jl") end
@testset "Abelian           " begin include("abelian.jl") end
@testset "Abelian spin-flip " begin include("abelian-spinflip.jl") end
@testset "Abelian reflection" begin include("abelian-reflection.jl") end
@testset "Time evolution    " begin include("time-evolution.jl") end


a2q(A,X) = giac(:a2q,A,X)
abcuv(a,b,c) = giac(:abcuv,a,b,c)
abcuv(a,b,c,var) = giac(:abcuv,a,b,c,var)
adjoint_matrix(A) = giac(:adjoint_matrix, A)

cot(x::giac) = giac(:cot, x)
sec(x::giac) = giac(:sec, x)
csc(x::giac) = giac(:csc, x)
#sech(x::giac) = giac(:sech, x) # not available in Giac
#csch(x::giac) = giac(:csch, x) # not available in Giac
asec(x::giac) = giac(:asec, x)
acsc(x::giac) = giac(:acsc, x)
acot(x::giac) = giac(:acot, x)

sign(x::giac) = giac(:sign,x)
log(::Irrational{:e}, x::giac) = log(x)
log(base, x::giac)  = giac(:logb, x, base)
log10(x::giac) = log(10, x)
round(x::giac) = giac(:round, x)
ceil(x::giac) = giac(:ceil, x)
floor(x::giac) = giac(:floor, x)
trunc(x::giac) = giac(:trunc, x)
factorial(x::giac) = giac(:factorial, x)
binomial(x::giac, y::giac) = giac(:binomial, x, y)
binomial(x::giac, y) = giac(:binomial, x, y)
binomial(x, y::giac) = giac(:binomial, x, y)
expm1(x::giac) = giac(:EXPM1, x)
log1p(x::giac) = giac(:LNP1, x)
Ei(x::giac) = giac(:Ei, x)
sinint(x::giac) = giac(:Si, x)
cosint(x::giac) = giac(:Ci, x)
Psi(x::giac) = giac(:Psi, x)
Psi(x::giac, y) = giac(:Psi, x, y)
Psi(x, y::giac) = giac(:Psi, x, y)
digamma(x::giac) = Psi(x)
polygamma(n, x::giac) = Psi(x, n)
polygamma(n::giac, x) = Psi(x, n)

erf(x::giac) = giac(:erf, x)
erfc(x::giac) = giac(:erfc, x)
gamma(x::giac) = giac(:Gamma, x) #  Giac/Gamma -> Julia/gamma
beta(x::giac, y::giac) = giac(:Beta, x, y) # Giac/Beta -> Julia/beta
beta(x::giac, y) = giac(:Beta, x, y) # Giac/Beta -> Julia/beta
beta(x, y::giac) = giac(:Beta, x, y) # Giac/Beta -> Julia/beta
zeta(x::giac) = giac(:Zeta, x)
airyai(x::giac) = giac(:Airy_Ai, x) # Giac/Airy_Ai does not work as expected
airybi(x::giac) = giac(:Airy_Bi, x)  # Giac/Airy_Bi does not work as expected
besselj(nu,x::giac) = giac(:BesselJ, nu, x) # Giac/BesselJ -> Julia/besselj
bessely(nu,x::giac) = giac(:BesselY, nu, x) # Giac/BesselY -> Julia/bessely

evaluate(x) = x
evaluatef(x) = x
simplify(x) = x

evaluate(ex::giac) = giac(:eval, ex)
evaluatef(ex::giac) = giac(:evalf, ex)
evaluatef(ex::giac, n::Int) = giac(:evalf, ex, n)
evalf = evaluatef
simplify(ex::giac) = giac(:simplify, ex)
factor(ex::giac) = giac(:factor, ex)
expand(ex::giac) = giac(:expand, ex)
collect(ex::giac) = giac(:collect, ex)
#collect(exs::Array{giac_SYMB}) = giac(:collect, giac(exs))
collect(exs::Array{giac}) = giac(:collect, giac(exs))
factor(ex::giac, sr)= giac(:factor, ex, sr)
diff(ex::giac, vars...) = giac(:derive, ex, vars...)
diff(ex::giac, vars::Array) = giac(:derive, ex, vars)
partfrac(ex) = giac(:partfrac, ex)
partfrac(ex, var) = giac(:partfrac, ex, var)
equal(left, right) = giac(:equal, left, right)
⩦(a,b) = equal(a, b)
right(eq) = giac(:right, eq)
left(eq) = giac(:left, eq)
denominator(ex::giac) = giac(:denom, ex) #Giac/denom -> Julia/denominator
numerator(ex::giac) = giac(:numer, ex) #Giac/numer -> Julia/numerator

subst(ex, var, val) = giac(:subst, ex, var, val)
subst(ex, sp::Pair...) = subst(ex, [v for (v,w) in sp], [w for (v,w) in sp])

limit(ex, var, val) = giac(:limit, ex, var, val)
limit(ex, var_val::Pair) = giac(:limit, ex, var_val[1], var_val[2])
limit(ex, var, val, dir) = giac(:limit, ex, var, val, dir)
limit(ex, var_val::Pair, dir) = giac(:limit, ex, var_val[1], var_val[2], dir)
series(ex, var, val) = giac(:series, ex, var, val)
series(ex, var, val, order) = giac(:series, ex, var, val, order)
series(ex, var_val::Pair) = giac(:series, ex, var_val[1], var_val[2])
series(ex, var_val::Pair, order) = giac(:series, ex, var_val[1], var_val[2], order)
series(ex, var, val, order, dir) = giac(:series, ex, var, val, order, dir)
series(ex, var_val::Pair, order, dir) = giac(:series, ex, var_val[1], var_val[2], order, dir)
sum(ex::giac) = giac(:sum, ex)
sum(ex::giac, x::giac) = giac(:sum, ex, x)
sum(ex::giac, x::giac, a::Union{giac,Number}, b::Union{giac,Number}) = giac(:sum, ex, x, a, b)
sum(ex::giac, x::giac, a::Union{giac,Number}, b::Union{giac,Number}, step::Union{giac,Number}) =
    giac(:sum, ex, x, a, b, step)
curl(exs, vars) = giac(:curl, exs, vars)
divergence(exs, vars) = giac(:divergence, exs, vars)
grad(ex, vars) = giac(:grad, ex, vars)
hessian(ex, vars) = giac(:hessian, ex, vars)
solve(eqs, var) = giac(:solve, eqs, var)
preval(ex, a, b) = giac(:preval, ex, a, b)
preval(ex, a, b, var) = giac(:preval, ex, a, b, var)
sum_riemann(expr, vars) = giac(:sum_riemann, expr, vars)
taylor(ex, var) = giac(:taylor, ex, var)
taylor(ex, var, order) = giac(:taylor, ex, var, order)
taylor(ex, var_val::Pair) = giac(:taylor, ex, var_val[1], var_val[2])
taylor(ex, var_val::Pair, order) = giac(:taylor, ex, var_val[1], order, var_val[2])
divpc(p1, p2, order) = giac(:divpc, p1, p2, order)
solve(eq) = giac(:solve, eq)
cSolve(eq) = giac(:cSolve, eq)
cSolve(eqs, var) = giac(:cSolve, eqs, var)
zeros(ex::giac) = giac(:zeros, ex)
zeros(exs, var) = giac(:zeros, exs, var)
cZeros(eq::giac) = giac(:cZeros, eq)
cZeros(exs, var) = giac(:cZeros, exs, var)
fSolve(eqs, vars, guess) = giac(:fsolve, eqs, vars, guess)
fSolve(eqs, var_guess::Pair) = giac(:fsolve, eqs, var_guess[1], var_guess[2])
fSolve(eqs, vars) = giac(:fsolve, eqs, vars)
fSolve(eqs) = giac(:fsolve, eqs)
deSolve(eq, var) = giac(:desolve, eq, var)
deSolve(eq, timevar, var) = giac(:desolve, eq, timevar, var)
linsolve(eqs, vars) = giac(:linsolve, eqs, vars)

logcollect(ex) = giac(:lncollect, ex)
powexpand(ex) = giac(:powexpand, ex)
texpand(ex) = giac(:texpand, ex)
exp2pow(ex) = giac(:exp2pow, ex)
pow2exp(ex) = giac(:pow2exp, ex)
exp2trig(ex) = giac(:exp2trig, ex)
expexpand(ex) = giac(:expexpand, ex)
asin2acos(ex) = giac(:asin2acos, ex)
asin2atan(ex) = giac(:asin2atan, ex)
sin2costan(ex) = giac(:sin2costan, ex)
acos2asin(ex) = giac(:acos2asin, ex)
acos2atan(ex) = giac(:acos2atan, ex)
cos2sintan(ex) = giac(:cos2sintan, ex)
atan2asin(ex) = giac(:atan2asin, ex)
atan2acos(ex) = giac(:atan2acos, ex)
tan2sincos(ex) = giac(:tan2sincos, ex)
halftan(ex) = giac(:halftan, ex)
trigsin(ex) = giac(:trigsin, ex)
trigcos(ex) = giac(:trigcos, ex)
trigtan(ex) = giac(:trigtan, ex)
atrig2log(ex) = giac(:atrig2ln, ex)
tlin(ex) = giac(:tlin, ex)
tcollect(ex) = giac(:tcollect, ex)
trigexpand(ex) = giac(:trigexpand, ex)
trig2exp(ex) = giac(:trig2exp, ex)

gbasis(polys, vars) = giac(:gbasis, polys, vars)
greduce(poly, basis, vars) = giac(:greduce, poly, basis, vars)

integrate(ex::giac) = giac(:integrate, ex)
integrate(ex::giac, var::giac) = giac(:integrate, ex, var)
integrate(ex::giac, var::giac, a::Union{giac,Number}, b::Union{giac,Number}) =
    giac(:integrate, ex, var, a, b)

head(ex::giac) = giac(:sommet, ex)
args(ex::giac) = giac(:feuille, ex)

# to be continued ...