using Test
using LinearAlgebra
using PolynomialBasis
using ImplicitDomainQuadrature
# using Revise
using CutCell
include("useful_routines.jl")

function exact_linear_solution(x, e11, e22)
    return [x[1] * e11, x[2] * e22]
end

function quadratic_displacement(x)
    u1 = x[1]^2 + 2 * x[1] * x[2]
    u2 = x[2]^2 + 3x[1]
    return [u1, u2]
end

function quadratic_body_force(lambda, mu)
    b1 = -2 * (lambda + 2mu)
    b2 = -(4lambda + 6mu)
    return [b1, b2]
end

function test_full_quadratic_displacement_bc()
    polyorder = 2
    numqp = 3
    nelmts = 11
    L = 2.0
    W = 1.0

    lambda, mu = 1.0, 2.0
    penalty = 1e1

    dx = 0.1
    e11 = dx / L
    e22 = -lambda / (lambda + 2mu) * e11
    dy = e22

    stiffness = CutCell.HookeStiffness(lambda, mu, lambda, mu)

    basis = TensorProductBasis(2, polyorder)
    levelset = InterpolatingPolynomial(1, basis)
    nf = CutCell.number_of_basis_functions(basis)
    mesh = CutCell.Mesh([0.0, 0.0], [L, W], [nelmts, nelmts], nf)

    normal = [1.0, 0.0]
    x0 = [1.0, 0.0]
    levelsetcoeffs =
        CutCell.levelset_coefficients(x -> plane_distance_function(x, normal, x0), mesh)

    cutmesh = CutCell.CutMesh(levelset, levelsetcoeffs, mesh)
    cellquads = CutCell.CellQuadratures(levelset, levelsetcoeffs, cutmesh, numqp)
    interfacequads = CutCell.InterfaceQuadratures(levelset, levelsetcoeffs, cutmesh, numqp)
    facequads = CutCell.FaceQuadratures(levelset, levelsetcoeffs, cutmesh, numqp)

    bilinearforms = CutCell.BilinearForms(basis, cellquads, stiffness, cutmesh)
    interfacecondition =
        CutCell.coherent_interface_condition(basis, interfacequads, stiffness, cutmesh, penalty)

    onbottomboundary(x) = x[2] ≈ 0.0 ? true : false
    onrightboundary(x) = x[1] ≈ L ? true : false
    ontopboundary(x) = x[2] ≈ W ? true : false
    onleftboundary(x) = x[1] ≈ 0.0 ? true : false
    onboundary(x) =
        onbottomboundary(x) || onrightboundary(x) || ontopboundary(x) || onleftboundary(x)

    displacementbc = CutCell.DisplacementCondition(
        x -> quadratic_displacement(x),
        basis,
        facequads,
        stiffness,
        cutmesh,
        onboundary,
        penalty,
    )

    sysmatrix = CutCell.SystemMatrix()
    sysrhs = CutCell.SystemRHS()

    CutCell.assemble_bilinear_form!(sysmatrix, bilinearforms, cutmesh)
    CutCell.assemble_interface_condition!(sysmatrix, interfacecondition, cutmesh)
    CutCell.assemble_body_force_linear_form!(
        sysrhs,
        x -> quadratic_body_force(lambda, mu),
        basis,
        cellquads,
        cutmesh,
    )
    CutCell.assemble_penalty_displacement_bc!(sysmatrix, sysrhs, displacementbc, cutmesh)

    matrix = CutCell.make_sparse(sysmatrix, cutmesh)
    rhs = CutCell.rhs(sysrhs, cutmesh)

    sol = matrix \ rhs
    disp = reshape(sol, 2, :)

    err = mesh_L2_error(disp, x -> quadratic_displacement(x), basis, cellquads, cutmesh)

    @test isapprox(norm(err), 0.0, atol = 1e-13)
end

test_full_quadratic_displacement_bc()


using FANN
using Base.Test

# test simple linear problem for which we know the solution
net = ANN([20, 1], [:linear], b=0.1)

X = randn(20, 5000)
y = mean(X, 1)

dset = DataSet(X, y)

# we just test these work
# ~~~~~~~~~~~~~~~~~~~~~~
#  ~~~ QUICKPROP ~~~~~
setup_qprop!(net; mu=1.7, decay=-0.00001)

# train
train!(net, dset,
	   max_epochs=1000,
	   epochs_between_reports=0,
	   desired_error=1e-8)

# ~~~~~~~~~~~~~~~~~~~~~~
#  ~~~~~ RPROP ~~~~~~~~~
setup_rprop!(net; increase_factor=1.5, decrease_factor=0.9, delta_min=0.0, delta_zero=0.001, delta_max=10.0)

# train
train!(net, dset,
	   max_epochs=1000,
	   epochs_between_reports=0,
	   desired_error=1e-8)

# ~~~~~~~~~~~~~~~~~~~~~~
#  ~~~~~ BATCH ~~~~~~~~~
setup_bprop!(net; batch=true, learning_rate=0.5, learning_momentum=0.1)

# train
train!(net, dset,
	   max_epochs=1000,
	   epochs_between_reports=0,
	   desired_error=1e-8)

# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#  ~~~~~ INCREMENTAL ~~~~~~~~~
setup_bprop!(net; batch=false, learning_rate=0.5, learning_momentum=0.1)

# train
train!(net, dset,
	   max_epochs=1000,
	   epochs_between_reports=0,
	   desired_error=1e-8)


"""
    MeanVariance <: AbstractMeanVariance

Mean Variance Formulation (Markowitz).

PS.: Equivalent to a single point uncertainty set.

Atributes:
- `predicted_mean::Array{Float64,1}`: Predicted mean of returns.
- `predicted_covariance::Array{Float64,2}`: Predicted covariance of returns.
"""
struct MeanVariance <: AbstractMeanVariance
    predicted_mean::Array{Float64,1}
    predicted_covariance::Array{Float64,2}
    number_of_assets::Int
end

function MeanVariance(;
    predicted_mean::Array{Float64,1},
    predicted_covariance::Array{Float64,2},
)
    number_of_assets = size(predicted_mean, 1)
    if number_of_assets != size(predicted_covariance, 1)
        error("size of predicted mean ($number_of_assets) different to size of predicted covariance ($(size(predicted_covariance, 1)))")
    end

    @assert issymmetric(predicted_covariance)
    return MeanVariance(
        predicted_mean, predicted_covariance, number_of_assets
    )
end

"""
    predicted_portfolio_return!(model::JuMP.Model, w, formulation::AbstractMeanVariance)

Return predicted portfolio return used in the formulation.

Arguments:
 - `model::JuMP.Model`: JuMP upper level portfolio optimization model.
 - `w`: Portfolio optimization investment variable ("weights").
"""
function predicted_portfolio_return!(::JuMP.Model, w, formulation::AbstractMeanVariance)
    return sum(formulation.predicted_mean'w)
end

"""
    portfolio_return!(model::JuMP.Model, w, formulation::AbstractMeanVariance; kwargs...)

Return worst case portfolio return of the Uncertainty set defined by the formulation.

Arguments:
 - `model::JuMP.Model`: JuMP upper level portfolio optimization model.
 - `w`: Portfolio optimization investment variable ("weights").
"""
function portfolio_return!(model::JuMP.Model, w, formulation::AbstractMeanVariance; kwargs...)
    return predicted_portfolio_return!(model, w, formulation; kwargs...)
end

"""
    predicted_portfolio_variance!(model::JuMP.Model, w, formulation::AbstractMeanVariance)

Return predicted portfolio variance used in the formulation:

Arguments:
 - `model::JuMP.Model`: JuMP upper level portfolio optimization model.
 - `w`: Portfolio optimization investment variable ("weights").
"""
function predicted_portfolio_variance!(::JuMP.Model, w, formulation::AbstractMeanVariance)
    return sum(w'formulation.predicted_covariance * w)
end

"""
    portfolio_variance!(model::JuMP.Model, w, formulation::AbstractMeanVariance; kwargs...)

Return worst case portfolio variance of the Uncertainty set defined by the formulation.

Arguments:
 - `model::JuMP.Model`: JuMP upper level portfolio optimization model.
 - `w`: Portfolio optimization investment variable ("weights").
"""
function portfolio_variance!(model::JuMP.Model, w, formulation::AbstractMeanVariance; kwargs...)
    return predicted_portfolio_variance!(model, w, formulation; kwargs...)
end

function _po_min_variance_limit_return_latex()
    return """
        ```math
        \\begin{aligned}
        \\min_{w} \\quad & V \\\\
        s.t. \\quad & R = (\\min r'w \\; | \\; r \\in \\Omega) \\\\
        & V = (\\max w ' \\Sigma w  \\; | \\; \\Sigma \\in \\Omega) \\\\
        & R \\geq R * W_0 \\\\
        & w \\in \\mathcal{X} \\\\
        \\end{aligned}
        ```
        """
end

"""
    po_min_variance_limit_return(formulation::AbstractPortfolioFormulation, R)

Mean-Variance Portfolio Allocation (with a risk free asset). Posed as a quadratic convex problem.
Minimizes the worst case portfolio variance (``V``) and limits the worst case portfolio return (``R``) in the uncertainty set (``\\Omega``)
to a minimal return parameter (``R_0``) normalized by current wealth (``W_0``).

$(_po_min_variance_limit_return_latex())

Where ``\\mathcal{X}`` represents the additional constraints defined in the model by the user
(e.g. a limit on maximum invested money).

Arguments:
 - `formulation::AbstractPortfolioFormulation`: Struct containing attributes of the formulation.
 - `minimal_return::Real`: Minimal normalized return accepted.

Optional Keywork Arguments:
 - `rf::Real = 0.0`: Risk-free return of money not invested.
 - `current_wealth::Real = 1.0`: Current available wealth to be invested.
 - `model::JuMP.Model = base_model(...)`: JuMP upper level portfolio optimization model. Defaults to a generic market specification.
 - `w`: Portfolio optimization investment variable reference ("weights").
 - `portfolio_return::Function = portfolio_return!`: Function that calculates ``R``.
 - `portfolio_variance::Function = portfolio_variance!`: Function that calculates ``V``.
"""
function po_min_variance_limit_return(formulation::AbstractPortfolioFormulation, minimal_return::Real;
    rf::Real = 0.0, current_wealth::Real = 1.0,
    model::JuMP.Model = base_model(formulation.number_of_assets; current_wealth=current_wealth),
    w=model[:w],
    portfolio_return::Function = portfolio_return!,
    portfolio_variance::Function = portfolio_variance!
)
    # auxilary variables
    @variable(model, R)
    sum_invested = create_sum_invested_variable(model, w)

    # model
    @constraint(model, R == portfolio_return(model, w, formulation) + rf * (current_wealth - sum_invested))
    @constraint(model, R >= minimal_return * current_wealth)
    @objective(model, Min, portfolio_variance(model, w, formulation))
    return model
end

function _po_max_return_limit_variance_latex()
    return """
        ```math
        \\begin{aligned}
        \\max_{w} \\quad & R \\\\
        s.t. \\quad & R = (\\min r'w \\; | \\; r \\in \\Omega) \\\\
        & V \\leq V_0 * W_0\\\\
        & V = (\\max w ' \\Sigma w  \\; | \\; \\Sigma \\in \\Omega) \\\\
        & w \\in \\mathcal{X} \\\\
        \\end{aligned}
        ```
        """
end

"""
    po_max_return_limit_variance(formulation::AbstractPortfolioFormulation, V_0)

Mean-Variance Portfolio Allocation (with a risk free asset). Posed as a quadratic convex problem.
Maximizes the worst case portfolio return (``R``) and limits the worst case portfolio variance (``V``) in the uncertainty set (``\\Omega``)
to a maximal risk parameter (``V_0``) normalized by current wealth (``W_0``).

$(_po_max_return_limit_variance_latex())

Where ``\\mathcal{X}`` represents the additional constraints defined in the model by the user
(e.g. a limit on maximum invested money).

Arguments:
 - `formulation::AbstractPortfolioFormulation`: Struct containing attributes of formulation.
 - `max_risk::Real`: Maximal normalized variance accepted.

Optional Keywork Arguments:
 - `rf::Real = 0.0`: Risk-free return of money not invested.
 - `current_wealth::Real = 1.0`: Current available wealth to be invested.
 - `model::JuMP.Model = base_model(...)`: JuMP upper level portfolio optimization model. Defaults to a generic market specification.
 - `w`: Portfolio optimization investment variable reference ("weights").
 - `portfolio_return::Function = portfolio_return!`: Function that calculates ``R``.
 - `portfolio_variance::Function = portfolio_variance!`: Function that calculates ``V``.
"""
function po_max_return_limit_variance(formulation::AbstractPortfolioFormulation, max_risk::Real;
    rf::Real = 0.0, current_wealth::Real = 1.0,
    model::JuMP.Model = base_model(formulation.number_of_assets; current_wealth=current_wealth),
    w=model[:w],
    portfolio_return::Function = portfolio_return!,
    portfolio_variance::Function = portfolio_variance!
)
    # auxilary variables
    @variable(model, R)
    sum_invested = create_sum_invested_variable(model, w)

    # model
    @constraint(model, R == portfolio_return(model, w, formulation) + rf * (current_wealth - sum_invested))
    @constraint(model, portfolio_variance(model, w, formulation) <= max_risk * current_wealth)
    @objective(model, Max, R)
    return model
end

###################### Not updated ######################
"""
Mean-Variance Portfolio Allocation With no risk free asset. Analytical solution.
"""
function mean_variance_noRf_analytical(formulation::MeanVariance, R_0)
    # parameters
    r̄ = formulation.predicted_mean
    Σ = formulation.predicted_covariance
    numA = formulation.number_of_assets
    # model
    invΣ = pinv(Σ, 1E-25)
    x = zeros(numA)
    A = sum(invΣ'r̄)
    B = sum(r̄'r̄)
    C = sum(invΣ)
    one = ones(size(r̄, 1))
    D = sum(r̄' * (invΣ'r̄))
    mu = (R_0 * C - A) / (C * D - A)
    x = (1 / C) * (invΣ * one) + mu * invΣ * (r̄ - one * (A / C))
    return x
end


abstract type ELMTypes end
struct Regression <: ELMTypes end
struct Classification <: ELMTypes
    nclass::Int
end

mutable struct ELM{T<:BaseType}
    W::Matrix{T}
    b::Matrix{T}
    beta::Matrix{T}
    L::Int
    mf::MappingFunctions
    elmtype::ELMTypes

    function ELM{T}(L::Int;
                 mf::MappingFunctions=Sigmoid(),
                 elmtype::ELMTypes=Regression()) where T<:BaseType
        net = new()
        net.L = L
        net.mf = mf
        net.elmtype = elmtype
        return net
    end
end

function train!(net::ELM{T}, X::AbstractMatrix{T}, Y::AbstractMatrix{S};
                tm::TrainingMethods=Batch(),
                init::Initializers=Uniform()) where {T<:BaseType, S<:Union{BaseType,Integer}}
    @assert(size(Y,2) == size(X,2))

    net.W = rand_arr(T, (net.L, size(X,1)), init)
    net.b = rand_arr(T, (net.L, 1), init)

    X, Y = preprocess(X, Y, net.elmtype)
    net.beta = train_core(net.W, net.b, X, Y, net.mf, tm)

    return net
end

function predict(net::ELM{T}, X::AbstractMatrix{T}) where {T<:BaseType}
    H = mapping(net.W, net.b, X, net.mf)
    Y = net.beta*H
    return postprocess(Y, net.elmtype)
end

function preprocess(X::AbstractMatrix{T}, Y::AbstractMatrix{T}, elmtype::Regression) where {T<:BaseType}
    return X, Y
end

function postprocess(Y::AbstractMatrix{T}, elmtype::Regression) where {T<:BaseType}
    return Y
end

function preprocess(X::AbstractMatrix{T}, Y::AbstractMatrix{S}, elmtype::Classification) where {T<:BaseType, S<:Integer}
    @assert(maximum(Y) < elmtype.nclass, "Class labels must be in {0,1,...,nclass-1}.")
    if elmtype.nclass == 2
        Yc = zeros(T, 1, size(Y,2))
        for i=1:size(Y,2)
            Yc[1,i] = Y[1,i] > zero(S) ? one(T) : -one(T)
        end
    else
        Yc = zeros(T, elmtype.nclass, size(Y,2))
        for i=1:size(Y,2)
            Yc[Y[1,i]+one(S),i] = one(T)
        end
    end
    return X, Yc
end

function postprocess(Yc::AbstractMatrix{T}, elmtype::Classification) where {T<:BaseType}
    Y = zeros(Int, 1, size(Yc,2))
    if elmtype.nclass == 2
        for i=1:size(Yc,2)
            Y[1,i] = real(Yc[i]) > zero(Int) ? one(Int) : zero(Int)
        end
    else
        for i=1:size(Yc,2)
            Y[1,i] = indmax(real(Yc[:,i])) - one(Int)
        end
    end
    return Y
end


#
# General systems
#

using DifferentialEquations

abstract type AbstractSystem{T} end

"""
    LinearSystem(A::Matrix[, Q::Matrix])
    LinearSystem(A::AbstractFloat[, Q::AbstractFloat])

Linear system of equations. Systems take a specific form depending on the type
of state they are used in conjunction with. With a `DiscreteState`, the system
takes on a discrete form

\$x_k = A x_{k-1} + w_{k-1} \\quad \\text{where} \\quad w_{k-1} \\sim N(0, Q)\$

With a `ContinuousState`, the system takes on a continous ODE form

\$\\dot{x}(t) = A x(t) + w(t)\$

A LinearSystem can be constructed with both matrix and scalar inputs. The
linear system will not contain process noise if the covaraince Q is
not provided.
"""
struct LinearSystem{T<:AbstractFloat} <: AbstractSystem{T}
    A::Matrix{T}
    Q::Covariance{T}

    function LinearSystem(A::Matrix{T}, Q::Covariance{T}) where T
        if size(A) != size(Q)
            throw(DimensionMismatch("Incompatible size of system matrices."))
        end
        if size(A,1) != size(A,2)
            throw(DimensionMismatch("System 'A' matrix must be square."))
        end
        new{T}(A, Q)
    end
end
LinearSystem{T<:AbstractFloat}(A::Matrix{T}) = LinearSystem(A, zeros(T,size(A)))
LinearSystem{T<:AbstractFloat}(A::T) =
    LinearSystem(reshape([A], 1, 1), zeros(T, 1, 1))
LinearSystem{T<:AbstractFloat}(A::T, Q::T) =
    LinearSystem(reshape([A], 1, 1), reshape([Q], 1, 1))

"""
    NonlinearSystem(F::Function, dF_dx::Function, Q::Covariance[, tolerances::Tuple])
    NonlinearSystem(F::Function, dF_dx::Function, Q::AbstractFloat[, tolerances::Tuple])

Nonlinear system of equations. Systems take a specific form depending on the
type of state they are used in conjunction with. With a `DiscreteState`, the
system takes on a discrete form

\$x_k = F(k-1, x_{k-1}) + w_{k-1} \\quad \\text{where} \\quad w_{k-1} \\sim N(0, Q)\$

With a `ContinuousState`, the system takes on a continous ODE form

\$\\dot{x}(t) = F(t, x(t)) + w(t) \\quad \\text{where} \\quad w(t) \\sim N(0, Q)\$

NonlinearSystem constructors require both the function,
`F(t::Number, x::Vector)`, and its Jacobian, `dF_dx(t::Number, x::Vector)`. Both
`F()` and `dF_dx()` must return a vector.

The user may optionally specify the integrator tolerances with a tuple of
`(abstol, reltol)`. The default tolerances are `(1e-2, 1e-2)`.
"""
struct NonlinearSystem{T<:AbstractFloat} <: AbstractSystem{T}
    F::Function
    dF_dx::Function
    Q::Covariance{T}
    abstol::T
    reltol::T

    function NonlinearSystem(F::Function, dF_dx::Function, Q::Covariance{T},
                             predict_tolerances=(1e-3, 1e-3)) where T
        new{T}(F, dF_dx, Q, predict_tolerances[1], predict_tolerances[2])
    end
end
NonlinearSystem(F::Function, dF_dx::Function, Q::AbstractFloat,
    predict_tolerances=(1e-3, 1e-3)) =
    NonlinearSystem(F, dF_dx, reshape([Q], 1, 1), predict_tolerances)
function NonlinearSystem(lin_sys::LinearSystem)
    F(t,x) = lin_sys.A*x
    dF_dx(t,x) = lin_sys.A
    return NonlinearSystem(F, dF_dx, lin_sys.Q)
end

"""
    assert_compatibility(sys::AbstractSystem{T}, state::AbstractState{T})

Require system dimensions to be compatible with input state.
"""
function assert_compatibility{T}(sys::LinearSystem{T},state::AbstractState{T})
    if size(sys.A, 2) != length(state.x)
        throw(DimensionMismatch("Linear system incompatible with input state."))
    end
end

"""
    state_transition_matrix(sys::LinearSystem{T}, state::AbstractState{T}, t)

Get state transition matrix for a LinearSystem over a specified time
span. Supports both discrete and continuous states in absolute and uncertain
forms. For discrete systems, t must be an integer.
"""
function state_transition_matrix{T}(sys::LinearSystem{T},
                                    state::DiscreteState{T}, t::Integer)
    assert_compatibility(sys, state)
    return sys.A^(t-state.t)
end
function state_transition_matrix{T}(sys::LinearSystem{T},
                                    state::UncertainDiscreteState{T},t::Integer)
    assert_compatibility(sys, state)
    return sys.A^(t-state.t)
end
function state_transition_matrix{T}(sys::LinearSystem{T},
                                    state::ContinuousState{T}, t::T)
    assert_compatibility(sys, state)
    return expm(sys.A*(t-state.t))
end
function state_transition_matrix{T}(sys::LinearSystem{T},
                                    state::UncertainContinuousState{T}, t::T)
    assert_compatibility(sys, state)
    return expm(sys.A*(t-state.t))
end
function state_transition_matrix{T,S<:UnionDiscrete{T}}(sys::NonlinearSystem{T},
                                                        state::S, t::Integer)
    x = state.x
    stm = eye(length(x))
    for t = state.t:t-1
        stm = sys.dF_dx(t, x)*stm
        x = sys.F(t, x)
    end
    #return DiscreteState(x, t), stm
    return stm
end
function state_transition_matrix{T,S<:UnionContinuous{T}}(
    sys::NonlinearSystem{T}, state::S, t::T)
    function stm_ode(du, u, p, t)
        du[:,1] .= sys.F(t, u[:,1])
        du[:,2:end] .= sys.dF_dx(t, u[:,1])*u[:,2:end]
    end
    initial_state = hcat(state.x, eye(T, length(state.x)))
    problem = ODEProblem(stm_ode, initial_state, (state.t, t))
    solution = DifferentialEquations.solve(problem, abstol=sys.abstol,
                                           reltol=sys.reltol)
    return solution.u[end][:, 2:end]
end

"""
    solve_linear_system(A, x0, dt)
    solve_linear_system(A, k, x0, dt)
"""
function solve_linear_system(A::AbstractMatrix, x0::AbstractVector,
                             dt::AbstractFloat)
    return expm(A*dt)*x0
end
function solve_linear_system(A::AbstractMatrix, k::AbstractVector,
                             x0::AbstractVector, dt::AbstractFloat)
    n = length(x0)
    A_expanded = zeros(2*n, 2*n)
    A_expanded[1:n, 1:n] .= A
    A_expanded[1:n, n+1:end] .= eye(n)
    return (expm(A_expanded*dt)*[x0; k])[1:n]
end

"""
    continuous_predict_cov(A::Matrix, Q::Matrix, P::Matrix, dt)

Predict covariance through a continuous linear system over a designated
time step.
"""
function continuous_predict_cov(A::Matrix, Q::Matrix, P::Matrix, dt)
    n = size(A,1)
    Ap = zeros(n*n, n*n)
    for i = 1:n
        shift = (i-1)*n
        @inbounds Ap[shift+1:shift+n, shift+1:shift+n] .= A
    end
    for i = 1:n*n, j = 1:n
        @inbounds Ap[(j-1)*n + (i-1)%n + 1, i] += A[j, 1+floor(Integer,(i-1)/n)]
    end

    return reshape(solve_linear_system(Ap, Q[:], P[:], dt), size(P))
end

"""
    predict(state::AbstractState, sys::AbstractSystem, t::Real)

Predict a state through an arbitrary system. The time step, `t`, must match the
input state in its type, i.e. for a DiscreteState, `t` must be an integer.

The `predict` method will advance both the state vector and, if necessary, its
covariance through the dynamics of the input system. For NonlinearSystems, the
covariance prediction is only a linear approximation.

Use `predict!` to update the input state with the predicted state in-place.
"""
function predict(::AbstractSystem) end
function predict{T}(state::AbstractState{T}, sys::AbstractSystem{T}, t::Real)
    out_state = deepcopy(state)
    predict!(out_state, sys, t)
    return out_state
end

"""
    predict!(state::AbstractState, sys::AbstractSystem, t::Real)

In-place prediction of a state through an arbitrary system.
"""
function predict!{T}(state::AbstractAbsoluteState{T}, sys::LinearSystem{T}, t::Real)
    state .= state_transition_matrix(sys, state, t) * state
    state.t = t
    return nothing
end
function predict!{T}(state::UncertainDiscreteState{T}, sys::LinearSystem{T},
                     t::Integer)
    assert_compatibility(sys, state)
    for idx = state.t:t-1
        state.x = sys.A*state.x
        state.P = sys.A*state.P*sys.A' + sys.Q
    end
    state.t = t
    return nothing
end
function predict!{T}(state::UncertainContinuousState{T}, sys::LinearSystem{T}, t::Real)
    state.x = state_transition_matrix(sys, state, t) * state.x
    state.P = continuous_predict_cov(sys.A, sys.Q, state.P, t-state.t)
    state.t = t
    return nothing
end
function predict!{T}(state::DiscreteState{T}, sys::NonlinearSystem{T},
                     t::Integer)
    for t_step = state.t:t-1
        state.x = sys.F(t_step, state.x)
    end
    state.t = t
    return nothing
end
function predict!{T}(state::UncertainDiscreteState{T}, sys::NonlinearSystem{T},
                     t::Integer)
    for t_step = state.t:t-1
        state.x = sys.F(t_step, state.x)
        jac = sys.dF_dx(t_step, state.x)
        state.P = jac*state.P*jac' + sys.Q
    end
    state.t = t
    return nothing
end
function predict!{T}(state::ContinuousState{T}, sys::NonlinearSystem{T}, t::Real)
    if t != state.t
        fcn(u, p, t) = sys.F(t, u)
        solution = DifferentialEquations.solve(
            ODEProblem(fcn, state.x, (state.t, t)),
            save_everystep=false, abstol=sys.abstol, reltol=sys.reltol)
        state.x .= solution.u[end]
        state.t = t
    end
    return nothing
end
function predict!{T}(state::UncertainContinuousState{T}, sys::NonlinearSystem{T}, t::Real)
    if t != state.t
        n = length(state.x)
        function combined_ode(x_in, p, t)
            x = x_in[1:n]
            P = reshape(x_in[n+1:end], n, n)
            xdot = sys.F(t, x)
            A = sys.dF_dx(t, x)
            Pdot = A*P + P*A' + sys.Q
            return [xdot; Pdot[:]]
        end

        combined_state = [state.x; state.P[:]]
        solution = DifferentialEquations.solve(
            ODEProblem(combined_ode, combined_state, (state.t, t)),
            save_everystep=false, abstol=sys.abstol, reltol=sys.reltol)

        state.x .= solution.u[end][1:n]
        state.P .= reshape(solution.u[end][n+1:end], n, n)
        state.t = t
    end
    return nothing
end

"""
    simulate(state::AbstractState, sys::AbstractSystem, t::Real)

Simulate a state prediction with initial state error and process noise. This
method returns an absolute state by sampling the initial state (if uncertain),
and propagating that state through the input system in the presence of process
noise.
"""
function simulate(::AbstractSystem) end
function simulate(state::AbstractAbsoluteState, sys::AbstractSystem, t::Real)
    return simulate(make_uncertain(state), sys, t)
end
function simulate(state::AbstractUncertainState, sys::LinearSystem, t::Real)
    return sample(predict(state, sys, t))
end
function simulate(state::UncertainDiscreteState, sys::NonlinearSystem, t::Integer)
    sampled_initial_state = sample(state)
    return sample(predict(make_uncertain(sampled_initial_state), sys, t))
end
function simulate{T}(state::UncertainContinuousState{T},
                     sys::NonlinearSystem{T}, t::T)
    sampled_initial_state = sample(state)
    if all(sys.Q .== 0)
        simulated_state = predict(sampled_initial_state, sys, t)
    else
        f(du, u, p, t) = du .= sys.F(t, u)
        g(du, u, p, t) = du .= chol(Hermitian(sys.Q))*u
        prob = SDEProblem(f, g, sampled_initial_state.x, (state.t, t))
        soln = DifferentialEquations.solve(prob)
        simulated_state = ContinuousState(soln.u[end], t)
    end
    return simulated_state
end


# A partial order

import Base: cmp, in, push!

struct Order{T}
    rel     :: Set{Pair{T}}
end
Order(T::Type) = Order(Set{Pair{T}}())

function push!(P::Order{T}, lt::Pair{T}) where {T}

    a, b = lt

    push!(P.rel, lt)
    push!(P.rel, a => a)
    push!(P.rel, b => b)

    # Update transitive closure:
    src, tgt = Set{T}(), Set{T}()
    for p in P.rel
        p.second == a && push!(src, p.first)
        p.first == b && push!(tgt, p.second)
    end
    for u in src, v in tgt
        push!(P.rel, u => v)
    end

    for (u, v) in P.rel
        (v => u) in P.rel && u ≠ v && error("irreflexive: $a vs $b")
    end

end

in(lt::Pair{T}, P::Order{T}) where {T} = lt in P.rel

function cmp(a::T, b::T, P::Order{T}) where {T}
    if a == b
        return 0
    elseif (a => b) ∈ P
        return -1
    elseif (b => a) ∈ P
        return +1
    else
        error("undefined: $a vs $b")
    end
end


function runbnntest()
  # Hide sampling progress.
  Turing.turnprogress(false);

  # Use reverse_diff due to the number of parameters in neural networks.
  Turing.setadbackend(:reverse_diff)

  # Number of points to generate.
  N = 80
  M = round(Int, N / 4)
  Random.seed!(1234)

  # Generate artificial data.
  x1s = rand(M) * 4.5; x2s = rand(M) * 4.5;
  xt1s = Array([[x1s[i] + 0.5; x2s[i] + 0.5] for i = 1:M])
  x1s = rand(M) * 4.5; x2s = rand(M) * 4.5;
  append!(xt1s, Array([[x1s[i] - 5; x2s[i] - 5] for i = 1:M]))

  x1s = rand(M) * 4.5; x2s = rand(M) * 4.5;
  xt0s = Array([[x1s[i] + 0.5; x2s[i] - 5] for i = 1:M])
  x1s = rand(M) * 4.5; x2s = rand(M) * 4.5;
  append!(xt0s, Array([[x1s[i] - 5; x2s[i] + 0.5] for i = 1:M]))

  # Store all the data for later.
  xs = [xt1s; xt0s]
  ts = [ones(2*M); zeros(2*M)]

  # Plot data points.
  function plot_data()
      x1 = map(e -> e[1], xt1s)
      y1 = map(e -> e[2], xt1s)
      x2 = map(e -> e[1], xt0s)
      y2 = map(e -> e[2], xt0s)

      Plots.scatter(x1,y1, color="red", clim = (0,1))
      Plots.scatter!(x2, y2, color="blue", clim = (0,1))
  end

  plot_data()

  function unpack(nn_params::AbstractVector)
      W₁ = reshape(nn_params[1:6], 3, 2);
      b₁ = reshape(nn_params[7:9], 3)

      W₂ = reshape(nn_params[10:15], 2, 3);
      b₂ = reshape(nn_params[16:17], 2)

      Wₒ = reshape(nn_params[18:19], 1, 2);
      bₒ = reshape(nn_params[20:20], 1)
      return W₁, b₁, W₂, b₂, Wₒ, bₒ
  end

  function nn_forward(xs, nn_params::AbstractVector)
      W₁, b₁, W₂, b₂, Wₒ, bₒ = unpack(nn_params)
      nn = Chain(Dense(W₁, b₁, tanh),
                 Dense(W₂, b₂, tanh),
                 Dense(Wₒ, bₒ, σ))
      return nn(xs)
  end

  alpha = 0.09
  sig = sqrt(1.0 / alpha)

  # Specify the probabalistic model.
  @model bayes_nn(xs, ts) = begin
      # Create the weight and bias vector.
      nn_params ~ MvNormal(zeros(20), sig .* ones(20))

      # Calculate predictions for the inputs given the weights
      # and biases in theta.
      preds = nn_forward(xs, nn_params)

      # Observe each prediction.
      for i = 1:length(ts)
          ts[i] ~ Bernoulli(preds[i])
      end
  end

  N = 5000
  ch = sample(bayes_nn(hcat(xs...), ts), HMC(0.05, 4), N)
  theta = ch[:nn_params].value.data

  # Plot the data we have.
  plot_data()

  # Find the index that provided the highest log posterior in the chain.
  _, i = findmax(ch[:lp].value.data)

  # Extract the max row value from i.
  i = i.I[1]

  # Plot the posterior distribution with a contour plot.
  x_range = collect(range(-6,stop=6,length=25))
  y_range = collect(range(-6,stop=6,length=25))
  Z = [nn_forward([x, y], theta[i, :])[1] for x=x_range, y=y_range]
  contour!(x_range, y_range, Z)
end


# --------------------------------------------------------------------------- #

# Setting an ip model of SPP
function setSPP(solverSelected, C, A)
  m, n = size(A)
  ip = Model(solver=solverSelected)
  @variable(ip, x[1:n], Bin)
  @objective(ip, Max, dot(C, x))
  @constraint(ip , cte[i=1:m], sum(A[i,j] * x[j] for j=1:n) <= 1)
  return ip, x
end


@testset "enumeration" begin
    @jyro function model()
        a ~ Bernoulli(0.5)
        b ~ Bernoulli(0.5)
        c ~ Bernoulli(0.5)
        # Use this as a deterministic function.
        s ~ DiscreteNonParametric([a + b + c], [1.0])
        return s
    end

    dist = discrete_enumeration(model, :_return)
    analytic_dist = DiscreteNonParametric([0, 1, 2, 3], [0.125, 0.375, 0.375, 0.125])
    @test dist == analytic_dist
    dist_s = discrete_enumeration(model, :s)
    @test dist_s == analytic_dist
end

@testset "helper functions" begin
    D = 2
    @jyro function model()
        x ~ Normal()
        y ~ MvNormal(D, 1)
        z ~ Normal()
    end

    condition!(model, Dict(:z => 0))
    param_info, num_params = Minijyro.get_param_info(model, ())

    @test num_params == (D + 1)
    @test !haskey(param_info, :z)
    @test param_info[:x][:is_scalar]
    @test !param_info[:y][:is_scalar]
    @test param_info[:y][:num_elems] == D
    @test param_info[:y][:shape] == (D,)

    params_dict = Dict()
    new_params = [0.0, 1.0, 1.0]
    Minijyro.load_params!(params_dict, param_info, new_params)

    params_dict2 = Dict()
    Minijyro.load_params!(params_dict2, param_info, new_params)

    # Because we loop through a dict we do not know the order of keys beforehand
    # but it should always be consistent.
    @test params_dict[:x] == params_dict2[:x]
    @test params_dict[:y] == params_dict2[:y]
end


##################################################
##
## Factorization

# Pencil is made up of two factorizations
# notation in paper is
# solve V - \lambda W
# or V*inv(W) - lambda I
# but V = QR, so consider
# Q*R*inv(W)
# to avoid (introduce?) confusion,we let R be V: and have
# Q * V * inv(W) with the R Factorization store both V and W
struct RFactorizationPencil{T, S} <: AbstractRFactorization{T, S}
V::RFactorizationRankOne{T, S}
W::RFactorizationRankOne{T, S}
end

Base.copy(RF::RFactorizationPencil) = RFactorizationPencil(copy(RF.V), copy(RF.W))
Base.size(RF::RFactorizationPencil) = size(RF.V)
function Base.Matrix(RF::RFactorizationPencil)
    V = Matrix(RF.V)
    W = Matrix(RF.W)
    V[1:end-1, 1:end-1] * inv(W[1:end-1, 1:end-1])
end

## VV = [Sym("v$i$j") for i in ("i","j","k"), j in ("i","j","k")] |> triu
## WW = [Sym("w$i$j") for i in ("i","j","k"), j in ("i","j","k")] |> triu
## VV * inv(WW)[:,3]
function Base.getindex(RF::RFactorizationPencil, l, k)
    Δ = k - l
    #@assert 0 <= Δ <= 2

    V, W = RF.V, RF.W
    l <= 0 &&  return zero(RF)
    if iszero(Δ)
        return V[k,k] / W[k,k]
    elseif Δ == 1
        j = l
        return - V[j,j] * W[j,k] / (W[j,j]*W[k,k]) + V[j,k]/W[k,k]
    else
        i, j = l, l+1
        return V[i,i] * (W[i,j] * W[j,k] - W[i,k] * W[j,j]) / (W[i,i] * W[j,j] * W[k,k])  - (V[i,j] *  W[j,k]) / ( W[j,j] * W[k,k])  + V[i,k]/W[k,k]
    end

end

## Constructor for Real coefficients; with pencil

function pencil_factorization(vs, ws)

    V = r_factorization(vs)
    W = r_factorization(ws)
    RFactorizationPencil(V,W)

end

## Pass a rotator through Rfactorization with Pencil from left or right
function passthrough!(RF::RFactorizationPencil, U::AbstractRotator)

    ## Pass Ut -> W (not W^{-1} <- U
    ## Then  passthrough V
    Ut = passthrough!(U', RF.W)
    U = passthrough!(RF.V, Ut')

    U
end

function passthrough!(U::AbstractRotator, RF::RFactorizationPencil)

    U = passthrough!(U, RF.V)
    Ut = passthrough!(RF.W, U')
    U = Ut'

    U
end

## When Ct and B are identical, we can update just one and leave U,V alone
function simple_passthrough!(RF::RFactorizationPencil, U, V)
    false  # might have room for improvement here; o/w consolidate methods
end

function simple_passthrough!(RF::RFactorizationPencil, U)
    false  # might have room for improvement here
end


import Plots, SearchBall
const sb = SearchBall
const pl = Plots

pl.gr()

function end_pos(elapsed::Number, v = 1.0, ω = π/16.0, maximumtime::Number=2π/ω)
    init = [1, 0]
    rot = elapsed*ω
    end_pos = (maximumtime - elapsed).*v .* sb.rotate2d(init, rot)
    end_pos
end

path = end_pos.(0:0.01:32)

x = getindex.(path, 1)
y = getindex.(path, 2)

pl.plot(x, y, aspect_ratio=:equal)

function spiral(θ::Number, a::Number=1, b::Number=1)
    a + b*θ
end

θ = 0:0.01:2π
pl.plot(θ, spiral.(θ, 0, 2.0), proj = :polar)

function spiral_param(φ::Number, a::Number=1)
    [a*φ*cos(φ), a*φ*sin(φ)]
end

φ = 0:0.01:2π
vals = spiral_param.(φ)
x_spiral = getindex.(vals, 1)
y_spiral = getindex.(vals, 2)
pl.plot(x_spiral, y_spiral, aspect_ratio=:equal)


include("../src/PowerFlowAnalysis.jl")

using JuMP
using Ipopt
using PowerModels
using PowerFlowAnalysis
using MATLAB
using PyPlot

casename = "case9" #Matpower test case
case = PowerModels.parse_file(join(["data/", casename, ".m"]))
pm = ACPPowerModel(case)
sol = run_ac_pf(case, IpoptSolver())
sys = PowerFlowSystem(pm, sol)
lf = LurieFormAd(sys)

# Choose between different choices of the u - constraints
net = Network(pm)
pqload = sys.p⁰[sys.pq]
nzload = find(pqload -> pqload < 0, pqload)
buspJ = nzload[[1,2]]##the bus ordering in this code
busp = [net.buses[k] for k in sys.pq[buspJ]] #the bus ordering in Matpower cases

npv = length(sys.pv)
nb = sys.nb

ratio = 1; #aspect ratio of two loads
ϕ = acot(ratio)
v⁰ = lf.sys.v⁰
R = create_loadability_R(lf, buspJ, [cos(ϕ), sin(ϕ)]) #loading scenario

lc = BaseLurieAdCertificate(lf, R)

Qlim = get_Qlims(lc, pm)
dV = 0.01 #voltage derivation limit
thermalrate = 2.
T = 0.01#angle separation limit
K = 0.01#logarithmic voltage difference limit

limitQQ = 1 #if reactive power constraints are considered
limitIQ = 1 #if reactive thermal constraints are considered

#core optimization problem for construction
mLP = brouwer_certificate_LP(lc, lc.B, lc.C, Qlim, thermalrate, T, K, dV, 100, limitQQ, limitIQ; region=:uniform)
sol_LP = solve(mLP)

Δ₁u = getvalue(getvariable(mLP, :Δ₁u))
Δ₁v = getvalue(getvariable(mLP, :Δ₁v))

#options for Continuation Power Flow
dV = findmax(Δ₁v[1,:])[1]
stepsize = 0.03
adaptive = 0

basep = -sys.p⁰[sys.pq[buspJ]] #the base point
resB = cal_box(R, Δ₁u, Δ₁v, npv, buspJ, basep, v⁰) #inner approx. region
nump = 4 #number of points to plot

polytope_plot(casename,busp,nump,resB,basep,nb,ratio,thermalrate,dV,stepsize,adaptive,limitQQ,limitIQ,Δ₁u)


using JuMP, EAGO

                     m = Model()

                     EAGO.register_eago_operators!(m)

                     @variable(m, -1 <= x[i=1:2] <= 1)

                     @variable(m, -3.892086739821988 <= q <= 1.133112254487742)

                     add_NL_constraint(m, :(swish(-0.9215225812660202 + -0.8659264200816277*swish(0.9754315885766114 + 0.9182647481159649*$(x[1]) + 0.2963572993024828*$(x[2])) + -0.7655479829179535*swish(0.7516798050016731 + -0.4187259277447679*$(x[1]) + -0.991762240929563*$(x[2]))) + swish(0.9152212659843197 + -0.09622429015475253*swish(0.9754315885766114 + 0.9182647481159649*$(x[1]) + 0.2963572993024828*$(x[2])) + 0.18727818139193309*swish(0.7516798050016731 + -0.4187259277447679*$(x[1]) + -0.991762240929563*$(x[2]))) - $q <= 0.0))

                     @objective(m, Min, q)

                     return m


using Test
using DataFrames
using Random
using LinearAlgebra

using LinRegOutliers

@testset "Apply function to columns" begin
    ones_vector = ones(Float64, 10)
    zeros_vector = zeros(Float64, 10)
    mat = hcat(ones_vector, zeros_vector)

    @test applyColumns(sum, mat) == [10.0, 0.0]
    @test applyColumns(mean, mat) == [1.0, 0.0]
end

@testset "Ordinary Least Squares" begin
    tol = 0.0001
    #  The model is exatly y = 5 + 5x
    var1 = Float64[1, 2, 3, 4, 5]
    X = hcat(ones(5), var1)
    betas = [5.0, 5.0]
    y = X * betas

    # OLS
    olsreg = ols(X, y)
    @test isapprox(coef(olsreg), betas, atol=tol)
    @test isapprox(residuals(olsreg), zeros(Float64, 5), atol=tol)
    @test isapprox(predict(olsreg), y, atol=tol)
end

@testset "Weighted Least Squares" begin
    tol = 0.0001
    n = 7
    #  The model is exatly y = 5 + 5x
    var1 = Float64[1, 2, 3, 4, 5, 6, 7]
    X = hcat(ones(n), var1)
    betas = [5.0, 5.0]
    y = X * betas
    y[n - 1] = 5000.0
    y[n] = 5000.0
    wts = [1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0]

    # WLS
    olsreg = wls(X, y, wts)
    @test isapprox(coef(olsreg), betas, atol=tol)
    @test isapprox(residuals(olsreg)[1:(n - 2)], zeros(Float64, n - 2), atol=tol)
    @test isapprox(predict(olsreg)[1:(n - 2)], y[1:(n - 2)], atol=tol)
end

@testset "Basis - createRegressionSetting, designMatrix, responseVector" begin
    dataset = DataFrame(
        x=[1.0, 2, 3, 4, 5],
        y=[2.0, 4, 6, 8, 10]
    )
    setting = createRegressionSetting(@formula(y ~ x), dataset)

    @test designMatrix(setting) == hcat(ones(5), dataset[:, "x"])
    @test responseVector(setting) == dataset[:, "y"]
    @test setting.formula == @formula(y ~ x)
end

@testset "dffit - single case" begin
    # Since this regression setting is deterministic,
    # omission of a single observation has not an effect on
    # the predicted response.
    myeps = 10.0^(-6.0)
    dataset = DataFrame(
        x=[1.0, 2, 3, 4, 5],
        y=[2.0, 4, 6, 8, 10]
    )
    setting = createRegressionSetting(@formula(y ~ x), dataset)
    n, _ = size(dataset)
    for i in 1:n
        mydiff = dffit(setting, i)
        @test abs(mydiff) < myeps
    end
end

@testset "dffit - for all observations" begin
    # Since this regression setting is deterministic,
    # omission of a single observation has not an effect on
    # the predicted response.
    myeps = 10.0^(-6.0)
    dataset = DataFrame(
        x=[1.0, 2, 3, 4, 5],
        y=[2.0, 4, 6, 8, 10]
    )
    setting = createRegressionSetting(@formula(y ~ x), dataset)
    stats = dffit(setting)
    for element in stats
        @test abs(element) < myeps
    end
end

@testset "hat matrix" begin
    X = [1.0 1.0; 1.0 2.0; 1.0 3.0; 1.0 4.0; 1.0 5.0];
    hats_real = X * inv(X' * X) * X'
    dataset = DataFrame(
        x=[1.0, 2, 3, 4, 5],
        y=[2.0, 4, 6, 8, 10]
    )
    setting = createRegressionSetting(@formula(y ~ x), dataset)
    hats_calculated = hatmatrix(setting)

    @test hats_real == hats_calculated
end

@testset "studentized residuals" begin
    dataset = DataFrame(
        x=Float64[1.0, 2.0, 3.0, 4.0, 5.0, 6.0],
        y=Float64[2.1, 3.9, 6.2, 7.8, 11.0, 18.0]
    )
    setting = createRegressionSetting(@formula(y ~ x), dataset)
    resi = studentizedResiduals(setting)
    @test abs(resi[1]) < 1.5
    @test abs(resi[2]) < 1.5
    @test abs(resi[3]) < 1.5
    @test abs(resi[4]) < 1.5
    @test abs(resi[5]) < 1.5
    @test abs(resi[6]) > 1.5
end

@testset "adjusted residuals" begin
    eps = 0.0001
    dataset = DataFrame(
        x=[1.0, 2, 3, 4, 5],
        y=[2.0, 4, 6, 8, 10]
    )
    setting = createRegressionSetting(@formula(y ~ x), dataset)
    resi = adjustedResiduals(setting)
    for element in resi
        @test abs(element) < eps
    end
end

@testset "Cook's distance - Phone data" begin
    eps = 0.00001
    setting = createRegressionSetting(@formula(calls ~ year), phones)
    knowncooks = [0.005344774190779771, 0.0017088194691033181, 0.00016624914057961155,
                    3.16444525831206e-5, 0.0005395058666404081, 0.0014375008774859539,
                    0.0024828140956511258, 0.0036279720445167277, 0.004357605989540906,
                    0.005288503758364767, 0.006313578057565415, 0.0076561205696857254,
                    0.009568574875389256, 0.009970039008782357, 0.02610396373381051,
                    0.029272523880917646, 0.05091236198400663, 0.08176555044049343,
                    0.14380266904640235, 0.26721539425047447, 0.051205153558783356,
                    0.13401084683481085, 0.16860324592350226, 0.2172819114905912]
    cookdists = cooks(setting)
    @test map((x, y) -> abs(x - y) < eps, cookdists, knowncooks) == trues(24)
end

@testset "Jacknifed standard error of regression" begin
    eps = 0.00001
    dataset = DataFrame(
        x=[1.0, 2, 3, 4, 5000],
        y=[2.0, 4, 6, 8, 1000]
    )
    setting = createRegressionSetting(@formula(y ~ x), dataset)
    @test jacknifedS(setting, 1) != 0
    @test jacknifedS(setting, 2) != 0
    @test jacknifedS(setting, 3) != 0
    @test jacknifedS(setting, 4) != 0
    @test jacknifedS(setting, 5) < eps
end

@testset "Mahalanobis squared distances" begin
    myeps = 10.0^(-6.0)
    x = [1.0, 2.0, 3.0, 4.0, 5.0]
    y = reverse(x)
    y[1] = y[1] * 10.0
    x[1] = x[1] * 10.0
    datamat = DataFrame(x=x, y=y)
    dmat = mahalanobisSquaredMatrix(datamat)
    d = diag(dmat)
    @test abs(d[1] - 3.2) < myeps
    @test abs(d[2] - 2.0) < myeps
    @test abs(d[3] - 0.4) < myeps
    @test abs(d[4] - 0.4) < myeps
    @test abs(d[5] - 2.0) < myeps
end

@testset "Find nonzero minimum" begin
    arr = [0.0, 2.0, 0.0, 0.0, 0.0, 9.0, 1.0]
    val = find_minimum_nonzero(arr)
    @test val == 1.0
end

@testset "Hadi & Simonoff 1993 - initial subset" begin
    # Create simple data
    rng = MersenneTwister(12345)
    n = 50
    x = collect(1:n)
    e = randn(rng, n) .* 2.0
    y = 5 .+ 5 .* x .+ e
    y[n] = y[n] * 2.0
    y[n - 1] = y[n - 1] * 2.0
    df = DataFrame(x=x, y=y)
    reg = createRegressionSetting(@formula(y ~ x), df)
    subset = hs93initialset(reg)
    @test !(49 in subset)
    @test !(50 in subset)
end

@testset "Hadi & Simonoff 1993 - basic subset" begin
    # Create simple data
    rng = MersenneTwister(12345)
    n = 50
    x = collect(1:n)
    e = randn(rng, n) .* 2.0
    y = 5 .+ 5 .* x .+ e
    y[n] = y[n] * 2.0
    y[n - 1] = y[n - 1] * 2.0
    df = DataFrame(x=x, y=y)
    reg = createRegressionSetting(@formula(y ~ x), df)
    initialsubset = hs93initialset(reg)
    basicsubset = hs93basicsubset(reg, initialsubset)
    @test !(49 in basicsubset)
    @test !(50 in basicsubset)
end

@testset "Hadi & Simonoff 1993 - Algorithm" begin
    # Create simple data
    rng = MersenneTwister(12345)
    n = 50
    x = collect(1:n)
    e = randn(rng, n) .* 2.0
    y = 5 .+ 5 .* x .+ e
    y[n] = y[n] * 2.0
    df = DataFrame(x=x, y=y)
    reg = createRegressionSetting(@formula(y ~ x), df)
    outset = hs93(reg)
    @test 50 in outset["outliers"]
end

@testset "Hadi & Simonoff 1993 - with Hadi & Simonoff random data" begin
    reg = createRegressionSetting(@formula(y ~ x1 + x2), hs93randomdata)
    outset = hs93(reg)
    @test 1 in outset["outliers"]
    @test 2 in outset["outliers"]
    @test 3 in outset["outliers"]
end

@testset "Kianifard & Swallow 1989 - Algorithm" begin
    df = phones
    reg = createRegressionSetting(@formula(calls ~ year), df)
    outset = ks89(reg, alpha=0.1)
    @test 15 in outset
    @test 16 in outset
    @test 17 in outset
    @test 18 in outset
    @test 19 in outset
    @test 20 in outset

    df2 = stackloss
    reg2 = createRegressionSetting(@formula(stackloss ~ airflow + watertemp + acidcond), stackloss)
    outset2 = ks89(reg2)
    @test 4 in outset2
    @test 21 in outset2
end

@testset "Sebert & Montgomery & Rollier 1998 - Algorithm" begin
    # Create simple data
    rng = MersenneTwister(12345)
    n = 50
    x = collect(1:n)
    e = randn(rng, n) .* 2.0
    y = 5 .+ 5 .* x .+ e
    y[n] = y[n] * 2.0
    y[n - 1] = y[n - 1] * 2.0
    y[n - 2] = y[n - 2] * 2.0
    df = DataFrame(x=x, y=y)
    reg = createRegressionSetting(@formula(y ~ x), df)
    outset = smr98(reg)
    @test 49 in outset
    @test 50 in outset
end

@testset "LMS - Algorithm - Random data" begin
    # Create simple data
    rng = MersenneTwister(12345)
    n = 50
    x = collect(1:n)
    e = randn(rng, n) .* 2.0
    y = 5 .+ 5 .* x .+ e
    y[n] = y[n] * 2.0
    y[n - 1] = y[n - 1] * 2.0
    y[n - 2] = y[n - 2] * 2.0
    x[n - 3] = x[n - 3] * 2.0
    df = DataFrame(x=x, y=y)
    reg = createRegressionSetting(@formula(y ~ x), df)
    outset = lms(reg)["outliers"]
    @test 48 in outset
    @test 49 in outset
    @test 50 in outset
end

@testset "LMS - Algorithm - Phone data" begin
    df = phones
    reg = createRegressionSetting(@formula(calls ~ year), df)
    outset = lms(reg)["outliers"]
    @test 15 in outset
    @test 16 in outset
    @test 17 in outset
    @test 18 in outset
    @test 19 in outset
    @test 20 in outset
    @test 21 in outset
end

@testset "LTS - Algorithm - Random data" begin
    # Create simple data
    rng = MersenneTwister(12345)
    n = 50
    x = collect(1:n)
    e = randn(rng, n) .* 2.0
    y = 5 .+ 5 .* x .+ e
    y[n] = y[n] * 2.0
    y[n - 1] = y[n - 1] * 2.0
    y[n - 2] = y[n - 2] * 2.0
    x[n - 3] = x[n - 3] * 2.0
    df = DataFrame(x=x, y=y)
    reg = createRegressionSetting(@formula(y ~ x), df)
    outset = lms(reg)["outliers"]
    @test 48 in outset
    @test 49 in outset
    @test 50 in outset
end

@testset "LTS - Algorithm - Phone data" begin
    df = phones
    reg = createRegressionSetting(@formula(calls ~ year), df)
    outset = lts(reg)["outliers"]
    @test 15 in outset
    @test 16 in outset
    @test 17 in outset
    @test 18 in outset
    @test 19 in outset
    @test 20 in outset
    @test 21 in outset
end

@testset "MVE - Algorithm - Phone data" begin
    df = phones
    outset = mve(df)["outliers"]
    @test 15 in outset
    @test 16 in outset
    @test 17 in outset
    @test 18 in outset
    @test 19 in outset
    @test 20 in outset
    @test 21 in outset
end

@testset "MCD - Algorithm - Phone data" begin
    df = phones
    outset = mcd(df)["outliers"]
    @test 15 in outset
    @test 16 in outset
    @test 17 in outset
    @test 18 in outset
    @test 19 in outset
    @test 20 in outset
    @test 21 in outset
end

@testset "MVE & LTS Plot - Algorithm - Phone data" begin
    df = phones
    reg = createRegressionSetting(@formula(calls ~ year), df)
    result = mveltsplot(reg, showplot=false)
    regulars = result["regular.points"]
    @test 1 in regulars
    @test 2 in regulars
    @test 3 in regulars
    @test 4 in regulars
    @test 5 in regulars
    @test 6 in regulars
    @test 7 in regulars
    @test 8 in regulars
    @test 9 in regulars
    @test 10 in regulars
    @test 11 in regulars
    @test 12 in regulars
    @test 13 in regulars
end

@testset "BCH - Algorithm - Hawkins & Bradu & Kass data" begin
    df = hbk
    reg = createRegressionSetting(@formula(y ~ x1 + x2 + x3), df)
    result = bch(reg)
    regulars = result["basic.subset"]
    for i in 15:75
        @test i in regulars
    end
end

@testset "PY95 - Algorithm - Hawkins & Bradu & Kass data" begin
    df = hbk
    reg = createRegressionSetting(@formula(y ~ x1 + x2 + x3), df)
    result = py95(reg)
    outliers = result["outliers"]
    for i in 1:14
        @test i in outliers
    end
end

@testset "satman2013 - Algorithm - Hawkins & Bradu & Kass data" begin
    df = hbk
    reg = createRegressionSetting(@formula(y ~ x1 + x2 + x3), df)
    result = satman2013(reg)
    outliers = result["outliers"]
    for i in 1:14
        @test i in outliers
    end
end

@testset "satman2015 - Algorithm - Phone data" begin
    @test dominates([1,2,4], [1,2,3])
    @test dominates([1,2,3], [0,1,2])
    @test !dominates([1,2,3], [1,2,3])
    @test !dominates([1,2,3], [1,2,4])

    df = phones
    reg = createRegressionSetting(@formula(calls ~ year), df)
    result = satman2015(reg)
    outliers = result["outliers"]
    for i in 15:20
        @test i in outliers
    end
end

@testset "asm2000 - Algorithm - Phone data" begin
    df = phones
    reg = createRegressionSetting(@formula(calls ~ year), df)
    result = asm2000(reg)
    outliers = result["outliers"]
    for i in 15:20
        @test i in outliers
    end
end

@testset "ccf - Algorithm - Phone data" begin
    eps = 0.0001
    df = phones
    reg = createRegressionSetting(@formula(calls ~ year), df)
    result = ccf(reg)
    outliers = result["outliers"]
    for i in 15:20
        @test i in outliers
    end

    @test all(isapprox.(result["betas"], [-63.4816, 1.30406], atol=eps, rtol=0.0))
end

@testset "LAD - Algorithm" begin
    eps = 0.0001
    df = phones
    reg = createRegressionSetting(@formula(calls ~ year), df)
    result = lad(reg)
    betas = result["betas"]
    @test betas[1] < 0
    @test betas[2] > 0

    df2 = DataFrame(
        x=[1,2,3,4,5,6,7,8,9,10],
        y=[2,4,6,8,10,12,14,16,18,1000]
    )
    reg2 = createRegressionSetting(@formula(y ~ x), df2)
    result2 = lad(reg2)
    betas2 = result2["betas"]
    @test abs(betas2[1] - 0) < eps
    @test abs(betas2[2] - 2) < eps
end

@testset "LAD with (X, y)" begin
    eps = 0.0001
    df2 = DataFrame(
        x=Float64[1,2,3,4,5,6,7,8,9,10],
        y=Float64[2,4,6,8,10,12,14,16,18,1000]
    )
    n = length(df2[!,"x"])
    X = hcat(ones(Float64, n), df2[!,"x"])
    y = df2[!, "y"]
    result2 = lad(X, y)
    betas2 = result2["betas"]
    @test abs(betas2[1] - 0) < eps
    @test abs(betas2[2] - 2) < eps
end

@testset "LTA - Algorithm - Phone data" begin
    eps = 0.0001
    df = phones
    reg = createRegressionSetting(@formula(calls ~ year), df)
    result = lta(reg, exact=true)
    betas = result["betas"]
    @test abs(betas[1] - -55.5) < eps
    @test abs(betas[2] -  1.15) < eps
end

@testset "Hadi 1992 - Algorithm - with several case" begin
    phones_matrix = hcat(phones[:, "calls"], phones[:, "year"])
    result = hadi1992(phones_matrix)
    outlier_indices = result["outliers"]
    for i in 15:20
        @test i in outlier_indices
    end

    hbk_matrix = hcat(hbk[:, "x1"], hbk[:, "x2"], hbk[:, "x3"])
    hbk_n, hbk_p = size(hbk_matrix)
    result = hadi1992(hbk_matrix)
    outlier_indices = result["outliers"]
    for i in 15:hbk_n
        @test !(i in outlier_indices)
    end

end

@testset "Hadi 1994 - Algorithm - with several case" begin
    phones_matrix = hcat(phones[:, "calls"], phones[:, "year"])
    result = hadi1994(phones_matrix)
    outlier_indices = result["outliers"]
    for i in 15:20
        @test i in outlier_indices
    end

    hbk_matrix = hcat(hbk[:, "x1"], hbk[:, "x2"], hbk[:, "x3"])
    hbk_n, hbk_p = size(hbk_matrix)
    result = hadi1994(hbk_matrix)
    outlier_indices = result["outliers"]
    for i in 15:hbk_n
        @test !(i in outlier_indices)
    end

end

@testset "gwcga - Algorithm - phone data" begin
    df = phones
    reg = createRegressionSetting(@formula(calls ~ year), df)
    result = gwcga(reg)
    clean_indices = result["clean.subset"]
    for i in 15:20
        @test !(i in clean_indices)
    end
end

@testset "Compact Genetic Algorithm" begin
    Random.seed!(12345)
    function fcost(bits)
        return sum(bits)
    end
    result = cga(chsize=10, costfunction=fcost, popsize=100)
    for element in result
        @test element == 0
    end
end

@testset "Floating-point Genetic Algorithm" begin
    Random.seed!(12345)
    fcost(genes) = (genes[1] - 3.14159265)^2 + (genes[2] - 2.71828)^2
    epsilon = 10.0^(-6.0)
    popsize = 30
    chsize = 2
    mins = [-100.0, -100.0]
    maxs = [100.0, 100.0]
    result = ga(popsize, chsize, fcost, mins, maxs, 0.90, 0.05, 1, 100)
    best::RealChromosome = result[1]

    @test length(result) == 30
    @test abs(best.genes[1] - 3.14159265) < epsilon
    @test abs(best.genes[2] - 2.71828) < epsilon
end

@testset "Satman(2012) (Csteps and GA based LTS) Algorithm - Phones data" begin
    Random.seed!(12345)
    epsilon = 10.0^(-3.0)
    df = phones
    reg = createRegressionSetting(@formula(calls ~ year), df)
    result = galts(reg)
    betas = result["betas"]
    @test abs(betas[1] - (-56.5219)) < epsilon
    @test abs(betas[2] - 1.16488) < epsilon
end

@testset "RANSAC (1981) Algorithm - Paper example" begin
    Random.seed!(12345)
    df = DataFrame(y=[0,1,2,3,3,4,10], x=[0,1,2,2,3,4,2])
    reg = createRegressionSetting(@formula(y ~ x), df)
    result = ransac(reg, t=0.8, w=0.85)
    @test result == [7]
end

@testset "covratio - phone data" begin
    eps = 0.00001
    knownvals = [1.2945913799871505, 1.2700457384289985, 1.247094858041991,
                1.2255082894941416, 1.2056307474474428, 1.1872216590773372,
                1.1702669047202972, 1.1544784969566473, 1.1415354222939595,
                1.129459077701347, 1.1188675334881035, 1.1092977882351849,
                1.1003495479103549, 1.1024131168300708, 1.0438195753346975,
                1.049139979063326, 1.0015616430250294, 0.9497339678816025,
                0.8530116434438035, 0.6899325070071758, 1.1297343653996992,
                1.0347237388802213, 1.039659281049157, 1.0393513929604028
    ]
    setting = createRegressionSetting(@formula(calls ~ year), phones)
    n = length(knownvals)
    for i in 1:n
        calculated = covratio(setting, i)
        @test abs(knownvals[i] - calculated) < eps
    end
end

@testset "dfbeta - phone data" begin
    eps = 0.00001
    reg = createRegressionSetting(@formula(calls ~ year), phones)
    n, p = size(phones)
    knownvalues = [9.6439157 -0.14686166; 5.3459460 -0.08092134;
                    1.6258961 -0.02443345; -0.6866294  0.01022725;
                    -2.7169197  0.04002009; -4.1910124  0.06085238;
                    -5.1029254  0.07267870; -5.5535000  0.07697356;
                    -5.2512176  0.06983887; -4.6721589  0.05791933;
                    -3.6868718  0.03945523; -2.3254391  0.01478033;
                    -0.5673087 -0.01652355; 1.4653937 -0.04958099;
                    -5.4474441  0.12867978; -8.6540162  0.18101030;
                    -14.6631749  0.28835085; -22.0168113  0.41708081;
                    -32.9443226  0.60863505; -49.0851269  0.89065754;
                    22.9820710 -0.41140246; 39.1639294 -0.69370540;
                    45.7655562 -0.80379984; 53.6862082 -0.93638735;]
    for i in 1:n
        for j in 1:p
            dfbetaresult = dfbeta(reg, i)
            @test abs(dfbetaresult[j] - knownvalues[i,j]) < eps
        end
    end
end

@testset "Atkinson 1994 - Algorithm" begin
    df = stackloss
    reg = createRegressionSetting(@formula(stackloss ~ airflow + watertemp + acidcond), stackloss)
    result = atkinson94(reg)
    @test result["outliers"] == [1, 3, 4, 21]
end

@testset "Hadi Measure" begin
    eps = 0.0001
    setting = createRegressionSetting(@formula(calls ~ year), phones)
    knowncooks = [0.19101337, 0.16141894, 0.13677220, 0.11673486, 0.10058688, 0.08815274, 0.07913586,
                 0.07353075, 0.06965672, 0.06906446, 0.07108612, 0.07605141, 0.08428584, 0.08612449,
                0.15005170, 0.15622736, 0.22082452, 0.29817740, 0.44310105, 0.72642163, 0.19942889,
                0.33018728, 0.36907744, 0.41937743 ]
    hm = hadimeasure(setting)["measure"]
    @test map((x, y) -> abs(x - y) < eps, hm, knowncooks) == trues(24)
end

@testset "Imon 2005 - Algorithm" begin
    Random.seed!(12345)
    reg = createRegressionSetting(@formula(y ~ x1 + x2 + x3), hbk)
    result = imon2005(reg)
    @test result["outliers"] == [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
end

@testset "BACON 2000 - Algorithm" begin
    df = stackloss
    reg = createRegressionSetting(@formula(stackloss ~ airflow + watertemp + acidcond), stackloss)
    result = bacon(reg, m=12)
    @test result == [1, 3, 4, 21]
end


#### This example shows the growth of the number of quadratic functions in the constrained case
#### The figure in the article was generated with "OPTIMIZE = false"

using EllZeroTrendFiltering
using DSP
using Plots
using Interpolations

### START TEST FUNCTIONS

t = 1:200
srand(1)

circle_segment(N) = sin.(acos.(range(-1, stop=1, length=N)))
linear_trend(N) = (0:N-1)

g_cll = [0.2*linear_trend(120) + circle_segment(120);
     -6 + 8*linear_trend(40);
     -6 + 8*linear_trend(40)]

g_cllc = [5*circle_segment(70);
    -5 + 4*linear_trend(30);
    -5 + 4*linear_trend(30);
     5*circle_segment(70)]

g_rand = () -> randn(200)

g_walk = () -> cumsum(randn(200))

g_square = range(0, stop=4, length=200).^2

g_cube = range(0, stop=4, length=200).^3

g_4 = range(0, stop=4, length=200).^4

g_5 = range(0, stop=4, length=200).^5

g_exp = exp.(range(0, stop=4, length=200))

g_super_exp = exp.(range(0, stop=3, length=200).^2)

g_super_exp2 = [exp.(range(0, stop=2, length=100).^2)   ;  exp.(range(2, stop=0.5, length=100).^2)]

g_super_exp3 = [exp.(range(0, stop=2, length=100).^2)   ; -exp.(range(2, stop=0.5, length=100).^2)]

g_super_exp4 = [exp.(range(2, stop=0.5, length=100).^2) ;  exp.(range(0, stop=2, length=100).^2)]

g_super_exp5 = [exp.(range(2, stop=0, length=100).^2)   ; 2-exp.(range(0, stop=0, length=100).^2)]

function randlin(N)
    t_grid = cumsum( rand(10:25, N) )
    y_grid = randn(N)
    g = interpolate((t_grid,), y_grid, Gridded(Linear()))[t]
    reverse!(g)
end

function randlinfilt1(N)
    t_grid = cumsum( rand(10:25, N) )
    y_grid = randn(N)
    g = interpolate((t_grid,), y_grid, Gridded(Linear()))[t]
    g = reverse(filt(ones(10), 1, reverse(filt(ones(10), 1, g))))
    reverse!(g)
end

function randlinfilt2(N)
    t_grid = cumsum( rand(10:25, N) )
    y_grid = randn(N)
    g = interpolate((t_grid,), y_grid, Gridded(Linear()))[t]
    g =filt(ones(40), 1, g)
end

function randlinfilt3(N)
    t_grid = cumsum( rand(10:25, N) )
    y_grid = randn(N)
    g = interpolate((t_grid,), y_grid, Gridded(Linear()))[t]
    g = filtfilt(ones(10), g)
end

g_data = [g_cll, g_cllc,
        g_exp, g_super_exp, g_super_exp2,
        g_super_exp3, g_super_exp4, g_super_exp5,
        g_square, g_cube, g_4, g_5,
        reverse!(g_cll), reverse!(g_cllc),
        reverse!(g_exp), reverse!(g_super_exp), reverse!(g_super_exp2),
        reverse!(g_super_exp3), reverse!(g_super_exp4), reverse!(g_super_exp5),
        reverse!(g_square), reverse!(g_cube), reverse!(g_4), reverse!(g_5),
        g_rand(), g_rand(), g_rand(), g_rand(), g_rand(), g_rand(), g_rand(), g_rand(), g_rand(), g_rand(),
        g_walk(), g_walk(), g_walk(), g_walk(), g_walk(), g_walk(), g_walk(), g_walk(), g_walk(), g_walk(),
        randlin(5), randlinfilt1(5), randlinfilt2(5), randlinfilt3(5),
        randlin(10), randlinfilt1(10), randlinfilt2(10), randlinfilt3(10),
        randlin(20), randlinfilt1(20), randlinfilt2(20), randlinfilt3(20),
        randlin(40), randlinfilt1(40), randlinfilt2(40), randlinfilt3(40),
        reverse!(randlin(5)), reverse!(randlinfilt1(5)), reverse!(randlinfilt2(5)), reverse!(randlinfilt3(5)),
        reverse!(randlin(10)), reverse!(randlinfilt1(10)), reverse!(randlinfilt2(10)), reverse!(randlinfilt3(10)),
        reverse!(randlin(20)), reverse!(randlinfilt1(20)), reverse!(randlinfilt2(20)), reverse!(randlinfilt3(20)),
        reverse!(randlin(40)), reverse!(randlinfilt1(40)), reverse!(randlinfilt2(40)), reverse!(randlinfilt3(40)),
        cumsum(randlin(5)), cumsum(randlinfilt1(5)), cumsum(randlinfilt2(5)), cumsum(randlinfilt3(5)),
        cumsum(randlin(10)), cumsum(randlinfilt1(10)), cumsum(randlinfilt2(10)), cumsum(randlinfilt3(10)),
        cumsum(randlin(20)), cumsum(randlinfilt1(20)), cumsum(randlinfilt2(20)), cumsum(randlinfilt3(20)),
        cumsum(randlin(40)), cumsum(randlinfilt1(40)), cumsum(randlinfilt2(40)), cumsum(randlinfilt3(40)),
        ]
### END TEST FUNCTIONS

nbr_prob = length(g_data)

V = zeros(length(t), nbr_prob)

g_save = zeros(length(t), nbr_prob)

for prob_i=1:nbr_prob
    g = g_data[prob_i]

    l = compute_discrete_transition_costs(g, t);
    cost_last = QuadraticPolynomial(0.0, 0.0, 0.0)

    Λ =  pwq_dp_constrained(l, cost_last, 15);

    l = [(isassigned(Λ,m,i) ? length(Λ[m,i]) : 0) for m = 1:size(Λ,1), i = 1:size(Λ,2)]

    V[:, prob_i] .= maximum(l, 1)[:]
    g_save[:, prob_i] .= g
end

gr()
plot(size=(800,400));
plot(t, V, lab="", linealpha=0.2, c=:black);
plot!(t, length(t) - t + 1, l=(:red, :dash), lab="")
savefig("complexity.png")
writecsv("complexity.csv", V)


module LossFunctions
#-----------------------------------------------------------
#PURPOSE
#	Colin T. Bowers module for loss functions
#NOTES
#	To add a new loss function, provide a new type definition and any constructors under the TYPE DEFINTIIONS heading, then add an appropriate loss method for scalar inputs.
#	If you want to aggregate your loss series in some way, you should probably look at using the Distances package before thinking about this package
#LICENSE
#	MIT License (see github repository for more detail: https://github.com/colintbowers/LossFunctions.jl.git)
#-----------------------------------------------------------

#Load any entire modules that are needed (use import ModuleName1, ModuleName2, etc)

#Load any specific variables/functions that are needed (use import ModuleName1.FunctionName1, ModuleName2.FunctionName2, etc)
import 	Base.string,
		Base.show

#Specify the variables/functions to export (use export FunctionName1, FunctionName2, etc)
export 	LossFunction, #Abstract supertype
		SymmetricLossFunction, #Abstract subtypes
		AsymmetricLossFunction, #Abstract subtypes
		SquaredLoss,
		AbsoluteLoss,
		MinkowskiLoss,
		HuberLoss,
		SquaredLogLoss,
		AbsoluteLogLoss,
		SquaredPropLoss,
		AbsolutePropLoss,
		QLIKE,
		QLIKEReverse,
		loss, #Generic method for calculating loss, ie L(x, y), or L(e) for the case e = x - y
		lossdiff #Generic method for calculating a loss differential, ie L(x-BaseCase) - L(y-BaseCase)

#******************************************************************************

#----------------------------------------------------------
#SET CONSTANTS FOR MODULE
#----------------------------------------------------------
const singleInputLossMethods = Union{SquaredLoss, AbsoluteLoss, MinkowskiLoss, HuberLoss}::Union

#----------------------------------------------------------
#LOSS FUNCTION TYPES
#----------------------------------------------------------
#Abstract types
abstract LossFunction
abstract SymmetricLossFunction <: LossFunction
abstract AsymmetricLossFunction <: LossFunction
#Loss function types
type DummyLoss <: LossFunction ; end #dummy type that does nothing
type SquaredLoss <: SymmetricLossFunction ; end
type AbsoluteLoss <: SymmetricLossFunction ; end
type MinkowskiLoss <: SymmetricLossFunction
	p::Float64
	MinkowskiLoss(p::Float64) = (p <= 0) ? error("Minkowski parameter must be strictly positive") : new(p)
end
MinkowskiLoss() = MinkowskiLoss(2.0) #Default is SquaredLoss
MinkowskiLoss(p::Number) = MinkowskiLoss(convert(Float64, p))
type SquaredLogLoss <: SymmetricLossFunction ; end
type AbsoluteLogLoss <: SymmetricLossFunction ; end
type SquaredPropLoss <: AsymmetricLossFunction ; end
type AbsolutePropLoss <: AsymmetricLossFunction ; end
type QLIKE <: AsymmetricLossFunction ; end
type QLIKEReverse <: AsymmetricLossFunction ; end
type HuberLoss <: SymmetricLossFunction
	p::Float64
	HuberLoss(p::Float64) = (p <= 0) ? error("Huber loss parameter must be strictly positive") : new(p)
end
HuberLoss() = HuberLoss(1.0)
HuberLoss(p::Number) = HuberLoss(convert(Float64, p))

#----------------------------------------------------------
#LOSS FUNCTION ADMIN METHODS
#----------------------------------------------------------
#string
string(lF::SquaredLoss) = "squaredLoss"
string(lF::AbsoluteLoss) = "absoluteLoss"
string(lF::MinkowskiLoss) = "minkowskiLoss"
string(lF::SquaredLogLoss) = "squaredLogLoss"
string(lF::AbsoluteLogLoss) = "absoluteLogLoss"
string(lF::SquaredPropLoss) = "squaredProportionalLoss"
string(lF::AbsolutePropLoss) = "absoluteProportionalLoss"
string(lF::QLIKE) = "QLIKE"
string(lF::QLIKEReverse) = "QLIKEReverse"
string(lF::HuberLoss) = "huberLoss"
#show
function show{T<:Union{MinkowskiLoss, HuberLoss}}(io::IO, lf::T)
	println(io, "Loss function = " * string(lF))
	println(io, "    p = " * string(lF.p))
end
show(io::IO, lf::LossFunction) = println(io, "Loss function = " * string(lF))
show(lf::LossFunction) = show(STDOUT, lf)
#tolossfunction    WARNING: NOT TYPE STABLE
function tolossfunction(s::Symbol)
	if s == :squaredLoss ; return(SquaredLoss())
	elseif s == :absoluteLoss ; return(AbsoluteLoss())
	elseif s == :minkowskiLoss ; return(MinkowskiLoss())
	elseif s == :squaredLogLoss ; return(SquaredLogLoss())
	elseif s == :absoluteLogLoss ; return(AbsoluteLogLoss())
	elseif s == :squaredPropLoss ; return(SquaredPropLoss())
	elseif s == :absolutePropLoss ; return(AbsolutePropLoss())
	elseif s == :QLIKE ; return(QLIKE())
	elseif s == :QLIKEReverse ; return(QLIKEReverse())
	elseif s == :huberLoss ; return(HuberLoss())
	else ; error("Invalid input symbol")
	end
end
tolossfunction(x::String) = tolossfunction(symbol(x))

#----------------------------------------------------------
# CORE METHODS FOR COMPUTING LOSS
#----------------------------------------------------------
#Methods that can be defined in terms of e = x - y, i.e. single input loss methods
loss(e::Number, lF::SquaredLoss) = e^2
loss(e::Number, lF::AbsoluteLoss) = abs(e)
loss(e::Number, lF::MinkowskiLoss) = abs(e)^lF.p
loss(e::Number, lF::HuberLoss) = abs(e) <= lF.p ? 0.5*e^2 : lF.p*(abs(e)-0.5*lF.p)
#Methods that cannot be defined in terms of e = x - y
loss{T<:singleInputLossMethods}(x::Number, y::Number, lF::T) = loss(x - y)
loss(e::Number, lF::SquaredLogLoss) = (log(x) - log(y))^2
loss(e::Number, lF::AbsoluteLogLoss) = abs(log(x) - log(y))
loss(e::Number, lF::SquaredPropLoss) = (x/y - 1)^2
loss(e::Number, lF::AbsolutePropLoss) = abs(x/y - 1)
loss(e::Number, lF::QLIKE) = x/y - log(x/y) - 1
loss(e::Number, lF::QLIKEReverse) = y/x - log(y/x) - 1
#Method for a loss differential (used a lot in forecast evaluation), i.e. L(x, basecase) - L(y, basecase)
lossdiff{T<:Number}(x1::T, x2::T, xBase::T, lF::LossFunction) = loss(x1, xBase, lF) - loss(x2, xBase, lF)

#----------------------------------------------------------
# ARRAY BASED EXTENSIONS TO CORE METHODS
#----------------------------------------------------------
loss{T1<:Number, T2<:singleInputLossMethods}(e::Vector{T}, lF::LossFunction) = T[ loss(e[n], lF) for n = 1:length(e) ]
loss{T<:Number}(x::Vector{T}, y::Vector{T}, lF::LossFunction) = length(x) == length(y) ? T[ loss(x[n], y[n], lF) for n = 1:length(x) ] : error("Length mismatch")
loss{T<:Number}(x::Matrix{T}, y::Matrix{T}, lF::LossFunction) = size(x) == size(y) ? T[ loss(x[n, m], y[n, m], lF) for n = 1:size(x, 1), m = 1:size(x, 2) ] : error("size mismatch")
loss{T<:Number}(x::Vector{T}, y::Matrix{T}, lF::LossFunction) = length(x) == size(y, 1) ? T[ loss(x[n], y[n, m], lF) for n = 1:size(y, 1), m = 1:size(y, 2) ] : error("size mismatch")
loss{T<:Number}(x::Matrix{T}, y::Vector{T}, lF::LossFunction) = size(x, 1) == length(y) ? T[ loss(x[n, m], y[n], lF) for n = 1:size(x, 1), m = 1:size(x, 2) ] : error("Size mismatch")
lossdiff{T<:Number}(x1::Vector{T}, x2::Vector{T}, xBase::Vector{T}, lF::LossFunction) = (length(xBase) == length(x1) == length(x2) ? T[ lossdiff(x1[n], x2[n], xBase[n], lF) for n = 1:length(xBase) ] : error("Length mismatch")
lossdiff{T<:Number}(x1::Matrix{T}, x2::Vector{T}, xBase::Vector{T}, lF::LossFunction) = (length(xBase) == size(x1, 1) == length(x2)) ? T[ lossdiff(x1[n, m], x2[n], xBase[n], lF) for n = 1:size(x1, 1), m = 1:size(x1, 2) ]
lossdiff{T<:Number}(x1::Vector{T}, x2::Matrix{T}, xBase::Vector{T}, lF::LossFunction) = -1 * lossdiff(x2, x1, xBase, lF)

end # module


"""
    cor(x::AbstractVector{Bool}, y)

    cor(y, x::AbstractVector{Bool})

Compute correlation between `Bool` and other types
"""
Statistics.cor(x::AbstractVector{Bool}, y::AbstractVector) = cor(y, Int.(x))
Statistics.cor(x::AbstractVector{Union{Bool, Missing}}, y::AbstractVector) = cor(y, passmissing(Int).(x))

"""
    dfcor(df::AbstractDataFrame, cols1=names(df), cols2=names(df), verbose=false)

Compute correlation in a DataFrames by specifying a set of columns `cols1` vs
another set `cols2`. The cartesian product of `cols1` and `cols2`'s correlation
will be computed
"""
function dfcor(df::AbstractDataFrame, cols1::Vector{T} = names(df), cols2::Vector{T} = names(df); verbose=false) where {T}
    k = 1
    l1 = length(cols1)
    l2 = length(cols2)
    res = Vector{Float32}(undef, l1*l2)
    names1 = Vector{T}(undef, l1*l2)
    names2 = Vector{T}(undef, l1*l2)
    for i in 1:l1
        icol = df[!, cols1[i]]

        if eltype(icol) >: String
            # do nothing
        else
            Threads.@threads for j in 1:l2
                if eltype(df[!, cols2[j]]) >: String
                    # do nothing
                else
                    if verbose
                        println(k, " ", cols1[i], " ", cols2[j])
                    end
                    df2 = df[:,[cols1[i], cols2[j]] |> unique] |> dropmissing
                    if size(df2, 1) > 0
                        res[k] = cor(df2[!,cols1[j]], df2[!, cols2[j]])
                        names1[k] = cols1[i]
                        names2[k] = cols2[j]
                        k+=1
                    end
                end
            end
        end
    end
    (names1[1:k-1], names2[1:k-1], res[1:k-1])
end


export OperationOnSequences

struct OperationOnSequences <: Sequence
  seq1
  seq2
  op
end

function sample(seq::OperationOnSequences, t::AbstractVector)
  seq.op.(sample(seq.seq1, t), sample(seq.seq2, t))
end

import Base.+, Base.-, Base.*
for op in (:+, :-, :*)
  @eval begin
    $op(s1::Sequence, s2::Sequence) = OperationOnSequences(s1, s2, $op)
    $op(s::Sequence, x::Number) = $op(s, Constant(x))
    $op(x::Number, s::Sequence) = $op(Constant(x), s)
  end
end


#=
# Create a recursive ordering
# Each state maps to a configuration of states with a rule and ordering
#
# For example, 2D Hilbert has 4 states: n, ], [, u labeled here as s1,s2,s3,s4
#
#       s1 s1
# s1 -> s2 s3  This state, s1, corresponds to rule [s2,s3,s1,s1] and ordering [1,3,4,2]
#
# Morton is the simplest with one rule [s1,s2,s3,s4] and ordering [1,2,3,4]
=#
export get_recursive_order, reorder_grid_recursive!
export HILBERT_ORDERING, MORTON_ORDERING

struct RecursiveOrdering
    states::Int                 # Number of possible states
    rules::Array{Array{Int,1}}  # Rules for each state mapping sectors to states
    orders::Array{Array{Int,1}} # Ordering for each state
end

HILBERT_ORDERING = "hilbert ordering";
MORTON_ORDERING = "morton ordering";

# Use this function to build an ordering for a specified grid size
function get_recursive_order(type, dim, griddim)
    nnodes = 1;
    for i=1:length(griddim)
        nnodes = nnodes*griddim[i];
    end

    if dim == 1
        # This doesn't make sense, to just keep regular ordering
        return 1:nnodes;

    elseif dim == 2
        # Find the smallest N such that 2^N >= max(griddim)
        maxgriddim = max(griddim[1],griddim[2]);
        N=0;
        while maxgriddim > (2^N)
            N = N+1;
        end
        twoN = 2^N;

        if type == HILBERT_ORDERING
            rorder_rules = hilbert_ordering_rules(2); # The RecursiveOrdering object
            rorder = build_ordering_2d(rorder_rules, N, false); # The ordering (false -> inverse)
        elseif type == MORTON_ORDERING
            rorder_rules = morton_ordering_rules(2); # The RecursiveOrdering object
            rorder = build_ordering_2d(rorder_rules, N, false); # The ordering (false -> inverse)
        end

        # Truncate ordering to match griddim size
        subrorder = zeros(Int,nnodes);
        subind = 1;
        for i=1:length(rorder)
            # Get coordinates in 2^N grid
            hind = rorder[i];
            hx = mod(hind-1,twoN) + 1;
            hy = Int(floor((hind-1) / (twoN))) + 1;

            if hx <= griddim[1] && hy <= griddim[2]
                gind = hx + griddim[1]*(hy-1);
                subrorder[subind] = gind;
                subind += 1;
            end
        end
        return subrorder;

    elseif dim == 3
        # Find the smallest N such that 2^N >= max(griddim)
        maxgriddim = max(griddim[1],max(griddim[2],griddim[3]));
        N=0;
        while maxgriddim > (2^N)
            N = N+1;
        end
        twoN = 2^N;

        if type == HILBERT_ORDERING
            rorder_rules = hilbert_ordering_rules(3); # The RecursiveOrdering object
            rorder = build_ordering_3d(rorder_rules, N, false); # The ordering (false -> inverse)
        elseif type == MORTON_ORDERING
            rorder_rules = morton_ordering_rules(3); # The RecursiveOrdering object
            rorder = build_ordering_3d(rorder_rules, N, false); # The ordering (false -> inverse)
        end

        # Truncate ordering to match griddim size
        subrorder = zeros(Int,nnodes);
        subind = 1;
        for i=1:length(rorder)
            # Get coordinates in 2^N grid
            hind = rorder[i];
            hx = mod(hind-1,twoN) + 1;
            hy = Int(floor(mod(hind-1,twoN*twoN) / (twoN))) + 1;
            hz = Int(floor((hind-1) / (twoN*twoN))) + 1;

            if hx <= griddim[1] && hy <= griddim[2] && hz <= griddim[3]
                gind = hx + griddim[1]*((hy-1) + griddim[2]*(hz-1));
                subrorder[subind] = gind;
                subind += 1;
            end
        end
        return subrorder;
    end
end

# Use this to reorder the nodes in a given grid.
function reorder_grid_recursive!(grid, griddim, type)
    dim = size(grid.allnodes,1);

    # get the ordering
    rordering = get_recursive_order(type, dim, griddim);
    # invert it for transfering the grid
    rordering = invert_ordering(rordering);

    reorder_grid_nodes!(grid, rordering);
    return grid;
end

#####################################################################################
# ordering rule sets

# Build a Morton Ordering
function morton_ordering_rules(dim)
    if dim == 1
        return RecursiveOrdering(1,[[1]],[[1]]);
    elseif dim == 2
        return RecursiveOrdering(1,[[1,1,1,1]], [[1,2,3,4]]);
    elseif dim == 3
        return RecursiveOrdering(1,[[1,1,1,1,1,1,1,1]], [[1,2,3,4,5,6,7,8]]);
    end
end

# Build a Hilbert Ordering
function hilbert_ordering_rules(dim)
    if dim == 1
        return RecursiveOrdering(1,[[1]], [[1]]);
    elseif dim == 2
        return RecursiveOrdering(4,
                                [[2,3,1,1], [1,2,4,2], [3,1,3,4], [4,4,2,3]], # state indices
                                [[1,4,2,3], [1,2,4,3], [3,4,2,1], [3,2,4,1]]);# indexing order
    elseif dim == 3
        # Oh boy rotations for each state
        rots = [[],
                [1,2],
                [1,2,1,2],
                [2],
                [1,2,2],
                [3],
                [1,1,1],
                [1,1,2],
                [3,3,3],
                [2,3],
                [2,1,1,1],
                [3,3],
                [2,2,2],
                [1],
                [1,1,3],
                [1,2,2,2],
                [1,3],
                [2,2],
                [1,3,3,3],
                [3,2,2,1],
                [1,1],
                [3,3,2],
                [2,2,1],
                [2,2,3]];
        first = [1,4,2,3,8,5,7,6];
        orders = [];
        for i=1:24
            push!(orders, rotations(first, rots[i]));
        end

        # state rules
        rules = [[2,12,3,3,20,12,17,17],
                [3,1,11,21,16,1,16,21],
                [1,18,19,19,2,10,2,10],
                [], [],[], [],[],[],  # These ones aren't used, so I'll just leave them empty
                [18,3,12,11,18,20,12,20],
                [17,17,21,12,2,10,2,10],
                [11,11,1,10,19,19,1,16],
                [], [],[],
                [18,2,12,2,18,17,12,19],
                [20,16,20,16,1,18,11,11],
                [21,10,3,3,21,16,17,17],
                [20,16,20,16,3,3,21,12],
                [10,1,10,21,17,1,19,21],
                [11,11,2,18,19,19,20,18],
                [], [],[]];

        return RecursiveOrdering(24, rules, orders);
    end
end

function rotation_3d(a,axis)
    r = [5 6 1 2 7 8 3 4 ; 2 6 4 8 1 5 3 7 ; 2 4 1 3 6 8 5 7];
    b = zeros(Int,length(a));
    for i=1:length(a)
        b[r[axis,i]] = a[i];
    end
    return b;
end

function rotations(a,rots)
    b = a;
    for i=1:length(rots)
        b = rotation_3d(b,rots[i]);
    end
    return b;
end
############################################################################################

# Build the ordering for a (2^N)^D grid
function build_ordering_2d(ord, N, invert = true)
    nnodes = (2^N)^2;
    lexorder = zeros(Int,nnodes,2);
    twoN = 2^N;
    for j=1:twoN
        for i=1:twoN
            ni = i + twoN*(j-1);
            lexorder[ni,1] = i;
            lexorder[ni,2] = j;
        end
    end

    bbox = zeros(4);
    bbox[1] = 1;
    bbox[3] = 1;
    bbox[2] = twoN;
    bbox[4] = twoN;

    center = [(bbox[1]+bbox[2])/2, (bbox[3]+bbox[4])/2];
    tmpbbox = zeros(4);
    tmpcenter = zeros(2);

    result = zeros(Int64, nnodes);
    for ni=1:nnodes
        vertex = lexorder[ni,:];
        for bi=1:4
            tmpbbox[bi] = bbox[bi];
        end
        tmpcenter[1] = center[1];
        tmpcenter[2] = center[2];

        # Find the index for this node
        index = Int64(0);
        state = 1;
        for i=1:N
            stupid = index;
            stupid = state;
            index = index<<2;
            # Which quadrant is it in?
            quad = 0;
            if vertex[1] > tmpcenter[1]
                quad = quad+1;
            end
            if vertex[2] > tmpcenter[2]
                quad = quad+2;
            end
            quad += 1; # 1-based index
            #println("ni="*string(ni)*" level="*string(i)*" quad="*string(quad));

            # Add this quad's ordering position to index
            index += ord.orders[state][quad] - 1;

            # Update the state to that quad's state according to rule for this one.
            state = ord.rules[state][quad];

            # Shrink the box to one quad
            for j=1:2
                if vertex[j] > tmpcenter[j]
                    tmpbbox[2*j-1] = tmpcenter[j];
                else
                    tmpbbox[2*j] = tmpcenter[j];
                end
            end
            tmpcenter = [(tmpbbox[1]+tmpbbox[2])/2, (tmpbbox[3]+tmpbbox[4])/2];
        end

        # add one for 1-based index
        index += 1;
        #println("index for ni="*string(ni)*" is "*string(index));
        if invert
            result[ni] = index;
        else
            result[index] = ni;
        end
    end

    return result;
end

# Build the ordering for a (2^N)^D grid
function build_ordering_3d(ord, N, invert = true)
    nnodes = (2^N)^3;
    lexorder = zeros(Int,nnodes,3);
    twoN = 2^N;
    for k=1:twoN
        for j=1:twoN
            for i=1:twoN
                ni = i + twoN*((j-1) + twoN*(k-1));
                lexorder[ni,1] = i;
                lexorder[ni,2] = j;
                lexorder[ni,3] = k;
            end
        end
    end

    bbox = zeros(6);
    bbox[1] = 1;
    bbox[3] = 1;
    bbox[5] = 1;
    bbox[2] = twoN;
    bbox[4] = twoN;
    bbox[6] = twoN;

    center = [(bbox[1]+bbox[2])/2, (bbox[3]+bbox[4])/2, (bbox[5]+bbox[6])/2];
    tmpbbox = zeros(6);
    tmpcenter = zeros(3);

    result = zeros(Int64, nnodes);
    for ni=1:nnodes
        vertex = lexorder[ni,:];
        for bi=1:6
            tmpbbox[bi] = bbox[bi];
        end
        tmpcenter[1] = center[1];
        tmpcenter[2] = center[2];
        tmpcenter[3] = center[3];

        # Find the index for this node
        index = Int64(0);
        state = 1;
        for i=1:N
            stupid = index;
            stupid = state;
            index = index<<3;
            # Which octant is it in?
            octant = 0;
            if vertex[1] > tmpcenter[1]
                octant = octant+1;
            end
            if vertex[2] > tmpcenter[2]
                octant = octant+2;
            end
            if vertex[3] > tmpcenter[3]
                octant = octant+4;
            end
            octant += 1; # 1-based index
            #println("ni="*string(ni)*" level="*string(i)*" oct="*string(octant));

            # Add this octant's ordering position to index
            index += ord.orders[state][octant] - 1;

            # Update the state to that octant's state according to rule for this one.
            state = ord.rules[state][octant];

            # Shrink the box to one octant
            for j=1:3
                if vertex[j] > tmpcenter[j]
                    tmpbbox[2*j-1] = tmpcenter[j];
                else
                    tmpbbox[2*j] = tmpcenter[j];
                end
            end
            tmpcenter = [(tmpbbox[1]+tmpbbox[2])/2, (tmpbbox[3]+tmpbbox[4])/2, (tmpbbox[5]+tmpbbox[6])/2];
        end

        # add one for 1-based index
        index += 1;
        #println("index for ni="*string(ni)*" is "*string(index));
        if invert
            result[ni] = index;
        else
            result[index] = ni;
        end
    end

    return result;
end

function invert_ordering(ord)
    iord = zeros(Int,length(ord));
    for i=1:length(ord)
        iord[ord[i]] = i;
    end
    return iord;
end


### A Pluto.jl notebook ###
# v0.12.21

using Markdown
using InteractiveUtils

# This Pluto notebook uses @bind for interactivity. When running this notebook outside of Pluto, the following 'mock version' of @bind gives bound variables a default value (instead of an error).
macro bind(def, element)
    quote
        local el = $(esc(element))
        global $(esc(def)) = Core.applicable(Base.get, el) ? Base.get(el) : missing
        el
    end
end

# ╔═╡ 0ad6e310-97e2-11eb-28bc-99ff80496ecb
using Revise, FourierTools, ImageCore, TestImages, PlutoUI, ImageShow

# ╔═╡ 19339dd6-97e2-11eb-2449-05f0a5ca2f9d
begin
	img = Float32.(testimage("fabio_512_gray"))
	z = zeros(Float32, (768, 768))
	FourierTools.center_set!(z, img)
end

# ╔═╡ 1e72338e-97e2-11eb-0ca1-374f5ab2d2da
@bind Δ Slider(-1024:1:1024)

# ╔═╡ 1aeea42c-97e2-11eb-3af3-d377949794ed
md"Δ=$Δ"

# ╔═╡ 3752a30c-97e2-11eb-3469-dbf857775870
Gray.(FourierTools.shear(z, Δ))

# ╔═╡ 39b3ba50-97e2-11eb-3153-9757657de483
Gray.(FourierTools.shear(img, Δ, 2, 1))

# ╔═╡ Cell order:
# ╠═0ad6e310-97e2-11eb-28bc-99ff80496ecb
# ╠═19339dd6-97e2-11eb-2449-05f0a5ca2f9d
# ╠═1aeea42c-97e2-11eb-3af3-d377949794ed
# ╠═1e72338e-97e2-11eb-0ca1-374f5ab2d2da
# ╠═3752a30c-97e2-11eb-3469-dbf857775870
# ╠═39b3ba50-97e2-11eb-3153-9757657de483


@testset "exgraph" begin
    let
        g = ExGraph()
        g = ExGraph(;ctx=[:foo => 42], x=1)
        @test g.ctx[:foo] == 42
        @test getvalue(g[1]) == 1
    end

    let
        g = ExGraph(:(z = (x + y)^2); x=1, y=1)
        @test length(g) == 5     # check that fuse_assigned() removed extra var
        @test varname(g[5]) == :z
        @test evaluate!(g) == 4

        g = ExGraph(:(z = (x .+ y).^2); x=ones(3), y=ones(3))
        @test evaluate!(g) == [4.0, 4.0, 4.0]
    end

    let
        ex = :(M = u'v)
        g = ExGraph(ex; u=rand(3), v=rand(3))
        @test getcategory(g[3]) == :call
        @test getexpr(g[3]) == :(transpose(u))
    end

    let
        g = ExGraph(:(x = u + v; z = 2x))
        nd = ExNode{:call}(:y, :(x - 1))
        insert!(g, 2, nd)
        @test varname(g[2]) == :y

        nds = ExGraph(:(t1 = u + x; t2 = t1 - x)).tape
        insert!(g, 3, nds)
        @test varname(g[3]) == :t1
        @test varname(g[4]) == :t2

        delete!(g, 3)
        @test length(g) == 5
        delete!(g, :t2)
        @test length(g) == 4

    end

    let
        # test graph with :opaque nodes
        g = ExGraph(:(y = 2x); x=rand(2))
        push!(g, ExNode{:opaque}(:z, :(x .* 3y)))
        rep_g = reparse(g)
        @test getvar(g[3]) == getvar(rep_g[3])
        @test evaluate!(g) == evaluate!(rep_g)

        rw_nd = rewrite(g[3], :(Z = X * Y), :(Z = X .* Y'); phs=[:X, :Y, :Z])
        @test getcategory(rw_nd) == :opaque
        @test getvalue(rw_nd) == nothing        # value should be cleared
    end
end

@testset "var renaming" begin

    ex = quote
        x = 0
        x = x + 1
        y = 4
        x = 2x + y
        y = 5
        z = x + y
    end

    g = ExGraph(ex)
    @test to_expr(g[end]) == :(z = x3 + y2)
end


using Graphs
using Base.Test

function setofsets(array_of_arrays)
    Set(map(Set, array_of_arrays))
end

function test_cliques(graph, expected)
    # Make test results insensitive to ordering
    setofsets(maximal_cliques(graph)) == setofsets(expected)
end

g = simple_adjlist(3, is_directed=false)
add_edge!(g, 1, 2)
@test test_cliques(g, Array[[1,2], [3]])
add_edge!(g, 2, 3)
@test test_cliques(g, Array[[1,2], [2,3]])

# Test for "pivotdonenbrs not defined" bug
h = simple_inclist(6, is_directed=false)
add_edge!(h, 1, 2)
add_edge!(h, 1, 3)
add_edge!(h, 1, 4)
add_edge!(h, 2, 5)
add_edge!(h, 2, 6)
add_edge!(h, 3, 4)
add_edge!(h, 3, 6)
add_edge!(h, 5, 6)

@test maximal_cliques(h) != []

# test for extra cliques bug

g2 = simple_inclist(7, is_directed=false)
add_edge!(g2,1,3)
add_edge!(g2,2,6)
add_edge!(g2,3,5)
add_edge!(g2,3,6)
add_edge!(g2,4,5)
add_edge!(g2,4,7)
add_edge!(g2,5,7)
@test test_cliques(g2, Array[[7,4,5], [2,6], [3,5], [3,6], [3,1]])