using Pkg # hideall
Pkg.activate("_literate/EX-wine/Project.toml")
Pkg.update()
macro OUTPUT()
    return isdefined(Main, :Franklin) ? Franklin.OUT_PATH[] : "/tmp/"
end;

# ## Initial data processing
#
# In this example, we consider the [UCI "wine" dataset](http://archive.ics.uci.edu/ml/datasets/wine)
#
# > These data are the results of a chemical analysis of wines grown in the same region in Italy but derived from three different cultivars. The analysis determined the quantities of 13 constituents found in each of the three types of wines.
#
# ### Getting the data
#
# Let's download the data thanks to the [UrlDownload.jl](https://github.com/Arkoniak/UrlDownload.jl) package and load it into a DataFrame:

using HTTP
using MLJ
using PyPlot
import DataFrames: DataFrame, describe
using UrlDownload
MLJ.color_off() # hide

url = "http://archive.ics.uci.edu/ml/machine-learning-databases/wine/wine.data"
header = ["Class", "Alcool", "Malic acid", "Ash", "Alcalinity of ash",
          "Magnesium", "Total phenols", "Flavanoids",
          "Nonflavanoid phenols", "Proanthcyanins", "Color intensity",
          "Hue", "OD280/OD315 of diluted wines", "Proline"]
data = urldownload(url, true, format=:CSV, header=header);

# The second argument to `urldownload` adds a progress meter for the download,
# the `format` helps indicate the format of the file and the `header` helps
# pass the column names which are not in the file.

df = DataFrame(data)
describe(df)

# the target is the `Class` column, everything else is a feature; we can
# dissociate the two  using the `unpack` function:

y, X = unpack(df, ==(:Class));

# ### Setting the scientific type
#
# Let's explore the scientific type attributed by default to the target and the features

scitype(y)

# this should be changed as it should be considered as an ordered factor. The
# difference is as follows:
#
# * a `Count` corresponds to an integer between 0 and infinity
# * a `OrderedFactor` however is a categorical object (there are finitely many options) with ordering (`1 < 2 < 3`).
#

yc = coerce(y, OrderedFactor);

# Let's now consider the features

scitype(X)

# So there are `Continuous` values (encoded as floating point) and `Count` values (integer).
# Note also that there are no missing value (otherwise one of the scientific type would have been a `Union{Missing,*}`).
# Let's check which column is what:

schema(X)

# The two variable that are encoded as `Count` can  probably be re-interpreted; let's have a look at the `Proline` one to see what it looks like

X[1:5, :Proline]

# It can likely be interpreted as a Continuous as well (it would be better to know precisely what it is but for now let's just go with the hunch).
# We'll do the same with `:Magnesium`:

Xc = coerce(X, :Proline=>Continuous, :Magnesium=>Continuous);

# Finally, let's have a quick look at the mean and standard deviation of each feature to get a feel for their amplitude:

describe(Xc, :mean, :std)

# Right so it varies a fair bit which would invite to standardise the data.
#
# **Note**: to complete such a first step, one could explore histograms of the various features for instance, check that there is enough variation among the continuous features and that there does not seem to be problems in the encoding, we cut this out to shorten the tutorial. We could also have checked that the data is balanced.
#
# ## Getting a baseline
#
# It's a multiclass classification problem with continuous inputs so a sensible start is  to test two very simple classifiers to get a baseline.
# We'll train two simple pipelines:
# - a Standardizer + KNN classifier and
# - a Standardizer + Multinomial classifier (logistic regression).

KNNC = @load KNNClassifier
MNC = @load MultinomialClassifier pkg=MLJLinearModels;

KnnPipe = Standardizer |> KNNC
MnPipe = Standardizer |> MNC

# Note the `|>` syntax, which is syntactic sugar for creating a linear `Pipeline` from components models.

# We can now fit this on a train split of the data setting aside 20% of the data for eventual testing.

train, test = partition(collect(eachindex(yc)), 0.8, shuffle=true, rng=111)
Xtrain = selectrows(Xc, train)
Xtest = selectrows(Xc, test)
ytrain = selectrows(yc, train)
ytest = selectrows(yc, test);

# Let's now wrap an instance of these models with data (all hyperparameters are set to default here):

knn = machine(KnnPipe, Xtrain, ytrain)
multi = machine(MnPipe, Xtrain, ytrain)

# Let's train a KNNClassifier with default hyperparameters and get a baseline misclassification rate using 90% of the training data to train the model and the remaining 10% to evaluate it:

opts = (resampling=Holdout(fraction_train=0.9), measure=cross_entropy)
res = evaluate!(knn; opts...)
round(res.measurement[1], sigdigits=3)

# Now we do the same with a MultinomialClassifier

res = evaluate!(multi; opts...)
round(res.measurement[1], sigdigits=3)

# Both methods seem to offer comparable levels of performance.
# Let's check the misclassification over the full training set:

mcr_k = misclassification_rate(predict_mode(knn, Xtrain), ytrain)
mcr_m = misclassification_rate(predict_mode(multi, Xtrain), ytrain)
println(rpad("KNN mcr:", 10), round(mcr_k, sigdigits=3))
println(rpad("MNC mcr:", 10), round(mcr_m, sigdigits=3))

# So here we have done no hyperparameter training and already have a misclassification rate below 5%.
# Clearly the problem is not very difficult.
#
# ## Visualising the classes
#
# One way to get intuition for why the dataset is so easy to classify is to project it onto a 2D space using the PCA and display the two classes to see if they are well separated; we use the arrow-syntax here (if you're on Julia <= 1.2, use the commented-out lines as you won't be able to use the arrow-syntax)

PCA = @load PCA
pca_pipe = Standardizer() |> PCA(maxoutdim=2)
pca = machine(pca_pipe, Xtrain)
fit!(pca)
W = transform(pca, Xtrain)

# Let's now display this using different colours for the different classes:

x1 = W.x1
x2 = W.x2

mask_1 = ytrain .== 1
mask_2 = ytrain .== 2
mask_3 = ytrain .== 3

figure(figsize=(8, 6))
plot(x1[mask_1], x2[mask_1], linestyle="none", marker="o", color="red")
plot(x1[mask_2], x2[mask_2], linestyle="none", marker="o", color="blue")
plot(x1[mask_3], x2[mask_3], linestyle="none", marker="o", color="magenta")

xlabel("PCA dimension 1", fontsize=14)
ylabel("PCA dimension 2", fontsize=14)
legend(["Class 1", "Class 2", "Class 3"], fontsize=12)
xticks(fontsize=12)
yticks(fontsize=12)

savefig(joinpath(@OUTPUT, "EX-wine-pca.svg")) # hide

# \figalt{PCA}{EX-wine-pca.svg}
#
# On that figure it now becomes quite clear why we managed to achieve such high scores with very simple classifiers.
# At this point it's a bit pointless to dig much deaper into parameter tuning etc.
#
# As a last step, we can report performances of the models on the test set which we set aside earlier:

perf_k = misclassification_rate(predict_mode(knn, Xtest), ytest)
perf_m = misclassification_rate(predict_mode(multi, Xtest), ytest)
println(rpad("KNN mcr:", 10), round(perf_k, sigdigits=3))
println(rpad("MNC mcr:", 10), round(perf_m, sigdigits=3))

# Pretty good for so little work!
PyPlot.close_figs() # hide


using DimArrays
using Test

a = DimVector([1,2,3,4], :aa, 100)

@test DimArrays.dname(a,1) == :aa
@test DimArrays.ifunc(a,1)(1) == 100
@test DimArrays.haslabel(a) == false

@test typeof(a') <: DimMatrix
@test ndims( (a')' )==1
@test DimArrays.dname( (a')' ,1) == :aa

@test typeof( map(sqrt, a) ) <: DimVector
@test typeof( [sqrt(i) for i in a] ) <: DimVector

b = DimArray([1.0 2; 3 4]; label=:c)

@test string(b) == "DimArray([1.0 2.0; 3.0 4.0]; label = :c)"

@test string(name!(selectdim(b, :col, 2), "one", "cont")) == "DimArray([2.0, 4.0], :one; label = :cont)"

@test permutedims(b, (:col, :row))[1,2] ≈ 3

using Statistics

@test sum(b; dims=[:row,:col])[1] ≈ 10
@test std(b; dims=:col)[1] ≈ 0.7071067811865476
@test typeof( mean(b; dims=2) ) <: DimMatrix

@test selectdim(b, :col, 1)[2] ≈ 3
@test typeof( selectdim(b, 2, 1) ) <: DimVector

@test (b .+ 99)[1,1] ≈ 100
# @test typeof(b .+ 99) <: DimMatrix

@test sum( hcat(DimArray([1,3]), [2,4]) ./ b ) ≈ 4

@test maximum( nest([DimArray([1,3]), [2,4]]) .- b ) ≈ 0

# aabb = vcat(hcat(a,a),b,b)

# @test size(aabb) == (8,2)
# @test DimArrays.dnames(aabb) == [:aa, :col]

push!(a, 55,66)
@test DimArrays.ifuncs(a)[1](length(a)) == 600

c = dictvector(ones(2), [:α, "β"], :ω, :N)

push!(c, 77, 88)
append!(c, [0,0])
push!(c, 99, :γ)

@test c isa DimVector
# @test sum(c, :ω)[1] ≈ 266 # todo: dims=: etc


using Maker
using Base.Test

x = [ 3.14159   6.28319   9.42478
      12.5664   15.708    18.8496
      21.9911   25.1327   28.2743 ]

writecsv("b.csv", x)

Maker.variable("a") do
    println("Generating `a`.")
    pi
end

Maker.variable("b", "b.csv") do
    println("Reading `b`.")
    readcsv("b.csv")
end

Maker.file("b.csv")

Maker.variable("c", ["a", "b"]) do
    println("Calculating `c`.")
    a * b
end

make("a")
make("b")
make("c")

@test a ≈ pi
@test b ≈ x
@test c ≈ pi * x

# redefine `a`
Maker.variable("a") do
    println("Regenerating `a`.")
    2pi
end

make("c", verbose = true)

@test a ≈ 2pi
@test b ≈ x
@test c ≈ 2pi * x

# redefine `a` again:

Maker.variable("a") do
    println("Generating `a`.")
    pi
end

make("a", verbose=true)

@test a ≈ pi
@test c ≈ 2pi * x

make("c", verbose=true)

@test a ≈ pi
@test c ≈ pi * x

# change `a`

a = -pi

make("c")

@test a ≈ pi
@test c ≈ pi * x

rm("b.csv")


### Statistics

function countmap(x::RLEVector{T1,T2}) where {T1,T2}
  tally = Dict{T1,T2}()
  for (v,w) in zip(values(x), widths(x))
    tally[v] = get(tally,v,0) + w
  end
  tally
end

function mode(x::RLEVector)
  tally = countmap(x)
  which_max = argmax(collect(values(tally)))
  max_val = collect(keys(tally))[ which_max ]
  max_val
end


using Test
using TypeClasses
using DataTypesBasic
using Suppressor

splitln(str) = split(strip(str), "\n")

# Combine
# =======

a = State(s -> ("hello $s", s+1))
b = State(s -> ("!", s+10))

@test (a ⊕ b)(4) == ("hello 4!", 15)

# FunctorApplicativeMonad
# =======================

g = State(s -> (s*2, s+1))
f = State(s -> (s*s, s))

# just function composition
fg = map(f, g)
@test fg(3) == ((36, 6), 4)

fPLUSg = mapn(f, g) do x1, x2
  x1 + x2
end

@test fPLUSg(3) == (9+6, 4)

fPRODg = @syntax_flatmap begin
  x1 = f
  x2 = g
  @pure x1 * x2
end

@test fPRODg(3) == (9*6, 4)

putget = @syntax_flatmap begin
  putstate(4)
  x = getstate
  @pure x * x
end
@test putget(()) == (16, 4)

# FlipTypes
# =========

# this only works because of the general `combine` implementation for State
# so be cautious as the eltypes need to support `combine` to not get a MethodError in runtime

a = State.([s -> (s, s), s -> (2s, s*s), s -> (s*s, 4)])
a2 = [map(x -> pure(Vector, x), v) for v in a]
@test flip_types(a)(3) == ([3, 6, 81], 4)


using VerilogFIR
using Test

@testset "Testing Auxiliary Functions" begin
    # maxrep
    for (i,j) in zip((2,3,4,5,6),(1,3,7,15,31))
        @test VerilogFIR.maxrep(i) == j
    end
    # minrep
    for (i,j) in zip((2,3,4,5,6),(2,4,8,16,32))
        @test VerilogFIR.minrep(i) == -j
    end
    # fminabs
    let
        t1 = (1.0, 10, -400, -0.1)
        @test VerilogFIR.fminabs(t1...) == 0
        # Test case when a division by 0 occurs, make sure result is still correct.
        t2 = (Inf, 1.1)
        @test VerilogFIR.fminabs(t2...) == 1
        t3 = (-Inf, Inf, -2.3)
        @test VerilogFIR.fminabs(t3...) == 2
    end
    # clog2
    let
        tests       = (1,2,3,2.5,100.23)
        expected    = (0,1,2,2,7)
        for (t,e) in zip(tests, expected)
            @test VerilogFIR.clog2(t) == e
        end
    end
    # flog2
    let
        tests       = (1,2,3,2.5,100.23)
        expected    = (0,1,1,1,6)
        for (t,e) in zip(tests, expected)
            @test VerilogFIR.flog2(t) == e
        end
    end
    # nbits_u
    let
        tests    = (6,7,8,9)
        expected = (3,3,4,4)
        for (t,e) in zip(tests, expected)
            @test VerilogFIR.nbits_u(t) == e
        end
    end
    # nbits_s
    let
        tests    = (7,8,9,-7,-8,-9)
        expected = (4,5,5,4,4,5)
        for (t,e) in zip(tests, expected)
            @test VerilogFIR.nbits_s(t) == e
        end
    end
    # tab
    let
        @test VerilogFIR.tab(1) == "    "
        @test VerilogFIR.tab(2) == "        "
    end
    # makedict
    let
        a = 1
        b = [10,20,30]
        c = "hello"
        d = 5
        f = 2.21
        dict = VerilogFIR.@makedict a b c d f
        expected = Dict(
            :a => 1,
            :b => [10,20,30],
            :c => "hello",
            :d => 5,
            :f => 2.21,
        )
        @test dict == expected
    end
end

#=
Tests for functions that scale coefficients.
=#
@testset "Testing Analysis Functions" begin
    # Test coefficient scaling
    let
        # Test with all positive numbers

        ## TEST 1
        coeffs = [2,1,1]
        w_coeff = 3
        @test VerilogFIR.scale_coefficients(coeffs, w_coeff) == ([2,1,1],0)

        ## TEST 2
        w_coeff = 4
        @test VerilogFIR.scale_coefficients(coeffs, w_coeff) == ([4,2,2],1)

        ## TEST 3
        # Test with negative numbers
        # With the same values as TEST 1, should scale since negative numbers
        # have one extra number to work with on the low end.
        coeffs = [-2,-1,-1]
        w_coeff = 3
        @test VerilogFIR.scale_coefficients(coeffs, w_coeff) == ([-4,-2,-2],1)

        ## TEST 4
        # Test with positive and negative numbers
        coeffs = [2,0,-2]
        w_coeff = 3
        @test VerilogFIR.scale_coefficients(coeffs, w_coeff) == ([2,0,-2],0)

        ## TEST 5
        w_coeff = 4
        @test VerilogFIR.scale_coefficients(coeffs, w_coeff) == ([4,0,-4],1)

        ## TEST 6
        coeffs = [1,0,-2]
        w_coeff = 3
        @test VerilogFIR.scale_coefficients(coeffs, w_coeff) == ([2,0,-4],1)
    end
    # Test accumulator width
    let
        coeffs = [1,-2,0,-1,3]
        w_input = 4
        # Get the maximum positive and negative values that can be represented
        # with 4 bits
        big     = 7
        little  = -8
        # Compute maximum and minimum values for the filter
        maxval = 7*(1+3) + -8*(-1-2)
        minval = -8*(1+3) + 7*(-1-2)
        # Get the number of bits for each
        bits_max = VerilogFIR.nbits_s(maxval)
        bits_min = VerilogFIR.nbits_s(minval)
        nbits = min(bits_max, bits_min)
        @test VerilogFIR.acc_width(coeffs, w_input) == nbits
    end
end

@testset "Testing complete run" begin
    generate_fir(stdout, ones(4)/4, w_input = 16, w_output = 16, w_coeff = 12)
    generate_fir(stdout, ones(4)/4, w_input = 8,  w_output = 8, w_coeff = 21)
    generate_fir(stdout, ones(4)/4, w_input = 16, w_output = 8, w_coeff = 9)
    # Test generating saturating logic
    generate_fir(ones(8))

    @test_throws InexactError generate_fir(stdout, ones(4)/4, w_coeff = 1)
    # Test with file
    file = "test15713209.v"
    generate_fir(file, ones(4)/4)
    rm(file)
end


import Base: rand,
             ∈

export ZeroSet,
       linear_map

"""
    ZeroSet{N<:Real} <: AbstractSingleton{N}

Type that represents the zero set, i.e., the set that only contains the origin.

### Fields

- `dim` -- the ambient dimension of this zero set
"""
struct ZeroSet{N<:Real} <: AbstractSingleton{N}
    dim::Int
end

# default constructor of type Float64
ZeroSet(dim::Int) = ZeroSet{Float64}(dim)

# --- AbstractSingleton interface functions ---

"""
    element(S::ZeroSet{N})::Vector{N} where {N<:Real}

Return the element of a zero set.

### Input

- `S` -- zero set

### Output

The element of the zero set, i.e., a zero vector.
"""
function element(S::ZeroSet{N})::Vector{N} where {N<:Real}
    return zeros(N, S.dim)
end

"""
    element(S::ZeroSet{N}, ::Int)::N where {N<:Real}

Return the i-th entry of the element of a zero set.

### Input

- `S` -- zero set
- `i` -- dimension

### Output

The i-th entry of the element of the zero set, i.e., 0.
"""
function element(S::ZeroSet{N}, ::Int)::N where {N<:Real}
    return zero(N)
end

# --- LazySet interface functions ---

"""
    dim(Z::ZeroSet)::Int

Return the ambient dimension of this zero set.

### Input

- `Z` -- a zero set, i.e., a set that only contains the origin

### Output

The ambient dimension of the zero set.
"""
function dim(Z::ZeroSet)::Int
    return Z.dim
end

"""
    σ(d::AbstractVector{N}, Z::ZeroSet{N}) where {N<:Real}

Return the support vector of a zero set.

### Input

- `d` -- direction
- `Z` -- a zero set, i.e., a set that only contains the origin

### Output

The returned value is the origin since it is the only point that belongs to this
set.
"""
function σ(d::AbstractVector{N}, Z::ZeroSet{N}) where {N<:Real}
    @assert length(d) == dim(Z) "the direction has the wrong dimension"
    return element(Z)
end

"""
    ρ(d::AbstractVector{N}, Z::ZeroSet{N}) where {N<:Real}

Evaluate the support function of a zero set in a given direction.

### Input

- `d` -- direction
- `Z` -- a zero set, i.e., a set that only contains the origin

### Output

`0`.
"""
function ρ(d::AbstractVector{N}, Z::ZeroSet{N}) where {N<:Real}
    @assert length(d) == dim(Z) "the direction has the wrong dimension"
    return zero(N)
end

"""
    ∈(x::AbstractVector{N}, Z::ZeroSet{N})::Bool where {N<:Real}

Check whether a given point is contained in a zero set.

### Input

- `x` -- point/vector
- `Z` -- zero set

### Output

`true` iff ``x ∈ Z``.

### Examples

```jldoctest
julia> Z = ZeroSet(2);

julia> ∈([1.0, 0.0], Z)
false
julia> ∈([0.0, 0.0], Z)
true
```
"""
function ∈(x::AbstractVector{N}, Z::ZeroSet{N})::Bool where {N<:Real}
    @assert length(x) == dim(Z)

    zero_N = zero(N)
    return all(i -> x[i] == zero_N, eachindex(x))
end

"""
    rand(::Type{ZeroSet}; [N]::Type{<:Real}=Float64, [dim]::Int=2,
         [rng]::AbstractRNG=GLOBAL_RNG, [seed]::Union{Int, Nothing}=nothing
        )::ZeroSet{N}

Create a zero set (note that there is nothing to randomize).

### Input

- `ZeroSet` -- type for dispatch
- `N`       -- (optional, default: `Float64`) numeric type
- `dim`     -- (optional, default: 2) dimension
- `rng`     -- (optional, default: `GLOBAL_RNG`) random number generator
- `seed`    -- (optional, default: `nothing`) seed for reseeding

### Output

The (only) zero set of the given numeric type and dimension.
"""
function rand(::Type{ZeroSet};
              N::Type{<:Real}=Float64,
              dim::Int=2,
              rng::AbstractRNG=GLOBAL_RNG,
              seed::Union{Int, Nothing}=nothing
             )::ZeroSet{N}
    rng = reseed(rng, seed)
    return ZeroSet{N}(dim)
end

"""
    linear_map(M::AbstractMatrix{N}, Z::ZeroSet{N}) where {N<:Real}

Concrete linear map of a zero set.

### Input

- `M` -- matrix
- `Z` -- zero set

### Output

The zero set whose dimension matches the output dimension of the given matrix.
"""
function linear_map(M::AbstractMatrix{N}, Z::ZeroSet{N}) where {N<:Real}
    @assert dim(Z) == size(M, 2) "a linear map of size $(size(M)) cannot be " *
                                 "applied to a set of dimension $(dim(Z))"

    return ZeroSet(size(M, 1))
end

"""
    translate(Z::ZeroSet{N}, v::AbstractVector{N}) where {N<:Real}

Translate (i.e., shift) a zero set by a given vector.

### Input

- `Z` -- zero set
- `v` -- translation vector

### Output

A singleton containing the vector `v`.
"""
function translate(Z::ZeroSet{N}, v::AbstractVector{N}) where {N<:Real}
    @assert length(v) == dim(Z) "cannot translate a $(dim(Z))-dimensional " *
                                "set by a $(length(v))-dimensional vector"
    return Singleton(v)
end


#=
This script runs step-by-step through md-RQA analysis in Julia. It uses DynamicalSystems.jl for delay embedding and analysis.
=#

# ----- this script relies on the following packages -----
using Pkg
Pkg.add("DataFrames");
Pkg.add("CSV");
Pkg.add("DynamicalSystems"); # RQA stuffs
Pkg.add("StatsBase");
Pkg.add("Plots"); # plotting
Pkg.add("ElasticArrays"); # efficent matrix iteration
Pkg.add("LinearAlgebra") # diag caculation
Pkg.add("Distances") # matrix pairwise distances

using DataFrames, Plots, CSV, DynamicalSystems, StatsBase, ElasticArrays, LinearAlgebra, Distances;

# you should also include the follwing functions from this repo
include("mdFnn.jl")
include("mdEmbed.jl")

# Step 1 ----- load in an example dataset
data = DataFrame(CSV.File("exampleData.csv"));
#data = DataFrame(CSV.File("lorenz.csv", header=false));

# turn to matrix (and in this case only use first 3 columns)
ts_names = names(data[:,1:3]) |> permutedims; # use this for ami plotting
data = Matrix(data)[:,1:3];

# get the dimensions (number of columns) of the data
ncol = size(data,2);

# Step 2 ----- find the delay (tau)

# parameters:
nbins = 10;
maxlag = 200;

# get delay for each dimension
lags = zeros(ncol);
plotMI = zeros(maxlag+1, ncol);
for dimension in 1:ncol
    plotMI[:,dimension] = selfmutualinfo(data[:,dimension],0:1:maxlag; nbins=nbins);
    lags[dimension] = estimate_delay(data[:,dimension], "mi_min", 0:1:maxlag; nbins=nbins);
end

τ = maximum(lags) |> round |> Int;

# Step 3 ------ find the embedding dimension
# parameters
tau = τ;
maxEmb = 10;
numSamples = 500;
Rtol = 10;
Atol = 2;
abs_threshold = 1;

fnnPerc = mdFnn(data[:,:], tau, maxEmb, numSamples, Rtol, Atol);

# embedding dimension

# simple version
D = findall(fnnPerc .< abs_threshold) |> minimum;

#= a more comple version would be to consider if the rate of FNN function is slowing. This would be useful for cases where the FNN function never goes below the threshold but does "level off"

ex:

# the minimum difference between FNN of two consecutive D's
diff_threshold = 3

D_below_absolute_threshold = findall(fnnPerc .< abs_threshold)

if size(D_below_absolute_threshold)[1]>0
    D = (findall(fnnPerc .< abs_threshold) |> minimum);
else
    # if no values are below the absolute threshold,
    # then use the difference threshold

    D = (findall(abs.(diff(fnnPerc)) .< diff_threshold) |> minimum) + 1
end
=#

# Step 4 ------ embed timeseries
embed_TS = mdEmbed(data, D, τ);

# Step 5 ------ construct Recurrence Matrix
R = RecurrenceMatrix(embed_TS, .05; fixedrate = true);

# Step 6 ------ Recurrence quantification
rqaOUT = rqa(R; lmin = 50, thieller = 1);

# Step 7 ------- Construct output data frame

# get the obtained parameters
parameters = (delay = tau, embed = D);

# join parameters and rqaOUT
df_out = [merge(parameters, rqaOUT)] # be careful not to duplicate column names

#optionally save to csv
CSV.write("rqa_output.csv", df_out)

# Step 8 ------ Diagnostic Plots

## DO NOT ACCEPT ANY VALUES WITHOUT LOOKING AT THE PLOTS!!!!

# plot of the ami function
ami_plot = plot(plotMI[:,:], label = ts_names);
plot!(ami_plot, [τ], seriestype="vline", linestyle=:dash,label = "delay selected")

# false nearest neighbors plot
fnn_plot = plot(fnnPerc, label = "%FNN");
plot!(fnn_plot, [D], seriestype="vline", linestyle=:dash,label = "embed selected")

# recurrence plot
xs, ys = coordinates(R);
rec_plot = scatter(xs, ys, markersize = 0.1, markercolor = :black, legend = false);

# combine plots
l = @layout [a{0.5w} [b; c]]
diag_plots = plot(rec_plot, ami_plot,fnn_plot, layout = l);
plot!(diag_plots, size=(800,400))
png(diag_plots, "simple_diagnostic_plots_filename.png")


@testset "SOCtoPSD" begin
    bridged_mock = MOIB.SOCtoPSD{Float64}(mock)
    mock.optimize! = (mock::MOIU.MockOptimizer) -> MOIU.mock_optimize!(mock, [1.0, 1/√2, 1/√2],
                          (MOI.VectorAffineFunction{Float64}, MOI.PositiveSemidefiniteConeTriangle) => [[√2/2, -1/2, √2/4, -1/2, √2/4, √2/4]],
                          (MOI.VectorAffineFunction{Float64}, MOI.Zeros)                            => [[-√2]])
    MOIT.soc1vtest(bridged_mock, config)
    MOIT.soc1ftest(bridged_mock, config)
    ci = first(MOI.get(bridged_mock, MOI.ListOfConstraintIndices{MOI.VectorAffineFunction{Float64}, MOI.SecondOrderCone}()))
    test_delete_bridge(bridged_mock, ci, 3, ((MOI.VectorAffineFunction{Float64}, MOI.PositiveSemidefiniteConeTriangle, 0),))
end

@testset "RSOCtoPSD" begin
    bridged_mock = MOIB.RSOCtoPSD{Float64}(mock)
    mock.optimize! = (mock::MOIU.MockOptimizer) -> MOIU.mock_optimize!(mock, [1/√2, 1/√2, 0.5, 1.0],
                          (MOI.SingleVariable,                MOI.EqualTo{Float64})       => [-√2, -1/√2],
                          (MOI.VectorAffineFunction{Float64}, MOI.PositiveSemidefiniteConeTriangle) => [[√2, -1/2, √2/8, -1/2, √2/8, √2/8]])
    MOIT.rotatedsoc1vtest(bridged_mock, config)
    mock.optimize! = (mock::MOIU.MockOptimizer) -> MOIU.mock_optimize!(mock, [1/√2, 1/√2],
                          (MOI.VectorAffineFunction{Float64}, MOI.PositiveSemidefiniteConeTriangle) => [[√2, -1/2, √2/8, -1/2, √2/8, √2/8]])
    MOIT.rotatedsoc1ftest(bridged_mock, config)
    ci = first(MOI.get(bridged_mock, MOI.ListOfConstraintIndices{MOI.VectorAffineFunction{Float64}, MOI.RotatedSecondOrderCone}()))
    test_delete_bridge(bridged_mock, ci, 2, ((MOI.VectorAffineFunction{Float64}, MOI.PositiveSemidefiniteConeTriangle, 0),))
end


import Base.rand

export Singleton

"""
    Singleton{N, VN<:AbstractVector{N}} <: AbstractSingleton{N}

Type that represents a singleton, that is, a set with a unique element.

### Fields

- `element` -- the only element of the set
"""
struct Singleton{N, VN<:AbstractVector{N}} <: AbstractSingleton{N}
    element::VN
end

isoperationtype(::Type{<:Singleton}) = false
isconvextype(::Type{<:Singleton}) = true

# --- AbstractSingleton interface functions ---

"""
    element(S::Singleton)

Return the element of a singleton.

### Input

- `S` -- singleton

### Output

The element of the singleton.
"""
function element(S::Singleton)
    return S.element
end

"""
    element(S::Singleton, i::Int)

Return the i-th entry of the element of a singleton.

### Input

- `S` -- singleton
- `i` -- dimension

### Output

The i-th entry of the element of the singleton.
"""
function element(S::Singleton, i::Int)
    return S.element[i]
end

# --- LazySet interface functions ---

"""
    rand(::Type{Singleton}; [N]::Type{<:Real}=Float64, [dim]::Int=2,
         [rng]::AbstractRNG=GLOBAL_RNG, [seed]::Union{Int, Nothing}=nothing)

Create a random singleton.

### Input

- `Singleton` -- type for dispatch
- `N`         -- (optional, default: `Float64`) numeric type
- `dim`       -- (optional, default: 2) dimension
- `rng`       -- (optional, default: `GLOBAL_RNG`) random number generator
- `seed`      -- (optional, default: `nothing`) seed for reseeding

### Output

A random singleton.

### Algorithm

The element is a normally distributed vector with entries of mean 0 and standard
deviation 1.
"""
function rand(::Type{Singleton};
              N::Type{<:Real}=Float64,
              dim::Int=2,
              rng::AbstractRNG=GLOBAL_RNG,
              seed::Union{Int, Nothing}=nothing)
    rng = reseed(rng, seed)
    element = randn(rng, N, dim)
    return Singleton(element)
end

"""
    translate(S::Singleton, v::AbstractVector)

Translate (i.e., shift) a singleton by a given vector.

### Input

- `S` -- singleton
- `v` -- translation vector

### Output

A translated singleton.

### Algorithm

We add the vector to the point in the singleton.
"""
function translate(S::Singleton, v::AbstractVector)
    @assert length(v) == dim(S) "cannot translate a $(dim(S))-dimensional " *
                                "set by a $(length(v))-dimensional vector"
    return Singleton(element(S) + v)
end

"""
    rectify(S::Singleton)

Concrete rectification of a singleton.

### Input

- `S` -- singleton

### Output

The `Singleton` that corresponds to the rectification of `S`.
"""
function rectify(S::Singleton)
    return Singleton(rectify(element(S)))
end

"""
    project(S::Singleton, block::AbstractVector{Int}; kwargs...)

Concrete projection of a singleton.

### Input

- `S`     -- singleton
- `block` -- block structure, a vector with the dimensions of interest

### Output

A set representing the projection of the singleton `S` on the dimensions
specified by `block`.
"""
function project(S::Singleton, block::AbstractVector{Int}; kwargs...)
    return Singleton(element(S)[block])
end

function permute(S::Singleton, p::AbstractVector{Int})
    e = element(S)[p]
    return Singleton(e)
end


"""
# BusinessModel
## Fields
- `t::Integer`: 1 if model was just initialized, 1 + the number of time periods elapsed since model initialization otherwise. Time is considered in an abstract sense, without units.
- `p::Vector{<:AbstractFloat}`: A vector of individual probabilities (0 <= p <= 1) linked to the access rights, representing the probability that this particular access right be required by a new user.
- `pσ::Number`: The standard deviation of the normal distribution from which random samples will be taken to make `p` evolve from time period to time period. Default: 1%.
- `a::Matrix{<:Bool}`: A matrix of access rights business requirements. Rows = Users. Columns = Access Rights.
"""
mutable struct BusinessModel
    t::Int
    p::Vector{<:Number}
    pσ::Number
    a::BitArray{2}
    function BusinessModel(u:: Int, r:: Int, pσ::Number = 0.01)
        t = 1; # The first instant of the model life is 1.
        p = rand(r); # Initialize right probabilities are randomely select from uniform distribution.
        a = BitArray{2}(undef, 0, r); # Initialize the original array without any users.
        m = new(t, p, pσ, a);
        add_users(m, u);
        return m;
    end;
end;
#function Base.show(io::IO, m::BusinessModel)
    #compact = get(io, :compact, false);
    #if compact
        #print("m(t: ", m.t, ", |u|: ", count_users(m), ", |r|: ", count_rights(m), "pσ: ", m.pσ, ")");
    #else
    #    println("BusinessModel");
    #    println("t: ", m.t);
    #    println("|u|: ", count_users(m));
    #    println("|r|: ", count_rights(m));
    #    println("pσ: ", m.pσ);
    #    println("p: ", m.p);
    #end;
#end;
#function Base.show(io::IO, ::MIME"text/plain", m::BusinessModel)
#    println("MIME: text/plain");
#    show(io, m);
#end;
function count_users(m::BusinessModel)
    return size(m.a)[1];
end;
function count_rights(m::BusinessModel)
    return size(m.a)[2];
end;
function add_users(m::BusinessModel, u::Int = 1)
    # Generate a matrix of random values,
    # that will be used to randomely determine which user is granted what access right.
    up = rand(u, count_rights(m));
    # Reshape the model access right probabilities to match the size of the above random matrix.
    mp = repeat(reshape(m.p, 1, length(m.p)),u);
    # For every matrix value where the user's random value is lower than the probability,
    # consider the right as granted, otherwise not granted.
    ua = broadcast(<, up, mp);
    # Merge the new user's rights into the model.
    m.a = cat(dims=1, m.a, ua);
end;
function sample1()
    m = BusinessModel(3,4,.01);
    count_users(m);
    count_rights(m);
    m;
    m.a;
end;
#sample1();


using Distributions
using JSON

a = 3
b = 6
c = 5
d = TriangularDist( a,b,c )

x = [ -5, -2.5, 0, 2.5, 5 ]

dmgf(t) = mgf(d, t )
y = map( dmgf, x )
println( y )

data = Dict([
	("a", a),
	("b", b),
	("c", c),
	("data", x),
	("expected", y)
])

outfile = open("./test/fixtures/partial.json", "w")
JSON.json(data)

write( outfile, JSON.json(data) )


function residuals(model::AModel, calibration)
    m = calibration[:exogenous]
    s = calibration[:states]
    x = calibration[:controls]
    p = calibration[:parameters]
    res = Dolo.arbitrage(model, m, s, x, m, s, x, p)
    S = Dolo.transition(model, m, s, x, m, p)
    return Dict(:arbitrage=>res, :transition=>S-s)
end

function residuals(model::AModel)
    return residuals(model, model.calibration)
end

function find_deterministic_equilibrium(model::AModel, calibration::ModelCalibration)
    m, p, s0, x0 = calibration[:exogenous, :parameters, :states, :controls]
    ns = length(s0)
    nx = length(x0)

    function obj!(sx)
        s = sx[1:ns]
        x = sx[ns+1:ns+nx]

        # update state part of residual
        s_new = transition(model, m, s, x, m, p)

        # now update control part
        resids = arbitrage(model, m, s, x, m, s, x, p)
        vcat(s_new - s, resids)
    end

    sol = nlsolve(obj!, vcat(s0, x0),  autodiff=:central, inplace=false)
    NLsolve.converged(sol) || error("Nonlinear solver failed to find steady state")

    # otherwise set controls
    out = deepcopy(calibration)
    out[:states] = sol.zero[1:ns]
    out[:controls] = sol.zero[ns+1:end]
    out
end

function find_deterministic_equilibrium(model::AModel)
    return find_deterministic_equilibrium(model, model.calibration)
end

# ---------- #
# Docstrings #
# ---------- #

"""
    find_deterministic_equilibrium(model::AModel, [calib::ModelCalibration])

Solve for the steady state equilibrium of `model`, data in `cailb` to fill
in parameter values and provide an initial guess for the states and controls.
When no calibration is passed `model.calibration` is used

The `exogenous` variables at time t-1 (`m`) and t (`M`) are set to
`calib[:exogenous]`.

The deterministic equilibrium is found by solving for vectors `s` and `x`, such
that

1. `s = transition(m, s, x, m, p)`
2. `0 = arbitrage(m, s, x, m, s, x, p)`

"""
find_deterministic_equilibrium

"""
    residuals(model::AModel, [calib::ModelCalibration])::Dict

Compute the steady state residuals for the aribtrage and transition equations
of `model`, when these functions are evaluated at the data in `calib`. If no
`calib` is provided, `model.calibration` will be used.

See the docstring for `find_deterministic_equilibrium` for more information
"""
residuals


using PtFEM

data = Dict(
  # Frame(nels, nn, ndim, nst, nip, finite_element(nod, nodof))
  :struc_el => Frame(3, 4, 3, 1, 1, Line(2, 3)),
  :properties => [
    4.0e6 1.0e6 0.3e6 0.3e6],
  :x_coords => [0.0, 5.0, 5.0, 5.0],
  :y_coords => [5.0, 5.0, 5.0, 0.0],
  :z_coords => [5.0, 5.0, 0.0, 0.0],
  :gamma => [0.0, 0.0, 90.0],             # Third element 90 degrees from x-y plane
  :g_num => [
    1 3 4;
    2 2 3],
  :support => [
    (1, [0 0 0 0 0 0]),
    (4, [0 0 0 0 0 0])
    ],
  :loaded_nodes => [
    (2, [0.0 -100.0 0.0 0.0 0.0 0.0])]
)

data |> display
println()

@time fem, dis_df, fm_df = p44(data)
println()

dis_df |> display
println()

println("Actions:")
fm_df |> display
println()


using Cairo
using ComplexVisual
@ComplexVisual.import_huge
using ComplexVisualGtk
@ComplexVisualGtk.import_huge

const fontface = cv_fontface("sans-serif")

function get_param_n()
    value_min, value_max = 1, 19
    slider_pos = CV_TranslateByOffset(Int)
    slider_pos.value = value_min
    set_slider_value_func = z -> begin
        slider_pos.value = round(UInt, max(min(z, value_max), value_min))
        return CV_Response(; redraw_flag=true)
    end
    label_style = cv_black → fontface → cv_fontsize(20)
    rulers=(CV_TickLabelAppearance(; label_style) ↦
                ("%.0f" ⇒ value_min:2:value_max),
            CV_Ruler("" ⇒  value_min+1:2:value_max),
            CV_TickLabelAppearance(; tick_length=0, gap=15, label_style) ↦
                (("n:" ⇒ 0.5),))

    return CV_ParamWithSlider(;
        slider_pos, value_min=value_min-0.1, value_max=value_max+0.1,
        rulers, set_slider_value_func)
end

param_n = get_param_n()

trafo1 = z -> 1/(1-z)
trafo2 = z -> abs(z) < 1 ?
    sum(k -> z^k, param_n.slider_pos.value:-1:1; init=1.0) :
    sum(k -> z^k, 1:param_n.slider_pos.value; init=1.0)

label_style = cv_black → fontface → cv_fontsize(20)
rulers=(CV_TickLabelAppearance(; label_style) ↦ ("%.0f" ⇒ [-1.0, 0.0, 1.0]),)
codomain1  = CV_Math2DCanvas(-1.5 + 1.5im, 1.5 - 1.5im, 150)
codomain2  = CV_Math2DCanvas(-1.5 + 1.5im, 1.5 - 1.5im, 150)

circle = (cv_black → cv_linewidth(3)) ↦ CV_LinePainter(
    cv_arc_lines(0.0, 2π, (1.0,)))

scene = cv_scene_comp_codomains_std((param_n, ), trafo1, trafo2,
    codomain1, codomain2; codomain1_re_rulers=rulers,
    painter1=CV_PortraitPainter(trafo1) → circle,
    painter2=CV_PortraitPainter(trafo2) → circle)
cv_get_redraw_func(scene)()

handler = cvg_visualize(scene)
cvg_wait_for_destroy(handler.window)

cvg_close(handler); cv_destroy(scene)

# vim:syn=julia:cc=79:fdm=marker:sw=4:ts=4:


# This file is part of TaylorSeries.jl, MIT licensed
#

using TaylorSeries
using Test

@testset "Testing an identity proved by Euler (8 variables)" begin
    make_variable(name, index::Int) = string(name, TaylorSeries.subscriptify(index))

    variable_names = String[make_variable("α", i) for i in 1:4]
    append!(variable_names, [make_variable("β", i) for i in 1:4])

    a1, a2, a3, a4, b1, b2, b3, b4 = set_variables(variable_names, order=4)

    lhs1 = a1^2 + a2^2 + a3^2 + a4^2
    lhs2 = b1^2 + b2^2 + b3^2 + b4^2
    lhs = lhs1 * lhs2

    rhs1 = (a1*b1 - a2*b2 - a3*b3 - a4*b4)^2
    rhs2 = (a1*b2 + a2*b1 + a3*b4 - a4*b3)^2
    rhs3 = (a1*b3 - a2*b4 + a3*b1 + a4*b2)^2
    rhs4 = (a1*b4 + a2*b3 - a3*b2 + a4*b1)^2

    rhs = rhs1 + rhs2 + rhs3 + rhs4

    @test lhs == rhs
    v = randn(8)
    @test evaluate( rhs, v) == evaluate( lhs, v)
end


## SCENARIO PARTITION
function GAPM(Ind_old, duals; α_val = 0.)
    dualRange = maximum(duals)-minimum(duals)
    MaxDiff = α_val*dualRange # Maximum difference between scenario duals

    NP = maximum(Ind_old)

    Scenarios = [duals Ind_old collect(1:N)]

    Ind,NsubParts = ones(N), 1

    for p in 1:NP
        DualPart = Scenarios[Scenarios[:,2].==p,:]
        DualPart=DualPart[sortperm(DualPart[:, 1]), :]
        sizePart = size(DualPart, 1)
        Ind[Integer(DualPart[1,3])] = NsubParts
        ref = DualPart[1,1]
        for n in 2:sizePart
            # Evaluating distance between duals
            if abs(DualPart[n,1] - ref) > MaxDiff
                NsubParts += 1
                ref = DualPart[n,1] # Reference (1st) value for the partition
            end
            Ind[Integer(DualPart[n,3])] = NsubParts
        end
        NsubParts += 1
    end
        return Integer.(Ind)
end


instructions = readlines("2a.txt")

struct Pos
    x
    z
end

function update_position(pos, direction, amount)
    if direction == "forward"
        Pos(pos.x+amount, pos.z)
    elseif direction == "up"
        Pos(pos.x, pos.z - amount)
    elseif direction == "down"
        Pos(pos.x, pos.z + amount)
    end
end

pos = Pos(0,0)
for instruction in instructions
    direction, amount = split(instruction)
    amount = parse(Int, amount)
    pos = update_position(pos, direction, amount)
end

pos.x*pos.z


function simple_volatility_given_returns(returns::Array{R,1}) where R<:Real
    sqrt(mean(returns .^ 2))
end

"""
Calculates volatility with the simple method with a specified time grid.
"""
function simple_volatility_with_grid(ts::SortedDataFrame, assets::Vector{Symbol}, time_grid)
    voldict = Dict{Symbol, eltype(ts.df[:,ts.value])}(assets .=> repeat([0.0], length(assets)))
    for a in assets
        at_times = time_grid[a]
        dd_compiled = latest_value(ts, at_times; assets = Array{Symbol}([a]))
        returns = get_returns(dd_compiled)
        vol = simple_volatility_given_returns(Array(returns[:,a]))
        av_tick_duration = (maximum(at_times) - minimum(at_times)) / (length(at_times) - 1)
        voldict[a] = vol / sqrt(av_tick_duration)
    end
    return voldict
end

"""
    simple_volatility(ts::SortedDataFrame, assets::Vector{Symbol} = get_assets(ts);
                      time_grid::Union{Missing,Dict} = missing , fixed_spacing::Union{Missing,Dict,<:Real} = missing,
                      use_all_obs::Bool = false, rough_guess_number_of_intervals::Integer = 5)

Calculates volatility with the simple method.
### Inputs
* `ts` - The tick data.
* `assets` - The assets you want to estimate volatilities for.
* `time_grid` - The grid with which to calculate returns. If missing one is generated with a fixed spacing (if that is provided) or a default spacing.
* `fixed_spacing` - A spacing used to calculate a time grid. Not used if a `time_grid` is input or if `use_all_obs = true`.
* `use_all_obs` - Use all observations to estimate volatilities. Not used if a `time_grid` is provided.
* `rough_guess_number_of_intervals` - A rough number of intervals to calculate a default spacing. Not used if a `time_grid` or `fixed_spacing` is provided or if `use_all_obs = true`.
### Returns
* A `Dict` with an estimated volatility for each asset.
"""
function simple_volatility(ts::SortedDataFrame, assets::Vector{Symbol} = get_assets(ts);
                           time_grid::Union{Missing,Dict} = missing , fixed_spacing::Union{Missing,Dict,<:Real} = missing,
                           use_all_obs::Bool = false, rough_guess_number_of_intervals::Integer = 5)
    if ismissing(time_grid)
        time_grid = Dict{Symbol,Vector{eltype(ts.df[:,ts.time])}}()
        if use_all_obs
            for a in assets
                time_grid[a] = ts.df[ts.groupingrows[a],ts.time]
            end
        elseif !ismissing(fixed_spacing)
            if isa(fixed_spacing, Dict)
                for a in assets
                    time_grid[a] = collect(minimum(ts.df[:,ts.time]):fixed_spacing[a]:maximum(ts.df[:,ts.time]))
                end
            else
                for a in assets
                    time_grid[a] = collect(minimum(ts.df[:,ts.time]):fixed_spacing:maximum(ts.df[:,ts.time]))
                end
            end
        else
            n_grid = default_spacing(ts; rough_guess_number_of_intervals = rough_guess_number_of_intervals)
            for a in assets
                time_grid[a] = collect(minimum(ts.df[:,ts.time]):n_grid[a]:maximum(ts.df[:,ts.time]))
            end
        end
    end
    return simple_volatility_with_grid(ts, assets, time_grid)
end

"""
    default_spacing(ts::SortedDataFrame; rough_guess_number_of_intervals::Integer = 5,
                    T = duration(ts; in_dates_period = false))

Calculates a default spacing between returns to use.
This comes from the equation at section 1.2.3 of Zhang, Mykland, Ait-Sahalia 2005.
### Inputs
* `ts` - The tick data.
* `rough_guess_number_of_intervals` - A rough estimate of how many intervals to split the tick data into. This is used in a first pass to estimate the optimal interval spacing.
* `T` - The duration of the tick data.
### Returns
* A scalar representing the optimal interval spacing.
"""
function default_spacing(ts::SortedDataFrame; rough_guess_number_of_intervals::Integer = 5,
                         T::Real = duration(ts; in_dates_period = false))
    rough_vol_guess, rough_micro_guess  = two_scales_volatility(ts; num_grids = rough_guess_number_of_intervals)
    n_guess = Dict{Symbol,eltype(ts.df[:,ts.value])}()
    for a in keys(rough_vol_guess)
        n_guess[a] = ( (T/(4* rough_micro_guess[a]^2)) * T * rough_vol_guess[a]^4 )^(1/3)
        if (rough_micro_guess[a] < 0.0000001)
            n_guess[a] = duration(ts; in_dates_period = false)/20
        end
    end
    return n_guess
end


export invariantmeasure, InvariantDistribution

import LinearAlgebra: norm

# Invariant measures (measures over the partition do not have to sum to 1)
abstract type InvariantMeasure end
abstract type AbstractRectangularInvariantMeasure <: InvariantMeasure end
abstract type AbstractTriangulationInvariantMeasure <: InvariantMeasure end

# Invariant distributions (measures over the partition must sum to 1)
abstract type AbstractInvariantDistribution{TO <: TransferOperatorApproximation, T} end

"""
    invariantmeasure(to::TransferOperatorApproximation) → iv::InvariantDistribution
    invariantmeasure(pts, method::SingleGrid) → iv::InvariantDistribution
    invariantmeasure(pts, method::SimplexPoint) → iv::InvariantDistribution
    invariantmeasure(pts, method::SimplexExact) → iv::InvariantDistribution

Compute an invariant probability distribution `iv` from an approximation to the
transfer operator over a partitioning of the input `pts` using one of the
available `method`s.

Alternatively, compute the distribution from a precomputed transfer operator approximation.

## Examples

### `SingleGrid` estimator

Use the `SingleGrid` estimator for long time series.

```julia
using DelayEmbeddings, PerronFrobenius
n = 20000
x = [cos(2π*(t + sin(t)/30)) for t in 1:n] .+ 0.2rand(n)
τs, js = (0, -2, -3), (1, 1, 1)
pts = genembed(x, τs, js);

# Direct approach
b = RectangularBinning(5)
invariantmeasure(pts, SingleGrid(b))

# Indirect approach
to = transferoperator(pts, SingleGrid(b))
invariantmeasure(to)
```

### Triangulation estimators

The `SimplexApprox` estimator can be useful for very short time series.
The `SimplexExact` estimator can also be useful for very short time series.
It computes transition probabilities from exact simplex intersections, so
is much slower than the `SimplexApprox` estimator.

```julia
using DelayEmbeddings, PerronFrobenius
n = 30
x = [cos(2π*(t + sin(t)/30)) for t in 1:n] .+ 0.2rand(n)
τs, js = (0, -2, -3), (1, 1, 1)
pts = genembed(x, τs, js);

# Use approximate simplex intersections. Sample each simplex using
# at least 300 points
invariantmeasure(pts, SimplexPoint(), n = 300)

# Use exact simplex intersections.
invariantmeasure(pts, SimplexExact())
```
"""
function invariantmeasure end

"""
    InvariantDistribution(to::TransferOperatorApproximation, dist, nonzero_inds)

An invariant distribution `dist` computed from the transfer operator approximation `to`.
`nonzero_inds` contains the indices of `dist` with nonzero entries.
"""
struct InvariantDistribution{TO <: TransferOperatorApproximation, T} <: AbstractInvariantDistribution{TO, T}
    to::TO
    dist::Vector{T} # Distribution over the simplices
    nonzero_inds::Vector{Int} # indices of nonzero entries
end

function Base.summary(iv::InvariantDistribution{TO, T}) where {TO, T}
    method = iv.to.g.method
    data = iv.to.g.pts
    s = size(iv.to.M, 1)
    l = length(iv.to.M)
    percent_nonzero = @sprintf("%.4f", count(iv.to.M .> 0.0)/length(iv.to.M) * 100)
    f = length(iv.nonzero_inds)/length(iv.dist)
    return "InvariantDistribution over $s partition elements ($(f*100)% with positive measure)"
end

function Base.show(io::IO, iv::InvariantDistribution{TO, T}) where {TO, T}
    s = summary(iv)
    print(io, s)
end

function invariantmeasure(to::TransferOperatorApproximation;
        N::Int = 200, tol::Float64 = 1e-8, δ::Float64 = 1e-8)
    M = to.M
    #=
    # Start with a random distribution `Ρ` (big rho). Normalise it so that it
    # sums to 1 and forms a true probability distribution over the simplices.
    =#
    Ρ = rand(Float64, 1, size(M, 1))
    Ρ .= Ρ ./ sum(Ρ, dims = 2)

    #=
    # Start estimating the invariant distribution. We could either do this by
    # finding the left-eigenvector of M, or by repeated application of M on Ρ
    # until the distribution converges. Here, we use the latter approach,
    # meaning that we iterate until Ρ doesn't change substantially between
    # iterations.
    =#
    distribution = Ρ * M

    distance = norm(distribution - Ρ) / norm(Ρ)

    check = floor(Int, 1 / δ)
    check_pts = floor.(Int, transpose(collect(1:N)) ./ check) .* transpose(collect(1:N))
    check_pts = check_pts[check_pts .> 0]
    num_checkpts = size(check_pts, 1)
    check_pts_counter = 1

    counter = 1
    while counter <= N && distance >= tol
        counter += 1
        Ρ = distribution

        # Apply the Markov matrix to the current state of the distribution
        distribution = Ρ * M

        if (check_pts_counter <= num_checkpts &&
           counter == check_pts[check_pts_counter])

            check_pts_counter += 1
            colsum_distribution = sum(distribution, dims = 2)[1]
            if abs(colsum_distribution - 1) > δ
                distribution = distribution ./ colsum_distribution
            end
        end

        distance = norm(distribution - Ρ) / norm(Ρ)
    end
    distribution = dropdims(distribution, dims = 1)

    # Do the last normalisation and check
    colsum_distribution = sum(distribution)

    if abs(colsum_distribution - 1) > δ
        distribution = distribution ./ colsum_distribution
    end

    # Find partition elements with strictly positive measure.
    Δ = δ/size(M, 1)
    positive_measure_inds = findall(distribution .> Δ)

    return InvariantDistribution(to, distribution, positive_measure_inds)
end


# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
#
# Desription
# ==============================================================================
#
#   Tests related to conversion from Euler angles to quaternion.
#
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #

# File: ./src/conversions/angle_to_quat.jl
# ========================================

# Functions: angle_to_quat
# ------------------------

@testset "Angle about an axis => Quaternion" begin
    for T in (Float32, Float64)
        # Sample an angle to use in all rotations.
        ang = _rand_ang(T)

        # Rotations
        # =========

        # X Axis
        # ------

        # Create a quaternion that rotates about X axis.
        q = angle_to_quat(ang, :X)
        @test eltype(q) === T

        # Create a vector that does not have X component.
        v = SVector(0, randn(T), randn(T))

        # Rotate the reference using the quaternion.
        v_r = vect(q \ v * q)
        @test eltype(v_r) === T

        # Get the sine of the angle between the representations.
        sin_ang = ((v_r × v) / norm(v)^2)[1]

        # Test the angle between the two representations.
        @test abs(sin(ang) - sin_ang) ≈ 0 atol = √(eps(T))

        # Y Axis
        # ------

        # Create a quaternion that rotates about Y axis.
        q = angle_to_quat(ang, :Y)
        @test eltype(q) === T

        # Create a vector that does not have Y component.
        v = SVector(randn(T), 0, randn(T))

        # Rotate the reference using the quaternion.
        v_r = vect(q \ v * q)
        @test eltype(v_r) === T

        # Get the sine of the angle between the representations.
        sin_ang = ((v_r × v) / norm(v)^2)[2]

        # Test the angle between the two representations.
        @test abs(sin(ang) - sin_ang) ≈ 0 atol = √(eps(T))

        # Z Axis
        # ------

        # Create a quaternion that rotates about Z axis.
        q = angle_to_quat(ang, :Z)
        @test eltype(q) === T

        # Create a vector that does not have Z component.
        v = SVector(randn(T), randn(T), 0)

        # Rotate the reference using the quaternion.
        v_r = vect(q \ v * q)
        @test eltype(v_r) === T

        # Get the sine of the angle between the representations.
        sin_ang = ((v_r × v) / norm(v)^2)[3]

        # Test the angle between the two representations.
        @test abs(sin(ang) - sin_ang) ≈ 0 atol = √(eps(T))
    end
end

@testset "Euler angles => Quaternion" begin
    for T in (Float32, Float64)
        # Two rotations
        # ======================================================================

        for rot_seq in valid_rot_seqs_2angles
            # Sample Euler angles.
            θ₁ = _rand_ang(T)
            θ₂ = _rand_ang(T)

            # Convert to DCM.
            q = angle_to_quat(θ₁, θ₂, rot_seq)
            @test eltype(q) === T

            # Create the DCM by composing the rotations.
            rot_seq_str = string(rot_seq)
            q₁ = angle_to_quat(θ₁, Symbol(rot_seq_str[1]))
            q₂ = angle_to_quat(θ₂, Symbol(rot_seq_str[2]))
            qe = q₁ * q₂

            # Compare.
            @test q ≈ qe
        end

        # Three rotations
        # ======================================================================

        for rot_seq in valid_rot_seqs
            # Sample Euler angles.
            ea = EulerAngles(_rand_ang(T), _rand_ang(T), _rand_ang(T), rot_seq)

            # Convert to quaternion.
            q = angle_to_quat(ea)
            @test eltype(q) === T

            # Create the quaternion by composing the rotations.
            rot_seq_str = string(rot_seq)

            q₁ = angle_to_quat(ea.a1, Symbol(rot_seq_str[1]))
            q₂ = angle_to_quat(ea.a2, Symbol(rot_seq_str[2]))
            q₃ = angle_to_quat(ea.a3, Symbol(rot_seq_str[3]))
            qe = q₁ * q₂ * q₃

            # Make sure q0 is positive.
            (qe.q0 < 0) && (qe = -qe)

            # Compare.
            @test q ≈ qe
        end
    end
end

@testset "Euler angles => Quaternion (Promotion)" begin
    # Check if promotion is working as intended.
    quat = angle_to_quat(Int64(1), 0.0f0, Float64(0))
    @test eltype(quat) === Float64

    quat = angle_to_quat(Int64(1), 0.0f0, 0.0f0)
    @test eltype(quat) === Float32

    quat = angle_to_quat(Int64(1), Int32(0), Float16(0))
    @test eltype(quat) === Float16
end

@testset "Euler angles => Quaternion (Errors)" begin
    @test_throws ArgumentError angle_to_quat(1, :V)
    @test_throws ArgumentError angle_to_quat(1, 2, :ZZ)
    @test_throws ArgumentError angle_to_quat(1, 2, 3, :ZZY)
end

# Functions: smallangle_to_quat
# -----------------------------

@testset "Small Euler angles => Quaternion" begin
    for T in (Float32, Float64)
        # Sample three small angles.
        θx = _rand_ang(T) * T(0.001)
        θy = _rand_ang(T) * T(0.001)
        θz = _rand_ang(T) * T(0.001)

        # Create the quaternion.
        q = smallangle_to_quat(θx, θy, θz)

        # Expected result.
        qe = Quaternion(1, θx / 2, θy / 2, θz / 2)
        qe = qe / norm(qe)

        @test qe ≈ q
    end
end

@testset "Small Euler angles => Quaternion (Promotion)" begin
    # Check if promotion is working as intended.
    quat = smallangle_to_quat(Int64(1), 0.0f0, Float64(0))
    @test eltype(quat) === Float64

    quat = smallangle_to_quat(Int64(1), 0.0f0, 0.0f0)
    @test eltype(quat) === Float32

    quat = smallangle_to_quat(Int64(1), Int32(0), Float16(0))
    @test eltype(quat) === Float16
end


using StatsBase
using ClusterValidityIndices
using Logging
using Plots

# Plotting options
theme(:dark)    # Plotting style
dpi = 300       # Plotting dots-per-inch
# gr()            # GR backend (default for Plots.jl)
unicodeplots()

# Include the library definitions
# include(projectdir("julia/lib_sim.jl"))

# Set the logging level to Info and standardize the random seed
LogLevel(Logging.Info)
# Random.seed!(0)
# data_dir(args...) = projectdir("work/data/meta_icvi", args...)
# results_dir(args...) = projectdir("work/results/meta_icvi", args...)
data_dir(args...) = joinpath("../data/training", args...)
results_dir(args...) = joinpath("../data/results", args...)

# Setup the data
data_paths = [
    data_dir("correct_partition.csv"),
    data_dir("over_partition.csv"),
    data_dir("under_partition.csv")
]
n_samples = 2000
n_data = 3

# Setup ICVIs
cvis = [
    # cSIL(),
    # PS()
    PS(),
    GD43()
]
n_cvis = length(cvis)

# Setup the correlation
n = 100                         # Window size
n_corr = n_samples - n + 1      # Number of correlations
corrs = zeros(n_corr, n_data)   # Preallocation of correlations
iter_corr = n:n_samples         # Iterator for the correlation generation

# Iterate over all partition types
for dx = 1:n_data
    data_path = data_paths[dx]

    # Load the training data
    data, labels = get_cvi_data(data_path)
    labels = relabel_cvi_data(labels)
    data_name = splitext(basename(data_path))[1]    # Get the data_name

    # Instantiate the icvis
    local_cvis = deepcopy(cvis)
    criterion_values = zeros(n_samples, n_cvis)

    # Run the ICVIs iteratively
    for i = 1:n_samples
        sample = data[:, i]
        label = labels[i]
        for j = 1:n_cvis
            criterion_values[i, j] = get_icvi!(local_cvis[j], sample, label)
        end
    end

    # Calculate the windowed correlations
    for i = iter_corr
        corrs[i-n+1, dx] = corspearman(criterion_values[i-n+1:i, 1], criterion_values[i-n+1:i, 2])
    end
end

p = plot(dpi=dpi, legend=:bottomleft)
for i = 1:n_data
    plot!(p, 1:n_corr, corrs[:, i], label=basename(data_paths[i]), size=(800,500))
end
# plot!(p, 1:n_samples, labels, label="Labels")
# plot!(p, 1:n_samples, data.train_y, label="True Labels")
# plot!(p, 1:n_corr, criterion_values_p, label="Porcelain")
# title!("Spearman, Data: " * basename(data_path))
title!("Partition ICVI Correlations")
xlabel!("Sample Index")
ylabel!("Spearman")
# xlims!(1, n_corr)
# ylims!(0, Inf)

# Save and show the plot
# png(p, results_dir("2_correlation_partition"))
display(p)
dest_file = results_dir("2_correlation_partition.txt")
savefig(p, dest_file)


using SparseArrays
using Statistics
using LinearAlgebra
using Arpack
"""
    (pm,v) = partition_fiedler(W; method, v)

Partition the vertices of a graph according to the Fiedler vector

### Input Arguments
* `W::SparseMatrixCSC{Float64,Int}`: the edge weight matrix
* `method::Symbol`: the parition method to be used (:L or :Lrw or else ...)
* `v::Vector{Float64}`: the Fiedler vector supplied if `v` is a null vector

### Output Arguments
* `pm::Vector{Int}`: a vector of 1's and -1's
* `v::Vector{Float64}`: the Fiedler vector used for graph partitioning

Copyright 2015 The Regents of the University of California

Implemented by Jeff Irion (Adviser: Dr. Naoki Saito)
Translated and revised by Naoki Saito, Feb. 9, 2017
"""
function partition_fiedler(W::SparseMatrixCSC{Float64,Int};
                           method::Symbol = :Lrw,
                           v::Vector{Float64} =Vector{Float64}(undef, 0))
#
# Easy case: the Fiedler vector is provided
#
    if !isempty(v)
        (pm, v) = partition_fiedler_pm(v)
        if size(W, 1) == length(v)
            # recovering the algebraic connectivity a(G)
            # Note: regardless of v coming from L or Lrw, the eigenvalue
            # computation below is fine: v'*L*v = a(G) or arw(G) because
            # for L: v'*L*v = \lambda v'*v = \lambda
            # for Lrw: vrw'*L*vrw = \lambda vrw'*D*vrw = \lambda
            # Note also, the result of the quadratic form is scalar
            # mathematically, but in julia, it's 1x1 array; so need to
            # convert it to a real scalar via `val = val[1]`.
            val = v' * (Diagonal(vec(sum(W, dims = 1))) - W) * v; val = val[1]
            pm = partition_fiedler_troubleshooting(pm, v, W, val)
        else
            pm = partition_fiedler_troubleshooting(pm)
        end
        return pm, v
    end

#
# Now, we need to compute the Fiedler vector here. Preliminaries first.
#
    N = size(W, 1)
    eigs_flag = 0

    sigma = eps()
    cutoff = 128 # this value could be changed...

    # handle the case when there are 2 nodes
    if N == 2
        pm = [1; -1]
        v = pm / sqrt(2)
        return pm, v
    end

    # Use L (unnormalized) if specified or edge weights are very small
    if method == :L || minimum(sum(W, dims = 1)) < 10^3 * sigma
        if N > cutoff # for a relative large W
            v0 = ones(N) / sqrt(N)
            try
                # MATLAB, [v,val,eigs_flag] = eigs(diag(sum(W))-W,2,sigma,opts);
                val, vtmp = eigs(sparse(Diagonal(vec(sum(W, dims = 1)))) - W,
                                 nev = 2, sigma = sigma, v0 = v0)
                # val = real.(val)
                # vtmp = real.(vtmp)
                if isa(val[1], Complex{Float64})
                    eigs_flag = 1
                end
            catch emsg
                @warn("Exception in eigs(L) occurred: ", emsg)
                eigs_flag = 2
            end
            if eigs_flag == 0   # no problem in `eigs` is detected.
                val, ind = findmax(val) # val is set to be a scalar.
                v = vec(vtmp[:, ind])   # This is the Fiedler vector!
                # In MATLAB, val was returned as a diagonal matrix, so we had to
                # do all these manipulations while julia, val is simply a vector.
                # [~,ind] = max(diag(val));
                # v = v(:,ind);
                # val = val(ind,ind);
            end
        end
        if N <= cutoff || eigs_flag != 0 # if W is small or eigs had a problem,
                                         # then use full svd.
            vtmp, val = svd(Matrix(Diagonal(vec(sum(W, dims = 1))) - W)) # v <-> U, val <-> S
            # Diagonal(vec(sum(W,1))) is Matrix{Float64} while W is SparseMatrix
            # But the results of the subtraction becomes Matrix{Float64}.
            # Also be careful here! val[end-2] == val[end-1] can happen.
            # If that is the case, vtmp[:,end-2] could also be the Fiedler vec.
            v = vec(vtmp[:, end - 1]) # v is set to be a vector.
            val = val[end - 1]       # val is set to a scalar.
            # In MATLAB, `full` was necessary to convert a sparse matrix format
            # to a usual matrix format as follows
            # [v,val,~] = svd(full(diag(sum(W))-W));
        end
    elseif method == :Lrw # Otherwise, use L_rw, which is normally preferred.
        if N > cutoff           # for a relatively large W
            v0 = ones(N) / sqrt(N)
            try
                # MATLAB: [v,val,eigs_flag] = ...
                #           eigs(diag(sum(W))-W,diag(sum(W)),2,sigma,opts);
                # This is L*v = \lambda*D*v case.
                temp = sparse(Diagonal(vec(sum(W, dims = 1))))
                val, vtmp = eigs(temp - W, temp,
                                 nev = 2, sigma = sigma, v0 = v0)
                # val = real.(val)
                # vtmp = real.(vtmp)
                if isa(val[1], Complex{Float64})
                    eigs_flag = 1
                end
            catch emsg
                @warn("Exception in eigs(Lrw) occurred: ", emsg)
                eigs_flag = 2
            end

            if eigs_flag == 0
                val, ind = findmax(val) # val is set to be a scalar.
                v = vtmp[:, ind]
                # MATLAB: v = (full(sum(W,2)).^(-0.5)) .* v
                # v = vec((sum(W, dims = 2).^(-0.5)) .* v) # This is the Fiedler vector!
            end
        end
        if N <= cutoff || eigs_flag != 0 # if W is small or eigs had a problem,
                                         # then use full svd
            colsumW = vec(sum(W, dims = 1))
            D = Diagonal(colsumW)
            D2 = Diagonal(colsumW.^(-0.5))
            vtmp, val = svd(Matrix(D2 * (D - W) * D2)) # SVD of Lsym
            # MATLAB: [v,val,~] = svd(full(...
            # bsxfun(@times,bsxfun(@times,full(sum(W,2)).^(-0.5),diag(sum(W))-W),
            #  full(sum(W,1)).^(-0.5)) ) );
            #
            # SVD is ordered in the order of nonincreasing singular values,
            # so, (end-1) entry corresponds to the algebraic connectivity.
            # Also be careful here! val[end-2] == val[end-1] can happen.
            # If that is the case, vtmp[:,end-2] could also be the Fiedler vec.
            val = val[end - 1]   # val is set to be a scalar.
            v = vtmp[:, end - 1] # v is set to be a vector.
            v = vec((sum(W, dims = 2).^(-0.5)) .* v) # This is the Fiedler vector of Lrw!
        end
    else                        # Unknown method is specified.
        error("Graph partitioning method :", method, " is not recognized!")
    end

    # partition using the Fiedler vector and troubleshoot any potential issues
    pm, v = partition_fiedler_pm(v)
    pm = partition_fiedler_troubleshooting(pm, v, W, val)
    return pm, v
end # of partition_fiedler

"""
    (pm, v) = partition_fiedler_pm(v::Vector{Float64})

    Partition an input Graph based on the Fiedler vector `v`

### Input Argument
* `v::Vector{Float64}`: the input Fiedler vector

### Output Arguments
* `pm::Vector{Int}`: the partition info
* `v::Vector{Float64}`: the output Fiedler vector if requested

Copyright 2015 The Regents of the University of California

Implemented by Jeff Irion (Adviser: Dr. Naoki Saito)
Translated and revised by Naoki Saito, Feb. 10, 2017
"""
function partition_fiedler_pm(v::Vector{Float64})
    # define a tolerance: if abs(x) < tol, then we view x = 0.
    tol = 10^3 * eps()

    # set the first nonzero element to be positive
    row = 1
    while abs(v[row]) < tol && row < length(v)
        row += 1
    end
    if v[row] < 0
        v = -v
    end

    # assign each point to either region 1 or region -1, and assign any zero
    # entries to the smaller region
    if sum(v .>= tol) > sum(v .<= -tol) # more (+) than (-) entries
        pm = 2*(v .>= tol) .- 1
    else
        pm = 2*(v .<= -tol) .- 1
    end

    # make sure the first point is assigned to region 1 (not -1)
    if pm[1] < 0
        pm = -pm
    end
    return pm, v
end # of partition_fiedler_pm

"""
    pm = partition_fiedler_troubleshooting(pm,v,W,val)

Troubleshoot potential issues with the partitioning

### Input Arguments
* `pm::Vector{Int}`: an input partition info (+1 or -1) of each node
* `v::Vector{Float64}`: the Fiedler vector of the graph
* `W::SparseMatrixCSC{Float64,Int}`: an input edge weight matrix
* `val::Float64`: the algebraic connectivity (i.e., ``\\lambda_2``)

### Ouput Arguments
* `pm::Vector{Int}`: a final partition info vector

Copyright 2015 The Regents of the University of California

Implemented by Jeff Irion (Adviser: Dr. Naoki Saito)
Translated and revised by Naoki Saito, Feb. 10, 2017
"""
function partition_fiedler_troubleshooting(pm::Vector{Int},v::Vector{Float64},
                                           W::SparseMatrixCSC{Float64,Int},
                                           val::Float64)
    # Initial setup
    N = length(pm)
    tol = 10^3 * eps()

    # If pm vector indicates no partition (single sign) or ambiguities via
    # 0 entries, then we'll trouble shoot.
    if sum(pm .< 0) == 0 || sum(pm .> 0) == 0 || sum(abs.(pm)) < N
        # Case 1: an input graph is not connected (i.e., alg. conn < tol)
        if val < tol
            pm = 2 .* (abs.(v) .> tol) .- 1
            while sum(pm .< 0) == 0 || sum(pm .> 0) == 0
                tol = 10 * tol    # relax the tolerance
                pm = 2 .* (abs.(v) .> tol) .- 1
                if tol > 1
                    pm[1:Int(ceil(N / 2))] .= 1
                    pm[(Int(ceil(N / 2)) + 1):N] .= -1
                end
            end
        # Case 2: it is connected, but something funny happened
        else
            pm = 2 .* (v .>= mean(v)) .- 1
            if sum(abs.(pm)) < N
                # assign the near-zero points based on the values of v
                # at their neighbor nodes
                pm0 = find(pm .== 0)
                pm[pm0] = (W[pm0, :] * v .> tol) - (W[pm0, :] * v .< -tol)
                # assign any remaining zeros to the group with fewer members
                pm[find(pm .== 0)] = (sum(pm .> 0) - sum(pm .< 0)) .>= 0
            end
        end
        # if one region has no points after all the above processing
        if sum(pm .< 0) == 0 || sum(pm .> 0) == 0
            pm[1:Int(ceil(N / 2))] .= 1
            pm[Int(ceil(N / 2) + 1):N] .= -1
        end
    end

# make sure that the first point is assigned as a 1
    if pm[1] < 0
        pm = -pm
    end
    return pm
end # of partition_fiedler_troubleshooting

"""
    pm = partition_fiedler_troubleshooting(pm)

Troubleshoot potential issues with the partitioning

### Input Arguments
* `pm::Vector{Int}`: an input partition info (+1 or -1) of each node

### Ouput Arguments
* `pm::Vector{Int}`: a final partition info vector

Copyright 2015 The Regents of the University of California

Implemented by Jeff Irion (Adviser: Dr. Naoki Saito)
Translated and revised by Naoki Saito, Feb. 10, 2017
"""
function partition_fiedler_troubleshooting(pm::Vector{Int})
    N = length(pm)
    if sum(pm .< 0) == 0 || sum(pm .> 0) == 0
        pm[1:Int(ceil(N / 2))] = 1
        pm[Int(ceil(N / 2) + 1):N] = -1
    end
    # make sure that the first point is assigned as a 1
    if pm[1] < 0
        pm = -pm
    end
    return pm
end


struct BootstrapEstimates
  Δhat::Vector{Float64}
  Δb::Matrix{Float64}     # each column represents a bootstrap estimate
end

function bootstrap(X, Y; bootSamples::Int64=300, estimSupport::Union{Vector{Vector{Int64}},Nothing}=nothing)
  nx, p = size(X)
  ny    = size(Y, 1)

  Sx = Symmetric( cov(X) )
  Sy = Symmetric( cov(Y) )

  eS = estimSupport === nothing ? __initSupport(Sx, Sy, X, Y) : estimSupport

  rp = div((p+1)*p, 2)
  # indS = Vector{Vector{Int64}}(undef, rp)

  Δhat = Vector{Float64}(undef, rp)
  Δb = Matrix{Float64}(undef, rp, bootSamples)

  bX = similar(X)
  bY = similar(Y)
  x_ind = Vector{Int64}(undef, nx)
  y_ind = Vector{Int64}(undef, ny)

  # obtain initial estimate
  it = 0
  for col=1:p
    for row=col:p
      it = it + 1
      indElem = sub2indLowerTriangular(p, row, col)
      # tmp = estimate(ReducedOracleNormal(), Sx, Sy, X, Y, indElem, eS[it])

      indS = eS[it]
      makeFirst!(indS, indElem)
      C = skron(Sx, Sy, indS)
      b = svec(Sy, indS) - svec(Sx, indS)
      Δ = C \ b

      Δhat[it] = Δ[1]
    end
  end

  for _b=1:bootSamples
    sample!(1:nx, x_ind)
    sample!(1:ny, y_ind)

    _fill_boot!(bX, X, x_ind)
    _fill_boot!(bY, Y, y_ind)

    bSx = Symmetric( cov(bX) )
    bSy = Symmetric( cov(bY) )

    it = 0
    for col=1:p
      for row=col:p
        it = it + 1
        indElem = sub2indLowerTriangular(p, row, col)
        # tmp = estimate(ReducedOracleNormal(), bSx, bSy, bX, bY, indexElementLinear, eS[it])

        indS = eS[it]
        makeFirst!(indS, indElem)
        C = skron(bSx, bSy, indS)
        b = svec(bSy, indS) - svec(bSx, indS)
        Δ = C \ b

        Δb[it, _b] = Δ[1]
        end
      end
    end

    BootstrapEstimates(Δhat, Δb), eS
end

function simulCI(straps::BootstrapEstimates, α::Float64=0.95)
    m, bootSamples = size(straps.Δb)

    infNormDist = Vector{Float64}(undef, bootSamples)
    CI = Matrix{Float64}(undef, m, 2)

    for b=1:bootSamples
        infNormDist[b] = norm_diff(straps.Δhat, view(straps.Δb, :, b), Inf)
    end
    x = quantile!(infNormDist, α)
    CI[:, 1] .= straps.Δhat .- x
    CI[:, 2] .= straps.Δhat .+ x

    CI
end

function simulCIstudentized(straps::BootstrapEstimates, α::Float64=0.95)
    m, bootSamples = size(straps.Δb)

    infNormDist = Vector{Float64}(undef, bootSamples)
    w = reshape(std(straps.Δb; dims = 2, corrected = false), :)

    CI = Matrix{Float64}(undef, m, 2)
    tmp = Vector{Float64}(undef, m)

    for b=1:bootSamples
        tmp .= (straps.Δhat .- straps.Δb[:, b]) ./ w
        infNormDist[b] = norm(tmp, Inf)
    end
    x = quantile!(infNormDist, α)
    @. CI[:, 1] = straps.Δhat - x * w
    @. CI[:, 2] = straps.Δhat + x * w

    CI
end

function _fill_boot!(bX, X, b_ind)
  n, p = size(X)
  for col=1:p
      for row=1:n
          bX[row, col] = X[b_ind[row], col]
      end
  end
end


## Packages
include("../src/StencilBuilder.jl")

using BenchmarkTools
using LoopVectorization
using CUDA
using .StencilBuilder

## CPU dynamics kernel.
function dynamics_w_kernel_cpu!(
    wt, u, v, w, s,
    visc,
    dxi, dyi, dzi, dzhi,
    is, ie, js, je, ks, ke)

    alpha = 9.81/300;

    @fast3d begin
        @fd (wt, u, v, w, s) wt += (
            - gradx(interpz(u) * interpx(w)) + visc * (gradx(gradx(w)))
            - grady(interpz(v) * interpy(w)) + visc * (grady(grady(w)))
            - gradz(interpz(w) * interpz(w)) + visc * (gradz(gradz(w)))
            + alpha*interpz(s) )
    end
end

## GPU dynamics kernel.
function dynamics_w_kernel_gpu!(
    wt, u, v, w, s,
    visc,
    dxi, dyi, dzi, dzhi,
    is, ie, js, je, ks, ke)

    i = (blockIdx.().x - 1) * blockDim().x + threadIdx().x + is - 1
    j = (blockIdx.().y - 1) * blockDim().y + threadIdx().y + js - 1
    k = blockIdx.().z + ks - 1

    alpha = 9.81/300;

    if i <= ie && j <= je && k <= ke
        @inbounds @fd (wt, u, v, w, s) wt += (
            - gradx(interpz(u) * interpx(w)) + visc * (gradx(gradx(w)))
            - grady(interpz(v) * interpy(w)) + visc * (grady(grady(w)))
            - gradz(interpz(w) * interpz(w)) + visc * (gradz(gradz(w)))
            + alpha*interpz(s) )
    end

    return
end

## User input
itot = 384; jtot = 384; ktot = 384;
igc = 1; jgc = 1; kgc = 1;

icells = itot + 2igc; jcells = jtot + 2jgc; kcells = ktot + 2kgc
is = igc+1; ie = igc+itot; js = jgc+1; je = jgc+jtot; ks = kgc+1; ke = kgc+ktot

wt = rand(icells, jcells, kcells)
u = rand(icells, jcells, kcells)
v = rand(icells, jcells, kcells)
w = rand(icells, jcells, kcells)
s = rand(icells, jcells, kcells)
dxi = rand()
dyi = rand()
dzi = rand(kcells)
dzhi = rand(kcells)
visc = 1.

## Copy data to GPU.
wt_gpu = CuArray(wt)
u_gpu = CuArray(u)
v_gpu = CuArray(v)
w_gpu = CuArray(w)
s_gpu = CuArray(s)
dzi_gpu = CuArray(dzi)
dzhi_gpu = CuArray(dzhi)

## Benchmark functions
function benchmark_cpu(
    wt, u, v, w, s,
    visc,
    dxi, dyi, dzi, dzhi,
    is, ie, js, je, ks, ke)

    dynamics_w_kernel_cpu!(
        wt, u, v, w, s,
        visc,
        dxi, dyi, dzi, dzhi,
        is, ie, js, je, ks, ke)
end

function benchmark_gpu(
    wt_gpu, u_gpu, v_gpu, w_gpu, s_gpu,
    visc,
    dxi, dyi, dzi_gpu, dzhi_gpu,
    is, ie, js, je, ks, ke)

    blocks_size = (32, 32, 1)
    blocks_num = (
        ceil(Int, (ie-is+1) / blocks_size[1]),
        ceil(Int, (je-js+1) / blocks_size[2]),
        (ke-ks+1) )

    CUDA.@sync begin
        @cuda threads=blocks_size blocks=blocks_num dynamics_w_kernel_gpu!(
            wt_gpu, u_gpu, v_gpu, w_gpu, s_gpu,
            visc,
            dxi, dyi, dzi_gpu, dzhi_gpu,
            is, ie, js, je, ks, ke)
    end
end

## Correctness test
benchmark_cpu(
    wt, u, v, w, s,
    visc,
    dxi, dyi, dzi, dzhi,
    is, ie, js, je, ks, ke)

benchmark_gpu(
    wt_gpu, u_gpu, v_gpu, w_gpu, s_gpu,
    visc,
    dxi, dyi, dzi_gpu, dzhi_gpu,
    is, ie, js, je, ks, ke)

wt_gpu_cpu = Array(wt_gpu)
println("Are wt and wt_gpu equal? ", isequal(wt, wt_gpu_cpu), " ", isapprox(wt, wt_gpu_cpu))

## Benchmarks
@btime benchmark_cpu(
    wt, u, v, w, s,
    visc,
    dxi, dyi, dzi, dzhi,
    is, ie, js, je, ks, ke)

@btime benchmark_gpu(
    wt_gpu, u_gpu, v_gpu, w_gpu, s_gpu,
    visc,
    dxi, dyi, dzi_gpu, dzhi_gpu,
    is, ie, js, je, ks, ke)


variable_primal(model) = x -> MOI.get(model, MOI.VariablePrimal(), x)

function heuristicCreate(model::Model)
    m = heuristicModel()
    m.model = model
    return m
end

function removeConstrains!(m::heuristicModel)
    model = m.model
    functions = m.functions
    sets = m.sets
    violation = m.violation
    functionValue = m.functionValue

    for (F, S) in list_of_constraint_types(model)
        if F != VariableRef
            for con in all_constraints(model, F, S)
                f = JuMP.MOI.get(model, JuMP.MOI.ConstraintFunction(), con)
                s = JuMP.MOI.get(model, JuMP.MOI.ConstraintSet(), con)
                push!(functions,f)
                push!(sets,s)
                push!(violation,Inf)
                push!(functionValue,0.0)
                delete(model,con)
                unregister(model,:con)
            end
        end
    end
    nothing
end

function updateFunctionValue!(m::heuristicModel)
    model = m.model
    functions = m.functions
    functionValue = m.functionValue

    for i in 1:length(functions)
        v = functions[i].constant
        for t in functions[i].terms
            coef = t.coefficient
            indx = t.variable_index.value
            v += coef * value(all_variables(model)[indx])
        end
        functionValue[i] = v
    end
    nothing
end

function updateViolation!(m::heuristicModel)
    sets = m.sets
    functionValue = m.functionValue
    violation = m.violation
    functions = m.functions
    sets = m.sets

    for i in 1:length(functions)
        violation[i] = MOD.distance_to_set(MOD.DefaultDistance(), functionValue[i], sets[i])
    end
    nothing
end


@add_problem affine function affine_negate_atom(
    handle_problem!,
    ::Val{test},
    atol,
    rtol,
    ::Type{T},
) where {T,test}
    x = Variable()
    p = minimize(-x, [x <= 0]; numeric_type = T)

    if test
        @test vexity(p) == AffineVexity()
    end
    handle_problem!(p)
    if test
        @test p.optval ≈ 0 atol = atol rtol = rtol
        @test evaluate(-x) ≈ 0 atol = atol rtol = rtol
    end
end

@add_problem affine function affine_kron_atom(
    handle_problem!,
    ::Val{test},
    atol,
    rtol,
    ::Type{T},
) where {T,test}
    x = ComplexVariable(3, 3)
    y = [1.0 2.0; 3.0 4.0]
    if test
        @test size(kron(x, y)) == (6, 6)
        @test size(kron(y, x)) == (6, 6)
    end
end

@add_problem affine function affine_multiply_atom(
    handle_problem!,
    ::Val{test},
    atol,
    rtol,
    ::Type{T},
) where {T,test}
    x = Variable(1)
    p = minimize(2.0 * x, [x >= 2, x <= 4]; numeric_type = T)

    if test
        @test vexity(p) == AffineVexity()
    end
    handle_problem!(p)
    if test
        @test p.optval ≈ 4 atol = atol rtol = rtol
        @test (evaluate(2.0x))[1] ≈ 4 atol = atol rtol = rtol
    end

    x = Variable(2)
    A = 1.5 * eye(2)
    p = minimize([2 2] * x, [A * x >= [1.1; 1.1]]; numeric_type = T)

    if test
        @test vexity(p) == AffineVexity()
    end
    handle_problem!(p)
    if test
        @test p.optval ≈ 2.93333 atol = atol rtol = rtol
        @test (evaluate([2 2] * x))[1] ≈ 2.93333 atol = atol rtol = rtol
        @test vec(evaluate(A * x)) ≈ [1.1; 1.1] atol = atol rtol = rtol
    end

    y = Variable(1)
    x = Variable(3)
    z = [1.0, 2.0, 3.0] * y
    k = -y * [1.0, 2.0, 3.0]
    c = [y <= 3.0, y >= 0.0, x >= ones(3), k <= x, x <= z]
    o = 3 * y
    p = Problem{T}(:minimize, o, c)
    if test
        @test vexity(p) == AffineVexity()
    end
    handle_problem!(p)
    if test
        @test p.optval ≈ 3 atol = atol rtol = rtol
    end

    p = Problem{T}(:minimize, o, c...)
    if test
        @test vexity(p) == AffineVexity()
    end
    handle_problem!(p)
    if test
        @test p.optval ≈ 3 atol = atol rtol = rtol
    end

    # Check #274
    x = ComplexVariable(2, 2)
    p = minimize(
        real([1.0im, 0.0]' * x * [1.0im, 0.0]),
        [x == [1.0 0.0; 0.0 1.0]];
        numeric_type = T,
    )

    handle_problem!(p)
    if test
        @test p.optval ≈ 1.0 atol = atol rtol = rtol
    end
end

@add_problem affine function affine_dot_atom(
    handle_problem!,
    ::Val{test},
    atol,
    rtol,
    ::Type{T},
) where {T,test}
    x = Variable(2)
    p = minimize(dot([2.0; 2.0], x), x >= [1.1; 1.1]; numeric_type = T)

    if test
        @test vexity(p) == AffineVexity()
    end
    handle_problem!(p)
    if test
        @test p.optval ≈ 4.4 atol = atol rtol = rtol
        @test (evaluate(dot([2.0; 2.0], x)))[1] ≈ 4.4 atol = atol rtol = rtol
    end
end

@add_problem affine function affine_dot_atom_for_matrix_variables(
    handle_problem!,
    ::Val{test},
    atol,
    rtol,
    ::Type{T},
) where {T,test}
    x = Variable(2, 2)
    p = minimize(dot(fill(2.0, (2, 2)), x), x >= 1.1; numeric_type = T)

    if test
        @test vexity(p) == AffineVexity()
    end
    handle_problem!(p)
    if test
        @test p.optval ≈ 8.8 atol = atol rtol = rtol
        @test (evaluate(dot(fill(2.0, (2, 2)), x)))[1] ≈ 8.8 atol = atol rtol =
            rtol
    end
end

@add_problem affine function affine_add_atom(
    handle_problem!,
    ::Val{test},
    atol,
    rtol,
    ::Type{T},
) where {T,test}
    x = Variable(1)
    y = Variable(1)
    p = minimize(x + y, [x >= 3, y >= 2]; numeric_type = T)

    if test
        @test vexity(p) == AffineVexity()
    end
    handle_problem!(p)
    if test
        @test p.optval ≈ 5 atol = atol rtol = rtol
        @test evaluate(x + y) ≈ 5 atol = atol rtol = rtol
    end

    x = Variable(1)
    p = minimize(x, [eye(2) + x >= eye(2)]; numeric_type = T)

    if test
        @test vexity(p) == AffineVexity()
    end
    handle_problem!(p)
    if test
        @test p.optval ≈ 0 atol = atol rtol = rtol
        @test evaluate(eye(2) + x) ≈ eye(2) atol = atol rtol = rtol
    end

    y = Variable()
    p = minimize(y - 5, y >= -1; numeric_type = T)

    if test
        @test vexity(p) == AffineVexity()
    end
    handle_problem!(p)
    if test
        @test p.optval ≈ -6 atol = atol rtol = rtol
        @test evaluate(y - 5) ≈ -6 atol = atol rtol = rtol
    end
end

@add_problem affine function affine_transpose_atom(
    handle_problem!,
    ::Val{test},
    atol,
    rtol,
    ::Type{T},
) where {T,test}
    x = Variable(2)
    c = ones(2, 1)
    p = minimize(x' * c, x >= 1; numeric_type = T)

    if test
        @test vexity(p) == AffineVexity()
    end
    handle_problem!(p)
    if test
        @test p.optval ≈ 2 atol = atol rtol = rtol
        @test (evaluate(x' * c))[1] ≈ 2 atol = atol rtol = rtol
    end

    X = Variable(2, 2)
    c = ones(2, 1)
    p = minimize(c' * X' * c, [X >= ones(2, 2)]; numeric_type = T)

    if test
        @test vexity(p) == AffineVexity()
    end
    handle_problem!(p)
    if test
        @test p.optval ≈ 4 atol = atol rtol = rtol
        @test (evaluate(c' * X' * c))[1] ≈ 4 atol = atol rtol = rtol
    end

    rows = 2
    cols = 3
    r = rand(rows, cols)
    r_2 = rand(cols, rows)
    x = Variable(rows, cols)
    c = ones(1, cols)
    d = ones(rows, 1)
    p = minimize(
        c * x' * d + d' * x * c' + (c * x''''' * d)',
        [x' >= r_2, x >= r, x''' >= r_2, x'' >= r];
        numeric_type = T,
    )
    if test
        @test vexity(p) == AffineVexity()
    end
    handle_problem!(p)
    s = sum(max.(r, r_2')) * 3
    if test
        @test p.optval ≈ s atol = atol rtol = rtol
        @test (evaluate(
            c * x' * d + d' * x * c' + (c * ((((x')')')')' * d)',
        ))[1] ≈ s atol = atol rtol = rtol
    end
end

@add_problem affine function affine_index_atom(
    handle_problem!,
    ::Val{test},
    atol,
    rtol,
    ::Type{T},
) where {T,test}
    x = Variable(2)
    p = minimize(x[1] + x[2], [x >= 1]; numeric_type = T)

    if test
        @test vexity(p) == AffineVexity()
    end
    handle_problem!(p)
    if test
        @test p.optval ≈ 2 atol = atol rtol = rtol
        @test (evaluate(x[1] + x[2]))[1] ≈ 2 atol = atol rtol = rtol
    end

    x = Variable(3)
    I = [true true false]
    p = minimize(sum(x[I]), [x >= 1]; numeric_type = T)

    if test
        @test vexity(p) == AffineVexity()
    end
    handle_problem!(p)
    if test
        @test p.optval ≈ 2 atol = atol rtol = rtol
        @test (evaluate(sum(x[I])))[1] ≈ 2 atol = atol rtol = rtol
    end

    rows = 6
    cols = 8
    n = 2
    X = Variable(rows, cols)
    A = randn(rows, cols)
    c = rand(1, n)
    p = minimize(c * X[1:n, 5:5+n-1]' * c', X >= A; numeric_type = T)

    if test
        @test vexity(p) == AffineVexity()
    end
    handle_problem!(p)
    s = c * A[1:n, 5:5+n-1]' * c'
    if test
        @test p.optval ≈ s[1] atol = atol rtol = rtol
        @test evaluate(c * (X[1:n, 5:(5+n)-1])' * c') ≈ s atol = atol rtol =
            rtol
    end
end

@add_problem affine function affine_sum_atom(
    handle_problem!,
    ::Val{test},
    atol,
    rtol,
    ::Type{T},
) where {T,test}
    x = Variable(2, 2)
    p = minimize(sum(x), x >= 1; numeric_type = T)

    if test
        @test vexity(p) == AffineVexity()
    end
    handle_problem!(p)
    if test
        @test p.optval ≈ 4 atol = atol rtol = rtol
        @test evaluate(sum(x)) ≈ 4 atol = atol rtol = rtol
    end

    x = Variable(2, 2)
    p = minimize(sum(x) - 2 * x[1, 1], x >= 1, x[1, 1] <= 2; numeric_type = T)

    if test
        @test vexity(p) == AffineVexity()
    end
    handle_problem!(p)
    if test
        @test p.optval ≈ 1 atol = atol rtol = rtol
        @test (evaluate(sum(x) - 2 * x[1, 1]))[1] ≈ 1 atol = atol rtol = rtol
    end

    x = Variable(10)
    a = rand(10, 1)
    p = maximize(sum(x[2:6]), x <= a; numeric_type = T)

    if test
        @test vexity(p) == AffineVexity()
    end
    handle_problem!(p)
    if test
        @test p.optval ≈ sum(a[2:6]) atol = atol rtol = rtol
        @test evaluate(sum(x[2:6])) ≈ sum(a[2:6]) atol = atol rtol = rtol
    end
end

@add_problem affine function affine_diag_atom(
    handle_problem!,
    ::Val{test},
    atol,
    rtol,
    ::Type{T},
) where {T,test}
    x = Variable(2, 2)
    p = minimize(sum(diag(x, 1)), x >= 1; numeric_type = T)

    if test
        @test vexity(p) == AffineVexity()
    end
    handle_problem!(p)
    if test
        @test p.optval ≈ 1 atol = atol rtol = rtol
        @test evaluate(sum(diag(x, 1))) ≈ 1 atol = atol rtol = rtol
    end

    x = Variable(4, 4)
    p = minimize(sum(diag(x)), x >= 2; numeric_type = T)

    if test
        @test vexity(p) == AffineVexity()
    end
    handle_problem!(p)
    if test
        @test p.optval ≈ 8 atol = atol rtol = rtol
        @test evaluate(sum(diag(x))) ≈ 8 atol = atol rtol = rtol
    end
end

@add_problem affine function affine_trace_atom(
    handle_problem!,
    ::Val{test},
    atol,
    rtol,
    ::Type{T},
) where {T,test}
    x = Variable(2, 2)
    p = minimize(tr(x), x >= 1; numeric_type = T)

    if test
        @test vexity(p) == AffineVexity()
    end
    handle_problem!(p)
    if test
        @test p.optval ≈ 2 atol = atol rtol = rtol
        @test evaluate(tr(x)) ≈ 2 atol = atol rtol = rtol
    end
end

@add_problem affine function affine_dot_multiply_atom(
    handle_problem!,
    ::Val{test},
    atol,
    rtol,
    ::Type{T},
) where {T,test}
    x = Variable(3)
    p = maximize(sum(dot(*)(x, [1, 2, 3])), x <= 1; numeric_type = T)

    if test
        @test vexity(p) == AffineVexity()
    end
    handle_problem!(p)
    if test
        @test p.optval ≈ 6 atol = atol rtol = rtol
        @test evaluate(sum((dot(*))(x, [1, 2, 3]))) ≈ 6 atol = atol rtol = rtol
    end

    x = Variable(3, 3)
    p = maximize(sum(dot(*)(x, eye(3))), x <= 1; numeric_type = T)

    if test
        @test vexity(p) == AffineVexity()
    end
    handle_problem!(p)
    if test
        @test p.optval ≈ 3 atol = atol rtol = rtol
        @test evaluate(sum((dot(*))(x, eye(3)))) ≈ 3 atol = atol rtol = rtol
    end

    x = Variable(5, 5)
    p = minimize(x[1, 1], dot(*)(3, x) >= 3; numeric_type = T)

    if test
        @test vexity(p) == AffineVexity()
    end
    handle_problem!(p)
    if test
        @test p.optval ≈ 1 atol = atol rtol = rtol
        @test (evaluate(x[1, 1]))[1] ≈ 1 atol = atol rtol = rtol
    end

    x = Variable(3, 1)
    p = minimize(sum(dot(*)(ones(3, 3), x)), x >= 1; numeric_type = T)

    if test
        @test vexity(p) == AffineVexity()
    end
    handle_problem!(p)
    if test
        @test p.optval ≈ 9 atol = atol rtol = rtol
        @test (evaluate(x[1, 1]))[1] ≈ 1 atol = atol rtol = rtol
    end

    x = Variable(1, 3)
    p = minimize(sum(dot(*)(ones(3, 3), x)), x >= 1; numeric_type = T)

    if test
        @test vexity(p) == AffineVexity()
    end
    handle_problem!(p)
    if test
        @test p.optval ≈ 9 atol = atol rtol = rtol
        @test (evaluate(x[1, 1]))[1] ≈ 1 atol = atol rtol = rtol
    end

    x = Variable(1, 3, Positive())
    p = maximize(sum(dot(/)(x, [1 2 3])), x <= 1; numeric_type = T)

    if test
        @test vexity(p) == AffineVexity()
    end
    handle_problem!(p)
    if test
        @test p.optval ≈ 11 / 6 atol = atol rtol = rtol
        @test evaluate(sum((dot(/))(x, [1 2 3]))) ≈ 11 / 6 atol = atol rtol =
            rtol
    end

    # Broadcast fusion works
    x = Variable(5, 5)
    a = 2.0 .* x .* ones(Int, 5)
    if test
        @test a isa Convex.DotMultiplyAtom
    end
end

@add_problem affine function affine_reshape_atom(
    handle_problem!,
    ::Val{test},
    atol,
    rtol,
    ::Type{T},
) where {T,test}
    A = rand(2, 3)
    X = Variable(3, 2)
    c = rand()
    p = minimize(sum(reshape(X, 2, 3) + A), X >= c; numeric_type = T)

    if test
        @test vexity(p) == AffineVexity()
    end
    handle_problem!(p)
    if test
        @test p.optval ≈ sum(A .+ c) atol = atol rtol = rtol
        @test evaluate(sum(reshape(X, 2, 3) + A)) ≈ sum(A .+ c) atol = atol rtol =
            rtol
    end

    b = rand(6)
    p = minimize(sum(vec(X) + b), X >= c; numeric_type = T)

    if test
        @test vexity(p) == AffineVexity()
    end
    handle_problem!(p)
    if test
        @test p.optval ≈ sum(b .+ c) atol = atol rtol = rtol
        @test evaluate(sum(vec(X) + b)) ≈ sum(b .+ c) atol = atol rtol = rtol
    end

    x = Variable(4, 4)
    c = ones(16, 1)
    reshaped = reshape(x, 16, 1)
    a = collect(1:16)
    p = minimize(c' * reshaped, reshaped >= a; numeric_type = T)

    if test
        @test vexity(p) == AffineVexity()
    end
    handle_problem!(p)
    # TODO: why is accuracy lower here?
    if test
        @test p.optval ≈ 136 atol = 10atol atol = atol rtol = rtol
        @test (evaluate(c' * reshaped))[1] ≈ 136 atol = 10atol atol = atol rtol =
            rtol
    end
end

@add_problem affine function affine_hcat_atom(
    handle_problem!,
    ::Val{test},
    atol,
    rtol,
    ::Type{T},
) where {T,test}
    x = Variable(4, 4)
    y = Variable(4, 6)
    p = maximize(
        sum(x) + sum([y fill(4.0, 4)]),
        [x y fill(2.0, (4, 2))] <= 2;
        numeric_type = T,
    )

    if test
        @test vexity(p) == AffineVexity()
    end
    handle_problem!(p)
    if test
        @test p.optval ≈ 96 atol = atol rtol = rtol
        @test evaluate(sum(x) + sum([y fill(4.0, 4)])) ≈ 96 atol = atol rtol =
            rtol
        @test evaluate([x y fill(2.0, (4, 2))]) ≈ fill(2.0, (4, 12)) atol = atol rtol =
            rtol
    end
end

@add_problem affine function affine_vcat_atom(
    handle_problem!,
    ::Val{test},
    atol,
    rtol,
    ::Type{T},
) where {T,test}
    x = Variable(4, 4)
    y = Variable(4, 6)

    # TODO: fix dimension mismatch [y 4*eye(4); x -ones(4, 6)]
    p = maximize(
        sum(x) + sum([y 4*eye(4); x -ones(4, 6)]),
        [x; y'] <= 2;
        numeric_type = T,
    )

    if test
        @test vexity(p) == AffineVexity()
    end
    handle_problem!(p)
    # TODO: why is accuracy lower here?
    if test
        @test p.optval ≈ 104 atol = 10atol atol = atol rtol = rtol
        @test evaluate(sum(x) + sum([y 4*eye(4); x -(ones(4, 6))])) ≈ 104 atol =
            10atol atol = atol rtol = rtol
        @test evaluate([x; y']) ≈ 2 * ones(10, 4) atol = atol rtol = rtol
    end
end

@add_problem affine function affine_single_hcat_atom(
    handle_problem!,
    ::Val{test},
    atol,
    rtol,
    ::Type{T},
) where {T,test}
    x = Variable(4, 4)
    p = maximize(tr(hcat(x)), hcat(x) <= 2; numeric_type = T)

    handle_problem!(p)
    if test
        @test p.optval ≈ 8 atol = atol rtol = rtol
    end
end

@add_problem affine function affine_single_vcat_atom(
    handle_problem!,
    ::Val{test},
    atol,
    rtol,
    ::Type{T},
) where {T,test}
    x = Variable(4, 4)
    p = maximize(tr(vcat(x)), vcat(x) <= 2; numeric_type = T)

    handle_problem!(p)
    if test
        @test p.optval ≈ 8 atol = atol rtol = rtol
    end
end

@add_problem affine function affine_Diagonal_atom(
    handle_problem!,
    ::Val{test},
    atol,
    rtol,
    ::Type{T},
) where {T,test}
    x = Variable(2, 2)
    if test
        @test_throws ArgumentError Diagonal(x)
    end

    x = Variable(4)
    p = minimize(sum(Diagonal(x)), x == [1, 2, 3, 4]; numeric_type = T)

    if test
        @test vexity(p) == AffineVexity()
    end
    handle_problem!(p)
    if test
        @test p.optval ≈ 10 atol = atol rtol = rtol
        @test all(abs.(evaluate(Diagonal(x)) - Diagonal([1, 2, 3, 4])) .<= atol)
    end

    x = Variable(3)
    c = [1; 2; 3]
    p = minimize(c' * Diagonal(x) * c, x >= 1, sum(x) == 10; numeric_type = T)

    if test
        @test vexity(p) == AffineVexity()
    end
    handle_problem!(p)
    if test
        @test p.optval ≈ 21 atol = atol rtol = rtol
    end

    x = Variable(3)
    p = minimize(sum(x), x >= 1, Diagonal(x)[1, 2] == 1; numeric_type = T)

    handle_problem!(p)
    if test
        @test p.status != MOI.OPTIMAL
    end
end

@add_problem affine function affine_conv_atom(
    handle_problem!,
    ::Val{test},
    atol,
    rtol,
    ::Type{T},
) where {T,test}
    x = Variable(3)
    h = [1, -1]
    p = minimize(sum(conv(h, x)) + sum(x), x >= 1, x <= 2; numeric_type = T)

    if test
        @test vexity(p) == AffineVexity()
    end
    handle_problem!(p)
    if test
        @test p.optval ≈ 3 atol = atol rtol = rtol
        @test evaluate(sum(conv(h, x))) ≈ 0 atol = atol rtol = rtol
    end

    x = Variable(3)
    h = [1, -1]
    p = minimize(sum(conv(x, h)) + sum(x), x >= 1, x <= 2; numeric_type = T)

    if test
        @test vexity(p) == AffineVexity()
    end
    handle_problem!(p)
    if test
        @test p.optval ≈ 3 atol = atol rtol = rtol
        @test evaluate(sum(conv(h, x))) ≈ 0 atol = atol rtol = rtol
    end
end

@add_problem affine function affine_satisfy_problems(
    handle_problem!,
    ::Val{test},
    atol,
    rtol,
    ::Type{T},
) where {T,test}
    x = Variable()
    p = satisfy(x >= 0; numeric_type = T)

    add_constraints!(p, x >= 1)
    add_constraints!(p, [x >= -1, x <= 4])
    handle_problem!(p)
    if test
        @test p.status == MOI.OPTIMAL
    end

    p = satisfy([x >= 0, x >= 1, x <= 2]; numeric_type = T)

    handle_problem!(p)
    if test
        @test p.status == MOI.OPTIMAL
    end

    p = maximize(1, [x >= 1, x <= 2]; numeric_type = T)

    handle_problem!(p)
    if test
        @test p.status == MOI.OPTIMAL
    end

    constr = x >= 0
    constr += x >= 1
    constr += x <= 10
    constr2 = x >= 0
    constr2 += [x >= 2, x <= 3] + constr
    p = satisfy(constr; numeric_type = T)

    handle_problem!(p)
    if test
        @test p.status == MOI.OPTIMAL
    end
end

@add_problem affine function affine_dualvalue(
    handle_problem!,
    ::Val{test},
    atol,
    rtol,
    ::Type{T},
) where {T,test}
    x = Variable()
    p = minimize(x, x >= 0; numeric_type = T)

    handle_problem!(p)
    if test
        @test p.constraints[1].dual ≈ 1 atol = atol rtol = rtol
    end

    x = Variable()
    p = maximize(x, x <= 0; numeric_type = T)

    handle_problem!(p)
    if test
        @test p.constraints[1].dual ≈ 1 atol = atol rtol = rtol
    end

    x = Variable()
    p = minimize(x, x >= 0, x == 2; numeric_type = T)

    handle_problem!(p)
    if test
        @test p.constraints[1].dual ≈ 0 atol = atol rtol = rtol
        @test abs.(p.constraints[2].dual) ≈ 1 atol = atol rtol = rtol
    end

    x = Variable(2)
    A = 1.5 * eye(2)
    p = minimize(dot([2.0; 2.0], x), [A * x >= [1.1; 1.1]]; numeric_type = T)

    handle_problem!(p)
    if test
        dual = [4 / 3; 4 / 3]
        @test all(abs.(p.constraints[1].dual - dual) .<= atol)
    end
end

@add_problem affine function affine_Partial_transpose(
    handle_problem!,
    ::Val{test},
    atol,
    rtol,
    ::Type{T},
) where {T,test}
    dims = [2, 3, 4]
    d = prod(dims)
    A = rand(ComplexF64, 2, 2)
    B = rand(ComplexF64, 3, 3)
    C = rand(ComplexF64, 4, 4)
    M = kron(A, B, C)
    Mt1 = kron(transpose(A), B, C)
    Mt2 = kron(A, transpose(B), C)
    Mt3 = kron(A, B, transpose(C))

    Rt1 = ComplexVariable(d, d)
    Rt2 = ComplexVariable(d, d)
    Rt3 = ComplexVariable(d, d)
    S = rand(ComplexF64, d, d)
    handle_problem!(
        satisfy(partialtranspose(Rt1, 1, dims) == S; numeric_type = T),
    )

    handle_problem!(
        satisfy(partialtranspose(Rt2, 2, dims) == S; numeric_type = T),
    )

    handle_problem!(
        satisfy(partialtranspose(Rt3, 3, dims) == S; numeric_type = T),
    )

    if test
        @test partialtranspose(M, 1, dims) ≈ Mt1 atol = atol rtol = rtol
        @test partialtranspose(M, 2, dims) ≈ Mt2 atol = atol rtol = rtol
        @test partialtranspose(M, 3, dims) ≈ Mt3 atol = atol rtol = rtol
        @test partialtranspose(S, 1, dims) ≈ evaluate(Rt1) atol = atol rtol =
            rtol
        @test partialtranspose(S, 2, dims) ≈ evaluate(Rt2) atol = atol rtol =
            rtol
        @test partialtranspose(S, 3, dims) ≈ evaluate(Rt3) atol = atol rtol =
            rtol
    end

    if test
        @test_throws ArgumentError partialtrace(rand(6, 6), 3, [2, 3])
        @test_throws ArgumentError partialtrace(rand(6, 6), 1, [2, 4])
        @test_throws ArgumentError partialtrace(rand(3, 4), 1, [2, 3])
    end
end

@add_problem affine function affine_permuteddims_matrix(
    handle_problem!,
    ::Val{test},
    atol,
    rtol,
    ::Type{T},
) where {T,test}
    #this function is used in the partial transpose
    for n in (2, 3, 4, 5)
        dims = ntuple(i -> rand(2:5), n)
        d = prod(dims)
        v = rand(d)
        p = randperm(n)
        out1 = vec(permutedims(reshape(v, dims), p))
        out2 = Convex.permutedims_matrix(dims, p) * v
        if test
            @test out1 ≈ out2 atol = atol rtol = rtol
        end
    end
end