#=
NOT INTENDED TO BE RUN AS A SCRIPT
=#
using GraphGLRM, LowRankModels, DataFrames, DataArrays

wdi = readtable("cleaned.csv")
edges = Tuple{Int64,Int64}[]
for i in 1:(nrow(wdi) - 1)
  if (wdi[:Country_Code][i] == wdi[:Country_Code][i+1]) && (wdi[:Year][i] + 1 == wdi[:Year][i+1])
    push!(edges, (i,i+1))
  end
end

xgraph = IndexGraph([1:nrow(wdi)...], edges)

"""
Converts DataFrame of floating-point numbers to a DataMatrix
"""
function dataMatrix(d::DataFrame)
  res = @data(zeros(size(d)))
  for j in 1:length(d)
    for i in 1:nrow(d)
      res[i,j] = d[i,j]
    end
  end
  res
end

isntna(A) = !isna(A)

#The raw DataMatrix from this DataFrame
wdiA = dataMatrix(wdi[:,3:end])
obsgini = wdi[:SI_POV_GINI] |> isntna
findfirst(names(wdi) .== :SI_POV_GINI) - 2

##############################Validation Set##################################
wdiDfVal = wdi[obsgini,:]

#Use the subset of the data where the gini index is defined
valedges = Tuple{Int64, Int64}[]
#Data is sorted by country first then by year.
for i in 1:(nrow(wdiDfVal) - 1)
  if (wdiDfVal[:Country_Code][i] == wdiDfVal[:Country_Code][i+1]) && (wdiDfVal[:Year][i] + 1 == wdiDfVal[:Year][i+1])
    push!(valedges, (i, i+1))
  end
end

xgraphval = IndexGraph([1:nrow(wdiDfVal)...], valedges)

wdiValidation = wdiA[obsgini, :]
wdival, m, s = standardize(wdiValidation)

#Knock out 1/2 of non-NA values at random because this is number knocked out of original
#This knocks out about 40% of the gini coefficient values, about the same as are missing originally.
knockout = deepcopy(wdival)
knockout[rand(1:length(knockout), length(knockout)÷2)] = NA
sum(isna(knockout[:,12])) #474
kidxs = isna(knockout[:,12])
tidxs = !kidxs
ginicol = 12 #This is the column index for the gini index, the desired variable

ggstrain = GGLRM(wdival, HuberLoss(), QuadReg(1.), QuadReg(1.), 20, obs=observations(wdival))

#Use standardized data for smaller objective values and to allow proper graph regularization
#Huber loss for robustness, two quadratic regularizers to reduce overfitting
#First try 20 dimensions
ggs20 = GGLRM(knockout, HuberLoss(), QuadReg(1.), QuadReg(1.), 20, obs=observations(knockout))
fit!(ggs20) #709 objective value
whole_objective(ggstrain, ggs20.X*ggs20.Y)

(((ggs20.X*ggs20.Y)[tidxs,12] - wdival[tidxs,12])./(wdival[tidxs,12])) |> abs2 |> mean
(((ggs20.X*ggs20.Y)[kidxs,12] - wdival[kidxs,12])./(wdival[kidxs,12])) |> abs2 |> mean

#Next try 10
ggs10 = GGLRM(knockout, HuberLoss(), QuadReg(1.), QuadReg(1.), 10, obs=observations(knockout))
fit!(ggs10) #657 objective value
whole_objective(ggstrain, ggs10.X*ggs10.Y)

(((ggs10.X*ggs10.Y)[tidxs,12] - wdival[tidxs,12])./(wdival[tidxs,12])) |> abs2 |> mean
(((ggs10.X*ggs10.Y)[kidxs,12] - wdival[kidxs,12])./(wdival[kidxs,12])) |> abs2 |> mean

#Next try 5
ggs5 = GGLRM(knockout, HuberLoss(), QuadReg(1.), QuadReg(1.), 5, obs=observations(knockout))
fit!(ggs5) #808 objective value
whole_objective(ggstrain, ggs5.X*ggs5.Y)

(((ggs5.X*ggs5.Y)[tidxs,12] - wdival[tidxs,12])./(wdival[tidxs,12])) |> abs2 |> mean
(((ggs5.X*ggs5.Y)[kidxs,12] - wdival[kidxs,12])./(wdival[kidxs,12])) |> abs2 |> mean

#Next try 2
ggs2 = GGLRM(knockout, HuberLoss(), QuadReg(1.), QuadReg(1.), 2, obs=observations(knockout))
fit!(ggs2) #Hits a much higher objective value than the others
whole_objective(ggstrain, ggs2.X*ggs2.Y)

(((ggs2.X*ggs2.Y)[tidxs,12] - wdival[tidxs,12])./(wdival[tidxs,12])) |> abs2 |> mean
(((ggs2.X*ggs2.Y)[kidxs,12] - wdival[kidxs,12])./(wdival[kidxs,12])) |> abs2 |> mean

#Mean l1 relative error
(((ggs10.X*ggs10.Y)[:,12] - wdival[:,12])./(wdival[:,12])) |> abs |> mean
#Mean l2 relative error
(((ggs10.X*ggs10.Y)[:,12] - wdival[:,12])./(wdival[:,12])) |> abs2 |> mean
#Mean l1 relative error over knocked out values
((ggs10.X*ggs10.Y)[kidxs,12] - wdival[kidxs,12])./(wdival[kidxs,12]) |> abs |> mean
#Mean l2 relative error over knocked out values
((ggs10.X*ggs10.Y)[kidxs,12] - wdival[kidxs,12])./(wdival[kidxs,12]) |> abs2 |> mean

#Mean squared error
((ggs10.X*ggs10.Y)[kidxs,12] - wdival[kidxs,12]) |> abs2 |> mean
#Median absolute error
((ggs10.X*ggs10.Y)[kidxs,12] - wdival[kidxs,12]) |> abs |> median

#mean absolute value
median(abs(wdival[:,12]))
#mean squared value
mean(abs2(wdival[:,12]))
#mean absolute value of knockouts
median(abs(wdival[kidxs,12]))
#mean squared value of knockouts
mean(abs2(wdival[kidxs,12]))

#########################GRAPH REGULARIZATION#################################
#Magnitude 0 graph regularization
graph_0 = GGLRM(knockout, HuberLoss(), GraphQuadReg(xgraphval, 0., 0.1), QuadReg(1.), 10, obs=observations(knockout))
fit!(graph_0, ProxGradParams(max_iter=1000))
whole_objective(graph_0, graph_0.X*graph_0.Y)
whole_objective(ggstrain, graph_0.X*graph_0.Y)

((graph_0.X*graph_0.Y)[kidxs,12] - wdival[kidxs,12])./(wdival[kidxs,12]) |> abs |> mean
((graph_0.X*graph_0.Y)[kidxs,12] - wdival[kidxs,12])./(wdival[kidxs,12]) |> abs2 |> mean
((graph_0.X*graph_0.Y)[tidxs,12] - wdival[tidxs,12])./(wdival[tidxs,12]) |> abs2 |> mean

#Magnitude 1 graph regularization
graph_1 = GGLRM(knockout, HuberLoss(), GraphQuadReg(xgraphval, 1., 0.1), QuadReg(1.), 10, obs=observations(knockout))
fit!(graph_1, ProxGradParams(max_iter=1000))
whole_objective(graph_1, graph_1.X*graph_1.Y)
whole_objective(ggstrain, graph_1.X*graph_1.Y)

((graph_1.X*graph_1.Y)[kidxs,12] - wdival[kidxs,12])./(wdival[kidxs,12]) |> abs |> mean
((graph_1.X*graph_1.Y)[kidxs,12] - wdival[kidxs,12])./(wdival[kidxs,12]) |> abs2 |> mean
((graph_1.X*graph_1.Y)[tidxs,12] - wdival[tidxs,12])./(wdival[tidxs,12]) |> abs2 |> mean

#Magnitude 2 graph regularization
graph_2 = GGLRM(knockout, HuberLoss(), GraphQuadReg(xgraphval, 2., 0.1), QuadReg(1.), 10, obs=observations(knockout))
fit!(graph_2, ProxGradParams(max_iter=1000))
whole_objective(graph_2, graph_2.X*graph_2.Y)
whole_objective(ggstrain, graph_2.X*graph_2.Y)

((graph_2.X*graph_2.Y)[kidxs,12] - wdival[kidxs,12])./(wdival[kidxs,12]) |> abs |> mean
((graph_2.X*graph_2.Y)[kidxs,12] - wdival[kidxs,12])./(wdival[kidxs,12]) |> abs2 |> mean
((graph_2.X*graph_2.Y)[tidxs,12] - wdival[tidxs,12])./(wdival[tidxs,12]) |> abs2 |> mean

#Magnitude 3 graph regularization
graph_3 = GGLRM(knockout, HuberLoss(), GraphQuadReg(xgraphval, 3., 0.1), QuadReg(1.), 10, obs=observations(knockout))
fit!(graph_3, ProxGradParams(max_iter=1000))
whole_objective(graph_3, graph_3.X*graph_3.Y)
whole_objective(ggstrain, graph_3.X*graph_3.Y)

((graph_3.X*graph_3.Y)[kidxs,12] - wdival[kidxs,12])./(wdival[kidxs,12]) |> abs |> mean
((graph_3.X*graph_3.Y)[kidxs,12] - wdival[kidxs,12])./(wdival[kidxs,12]) |> abs2 |> mean
((graph_3.X*graph_3.Y)[tidxs,12] - wdival[tidxs,12])./(wdival[tidxs,12]) |> abs2 |> mean

#Magnitude 4 graph regularization
graph_4 = GGLRM(knockout, HuberLoss(), GraphQuadReg(xgraphval, 4., 0.1), QuadReg(1.), 10, obs=observations(knockout))
fit!(graph_4, ProxGradParams(max_iter=1000))
whole_objective(graph_4, graph_4.X*graph_4.Y)
whole_objective(ggstrain, graph_4.X*graph_4.Y)

((graph_4.X*graph_4.Y)[kidxs,12] - wdival[kidxs,12])./(wdival[kidxs,12]) |> abs |> mean
((graph_4.X*graph_4.Y)[kidxs,12] - wdival[kidxs,12])./(wdival[kidxs,12]) |> abs2 |> mean
((graph_4.X*graph_4.Y)[tidxs,12] - wdival[tidxs,12])./(wdival[tidxs,12]) |> abs2 |> mean

#Magnitude 5 graph regularization
graph_5 = GGLRM(knockout, HuberLoss(), GraphQuadReg(xgraphval, 5., 0.1), QuadReg(1.), 10, obs=observations(knockout))
fit!(graph_5, ProxGradParams(max_iter=1000))
whole_objective(graph_5, graph_5.X*graph_5.Y)
whole_objective(ggstrain, graph_5.X*graph_5.Y)

((graph_5.X*graph_5.Y)[kidxs,12] - wdival[kidxs,12])./(wdival[kidxs,12]) |> abs |> mean
((graph_5.X*graph_5.Y)[kidxs,12] - wdival[kidxs,12])./(wdival[kidxs,12]) |> abs2 |> mean
((graph_5.X*graph_5.Y)[tidxs,12] - wdival[tidxs,12])./(wdival[tidxs,12]) |> abs2 |> mean

#writedlm("validationmatrix.txt", wdival)
#writedlm("validationerror.txt", wdival - graph_3.X*graph_3.Y)
testset1, testset2 = deepcopy(wdival), deepcopy(wdival)
testset1[rand(1:length(testset1), length(testset1)÷2)] = NA
testset2[rand(1:length(testset2), length(testset2)÷2)] = NA
testidx1 = isna(testset1[:,12])
testidx2 = isna(testset2[:,12])
tridx1 = !testidx1
tridx2 = !testidx2

graph_test1 = GGLRM(testset1, HuberLoss(), GraphQuadReg(xgraphval, 3., 0.1), QuadReg(1.), 10, obs=observations(testset1))
graph_test2 = GGLRM(testset2, HuberLoss(), GraphQuadReg(xgraphval, 3., 0.1), QuadReg(1.), 10, obs=observations(testset2))
fit!(graph_test1, ProxGradParams(max_iter=1000))
fit!(graph_test2, ProxGradParams(max_iter=1000))

graph_base1 = GGLRM(testset1, HuberLoss(), QuadReg(1.), QuadReg(1.), 10, obs=observations(testset1))
graph_base2 = GGLRM(testset2, HuberLoss(), QuadReg(1.), QuadReg(1.), 10, obs=observations(testset2))
fit!(graph_base1, ProxGradParams(max_iter=1000))
fit!(graph_base2, ProxGradParams(max_iter=1000))

((graph_test1.X*graph_test1.Y)[testidx1,12] - wdival[testidx1,12])./(wdival[testidx1,12]) |> abs2 |> mean
((graph_test1.X*graph_test1.Y)[tridx1,12] - wdival[tridx1,12])./(wdival[tridx1,12]) |> abs2 |> mean

((graph_test2.X*graph_test2.Y)[testidx2,12] - wdival[testidx2,12])./(wdival[testidx2,12]) |> abs2 |> mean
((graph_test2.X*graph_test2.Y)[tridx2,12] - wdival[tridx2,12])./(wdival[tridx2,12]) |> abs2 |> mean

((graph_base1.X*graph_base1.Y)[testidx1,12] - wdival[testidx1,12])./(wdival[testidx1,12]) |> abs2 |> mean
((graph_base1.X*graph_base1.Y)[tridx1,12] - wdival[tridx1,12])./(wdival[tridx1,12]) |> abs2 |> mean

((graph_base2.X*graph_base2.Y)[testidx2,12] - wdival[testidx2,12])./(wdival[testidx2,12]) |> abs2 |> mean
((graph_base2.X*graph_base2.Y)[tridx2,12] - wdival[tridx2,12])./(wdival[tridx2,12]) |> abs2 |> mean


"""
    Noise(data, level; scale = :percent_max, type = :gaussian )

Adds noise (uniform, or gaussian) to an array of data.
The scale can be made relative to the percent_max, percent_average, or the absolute level values.

Note: for Gaussian noise it is likely convenient to scale by an inverse quantile(say 0.999)
"""
function Noise(data, level; scale = :percent_max, type = :gaussian )
    (maxes, minis) = ( [], [] )
    noise = []
    if type == :gaussian
        noise = randn( size( data ) )
    elseif type == :uniform
        noise = rand( Float64, size( data ) ) .- 0.5
    else #ToDo: Throw exception
        return
    end
    if scale == :percent_max
        if length( size( data ) ) > 1
            (maxes, minis) = ( reduce( max, data, dims = 2 ), reduce( min, data, dims = 2 ) )
            return ( noise .* (maxes .- minis) .* level ) .+ data
        else
            (maxes, minis) = ( reduce( max, data ), reduce( min, data ) )
            return ( noise .* (maxes .- minis) .* level ) .+ data
        end
    elseif scale == :percent_average
        if length(size(data)) > 1
            (maxes, minis) = ( reduce( max, data, dims = 2 ), reduce( min, data, dims = 2 ) )
            avg = mean( (data .- minis) ./ (maxes .- minis), dims = 2 )
            return ( noise .* avg .* level ) .+ data
        else
            (maxes, minis) = ( reduce( max, data ), reduce( min, data ) )
            avg = mean( (data .- minis) ./ (maxes .- minis) )
            return ( noise .* avg .* level ) .+ data
        end
    elseif scale == :absolute
        return ( noise .* level ) .+ data
    end
end

"""
    StandardNormalVariate(X)

Scales the columns of `X` by the mean and standard deviation of each row. Returns the scaled array.
"""
StandardNormalVariate(X) = ( X .- Statistics.mean(X, dims = 2) ) ./ Statistics.std(X, dims = 2)

"""
    Scale1Norm(X)

Scales the columns of `X` by the 1-Norm of each row. Returns the scaled array.
"""
Scale1Norm(X) = X ./ sum(abs.(X), dims = 2)

"""
    Scale2Norm(X)

Scales the columns of `X` by the 2-Norm of each row. Returns the scaled array.
"""
Scale2Norm(X) = X ./ sqrt.(sum(X .^ 2, dims = 2))

"""
    ScaleInfNorm(X)

Scales the columns of `X` by the Inf-Norm of each row. Returns the scaled array.
"""
ScaleInfNorm(X) = X ./ reduce(max, X, dims = 2)

"""
    ScaleFNorm(X)

Scales EACH entry of `X` by the Frobenius Norm of. Returns the scaled array.
"""
ScaleFNorm(X) = X ./ FNorm( X )

"""
    ScaleMinMax(X)

Scales the columns of `X` by the Min and Max of each row such that no observation is greater than 1 or less than zero.
Returns the scaled array.
"""
function ScaleMinMax(X)
    mini = reduce(min, X, dims = 2)
    maxi = reduce(max, X, dims = 2)
    return (X .- mini) ./ (maxi .- mini)
end

"""
    ScaleByIntensity(X, index)

Scales the columns of `X` by the value of a peak at a specified `index` in each row.
Returns the scaled array.
"""
ScaleByIntensity(X, index) = X ./ X[:,index]

"""
    offsetToZero(X)

Ensures that no observation(row) of Array `X` is less than zero, by ensuring the minimum value of each row is zero.
"""
offsetToZero(X) = X .- reduce(min, X, dims = 2)

"""
    boxcar(X; windowsize = 3, fn = mean)

Applies a boxcar function (`fn`) to each window of size `windowsize` to every row in `X`.
*Note: the function provided must support a dims argument/parameter.*
"""
function boxcar(X; windowsize = 3, fn = mean)
    X = forceMatrixT(X)
    (obs, vars) = size(X)
    @assert windowsize <= vars
    result = zeros(obs, vars - windowsize + 1)
    for v in 1 : (vars - windowsize + 1 )
        result[:, v] = fn(X[:, v : (v + windowsize - 1) ], dims = 2)
    end
    return result
end

"""
    ALSSmoother(X; lambda = 100, p = 0.001, maxiters = 10)

Applies an assymetric least squares smoothing function to a 2-Array `X`. The `lambda`, `p`, and `maxiters`
parameters control the smoothness. See the reference below for more information.

Paul H. C. Eilers, Hans F.M. Boelens. Baseline Correction with Asymmetric Least Squares Smoothing.  2005
"""
function ALSSmoother(X; lambda = 100, p = 0.001, maxiters::Int = 10)
    X = forceMatrixT(X)
    (obs, vars) = size(X)
    output = zeros(obs,vars)
    for r in 1:obs
        D = SecondDerivative( sparse( I, vars, vars )' )';
        w = ones(vars);
        z = zeros(vars)
        for it in 1 : maxiters
            W = spdiagm(0 => w);
            C = cholesky(W + lambda * D' * D).U;
            z = C \ (C' \ (w .* X[r,:]));
            w[ X[r,:] .> z ] .= p;
            w[ X[r,:] .< z ] .+= 1.0 - p;
        end
        output[r,:] = z
    end
    return output
end

"""
    PerfectSmoother(X; lambda = 100)

Applies an assymetric least squares smoothing function to a a 2-Array `X`. The `lambda`
parameter controls the smoothness. See the reference below for more information.

Paul H. C. Eilers. "A Perfect Smoother". Analytical Chemistry, 2003, 75 (14), pp 3631–3636.
"""
function PerfectSmoother(X; lambda = 100)
    X = forceMatrixT(X)
    (obs, vars) = size(X)
    output = zeros(obs,vars)
    for r in 1:obs
        D = SecondDerivative( sparse( I, vars, vars )' )';
        w = spdiagm(0 => ones(vars));
        C = cholesky(w + lambda * D' * D).U
        output[r,:] = C \ (C' \ (w * X[r,:]))
    end
    return output
end

#Pretty sure this is reversible like a transform, but don't have time to solve the inverse yet...
struct MultiplicativeScatterCorrection
    BiasedMeans
    Bias
    Coefficients
end

"""
    MultiplicativeScatterCorrection(Z::Array)

Creates a MultiplicativeScatterCorrection object from the data in Z

Martens, H. Multivariate calibration. Wiley
"""
function MultiplicativeScatterCorrection(Z::Array)
    BiasedMeans = hcat( ones( ( size(Z)[2], 1) ) , StatsBase.mean( Z, dims = 1 )[1,:] )
    Coeffs = ( BiasedMeans' * BiasedMeans ) \ ( Z * BiasedMeans )'
    MultiplicativeScatterCorrection( BiasedMeans, Coeffs[1,:], Coeffs[2,:] )
end

"""
    (T::MultiplicativeScatterCorrection)(Z)

Applies MultiplicativeScatterCorrection from a stored object `T` to Array `Z`.
"""
function (T::MultiplicativeScatterCorrection)(Z)
    Coeffs = ( T.BiasedMeans' * T.BiasedMeans ) \ ( Z * T.BiasedMeans )'
    return (Z .- Coeffs[1,:]) ./ Coeffs[2,:]
end

"""
    FirstDerivative(X)

Uses the finite difference method to compute the first derivative for every row in `X`.
*Note: This operation results in the loss of a column dimension.*
"""
function FirstDerivative(X)
    X = forceMatrixT(X)
    Xsize = size(X)
    XNew = zeros( Xsize[ 1 ] , Xsize[ 2 ] - 1)
    for c in 1 : ( Xsize[ 2 ] - 1 )
        XNew[ :, c ] = X[ :, c + 1 ] .- X[ :, c ]
    end
    return XNew
end

"""
    FirstDerivative(X)

Uses the finite difference method to compute the second derivative for every row in `X`.
*Note: This operation results in the loss of two columns.*
"""
function SecondDerivative(X)
    X = forceMatrixT(X)
    Xsize = size(X)
    XNew = zeros( Xsize[ 1 ], Xsize[ 2 ] - 2 )
    for c in 1 : ( Xsize[ 2 ] - 2 )
        XNew[ :, c ] = (X[ :, c + 2 ] .- X[ :, c + 1 ]) - (X[ :, c + 1 ] .- X[ : , c ] )
    end
    return XNew
end

"""
    FractionalDerivative(Y, X = 1 : length(Y); Order = 0.5)

Calculates the Grunwald-Leitnikov fractional order derivative on every row of Array Y.
Array `X` is a vector that has the spacing between column-wise entries in `Y`. `X` can be a scalar if that is constant (common in spectroscopy).
`Order` is the fractional order of the derivative.
*Note: This operation results in the loss of a column dimension.*

The Fractional Calculus, by Oldham, K.; and Spanier, J. Hardcover: 234 pages. Publisher: Academic Press, 1974. ISBN 0-12-525550-0
"""
function FractionalDerivative(Y, X = 1 : length(Y); Order = 0.5)
    Y = forceMatrixT(Y)
    (Obs,Vars) = size(Y)
    ddy = zeros(Obs,Vars-1)
    w = zeros(Vars)
    w[1] = 1.0
    for var in 2:Vars
        w[ var ] = w[ var - 1 ] * ( 1.0 - ( Order + 1.0 ) / ( var - 1.0 ) )
    end
    for var in 2:Vars
        h = (length(X) > 1) ? X[ var ] - X[ var - 1 ] : X
        for obs in 1 : Obs#This could definitely be broadcasted
            ddy[ obs, var-1 ] = w[ 1 : var ]' * Y[obs, var : -1: 1 ] ./ (h^Order)
        end
    end
    return ddy
end

"""
    ConvFilter1DFFT(a, filter)

Performs a 1D convolution of vector `filter` onto `a` via the FFT definition of a
 convolution. This method implicitly zeropads and truncates the result.

Note: This isn't highly optimized for performance. It was made when DSP.jl broke,
and broke the ChemometricsTools.jl package.
"""
function ConvFilter1DFFT(a, filter)
    @assert(length(filter) <= length(a), "Length of filter should be less than the length of the vector." )
    NewSize = length(a) + length(filter) - 1;
    ADiff = Int(round((NewSize - length(a)) ))
    ADiffhalf = Int(floor(ADiff / 2))
    filterDiff = Int(round((NewSize - length(filter)) ))
    X = fft( vcat( a, zeros( ADiff ) ) )
    H = fft( vcat( filter, zeros( filterDiff ) ) )
    return real.(ifft( X .* H ))[ (ADiffhalf + 1):(end-ADiffhalf) ]
end

"""
    SavitzkyGolay(X, Delta, PolyOrder, windowsize)

Performs SavitskyGolay smoothing across every row in an Array `X`.
The `window size` is the size of the convolution filter, `PolyOrder` is the order of the polynomial,
and `Delta` is the order of the derivative.

Savitzky, A.; Golay, M.J.E. (1964). "Smoothing and Differentiation of Data by Simplified Least Squares Procedures". Analytical Chemistry. 36 (8): 1627–39. doi:10.1021/ac60214a047.
"""
function SavitzkyGolay(X, DerivOrder, PolyOrder, windowsize::Int)
    @assert( (windowsize % 2) == 1, "Window size must be an odd number" )
    @assert( PolyOrder < (windowsize-1), "Polynomial order must be less than the window size." )
    X = forceMatrixT(X)
    (Obs,Vars) = size(X)
    windowspan = (windowsize - 1) / 2
    basis = ones( windowsize, PolyOrder+1 )
    basis[:,2:end] = ( (-windowspan:windowspan)' .^ ( 1 : PolyOrder ))'
    DerivFac = factorial( DerivOrder )
    FIR = inv( basis' * basis) * basis'
    output = DerivFac * ConvFilter1DFFT(X[1,:], FIR[DerivOrder+1,: ])'
    for r in 2:Obs
        sg = DerivFac * ConvFilter1DFFT(X[r,:], FIR[DerivOrder+1,: ])'
        output = vcat( output, sg )
    end
    offset = Int( windowspan )
    return output[:, (offset + 1) : (end - offset)]
end

"""
    LinearResample(X, newsize)

Resamples a vector `X` to be of size `newsize` via linear interpolation.
"""
function LinearResample(X, newsize)
   cursize = length(X)
   if cursize == newsize
      return X
   end
   newspacing = ( (cursize-1) / (newsize-1) )
   newsampling = 1 : newspacing : cursize
   interpolated = zeros(newsize)
   for i in 1 : ( cursize - 1 )
      #For large arrays this is costly - could change to track the indices
      #assuming uniform spacing.
      inds = findall( i .<= newsampling .<= ( i + 1 ) )
      slope = ( X[i+1] - X[i] )
      offset = X[i] - (slope*i)
      interpolated[inds] = (newsampling[inds] .* slope) .+ offset
   end
   return interpolated
end

"""
    SincInterpolation(Y, S, Up)
    Y - vector of a line shape
    S - Sampled domain of Y
    Up - Upsampled X vector

Refactored from: https://gist.github.com/endolith/1297227#file-sinc_interp-m
"""
SincInterpolation(Y, S, Up) = sinc.( (Up .- S') ./ (S[2] - S[1]) ) * Y

"""
    FourierUpsample(X, newlength)

zero pads a vector `X` in the frequency domain then converts it back to the time domain.

"""
function FourierUpsample(X, newlength)
    len = length(x)
    @assert(newlength > len, "New length must be greater than the vector length to upsample!")
    xfft = fft(x)
    left = Int( floor( (len+1) / 2 ) )
    return real.(ifft( vcat(xfft[1:left], zeros(newlength - len), xfft[(left+1):end] ) )) .* (newlength/len)
end

struct DirectStandardizationXform
    pca::PCA
    TransferMatrix::Array
end

"""
    DirectStandardization(InstrumentX1, InstrumentX2; Factors = minimum(collect(size(InstrumentX1))) - 1)

Makes a DirectStandardization object to facilitate the transfer from Instrument #2 to Instrument #1 .
The returned object can be used to transfer unseen data to the approximated space of instrument 1.
The number of `Factors` used are those from the internal orthogonal basis.

Yongdong Wang and Bruce R. Kowalski, "Calibration Transfer and Measurement Stability of Near-Infrared Spectrometers," Appl. Spectrosc. 46, 764-771 (1992)
"""
function DirectStandardization(InstrumentX1, InstrumentX2; Factors = minimum(collect(size(InstrumentX1))) - 1)
    pcamodel = PCA(InstrumentX1; Factors = Factors)
    InstrumentX2_DS = pcamodel(InstrumentX2; Factors = Factors, inverse = false)
    TransferMatrix = LinearAlgebra.pinv(InstrumentX2_DS) * pcamodel.Scores[:, 1:Factors]
    return DirectStandardizationXform(pcamodel, TransferMatrix)
end

"""
    (DSX::DirectStandardizationXform)(X; Factors = length(DSX.pca.Values))

Applies a the transform from a learned direct standardization object `DSX` to new data `X`.
"""
function (DSX::DirectStandardizationXform)(X; Factors = length(DSX.pca.Values))
    Into = DSX.pca(X; Factors = Factors)
    Bridge = Into * DSX.TransferMatrix[1:Factors,1:Factors]
    return DSX.pca(Bridge; Factors = Factors, inverse = true)
end

struct OrthogonalSignalCorrection
    Weights::Array
    Loadings::Array
    Residuals::Array
end

"""
    OrthogonalSignalCorrection(X, Y; Factors = 1)

Performs Thomas Fearn's Orthogonal Signal Correction to an endogenous `X` and exogenous `Y`.
The number of `Factors` are the number of orthogonal components to be removed from `X`.
This function returns an OSC object.

Tom Fearn. On orthogonal signal correction. Chemometrics and Intelligent Laboratory Systems. Volume 50, Issue 1, 2000, Pages 47-52.
"""
function OrthogonalSignalCorrection(X, Y; Factors = 1)
    (Obs,Vars) = size(X)
    W = zeros(Vars, Factors)
    Loadings = zeros( Vars, Factors)
    Residuals = copy(X)
    M = LinearAlgebra.Diagonal(ones(Vars)) .- ( X' * Y * Base.inv(Y' * X * X' * Y) * Y' * X )
    P = svd( X * M )
    for factor in 1 : Factors
        Projection = M * Residuals' * ( P.U[ : , factor ] ./ P.S[ factor ] )
        W[ : , factor ] = Projection / sqrt(Projection' * Projection)
        t = Residuals * Projection
        p = Residuals' * (t ./ ( t' * t ))
        Loadings[ : , factor ] = p
        Residuals = Residuals .- (t * p')
    end
    return OrthogonalSignalCorrection(W, Loadings, Residuals)
end

"""
    (OSC::OrthogonalSignalCorrection)(Z; Factors = 2)

Applies a the transform from a learned orthogonal signal correction object `OSC` to new data `Z`.
"""
function (OSC::OrthogonalSignalCorrection)(Z; Factors = 2)
    X = copy(Z)
    for factor in 1 : Factors
        X .-= (X * OSC.Weights[ : , factor ] * OSC.Loadings[:, factor ]')
    end
    return X
end

struct TransferByOrthogonalProjection
    Factors::Int
    vars::Int
    pca::PCA
end

"""
    TransferByOrthogonalProjection(X1, X2; Factors = 1)

Performs Thomas Fearns Transfer By Orthogonal Projection to facilitate transfer from `X1` to `X2`. Returns a TransferByOrthogonalProjection object.

Anne Andrew, Tom Fearn. Transfer by orthogonal projection: making near-infrared calibrations robust to between-instrument variation. Chemometrics and Intelligent Laboratory Systems. Volume 72, Issue 1, 2004, Pages 51-56,
"""
function TransferByOrthogonalProjection(X1, X2; Factors = 1)
    (Obs,Vars) = size(X1)
    OneToTwo = PCA(X1 .- X2; Factors = Factors)
    return TransferByOrthogonalProjection(Factors, Vars, OneToTwo)
end

"""
    (TbOP::TransferByOrthogonalProjection)(X1; Factors = TbOP.Factors)

Applies a the transform from a learned transfer by orthogonal projection object `TbOP` to new data `X1`.
"""
function (TbOP::TransferByOrthogonalProjection)(X1; Factors = TbOP.Factors)
    return X1 * (LinearAlgebra.Diagonal( ones( TbOP.vars ) ) .- (TbOP.pca.Loadings[1:Factors,:]' * TbOP.pca.Loadings[1:Factors,:]))
end

struct CORAL
    coralmat::Array
end
"""
    CORAL(X1, X2; lambda = 1.0)

Performs CORAL to facilitate covariance based transfer from `X1` to `X2` with regularization parameter `lambda`. Returns a CORAL object.

Correlation Alignment for Unsupervised Domain Adaptation. Baochen Sun, Jiashi Feng, Kate Saenko. https://arxiv.org/abs/1612.01939
"""
function CORAL(X1, X2; lambda = 1.0)
    (Obs1, vars) = size(X1)
    (Obs2, vars) = size(X2)
    d = lambda .* LinearAlgebra.Diagonal(repeat([1], vars))
    c1 = (1.0 / Obs1) * (X1' * X1) .+ d
    c2 = (1.0 / Obs2) * (X2' * X2) .+ d
    CORALXfer = c1 ^ (-1/2) * c2 ^ (1/2)
    return CORAL(CORALXfer)
end

"""
    (C::CORAL)(Z)

Applies a the transform from a learned `CORAL` object to new data `Z`.
"""
(C::CORAL)(Z) = Z * C.coralmat


# Copyright 2020 Digital Domain 3.0
#
# Licensed under the Apache License, Version 2.0 (the "Apache License")
# with the following modification; you may not use this file except in
# compliance with the Apache License and the following modification to it:
# Section 6. Trademarks. is deleted and replaced with:
#
# 6. Trademarks. This License does not grant permission to use the trade
#    names, trademarks, service marks, or product names of the Licensor
#    and its affiliates, except as required to comply with Section 4(c) of
#    the License and to reproduce the content of the NOTICE file.
#
# You may obtain a copy of the Apache License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the Apache License with the above modification is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the Apache License for the specific
# language governing permissions and limitations under the Apache License.

using Test
using Blades
using KVectors
using KForms
using LinearAlgebra

module G2
  using Blades
  # need last param to be true for differential forms
  @generate_basis("++",false,true,true)
end

module G3
  using Blades
  @generate_basis("+++",false,true,true)
end

using .G2
using .G3

zerofun(x,y...) = zero(x)

@testset "CMU DEC course G2" begin
  e₁, e₂, e₁₂ = alle( G2, 2)
  e¹, e², e¹² = alld( G2, 2)
  𝐼 = 1e₁₂

  # http://brickisland.net/DDGSpring2019/wp-content/uploads/2019/01/DDG_458_SP19_Lecture04_kForms.pdf
  # 1-form example in coordinates
  v = 2e₁ + 2e₂
  α = -2e¹ + 3e²
  # could repeat this test in multivectors when * is defined, here we use ⋅ instead of *
  @test α⋅v == 2 == α⋅reverse(v) # == apply(α,v)

  # 2-form example in coordinates
  u = 2e₁ + 2e₂; v = -2e₁ + 2e₂
  α = 1e¹ + 3e²; β = 2e¹ + 1e²
  @test (α∧β)*reverse(u∧v) == -40 == kform(α∧β,u∧v)

  # lecture 6 slides: α(x,y)=xdx + ydy
  ∂αᵢ∂e = [1 0; 0 1]
  α = 1e₁+1e₂ # evaluate anywhere, so choose 1,1
  @test 𝑑(α, ∂αᵢ∂e) == 0
  # negative x row is in dual basis
  star_∂αᵢ∂e = [0 1; 1 0]
  @test 𝑑(⋆α, star_∂αᵢ∂e) == 2e₁₂

  # use ZForm type
  α = e₁(ZForm{2}((x,y)->x, [(x,y)->1, (x,y)->0])) +
      e₂(ZForm{2}((x,y)->y, [(x,y)->0, (x,y)->1]))
  @test apply(α, KVector(2e₂), 1e₁+1e₂) == 2
  @test apply(α, KVector(3e₂+2e₂), 1e₁+1e₂) == 5
  @test apply(𝑑(α), 1e₁+1e₂, KVector(1e₁₂)) == 0
  @test apply(𝑑(⋆α), 1e₁+1e₂, KVector(1e₁₂)) == 2.0
  @test apply(𝑑(⋆⋆α), 1e₁+1e₂, KVector(1e₁₂)) == 0
  @test apply(𝑑(⋆⋆⋆α), 1e₁+1e₂, KVector(1e₁₂)) == -2.0

  # a naked zero form: ϕ(x,y) = ½e⁻⁽ˣ²⁺ʸ²)
  # dϕ = -2*ϕ*(xdx + ydy)
  # expected result: -2*ϕ*x*e₁ + -2*ϕ*y*e₁
  # -2*ϕ(0.2,0.3)*0.2*e₁ + -2*ϕ*(0.2,0.3)*0.3*e₂
  xy = 0.2e₁+0.3e₂

  ϕ(x,y) = 0.5*exp(-(x^2+y^2))

  # evaluate at 0.2,0.3
  α = ZForm{2}(ϕ, [(x,y)->-2.0*ϕ(x,y)*x,
                (x,y)->-2.0*ϕ(x,y)*y])
  dp = 𝑑(α, 𝐼)
  dp = sortbasis(dp)
  @test untype(dp[1]) == e₁
  @test untype(dp[2]) == e₂
  @test dp[1].x.f(0.2,0.3) == -2.0*ϕ(coords(xy)...)*xy[1].x
  @test dp[2].x.f(0.2,0.3) == -2.0*ϕ(coords(xy)...)*xy[2].x
  @test apply(dp, xy, 1.0e₁+1.0e₂) ==  -2.0*ϕ(coords(xy)...)*xy[1].x + -2.0*ϕ(coords(xy)...)*xy[2].x

  # http://brickisland.net/DDGSpring2019/wp-content/uploads/2019/02/DDG_458_SP19_Lecture07_ExteriorIntegration.pdf
  # Integration of differential 2-forms: - Example

  ϕ(x,y) = x + x*y
  ω = ZForm{2}(ϕ, [(x,y)->1.0+y, (x,y)->x])∧e₁₂
  dx = 0.005; dy= 0.005;  dA = (dx*dy)e₁₂; Ω = [(xᵢ,yᵢ) for xᵢ in 0:dx:1.0 for yᵢ in 0.0:dy:1.0];
  integ_ω = mapreduce(xy->apply(ω, xy[1]*e₁ + xy[2]*e₂, KVector(dA)),+,Ω)
  # as dA -> 0 ∑ω(p)dA = ∫ωdA = 0.75
  @test abs(integ_ω - 3/4) < 0.01;

  # inner product of 1-forms - Example
  α = ZForm{2}((x,y)->1.0, [(x,y)->0.0, (x,y)->0.0])∧e₁
  βₓ = ZForm{2}((x,y)->x, [(x,y)->1.0, (x,y)->0.0])∧e₁
  βᵥ = ZForm{2}((x,y)->-y, [(x,y)->0.0, (x,y)->-1.0])∧e₂
  ω = ⋆(α)∧(βₓ+βᵥ)
  integ_ω = mapreduce(xy->apply(ω, xy[1]*e₁ + xy[2]*e₂, KVector(dA)),+,Ω)
  @test abs(integ_ω - (-1/2)) < 0.01;

  # http://brickisland.net/DDGSpring2019/wp-content/uploads/2019/02/ExteriorCalculus-1.pdf
  # Exercize 13
  ϕ(x,y) = x*y+2*(y^2)
  α = ZForm{2}(ϕ, [(x,y)->y, (x,y)->x + 4y], [(x,y)->0 (x,y)->1 ; (x,y)->1 (x,y)->4])
  @test apply(KForms.Δ(α), 1e₁+1e₂) == 4

  # http://brickisland.net/DDGSpring2019/wp-content/uploads/2019/02/DDG_458_SP19_Lecture08_DiscreteDifferentialForms-1.pdf
  # Integrating a 1-Form over and Edge - Example
  α = ZForm{2}((x,y)->x*y, [(x,y)->y, (x,y)->x])∧e₁ + ZForm{2}((x,y)->-x*x, [(x,y)->-2.0x, zerofun])∧e₂

  p₁ = -1.0e₁ + 2.0e₂
  p₂ = 3.0e₁ + 1.0e₂
  L = sqrt(17.0)
  T = (p₂-p₁)/L
  p(s) = p₁ + (s/L)*(p₂-p₁)
  ds = 0.01
  p0L = map(s->p(s),0:ds:L);
  αT = apply(α, T)
  @test round(mapreduce(pᵢ->apply(αT, pᵢ),+,p0L)*ds) == 7.0

  # Exterior Derivative on Vector-Valued Forms
  # http://brickisland.net/DDGSpring2019/wp-content/uploads/2019/02/DDG_458_SP19_Lecture10_SmoothCurves.pdf
  ϕ = ZForm{2}((x,y)->[x^2,x*y], [(x,y)->[2x,y], (x,y)->[0.0,x]])∧e₁ +
      ZForm{2}((x,y)->[x*y,y^2], [(x,y)->[y,0.0], (x,y)->[x,2y]])∧e₂

  @test apply(𝑑(ϕ), 3.0e₁+5.0e₂, 1.0e₁₂) == [5.0, -3.0]

end

@testset "CMU DEC course G3" begin

  e₁, e₂, e₃, e₁₂, e₁₃, e₂₃, e₁₂₃ = alle( G3, 3)

  @test (1.0e₂∧1.0e₃)∧⋆(1.0e₂∧1.0e₃) == 1.0e₁₂₃
  @test (1.0e₂)∧⋆(1.0e₂) == 1.0e₁₂₃
  @test (-2.0e₂)∧⋆(-2.0e₂) == 4.0e₁₂₃

  u = 2.0e₁ + 2.0e₂; v = -2.0e₁ + 2.0e₂
  detuv = det(hcat(vcat(magnitude.(u),0.0), vcat(magnitude.(v),0.0), [0.0,0.0,magnitude.(⋆(u∧v))]))
  @test detuv > 0
  #two orthonormal vectors u1, u2, we ask that det(u1, u2, ⋆(u1 ∧ u2)) = 1
  u = normalize(u); v = normalize(v)
  detuv = det(hcat(vcat(magnitude.(u),0.0), vcat(magnitude.(v),0.0), [0.0,0.0,magnitude.(⋆(u∧v))]))
  @test detuv ≈ 1

  u = -2.0e₂; v = 1.0e₃; uv = u∧v
  detuv = det(hcat([0.0,magnitude.(u),0.0], [0.0,0.0,magnitude.(v)], [magnitude.(⋆(uv)),0,0]))
  @test detuv > 0

  @test ⋆(-1.0e₁∧1.0e₂ - 1.0e₁∧1.0e₃ - 2.0e₂∧1.0e₃) == -2.0e₁ + 1.0e₂ - 1.0e₃

end

@testset "differential forms" begin
  e₁, e₂, e₁₂ = alle( G2, 2)
  𝐼 = 1e₁₂

  α = ZForm{2}((x,y)->x^3, [(x,y)->3.0*x^2, (x,y)->0.0],[(x,y)->6.0x (x,y)->0.0 ; (x,y)->0.0 (x,y)->0.0])
  @test apply(𝑑(𝑑(α, 𝐼)), 1.0e₁+1.0e₂, 1.0e₁₂) == 0.0

  α = ZForm{2}((x,y)->y*x^3 + x*y^2,
               [(x,y)->y*3.0x^2 + y^2,
                (x,y)->x^3 +      x*2.0y],
               [(x,y)->y*6.0x     (x,y)->3.0x^2 + 2.0y ;
                (x,y)->3.0x^2 + 2.0y  (x,y)->2.0])

  @test apply(𝑑(𝑑(α, 𝐼)), 1.0e₁+1.5e₂, 1.0e₁₂) == 0.0
  @test apply(𝑑(⋆𝑑(α, 𝐼)), 1.0e₁+1.5e₂, 1.0e₁₂) != 0.0
end


using DataFrames, MLDataUtils
using Clustering, Distances
using CSV
using Random
using Logging

# Set up Logging - we recommend to use this command to avoid package warnings during the model training process.
logger = Logging.SimpleLogger(stderr,Logging.Warn);
global_logger(logger);

#### Set parameters for the learners
cr = :dunnindex
method = "ICOT_local"
warm_start = :none;
geom_search = false
threshold = 0.0
seed = 1
gridsearch = false
num_tree_restarts = 100
complexity_c = 0.0
min_bucket = 10
maxdepth = 5

###### Step 1: Prepare the data
# Read the data - recommend the use of the (deprecated) readtable() command to avoid potential version conflicts with the CSV package.
data = readtable("../data/ruspini.csv");

# Convert the dataset to a matrix
data_array = convert(Matrix{Float64}, data);
# Get the number of observations and features
n, p = size(data_array)
data_t = data_array';

##### Step 2: Fit K-means clustering on the dataset to generate a warm-start for ICOT
#Fix the seed
Random.seed!(seed);

# The ruspini dataset has pre-defined clusters, which we will use to select the cluster count (K) for the K-means algorithm.
# In an unsupervised setting (with no prior-known K), the number of clusters for K means can be selected using the elbow method.
K = length(unique(data_array[:,end]))

# Run k-means and save the assignments
kmeans_result = kmeans(data_t, K);
assignment = kmeans_result.assignments;

data_full = DataFrame(hcat(data, assignment, makeunique=true));
names!(data_full, [:x1, :x2, :true_labels, :kmean_assign]);

# Prepare data for ICOT: features are stored in the matrix X, and the warm-start labels are stored in y
X = data_full[:,1:2]; y = data_full[:,:true_labels];

##### Step 3a. Before running ICOT, start by testing the IAI license
lnr_oct = ICOT.IAI.OptimalTreeClassifier(localsearch = false, max_depth = maxdepth,
													 minbucket = min_bucket,
													 criterion = :misclassification
													 )
grid = ICOT.IAI.GridSearch(lnr_oct)
ICOT.IAI.fit!(grid, X, y)
ICOT.IAI.showinbrowser(grid.lnr)

##### Step 3b. Run ICOT

# Run ICOT with no warm-start:
warm_start= :none
lnr_ws_none = ICOT.InterpretableCluster(ls_num_tree_restarts = num_tree_restarts, ls_random_seed = seed, cp = complexity_c, max_depth = maxdepth,
	minbucket = min_bucket, criterion = cr, ls_warmstart_criterion = cr, kmeans_warmstart = warm_start,
	geom_search = geom_search, geom_threshold = threshold);
run_time_icot_ls_none = @elapsed ICOT.fit!(lnr_ws_none, X, y);

ICOT.showinbrowser(lnr_ws_none)

score_ws_none = ICOT.score(lnr_ws_none, X, y, criterion=:dunnindex);
score_al_ws_none = ICOT.score(lnr_ws_none, X, y, criterion=:silhouette);

# Run ICOT with an OCT warm-start: fit an OCT as a supervised learning problem with labels "y" and use this as the warm-start
warm_start= :oct
lnr_ws_oct = ICOT.InterpretableCluster(ls_num_tree_restarts = num_tree_restarts, ls_random_seed = seed, cp = complexity_c, max_depth = maxdepth,
	minbucket = min_bucket, criterion = cr, ls_warmstart_criterion = cr, kmeans_warmstart = warm_start,
	geom_search = geom_search, geom_threshold = threshold);
run_time_icot_ls_oct = @elapsed ICOT.fit!(lnr_ws_oct, X, y);

score_ws_oct = ICOT.score(lnr_ws_oct, X, y, criterion=:dunnindex);
score_al_ws_oct = ICOT.score(lnr_ws_oct, X, y, criterion=:silhouette);

##### Comments
# Note that in this example, the OCT tree and ICOT (OCT warm-start, or no warm-start) result in the same solution.
# This is not generally the case, but can occur when the data is easily separated (as in the ruspini dataset)

# The score printed in the browser view is negative to reflect the formulation as a minimization problem.
# The positive score returned by the ICOT.score() functions reflect the "correct" interpretation of the score,
# in which we seek to maximize the criterion (with a maximum value of 1).

# For larger datasets, we recommend setting warm_start = :oct and threshold = 0.99 to improve the solve time.


export VrPCA,
        reduceToRank,
        getindex,
        SvdApprox,
        isConverged

"""
    # enum SvdApprox

. Lrapprox encodes for the initialization of vrpca with the LowRankApprox package

. Ifsvd  encodes for the initialization of vrpca with incremental svd of Vempala-Kunnan

"""
@enum SvdApprox begin
    lrapprox
    ifsvd
end

"""
# VrPCA

returns an orthonormal matrix W with columns vectors the first
k left singular vectors  spanning the range of X.
**k is supposed to be small compared to d**.

If X = U S Vt, the methode computes in W the matrix U[:, 1:k]
This structure gathers data to do stochastic svd as described in :
***Fast Stochastic Algorithms for SVD and PCA : convergences Properties and Convexity
Ohad Shamir 2015***

FIELDS
------

 - X  the (d,n) matrix we want to reduce.
 - k     the dimension into which we want to reducte the data Matrix
         ***Note :  If the obtained rank is less than asked for value of k is reset!***

 - W     orthogonal (d,k) matrix
 - eta   step size
 - m     epoch length

 - U : Union{ Some{Matrix{Float64}}, Nothing} the matrix of left singular vectors if any
 - S : Union{ Some{Vector{Float64}}, Nothing} the vecor of singular values if any
 - Vt: Union{ Some{Matrix{Float64}}, Nothing} the transposed matrix of right singular vectors if any

"""
mutable struct VrPCA
    X::Matrix{Float64}
    k::Int64
    # orthogonal matrix (d,k)
    W::Matrix{Float64}
    converged::Bool
    relerr::Float64
    #
    U::Union{ Some{Matrix{Float64}}, Nothing}
    S::Union{ Some{Vector{Float64}}, Nothing}
    Vt::Union{ Some{Matrix{Float64}}, Nothing}
end

"""
# function VrPCA(data::Matrix{Float64}, k::Int64)

This function initialize a VrPCA problem with the LowRankApprox package

## Args
. data : the data matrix with data vectors in column
. k : the rank we want as output
"""
function VrPCA(data::Matrix{Float64}, k::Int64)
        @debug  "VrPCA initialized by LowRankApprox and rank target"
        # get an initial W0, we do not need very good precision
        opts = LowRankApprox.LRAOptions(rank=k, rtol = 0.001)
        # get W such that Mdata ~ W*W'*Mdata  in Froebenius norm
        W = prange(data, opts)
        if size(W)[2] < k
            @warn "VrPCA initialization got less than asked rank, got : " size(W)[2]
        end
        @debug "got rank " size(W)[2]
        VrPCA(data, size(W)[2] , W, false, 1., nothing,nothing,nothing)
end

"""
# function VrPCA(data::Matrix{Float64}, epsil::Float64)

This function initialize a VrPCA problem with the LowRankApprox package

## Args
 - data : the data matrix with data vectors in column
 - epsil: the precision we ask at initialisation, the VrPCA iteration will further reduce error
         when iterating with the rank obtained at initialization
"""
function VrPCA(data::Matrix{Float64}, epsil::Float64)
    @debug "VrPCA initialized by Vempala's LowRankApproxMc and epsil target"
    lrmc = LowRankApproxMc(data)
    samplingSize = 100
    reduceToEpsil(lrmc, samplingSize, epsil)
    U = PartialSvdStoch.getindex(lrmc, :U)
    if U === nothing
        throw("initialization of VrPCA with epsil failed")
    end
    W = something(U)
    @debug "initialization got rank " size(W)[2]
    VrPCA(data, size(W)[2], W, false, 1. , nothing, nothing, nothing)
end

"""
# function iterateVrPCA(pcaPb::VrPCA, eta::Float64, m::Int64)

## Args
 - eta : the gradient step. Should be sufficiently small to ensure inversibility of the (k,k) matrix
         W*transpose(W) obtained during iterations on W(d,k)

 - m   : the batch size. Roughly inversely proportional to eta.

    eta = 0.01 and m = 20 is a good compromises. m does not need to be large
        as sampling is weighted by L2-norm of data column.

 Output : The function does not return any output.
    It fills the fields U, S, Vt of the structure VrPCA.
    The fields can be retrieved by the function getIndex.
"""
function iterateVrPCA(pcaPb::VrPCA, eta::Float64, m::Int64)
    @debug "entering VrPCA reduce eta m block version" eta m
    small = 1.E-5
    d,n = size(pcaPb.X)
    k = pcaPb.k
    # allocates once and for all 3 temproray matrices
    Us = zeros(Float64, d, k)  # is (d,k)
    B_t = zeros(Float64, k, k)   # Bt_1 is Wt*W so it is a (k,k) matrix
    W_tmp = zeros(Float64, d, k)
    pcaPb.converged = false
    s = 1
    W_s = pcaPb.W
    W_t = pcaPb.W
    deltaWIter = Vector{Float64}()
    # get L2 proba per column
    probaC = Vector{Float64}(undef,n)
    normC = Vector{Float64}(undef,n)
    normX = 0
    Threads.@threads for i = 1:n
        normC[i] = norm(pcaPb.X[:,i]) * norm(pcaPb.X[:,i])
    end
    for i = 1:n
        probaC[i] = i > 1 ? probaC[i-1] + normC[i] : normC[i]
        normX += normC[i]
    end
    if probaC[end] <= 0
        @debug "\n non positive proba for last slot=  :  " probaC[end]
        error("cannot sample column")
    end
    map!(x -> x/probaC[end], probaC, probaC)
    #
    @debug " W_s dim" size(W_s)
    more = true
    while more
        fill!(Us,0.)
        # split in block the update of Us to get speed from BLAS.gemm
        blockSize = 1000
        y = zeros(k, blockSize)
        nbblocs = floor(Int64, n / blockSize)
        for numbloc = 1:nbblocs
            blocfirst = 1 + (numbloc -1) * blockSize
            bloclast = min(blockSize * numbloc, n)
            # get  W_s' * pcaPb.X[:,i] in y
            BLAS.gemm!('T', 'N', 1., W_s , pcaPb.X[:,blocfirst:bloclast], 0., y)
            # Us = Us + pcaPb.X[:,i] * y'  (= transpose(pcaPb.X[:,i]) * W_s)
            for i in 1:blockSize
                j = (numbloc - 1) * blockSize + i
                BLAS.ger!(1.,pcaPb.X[:,j], y[:,i], Us)
            end
            # Us = Us + pcaPb.X[:,i] * (transpose(pcaPb.X[:,i]) * W_s)
        end
        # do not forget the residual part of block splitting!!
        if n % blockSize > 0
            y = zeros(k)
            for i in 1 + nbblocs * blockSize:n
                BLAS.gemm!('T', 'N', 1., W_s , pcaPb.X[:,i], 0., y)
                BLAS.ger!(1.,pcaPb.X[:,i], y, Us)
            end
        end
        Us = Us / n
        # stochastic update pass
        # we need to store Wt W_{s-1} (Ws1 in the code) W_{t-1} (Wt in the code)
        for t = 1:m
            # compute in place to avoid allocator transpose(W_{t_1}) * W_{s-1}, B_t is (k,k)
            BLAS.gemm!('T', 'N', 1., W_t, W_s, 0., B_t)
            # compute svd of transpose(W_{t_1}) * W_{s-1}    i.e do a svd of a (k,k) matrix
            U,S,Vt = LAPACK.gesvd!('A', 'A', B_t)
            # compute B_t as V*Ut
            BLAS.gemm!('T', 'T', 1., Vt, U, 0., B_t)
            #
            # biased sampling instead of it = rand(1:n)
            xsi = rand(Float64)
            it = searchsortedfirst(probaC, xsi)
            s_weight = 1. / (n * probaC[it])
            # update Wt, vaux1 is  (1,k)
            vaux1 = transpose(pcaPb.X[:,it]) * W_t - (transpose(pcaPb.X[:,it]) * W_s) * B_t
            # vaux2 is (d,k)
            vaux2 = s_weight * pcaPb.X[:,it] * vaux1 + BLAS.gemm('N', 'N', Us, B_t)
            # @debug "norm vaux2" norm(vaux2)
            # W_s1 * B_t1 is (d,k), so W_t1 and W_s1. W_tmp _s (d,k)
            W_tmp = W_t + eta * vaux2
            #
            # final update of W_t, compute sqrt(transpose(W_t1)*W_{t}),  (k,k)
            # in fact we want to orthonormalize U and Vt are (k,k)
            U,S,Vt = LAPACK.gesvd!('A', 'A', transpose(W_tmp) * W_tmp)
            # @debug "\n size U , Vt" size(U), size(Vt)
            # check for null values of S
            nbsmall = count( x -> x < small, S)
            if nbsmall > 0
                @debug "too small values " S
                @warn "could not do W update, ||W|| " norm(W_tmp, 2)
            end
            map!(x -> 1/sqrt(x) , S, S)
            # compute the inverse, BLAS.gemm does not seem to spped up things here
            Wnorminv = transpose(Vt) * diagm(0 => S) * transpose(U)
            # orthonormalize W. compute:  W_t / sqrt(transpose(W_t)*W_t)
            # @debug "\n size W_tmp Wnorminv" size(W_tmp) size(Wnorminv)
            W_t = BLAS.gemm('N', 'N', W_tmp, Wnorminv)
            # Cshould check W_t orthogonality : OK!
            # @debug "\n W orthogonality check " norm(transpose(W_t) * W_t - Matrix{Float64}(I,k,k))
        end
        #
        # convergence detection
        normWt = norm(W_t) * norm(W_t)
        deltaW = norm(W_t - W_s) * norm(W_t - W_s)
        lastDeltaW = 0
        if length(deltaWIter) > 0
            lastDeltaW = deltaWIter[end]
        end
        push!(deltaWIter, deltaW)
        @debug " iter ,  ||W_t - W_s||_2 , ||W_t||_2" s deltaW normWt
        W_s = W_t
        #
        # do a convergence test
        if normWt > 0 && deltaW/normWt < 1.E-3
            pcaPb.W = W_t
            pcaPb.relerr = deltaW/normWt
            more = false
            pcaPb.converged = true
        else
            if s >= 3
                if deltaW >= 0.8 * lastDeltaW || deltaW <= 0.05 * deltaWIter[1]
                    @warn "\n reached stationarity  s deltaW  ||W|| ,  exiting :  " s  deltaW  normWt
                    pcaPb.relerr =  deltaW/normWt
                    pcaPb.W = W_t
                    pcaPb.converged = true
                    more  = false
                elseif s > 5
                    # we stop without setting pcaPb.converged...
                    @warn "\n stopping iteration" s  deltaW  normWt
                    pcaPb.relerr =  deltaW/normWt
                    pcaPb.W = W_t
                    more = false
                end
            end
            s = s+1
        end
    end # end of while
    # now we have a converged matrix W (d,k) consisting in k left singular vectors
    # compute singular values and right singular vectors
    #                 = (k,d) * (d,n)
    #      Vt = transpose(W_t) * pcaPb.X
    Vt = Matrix{Float64}(undef, k, n)
    BLAS.gemm!('T', 'N', 1., W_t, pcaPb.X, 0., Vt)
    S = map(i -> norm(Vt[i,:]), (1:size(Vt)[1]))
    for i in 1:length(S)
        Vt[i,:] /= S[i]
    end
    pcaPb.U = Some(W_t)
    pcaPb.S = Some(S)
    pcaPb.Vt = Some(Vt)
end

"""
# function getindex(VrPCA, Symbol)

returns U,S,Vt,V or rank according to Symbol :S , :U, :Vt , V or :k

"""
function getindex(pcaPb::VrPCA, d::Symbol)
    @match d begin
    :S    =>  pcaPb.S
    :U    =>  pcaPb.U
    :V    =>  pcaPb.Vt'
    :Vt   =>  pcaPb.Vt
    :k    =>  length(pcaPb.S)
    _     =>  throw(KeyError(d))
    end
end

"""
# function getResidualError(vrpca::VrPCA)

if X is the data matrix and U computation succeeded
    return the 2-uple (error, relerr) with:
```math
        error = norm(X - U * transpose(U) * X)
```
and
```math
        relerror = norm(X - U * transpose(U) * X)/norm(X)
```

else throws an error
"""
function getResidualError(vrpca::VrPCA)
    U = something(vrpca.U)
    resErr = norm(vrpca.X - U* BLAS.gemm('T', 'N', U, vrpca.X))
    return resErr,resErr/norm(vrpca.X)
end

"""
# function isConverged(pcaPb::VrPCA)

return true if stationarity criteria were satisfied during iterations.
"""
function isConverged(pcaPb::VrPCA)
    return pcaPb.converged
end


# Different cooldown schedules. Internally, a cooldown function
# accepts the current temperature as its first argument and the
# current energy as its second argument and return the new temperature
# of the system.

@doc raw"""
    exponential_cooldown(λ :: Float64)

Create an exponential cooldown schedule. Temperature on step $n + 1$
is calculated as $T_{n + 1} = \lambda T_{n}$.
"""
exponential_cooldown(λ :: Float64 = 0.999999) = (T :: Float64, :: Float64) -> λ * T

"""
    aarts_korst_cooldown(;n = 15, λ = 0.01)

Make the Aarts-Korst cooldown schedule. The temperature decreases
each `n` steps of annealing algorithm. The higer `λ` is the faster
decreases the temperature.

For more information, see Aarts, E.H.L. and Korst, J.H.M. (1989)
Simulated Annealing and Boltzmann Machines: A Stochastic Approach to
Combinatorial Optimization and Neural Computing. John Wiley & Sons,
Chichester.
"""
function aarts_korst_cooldown(;n :: Integer = 15, λ :: Float64 = 0.01)
    costs = Vector{Float64}(undef, n)
    counter = 1

    function f(T :: Float64, cost :: Float64)
        costs[counter] = cost
        counter = mod1(counter + 1, n)

        newT = T
        if counter == 1
            σ = std(costs)
            newT = T * σ / (σ + λ*T)
        end

        return newT
    end

    return f
end

"""
    frost_heineman_cooldown(;n = 250, λ = 0.01)

Make the Frost-Heineman cooldown schedule. The temperature decreases
each `n` steps of annealing algorithm or until some "target" energy is
reached. The target energy is based on standard deviation of the cost
function and a parameter `λ`.

For more information, see R. Frost, P. Heineman "Simulated Annealing:
A Heuristic for Parallel Stochastic Optimization" (1997)
"""
function frost_heineman_cooldown(;n :: Integer = 250, λ :: Float64 = 0.01)
    costs = Vector{Float64}(undef, 0)
    sizehint!(costs, n)
    target = Inf

    function f(T :: Float64, cost :: Float64)
        push!(costs, cost)
        newT = T

        μ = mean(costs)
        if length(costs) == n && μ < target
            σ = std(costs)
            empty!(costs)

            target_prev = isinf(target) ? μ : target
            target = μ - λ*σ
            newT = T + (target - target_prev) * (T/σ)^2
        end

        return newT
    end

    return f
end


using GeometricFlux
using Graphs
using SparseArrays
using Plots
using GraphPlot

using TSne
using Cairo, Compose

function alias_sampling_example()
    J, q = alias_setup([0.05, 0.1, 0.2, 0.3, 0.2, 0.1, 0.05])
    samples = [alias_sample(J, q) for _ in 1:1000]
    counts = countmap(samples)
    for i in 1:7
        println(i, ": ", counts[i])
    end
end

clusters = Dict(
3 => 1,
14 => 1,
20 => 1,
8 => 1,
2 => 1,
1 => 1,
4 => 1,
18 => 1,
13 => 1,
22 => 1,
12 => 1,
5 => 1,
6 => 1,
11 => 1,
7 => 1,
17 => 1,
32 => 2,
25 => 2,
26 => 2,
28 => 2,
29 => 2,
24 => 2,
9 => 2,
19 => 2,
33 => 2,
34 => 2,
21 => 2,
30 => 2,
31 => 2,
16 => 2,
27 => 2,
15 => 2,
23 => 2,
10 => 2,
)
int2col(x::Int) = x==1 ? "blue" : "red"

g = smallgraph(:karate)
vectors = node2vec(g; walks_per_node=100, len=5, p=0.5, q=0.5)
points = tsne(vectors', 2, 75, 1000, 5.0)
point_tuples = [tuple(row...) for row in eachrow(points)]

draw(
    PDF("karateclub.pdf", 16cm, 16cm),
    gplot(g, nodefillc=[int2col(clusters[i]) for i in 1:34])
)

# alias_sampling_example()
# sample_random_walk()


function lineplot(x::AbstractArray, y::AbstractArray)
    x, y = vec(x), vec(y)

    # Sanity checking
    N = length(x)
    if N != length(y)
        error("x and y must have the same length")
    end

    # Resolution along x and y dimensions
    res_x, res_y = 60, 20

    # Sort the points to make slopes easier to calculate
    ind = sortperm(x)
    x = x[ind]
    y = y[ind]

    # Standarize data scale
    minx = minimum(x)
    maxx = maximum(x)
    miny = minimum(y)
    maxy = maximum(y)
    x = x .- minx
    x = x / maximum(x)
    y = y .- miny
    y = y / maximum(y)

    # Snap data points to a grid
    xi = floor.(Integer, x * (res_x - 1)) .+ 1
    yi = floor.(Integer, y * (res_y - 1)) .+ 1

    # Compute slope and non-empty points
    dy = diff([y; 2 * y[end] - y[end - 1]]) ./
        max.(diff([x; 2 * x[end] - x[end - 1]]), 1 / N)
    S = zeros(res_y, res_x)
    A = zeros(res_y, res_x)
    for i in 1:N
        S[yi[i], xi[i]] = dy[i]
        A[yi[i], xi[i]] = 1
    end

    io = IOBuffer()

    print(io, "\n")

    # Top grid line
    print(io, "\t")
    for j = 1:(res_x + 1)
        print(io, "-")
    end
    print(io, "\n")

    for i = res_y:-1:1
        # Left grid line
        print(io, "\t|")

        # Data points
        for j = 1:res_x
            if A[i, j] == 1
                if S[i, j] < -0.5
                    print(io, "\\")
                elseif S[i, j] < 0.5
                    print(io, "-")
                else
                    print(io, "/")
                end
            else
                print(io, " ")
            end
        end

        # Right grid line + Y tick marks
        if i == res_y
            @printf io "| %2.2f\n" maxy
        elseif i == 1
            @printf io "| %2.2f\n" miny
        else
            print(io, "|\n")
        end
    end

    # Bottom grid line
    print(io, "\t")
    for j = 1:(res_x + 1)
        print(io, "-")
    end
    print(io, "\n")

    # X tick marks
    @printf io "\t%2.2f" minx
    for j = 1:(res_x - 8)
        print(io, " ")
    end
    @printf io "%2.2f" maxx
    print(io, "\n")

    return ASCIIPlot(String(take!(io)))
end

lineplot(y::AbstractArray) = lineplot([1:length(y);], y)