using LinearAlgebra

mutable struct RadialBasis{F,Q,X,Y,L,U,C,S} <: AbstractSurrogate
    phi::F
    dim_poly::Q
    x::X
    y::Y
    lb::L
    ub::U
    coeff::C
    scale_factor::S
end

mutable struct RadialFunction{Q,P}
    q::Q
    phi::P
end

linearRadial = RadialFunction(0,z->norm(z))
cubicRadial = RadialFunction(1,z->norm(z)^3)
multiquadricRadial = RadialFunction(1,z->sqrt(norm(z)^2+1))
thinplateRadial = RadialFunction(2,z->norm(z)^2*log(norm(z)))

"""
    RadialBasis(x,y,lb::Number,ub::Number; rad::RadialFunction = linearRadial,scale::Real=1.0)

Constructor for RadialBasis surrogate.
"""
function RadialBasis(x, y, lb::Number, ub::Number; rad::RadialFunction=linearRadial, scale_factor::Real=1.0)
    q = rad.q
    phi = rad.phi
    coeff = _calc_coeffs(x, y, lb, ub, phi, q,scale_factor)
    return RadialBasis(phi, q, x, y, lb, ub, coeff,scale_factor)
end

"""
RadialBasis(x,y,lb,ub,rad::RadialFunction, scale_factor::Float = 1.0)

Constructor for RadialBasis surrogate
"""
function RadialBasis(x, y, lb, ub; rad::RadialFunction = linearRadial, scale_factor::Real=1.0)
    q = rad.q
    phi = rad.phi
    coeff = _calc_coeffs(x, y, lb, ub, phi, q, scale_factor)
    return RadialBasis(phi, q, x, y, lb, ub, coeff,scale_factor)
end

function _calc_coeffs(x, y, lb, ub, phi, q, scale_factor)
    D = _construct_rbf_interp_matrix(x, first(x), lb, ub, phi, q, scale_factor)
    Y = _construct_rbf_y_matrix(y, first(y), length(y) + q)
    coeff = D \ Y
    return coeff
end

function _construct_rbf_interp_matrix(x, x_el::Number, lb, ub, phi, q, scale_factor)
    n = length(x)
    m = n + q
    D = zeros(eltype(x_el), m, m)

    @inbounds for i = 1:n
        for j = 1:n
            D[i,j] = phi( (x[i] .- x[j]) ./ scale_factor )
        end
        if i <= n
            for k = 1:q
                D[i,n+k] = _scaled_chebyshev(x[i], k-1, lb, ub)
            end
        end
    end
    D_sym = Symmetric(D, :U)
    return D_sym
end

function _construct_rbf_interp_matrix(x, x_el, lb, ub, phi, q, scale_factor)
    n = length(x)
    nd = length(x_el)
    central_point = _center_bounds(x_el, lb, ub)
    sum_half_diameter = sum((ub[k]-lb[k])/2 for k = 1:nd)
    mean_half_diameter = sum_half_diameter / nd
    m = n+q
    D = zeros(eltype(x_el), m, m)

    @inbounds for i = 1:n
        for j = 1:n
            D[i,j] = phi( (x[i] .- x[j]) ./ scale_factor)
        end
        if i < n + 1
            for k = 1:q
                D[i,n+k] = centralized_monomial(x[i], k-1, mean_half_diameter, central_point)
            end
        end
    end
    D_sym = Symmetric(D, :U)
    return D_sym
end

_construct_rbf_y_matrix(y, y_el::Number, m) = [i <= length(y) ? y[i] : zero(y_el) for i = 1:m]
_construct_rbf_y_matrix(y, y_el, m) = [i <= length(y) ? y[i][j] : zero(first(y_el)) for i=1:m, j=1:length(y_el)]

"""
    centralized_monomial(vect,alpha,mean_half_diameter,central_point)

Returns the value at point vect[] of the alpha degree monomial centralized.

#Arguments:
-'vect': vector of points i.e [x,y,...,w]
-'alpha': degree
-'mean_half_diameter': half diameter of the domain
-'central_point': central point in the domain
"""
function centralized_monomial(vect,alpha,mean_half_diameter,central_point)
    if iszero(alpha) return one(eltype(vect)) end
    centralized_product = prod(vect .- central_point)
    return (centralized_product / mean_half_diameter)^alpha
end

"""
Calculates current estimate of value 'val' with respect to the RadialBasis object.
"""
function (rad::RadialBasis)(val)
    approx = _approx_rbf(val, rad)
    return _match_container(approx, first(rad.y))
end

function _approx_rbf(val::Number, rad)
    n = length(rad.x)
    q = rad.dim_poly
    lb = rad.lb
    ub = rad.ub
    approx = zero(rad.coeff[1, :])
    for i = 1:n
        approx += rad.coeff[i, :] * rad.phi( (val .- rad.x[i]) / rad.scale_factor)
    end
    for k = 1:q
        approx += rad.coeff[n+k, :] * _scaled_chebyshev(val, k-1, lb, ub)
    end
    return approx
end
function _approx_rbf(val, rad)
    n = length(rad.x)
    d = length(rad.x[1])
    q = rad.dim_poly
    lb = rad.lb
    ub = rad.ub
    sum_half_diameter = sum((ub[k]-lb[k])/2 for k = 1:d)
    mean_half_diameter = sum_half_diameter/d
    central_point = _center_bounds(first(rad.x), lb, ub)

    approx = zero(rad.coeff[1, :])
    @views approx += sum( rad.coeff[i, :] * rad.phi( (val .- rad.x[i]) ./rad.scale_factor) for i = 1:n)
    for k = 1:q
        @views approx += rad.coeff[n+k, :] .* centralized_monomial(val, k-1, mean_half_diameter, central_point)
    end
    return approx
end

_scaled_chebyshev(x, k, lb, ub) = cos(k*acos(-1 + 2*(x-lb)/(ub-lb)))
_center_bounds(x::Tuple, lb, ub) = ntuple(i -> (ub[i] - lb[i])/2, length(x))
_center_bounds(x, lb, ub) = (ub .- lb) ./ 2

"""
    add_point!(rad::RadialBasis,new_x,new_y)

Add new samples x and y and update the coefficients. Return the new object radial.
"""
function add_point!(rad::RadialBasis,new_x,new_y)
    if (length(new_x) == 1 && length(new_x[1]) == 1) || ( length(new_x) > 1 && length(new_x[1]) == 1 && length(rad.lb)>1)
        push!(rad.x,new_x)
        push!(rad.y,new_y)
    else
        append!(rad.x,new_x)
        append!(rad.y,new_y)
    end
    rad.coeff = _calc_coeffs(rad.x,rad.y,rad.lb,rad.ub,rad.phi,rad.dim_poly, rad.scale_factor)
    nothing
end


function weighted_mean{T<:Real,W<:Real}(data::Array{T,1}, weights::Array{W,1})
    weights = float(weights) / sum(weights)
    first(float(data)' * weights)
end


mutable struct Node
    depth
    is_leaf
    prediction
    feature_index
    thresh
    left
    right
end

function Node(depth, is_leaf, prediction)
    Node(depth, is_leaf, prediction, nothing, nothing, nothing, nothing)
end
struct WeightedDecisionStump
    root_node

    function WeightedDecisionStump(X, y, weights)
        node = Node(1, false, nothing)
        feature_index, thresh, left_prediction = split(X, y, weights)
        node.feature_index = feature_index
        node.thresh = thresh
        node.left = Node(2, true, left_prediction)
        node.right = Node(2, true, -left_prediction)

        new(node)
    end
end

function split(X, y, weights)
    best_feature = 1
    best_thresh = minimum(X[:, 1])
    best_left_prediction = 1
    error_val = Inf

    for (feature_index, feature_array) in enumerate(eachcol(X))
        sorted_indices = sortperm(feature_array)
        X_sorted = X[sorted_indices, :]
        y_sorted = y[sorted_indices]
        weights_sorted = weights[sorted_indices]
        predictions = zeros(length(y_sorted))

        for left_prediction = (-1, 1)
            for i = 1:(length(y_sorted) - 1)
                predictions[1:i] .= left_prediction
                predictions[i+1:end] .= -left_prediction
                temp_error_val = misclassification_rate(predictions, y_sorted, weights_sorted)
                if temp_error_val < error_val
                    error_val = temp_error_val
                    best_feature = feature_index
                    best_thresh = (X_sorted[i, feature_index] + X_sorted[i+1, feature_index]) / 2
                    best_left_prediction = left_prediction
                end
            end
        end
    end

    return best_feature, best_thresh, best_left_prediction
end

function misclassification_rate(predictions, y, weights)
    sum((predictions .!= y) .* weights)
end

struct AdaBoostModel
    num_models
    models
    alphas

    function AdaBoostModel(X, y, num_models)
        # AdaBoost model requires labels being -1 or 1
        @assert Set(y) == Set([-1, 1])
        weights = fill(1.0 / length(y), length(y))
        models = WeightedDecisionStump[]
        alphas = Float64[]

        for _ = 1:num_models
            model = WeightedDecisionStump(X, y, weights)
            predictions = predict(model, X)
            mistakes = predictions .!= y
            error = sum(weights .* mistakes) / sum(weights)
            alpha = error == 0.0 ? 1.0 : log((1-error)/error)
            weights = weights .* exp.(alpha .* mistakes)
            weights = weights ./ sum(weights)

            push!(models, model)
            push!(alphas, alpha)
        end

        new(num_models, models, alphas)
    end
end

function predict(model::WeightedDecisionStump, X)
    root_node = model.root_node
    feature_index = root_node.feature_index
    thresh = root_node.thresh
    predictions = zeros(size(X)[1])

    left_indices = X[:, feature_index] .< thresh
    predictions[left_indices] .= root_node.left.prediction
    predictions[.!left_indices] .= root_node.right.prediction
    return predictions
end

function predict(model::AdaBoostModel, X)
    models = model.models
    alphas = model.alphas
    predictions = zeros(size(X)[1])
    votes = zeros(size(X)[1])

    for (alpha, model) in zip(alphas, models)
        predictions = predict(model, X)
        votes .+= alpha.*predictions
    end

    return sign.(sum(votes, dims=2))
end


#
# Prepare cost, proximal maps and differentials
M = PowerManifold(pixelM, NestedPowerRepresentation(), size(f)...)
N = TangentBundle(M)
fidelity(M, x) = 1 / 2 * distance(M, x, f)^2
Λ(M, x) = ProductRepr(x, forward_logs(M, x))
prior(M, x) = norm(norm.(Ref(pixelM), x, submanifold_component(N, Λ(M, x), 2)), 1)
cost(M, x) = (1 / α) * fidelity(M, x) + prior(M, x)

prox_F(M, λ, x) = prox_distance(M, λ / α, f, x, 2)
function prox_G_dual(N, n, λ, ξ)
    return ProductRepr(
        submanifold_component(N, ξ, 1),
        project_collaborative_TV(
            base_manifold(N),
            λ,
            submanifold_component(N, n, 1),
            submanifold_component(N, ξ, 2),
            Inf,
            Inf,
            1.0,
        ),
    )
end

DΛ(M, m, X) = ProductRepr(zero_vector(M, m), differential_forward_logs(M, m, X))
adjoint_DΛ(N, m, n, ξ) = adjoint_differential_forward_logs(N.manifold, m, ξ[N, :vector])


# parameters used for distance calculations and coordinate system transforms

#
# Ellipsoids
#
immutable Ellipsoid
    a::Float64                          # Semi-major axis
    b::Float64                          # Semi-minor axis
    e²::Float64                         # Eccentricity squared
    e′²::Float64                         # Second eccentricity squared
end

function Ellipsoid(; a::AbstractString="", b::AbstractString="", f_inv::AbstractString="")
    if isempty(a) || isempty(b) == isempty(f_inv)
        throw(ArgumentError("Specify parameter 'a' and either 'b' or 'f_inv'"))
    end
    if isempty(b)
        _ellipsoid_af(parse(BigFloat, a), parse(BigFloat, f_inv))
    else
        _ellipsoid_ab(parse(BigFloat, a), parse(BigFloat, b))
    end
end

function _ellipsoid_ab(a::BigFloat, b::BigFloat)
    e² = (a^2 - b^2) / a^2
    e′² = (a^2 - b^2) / b^2

    Ellipsoid(a, b, e², e′²)
end
function _ellipsoid_af(a::BigFloat, f_inv::BigFloat)
    b = a * (1 - inv(f_inv))
    _ellipsoid_ab(a, b)
end

#
# Predefined Ellipses
#

# A few common Ellipsoids (TODO: steal values from Proj4!)
const eWGS84      = Ellipsoid(a = "6378137.0", f_inv = "298.257223563")
const eGRS80      = Ellipsoid(a = "6378137.0", f_inv = "298.257222100882711243") # f_inv derived
const eHayford    = Ellipsoid(a = "6378388.0", f_inv = "297.0")
const eClarke1866 = Ellipsoid(a = "6378206.4",   b = "6356583.8")
const eAiry       = Ellipsoid(a = "6377563.396", b = "6356256.909")
const eNZGD2000   = Ellipsoid(a = "6378137.0", b = "6356752.3141")

#
# Generate Ellipse types
#

# build known ellipses
abstract Ellipse <: KnownDatum

# custom type
immutable CustomEllipse{T <: Ellipsoid} <: Ellipse end
ellipsoid{T <: Ellipsoid}(::Type{CustomEllipse{T}}) = T # grab the ellipse from the type

# A few common ellipses
immutable WGS84_ELLIPSE <: Ellipse end
ellipsoid(::Type{WGS84_ELLIPSE}) = eWGS84

immutable GRS80_ELLIPSE <: Ellipse end
ellipsoid(::Type{GRS80_ELLIPSE}) = eGRS80

immutable HAYFORD_ELLIPSE <: Ellipse end
ellipsoid(::Type{HAYFORD_ELLIPSE}) = eHayford

immutable AIRY_ELLIPSE <: Ellipse end
ellipsoid(::Type{AIRY_ELLIPSE}) = eAiry

immutable CLARKE66_ELLIPSE <: Ellipse end
ellipsoid(::Type{CLARKE66_ELLIPSE}) = eClarke1866

immutable NZGD2000_ELLIPSE <: Ellipse end
ellipsoid(::Type{NZGD2000_ELLIPSE}) = eNZGD2000


export lanczos

# this finds the OPs and recurrence for a
function lanczos(w,N)
    x = identity_fun(space(w))

    f1=Fun(1./sqrt(sum(w)),space(x))

    P = Array(Fun,N + 1)
    β = Array(eltype(w),N)
    γ = Array(eltype(w),N)

    P[1] = f1

    v = x.*P[1]
    β[1] = sum(w.*v.*P[1])

    v = v - β[1]*P[1]
    γ[1] = sqrt(sum(w.*v.^2))

    P[2] = v/γ[1]

    for k = 2:N
        v = x.*P[k] - γ[k-1]*P[k-1]
        β[k] = sum(w.*v.*P[k])
        v = v - β[k]*P[k]
        γ[k] = sqrt(sum(w.*v.^2))
        P[k+1] = v/γ[k]
    end

    P,β,γ
end


using PooksoftAssetModelingKit
using CSV
using DataFrames
using Test
using Statistics
using KernelDensity
using Dates

function estimate_random_walk_model_test()

    # test setup -
    ticker_symbol =
    path_test_data_file = "./data/Test.csv"

    # load the data file -
    df = CSV.read(path_test_data_file,DataFrame)

    # what dates are we going to look at?
	date_start = Date("2016-01-04")
    date_stop = Date("2020-10-09")

    # filter the date range we want -
	tmp = filter(r->(r.timestamp >= date_start && r.timestamp <= date_stop),df)

    # compute the difference for each ticker -
	result = compute_linear_return_array(tmp)
	if (isa(result.value, Exception) == true)
		return false
	end
    price_return_array = result.value

    # grab the model - is this the right type?
	model = kde_lscv(price_return_array)
    if (isa(model, UnivariateKDE) == false)
        return false
    end

    # we got here - all is ok
    return true
end

function run_gbm_model_simulation()

    # test setup -
    path_test_data_file = "./data/Test.csv"

    # load the data file -
    data_table = CSV.read(path_test_data_file,DataFrame)

    # estimated the average return
    result = compute_linear_return_array(data_table)
    if (isa(result.value,Exception) == true)
        return false
    end
    μ = mean(result.value)

    # estimate the average volatility -
    result = compute_return_volatility(data_table)
    if (isa(result.value,Exception) == true)
        return false
    end
    σ = result.value

    # build the model object -
    result = build_geometric_brownian_motion_model(μ,σ)
    if (isa(result.value,Exception) == true)
        return false
    end
    model = result.value

    # setup and run the simulation -
    tspan = (0.0,14.0)
    stepSize = 0.0001
    initial_condition = 83.81
    number_of_trials = 100

    # run -
    result = evaluate(model,initial_condition,tspan,stepSize;number_of_trials=number_of_trials)
    if (isa(result.value,Exception) == true)
        return false
    end
    simulation_result = result.value

    # check types on the return -
    if (haskey(simulation_result,:X) == true && haskey(simulation_result,:T) == true)
        return true
    end

    # default is to fail (just like everyhting I do in life ...)
    return false
end

@testset "asset_modeling_test_set" begin
    @test run_gbm_model_simulation() == true
    @test estimate_random_walk_model_test() == true
end


using Revise
using Dice

# Convert (value, weight) tuples to input for dice discrete distribution
# Example: unpack_sparse([(0, 0.2), (3, 0.8)]) => [0.2, 0, 0, 0.8]
function unpack_sparse(sparse::Vector{Tuple{Int,Float64}})
    dice_disc = zeros(maximum(val for (val, _) in sparse) + 1)
    for (val, weight) in sparse
        dice_disc[val + 1] = weight
    end
    return dice_disc
end

# Naively calculate P[gcd(X, Y) = res] given distributions of X and Y
function gcd_enumeration(
    sparse1::Vector{Tuple{Int,Float64}},
    sparse2::Vector{Tuple{Int,Float64}},
    res::Int
)
    total_weight = 0
    for (val1, weight1) in sparse1
        for (val2, weight2) in sparse2
            if gcd(val1, val2) == res
                total_weight += weight1 * weight2
            end
        end
    end
    return total_weight
end

# Generate Dice code for P[gcd(X, Y) = res] given distributions of X and Y
function code_gcd(
    sparse1::Vector{Tuple{Int,Float64}},
    sparse2::Vector{Tuple{Int,Float64}},
    res::Int
)
    discrete_dist1 = unpack_sparse(sparse1)
    discrete_dist2 = unpack_sparse(sparse2)
    @dice begin
        function discrete(p::Vector{Float64})
            mb = length(p)
            v = Vector(undef, mb)
            sum = 1
            for i=1:mb
                v[i] = p[i]/sum
                sum = sum - p[i]
            end

            ans = DistInt(dicecontext(), mb-1)
            for i=mb-1:-1:1
                ans = if flip(v[i]) DistInt(dicecontext(), i-1) else ans end
            end
            return ans
        end

        # Subtraction-based GCD
        # https://en.wikipedia.org/wiki/Euclidean_algorithm#Implementations
        function dice_gcd(a::Tuple{DistInt, DistBool}, b::Tuple{DistInt, DistBool})
            # Swap if a < b to handle cases where a=0 and b!=0
            lt = b[1] > a[1]
            t = a
            a = Dice.ifelse(lt, b[1], a[1]), Dice.ifelse(lt, b[2], a[2])
            b = Dice.ifelse(lt, t[1], b[1]), Dice.ifelse(lt, t[2], b[2])

            for _ = 0:100  # TODO: better bound
                gt = a[1] > b[1]
                lt = b[1] > a[1]
                amb = a[1] - b[1]
                bma = b[1] - a[1]
                a = Dice.ifelse(gt, amb[1], a[1]), a[2] | (amb[2] & gt)
                b = Dice.ifelse(lt, bma[1], b[1]), b[2] | (bma[2] & lt)
            end
            return a[1], a[2]
        end

        a = discrete(discrete_dist1), flip(0)
        b = discrete(discrete_dist2), flip(0)
        g = dice_gcd(a, b)
        prob_equals(g[1], res) & !g[2]
    end
end

all_pass = true
function test(sparse::Vector{Tuple{Int,Float64}})
    for res = 0:10
        code = code_gcd(sparse, sparse, res)
        bdd = compile(code)
        dice_p = infer(code, :bdd)
        naive_p = gcd_enumeration(sparse, sparse, res)
        if !isapprox(dice_p, naive_p)
            global all_pass = false
            println("FAIL. ", sparse, " ", res, " Expected: ", naive_p, " Got: ", dice_p)
        end
    end
end

sparse1 = [(4, 0.3), (6, 0.2), (9, 0.5)] # 4 has probability 0.3, etc...
sparse2 = [(0, 0.2), (4, 0.3), (6, 0.5)]
test(sparse1)
test(sparse2)
if all_pass
    println("ALL TESTS PASSED")
end


module StreamStats

using OffsetArrays
using Statistics
using StatsBase
using JSON

function monte_carlo_pi(data)
    """
    Calculate Monte Carlo Pi approximation for a given byte array.
    Keyword arguments:
    data -- data bytes
    """
    set_length::Int32 = round((length(data) / 2) - 1)
    r_square::Int16 = 128^2
    circle_surface::Float16 = 0

    for i in 1:set_length
        (data[i*2] - 128)
        if ((data[i*2] - 128) ^ 2 + (data[i*2+1] - 128) ^ 2) <= r_square
            circle_surface += 1
        end
    end
    return 4 * circle_surface / set_length
end

function pi_deviation(pi_value)
    """
    Returns an absolute percentage of difference between the provided Pi
    value and canonic.
    Keyword arguments:
    pi_value -- Pi value to be tested for difference
    """
    return abs(100 - (pi_value * 100 / pi))
end

function chi_squared(data)
    """Calculate Chi-Squared value for a given byte array.
    Keyword arguments:
    data -- data bytes
    """
    expected = length(data) / 256
    observed = OffsetVector([0 for i in 0:255], 0:255)

    for b in data
        observed[b] += 1
    end

    chi_squared = 0
    for o in observed
        chi_squared += (o - expected) ^ 2 / expected
    end

    return chi_squared
end

function get_all(data)
    """
    Returns a set of Statistical Information about the data
    Keywird arguments:
    data -- data bytes
    """
    carlo = monte_carlo_pi(data)
    dist = StatsBase.countmap(data)
    ddist = Dict{Int32,Int32}()
    for i in 0:255
        if Base.haskey(dist,i)
          ddist[i] = dist[i]
        else
          ddist[i] = 0
        end
    end

    orderd_dist =  sort(collect(dist), by=x->x[1])

    return JSON.json(Dict{String, Any}([
                "length"         => length(data),
                "chi-squared"    => chi_squared(data),
                "min"            => Int(Statistics.minimum(data)),
                "max"            => Int(Statistics.maximum(data)),
                "mean"           => Statistics.mean(data),
                "median"         => Statistics.median(data),
                "middle"         => Statistics.middle(data),
                "standard-dev"   => StatsBase.std(data),
                "kurtosis"       => StatsBase.kurtosis(data),
                "cor"            => Int(StatsBase.cor(data)),
                "skewness"       => StatsBase.skewness(data),
                "variation"      => StatsBase.variation(data),
                "sem"            => StatsBase.sem(data),
                "entropy"        => StatsBase.entropy(data),
                "cov"            => StatsBase.cov(data),
                "mad"            => StatsBase.mad(data, normalize=false),
                "monte-carlo-pi" => carlo,
                "pi-deviation"   => pi_deviation(carlo),
                "distribution"   => orderd_dist
                ])
            )
end

Base.@ccallable function julia_main()::Cint
    try
        main()
    catch
        Base.invokelatest(Base.display_error, Base.catch_stack())
        return 1
    end
    return 0
end

function main()
    println(ARGS)
    for file in ARGS
        if !isfile(file)
            error("could not find file $file")
        end
        println(get_all(read(file)))
    end
end

if abspath(PROGRAM_FILE) == @__FILE__
    main()
end

end


function func_U(t)
    e = exp(1)
    return 2.0*((e^(3.0*t))-(e^3))/((e^(3.0*t/2.0))*(2.0+(e^3)))
end

function func(X,t)
    F = [0.5*X[1]+ func_U(t), X[1]*X[1]+0.5*(func_U(t)*func_U(t))]
    return F
end

function func_adj(BF, X)
    BX = [0.5*BF[1]+2.0*X[1]*BF[2], 0.0]
    return BX
end

function advance(F_H,t,h)
    k0 = func(F_H,t)
    G = [F_H[1] + h/2.0*k0[1], F_H[2] + h/2.0*k0[2]]
    k1 = func(G,t+h/2.0)
    F = [F_H[1] + h*k1[1], F_H[2] + h*k1[2]]
    return F
end

function adjoint(F_H,L_H,F,L,t,h)
    k0 = func(F_H,t)
    G = similar(F_H)
    Bk1 = similar(L_H)
    G = [F_H[1] + h/2.0*k0[1], F_H[2] + h/2.0*k0[2]]
    k1 = func(G,t+h/2.0)
    L = [L_H[1], L_H[2]]
    Bk1 = [h*L_H[1], h*L_H[2]]
    BG = func_adj(Bk1,G)
    Bk0 = similar(BG)
    L = [L[1] + BG[1], L[2] + BG[2]]
    Bk0 = [h/2.0*BG[1], h/2.0*BG[2]]
    BH = func_adj(Bk0,F_H)
    L = [L[1] + BH[1], L[2] + BH[2]]
    return L
end

function opt_sol(Y,t)
    e = exp(1)
    Y[1] = (2.0*e^(3.0*t)+e^3)/(e^(3.0*t/2.0)*(2.0+e^3))
    Y[2] = (2.0*e^(3.0*t)-e^(6.0-3.0*t)-2.0+e^6)/((2.0+e^3)^2)
    return
end

function opt_lambda(L,t)
    e = exp(1)
    L[1] = (2.0*e^(3-t)-2.0*e^(2.0*t))/(e^(t/2.0)*(2+e^3))
    L[2] = 1.0
    return
end


module _Sine

using Brainstorm._Math._MPFR: add!, precision!

const guard_digits = 5
const bits_per_digit = log2(10)

pi(digits::T) where {T<:Integer} = begin
    setprecision(8) do
        target_prec = ceil(T, bits_per_digit * (digits + guard_digits))
        _pi(target_prec)
    end
end

# http://numbers.computation.free.fr/Constants/Pi/iterativePi.html
_pi(target_prec) = begin
    prec = 20
    setprecision(prec)
    α = big(355/113)

    while prec < target_prec
        prec = min(3*prec, target_prec)
        precision!(α, prec); setprecision(prec)

        add!(α, sin(α))
    end
    α
end

end


using StructuredProximalOperators
using LinearAlgebra
using Random
using Manifolds

function get_Ṡ(::FixedRankMatrices{m,n,k,ℝ}, x, ξ) where {m,n,k}
    return Vector([ξ.M[i, i] for i in 1:k])
end

function get_U̇(::FixedRankMatrices{m,n,k,ℝ}, x, ξ) where {m,n,k}
    F = zeros(k, k)
    for i in 1:k, j in 1:k
        if x.S[i] != x.S[j]
            F[i, j] = 1 / (x.S[j]^2 - x.S[i]^2)
        end
    end

    return x.U * (F .* (ξ.M * Diagonal(x.S) + Diagonal(x.S) * ξ.M')) +
           ξ.U * Diagonal(x.S .^ (-1))
end
function get_V̇t(::FixedRankMatrices{m,n,k,ℝ}, x, ξ) where {m,n,k}
    F = zeros(k, k)
    for i in 1:k, j in 1:k
        if x.S[i] != x.S[j]
            F[i, j] = 1 / (x.S[j]^2 - x.S[i]^2)
        end
    end

    return (-F .* (Diagonal(x.S) * ξ.M + ξ.M' * Diagonal(x.S))) * x.Vt +
           Diagonal(x.S .^ (-1)) * ξ.Vt
end

function main()
    m, n = 5, 6
    k = 3
    M = FixedRankMatrices{m,n,k,ℝ}()

    Random.seed!(152)
    x = randomMPoint(M)
    Random.seed!(4158)
    ξ = randomTVector(M, x)

    x_emb = embed(M, x)
    ξ_emb = embed(M, x, ξ)

    γ(t) = retract(M, x, t * ξ)
    function errors_retraction(t)
        return norm(embed(M, γ(t)) - x_emb - t * ξ_emb)
    end
    function errors_proj_retraction(t)
        return norm(project(M, x, embed(M, γ(t)) - x_emb - t * ξ_emb))
    end

    #
    ## Symmetric term
    #
    S = x.S
    Ṡ = get_Ṡ(M, x, ξ)
    function errorsS(t)
        F = svd(embed(M, γ(t)))
        return norm(F.S[1:k] - (S + t * Ṡ))
    end

    #
    ## U term
    #
    U = x.U
    U̇ = get_U̇(M, x, ξ)
    function errorsU(t)
        F = svd(embed(M, γ(t)))
        return norm(F.U[:, 1:k] - (U + t * U̇))
    end

    #
    ## Vt term
    #
    Vt = x.Vt
    V̇t = get_V̇t(M, x, ξ)
    function errorsVt(t)
        F = svd(embed(M, γ(t)))
        return norm(F.Vt[1:k, :] - (Vt + t * V̇t))
    end

    comparison = compare_curves(
        errors_retraction,
        errors_proj_retraction,
        errorsU,
        errorsS,
        errorsVt,
    )
    return display_curvescomparison(comparison)
end

main()


using GeodesicLearning

## energy function
f(x) = sum( 3*(x .- 5).^2 )

pop = initialize_pop(f, 20, dim = 3) # initialize_pop with 20 points
##
x = GeodesicLearning.least_squares(pop)
entropy_invert(pop, log(20)/100, x)


abstract type AbstractTransform end

function logjacobian(g::AbstractTransform, λ::Tuple, y)
    return sum(x -> log∂(g, λ, x), y)
end

struct DoNothing <: AbstractTransform end

transform(::DoNothing, y) = y
log∂(::DoNothing, _) = 0

struct BoxCox <: AbstractTransform end

transform(::BoxCox, λ::Number, y) = λ == 0 ? log(y) : expm1(log(y) * λ) / λ
log∂(::BoxCox, λ::Number, y) = (λ - 1) * log(y)

struct YeoJohnson <: AbstractTransform end

function transform(::YeoJohnson, λ::Number, y)
    λ′ = y >= 0 ? λ : 2 - λ
    return sign(y) * (λ′ == 0 ? log1p(abs(y)) : expm1(log1p(abs(y)) * λ′) / λ′)
end
log∂(::YeoJohnson, λ::Number, y) = sign(y) * (λ - 1) *  log(abs(y))

struct SumTanh <: AbstractTransform end

function transform(::SumTanh, α::Vector, β::Vector, c::Vector, y)
    return sum(α .* tanh.(β .* (y .+ c)))
end
function log∂(::SumTanh, α::Vector, β::Vector, c::Vector, y)
    return sum(α .* β .* sech.(β .* (y .+ c)) .^ 2)
end


using SparseArrays

export boundary1
function boundary1(mesh::DECMesh)
    δ = spzeros(length(mesh.tri.edges), length(mesh.tri.vertices))
    edges = mesh.tri.edges
    for i = 1:length(edges)
        e = edges[i]
        δ[i, e[1]] = -1
        δ[i, e[2]] = 1
    end
    return δ
end

export boundary2d
function boundary2d(mesh::DECMesh)
    polyns = mesh.dtri.dvertices
    edges = mesh.dtri.dedges
    δ = spzeros(length(polyns), length(edges))
    for i = 1:length(polyns)
        p = polyns[i]
        e = vcat([[p[i - 1], p[i]] for i = 2:length(p)], [[p[end], p[begin]]])
        for j = 1:length(e)
            if e[j] in edges
                k = findfirst(isequal(e[j]), edges)
                δ[i, k] = 1
            elseif reverse!(e[j]) in edges
                k = findfirst(isequal(e[j]), edges)
                δ[i, k] = -1
            end
        end
    end
    return δ
end

export hodgestar1
function hodgestar1(mesh::DECMesh)
    vertices = mesh.tri.vertices
    edges = mesh.tri.edges
    dtriangles = mesh.dtri.dtriangles
    dedges = mesh.dtri.dedges
    star = spzeros(length(edges), length(edges))
    for i = 1:length(edges)
        edge = edges[i]
        dedge = dedges[i]
        star[i, i] = longitude(dtriangles[dedge]...) / longitude(vertices[edge]...)
    end
    return star
end

export hodgestar2
function hodgestar2(mesh::DECMesh)
    dtriangles = mesh.dtri.dtriangles
    dvertices = mesh.dtri.dvertices
    star = spzeros(length(mesh.tri.vertices), length(mesh.tri.vertices))
    for i = 1:length(mesh.tri.vertices)
        star[i, i] = 1 / area(dtriangles[dvertices[i]])
    end
    return star
end

export laplacian
function laplacian(mesh::DECMesh)
    d₀ = boundary1(mesh)
    d₁ = boundary2d(mesh)
    *₁ = hodgestar1(mesh)
    *₂ = hodgestar2(mesh)

    Δf =  *₂  * d₁ *  *₁  * d₀
end


# MIT license
# Copyright (c) Microsoft Corporation. All rights reserved.
# See LICENSE in the project root for full license information.

function bsplinecurve()
    knots = KnotVector{Float64}([0.0, 0.0, 0.0, 0.0, 0.3, 0.6, 1.0, 1.0, 1.0, 1.0])
    points = Array{MVector{2,Float64},1}([(0.05, 0.05), (0.05, 0.8), (0.6, 0.0), (0.7, 0.5), (0.5, 0.95), (0.95, 0.5)])
    curve = BSplineCurve{OpticSim.Euclidean,Float64,2,3}(knots, points)
    return curve
end

function homogeneousbsplinecurve()
    knots = KnotVector{Float64}([0.0, 0.0, 0.0, 0.0, 0.3, 0.6, 1.0, 1.0, 1.0, 1.0])
    points = Array{MVector{3,Float64},1}([(0.05, 0.0, 1.0), (0.05, 0.8, 1.0), (0.6, 0.0, 1.0), (0.7, 0.5, 1.0), (0.5, 0.95, 1.0), (0.95, 0.5, 1.0)])
    curve = BSplineCurve{OpticSim.Rational,Float64,3,3}(knots, points)
    return curve
end

function beziercurve()
    orig = onespanspline()
    return BezierCurve{OpticSim.Euclidean,Float64,2,3}(tobeziersegments(orig)[1])
end

function onespanspline()
    knots = KnotVector{Float64}([0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0])
    points = Array{MVector{2,Float64},1}([(0.05, 0.05), (0.05, 0.95), (0.5, 0.95), (0.95, 0.5)])
    curve = BSplineCurve{OpticSim.Euclidean,Float64,2,3}(knots, points)
    return curve
end


# p = (1, 0, 0)
p = zeros(4)
p[2] = 1.0
p[3] = 0.0
p[4] = 0.0

# v = (0, 0, 1)
v = zeros(3)
v[3] = 1.0

# theta
# 右手系と逆になる
# Mz > 0 ならば負の方向に回転
theta = -pi/2

# q = (cos(t/2),  vx*sin(t/2),  vy*sin(t/2),  vz*sin(t/2))
q = zeros(4)

q[1] = cos(theta/2)
q[2] = v[1]*sin(theta/2)
q[3] = v[2]*sin(theta/2)
q[4] = v[3]*sin(theta/2)

A = zeros(3,3)

A[1,1] = 1 - 2*q[3]^2 - 2*q[4]^2
A[1,2] = 2*q[2]*q[3] + 2*q[1]*q[4]
A[1,3] = 2*q[2]*q[4] - 2*q[1]*q[3]

A[2,1] = 2*q[2]*q[3] - 2*q[1]*q[4]
A[2,2] = 1 - 2*q[4]^2 - 2*q[2]^2
A[2,3] = 2*q[3]*q[4] + 2*q[1]*q[2]

A[3,1] = 2*q[2]*q[4] + 2*q[1]*q[3]
A[3,2] = 2*q[3]*q[4] - 2*q[1]*q[2]
A[3,3] = 1 - 2*q[2]^2 - 2*q[3]^2

newx = A[1,1]*p[2] + A[1,2]*p[3] + A[1,3]*p[4]
newy = A[2,1]*p[2] + A[2,2]*p[3] + A[2,3]*p[4]
newz = A[3,1]*p[2] + A[3,2]*p[3] + A[3,3]*p[4]

println(newx)
println(newy)
println(newz)

p[2] = newx
p[3] = newy
p[4] = newz

newx = A[1,1]*p[2] + A[1,2]*p[3] + A[1,3]*p[4]
newy = A[2,1]*p[2] + A[2,2]*p[3] + A[2,3]*p[4]
newz = A[3,1]*p[2] + A[3,2]*p[3] + A[3,3]*p[4]

println(newx)
println(newy)
println(newz)


using StringUtils

function main()
    product = 0
    for a = 1:998
        for b = (a+1):999
            c = 1000 - a - b
            if c > a && c > b && c^2 == a^2 + b^2
                @printf("(a,b,c) = (%d, %d, %d)\n", a, b, c)
                product = a * b * c
            end
        end
    end

    @printf("The Pythagorean triplet product is %d\n", product)
end

main()


using JuliaSets
using Base.Test

# Define function
function R{T <: Real}(a::Complex{T})
    a^2 - 0.75
end

# Run test problem
x = [-2:.01:2]
y = [-2:.01:2]
A = visJuliaSet(R, x, y)

@test A[1, 1] < 10


using Plots
using Random

# Number of observations
T = 50

# Variances
Q = 0.1
R = 0.5

# Define model with prior N(0, 1000)
model = LocalLevel(zeros(T), Q, R)

# Simulate
model.y, x = simulate(model)

# Filter
kf = kalmanfilter(model)

# Smoother
ks = kalmansmoother(model, kf.priors, kf.posteriors)

# Plot
scatter(y, label = "Measurements", legend = :topright, title = "Univariate local level model",
        markerstrokewidth = 0, markercolor = "black", foreground_color_legend = nothing)
plot!(kf.posteriors.μ, label = "Filter", linewidth = 2, color = "red")
plot!(ks.μ, label = "Smoother", linewidth = 2, color = "green")


# Fitting

@doc raw"""
Calculate
```math
\int_{[k_{j+n-1}, k_{j+n}]} B_{(i,p,k)}(t) B_{(j,p,k)}(t) dt
```
Assumption:
* ``i ≤ j``
* ``1 ≤ n ≤ p-j+i+1``
"""
function _b_b_int(P::AbstractBSplineSpace, i, j, n, gl)
    p = degree(P)
    I = knots(P)[j+n-1]..knots(P)[j+n]

    f(t) = bsplinebasis(P, i, t) * bsplinebasis(P, j, t)
    return integrate(f, I, gl)
end

@doc raw"""
Calculate
```math
\begin{aligned}
&\int_{\mathbb{R}} B_{(i,p,k)}(t) B_{(j,p,k)}(t) dt \\
={}&
\begin{cases}
\displaystyle \int_{[k_{j}, k_{i+p+1}]} B_{(i,p,k)}(t)  B_{(j,p,k)}(t) dt & (i \le j) \\
\displaystyle \int_{[k_{i}, k_{j+p+1}]} B_{(i,p,k)}(t)  B_{(j,p,k)}(t) dt & (j \le i)
\end{cases}
\end{aligned}
```
"""
function _b_b_int_R(P::AbstractBSplineSpace, i, j, gl)
    p = degree(P)
    k = knots(P)
    Δ = j - i
    if Δ < -p
        return 0.0
    elseif Δ > p
        return 0.0
    elseif Δ ≥ 0  # i ≤ j
        s = 0.0
        for n in 1:p-j+i+1
            s += _b_b_int(P, i, j, n, gl)
        end
        return s
    else  # j < i
        s = 0.0
        for n in 1:p-i+j+1
            s += _b_b_int(P, j, i, n, gl)
        end
        return s
    end
end

@doc raw"""
Calculate
```math
\begin{aligned}
&\int_{I} B_{(i,p,k)}(t) B_{(j,p,k)}(t) dt \\
\end{aligned}
```
"""
function _b_b_int_I(P::AbstractBSplineSpace, i, j, gl)
    p = degree(P)
    k = knots(P)
    n = dim(P)
    Δ = j - i
    if Δ < -p
        return 0.0
    elseif Δ > p
        return 0.0
    elseif Δ ≥ 0  # i ≤ j
        s = 0.0
        for m in max(p-j+2,1):min(n-j+1,p-j+i+1)
            s += _b_b_int(P, i, j, m, gl)
        end
        return s
    else  # j < i
        s = 0.0
        for m in max(p-i+2,1):min(n-i+1,p-i+j+1)
            s += _b_b_int(P, j, i, m, gl)
        end
        return s
    end
end

function innerproduct_R(P::AbstractBSplineSpace)
    p = degree(P)
    n = dim(P)
    nip = p + 1
    gl = GaussLegendre(nip)
    return [_b_b_int_R(P, i, j, gl) for i in 1:n, j in 1:n]
end

function innerproduct_I(P::AbstractBSplineSpace)
    p = degree(P)
    n = dim(P)
    nip = p + 1
    gl = GaussLegendre(nip)
    return [_b_b_int_I(P, i, j, gl) for i in 1:n, j in 1:n]
end

function _f_b_int_R(func, i1, P1::AbstractBSplineSpace, gl1)
    k1 = knots(P1)
    p1 = degree(P1)
    F(t1) = bsplinebasis(P1, i1, t1) * func(t1)

    f1,l1 = i1, i1+p1

    S1 = integrate(F, k1[f1]..k1[f1+1], gl1)
    for j1 in f1+1:l1
        S1 += integrate(F, k1[j1]..k1[j1+1], gl1)
    end
    return S1
end

function _f_b_int_I(func, i1, P1::AbstractBSplineSpace, gl1)
    k1 = knots(P1)
    m1 = length(k1)
    p1 = degree(P1)
    F(t1) = bsplinebasis(P1, i1, t1) * func(t1)

    f1,l1 = max(i1, p1+1), min(i1+p1, m1-p1-1)

    S1 = integrate(F, k1[f1]..k1[f1+1], gl1)
    for j1 in f1+1:l1
        S1 += integrate(F, k1[j1]..k1[j1+1], gl1)
    end
    return S1
end

function _f_b_int_R(func, i1, i2, P1::AbstractBSplineSpace, P2::AbstractBSplineSpace, gl1, gl2)
    k1, k2 = knots(P1), knots(P2)
    p1, p2 = degree(P1), degree(P2)
    F(t1, t2) = bsplinebasis(P1, i1, t1) * bsplinebasis(P2, i2, t2) * func(t1, t2)

    f1, l1 = i1, i1+p1
    f2, l2 = i2, i2+p2

    S2 = integrate(F, k1[f1]..k1[f1+1], k2[f2]..k2[f2+1], gl1, gl2)
    for j2 in f2+1:l2
        S2 += integrate(F, k1[f1]..k1[f1+1], k2[j2]..k2[j2+1], gl1, gl2)
    end
    S1 = S2
    for j1 in f1+1:l1
        S2 = integrate(F, k1[j1]..k1[j1+1], k2[f2]..k2[f2+1], gl1, gl2)
        for j2 in f2+1:l2
            S2 += integrate(F, k1[j1]..k1[j1+1], k2[j2]..k2[j2+1], gl1, gl2)
        end
        S1 += S2
    end
    return S1
end

function _f_b_int_I(func, i1, i2, P1::AbstractBSplineSpace, P2::AbstractBSplineSpace, gl1, gl2)
    k1, k2 = knots(P1), knots(P2)
    m1, m2 = length(k1), length(k2)
    p1, p2 = degree(P1), degree(P2)
    F(t1, t2) = bsplinebasis(P1, i1, t1) * bsplinebasis(P2, i2, t2) * func(t1, t2)

    f1, l1 = max(i1, p1+1), min(i1+p1, m1-p1-1)
    f2, l2 = max(i2, p2+1), min(i2+p2, m2-p2-1)

    S2 = integrate(F, k1[f1]..k1[f1+1], k2[f2]..k2[f2+1], gl1, gl2)
    for j2 in f2+1:l2
        S2 += integrate(F, k1[f1]..k1[f1+1], k2[j2]..k2[j2+1], gl1, gl2)
    end
    S1 = S2
    for j1 in f1+1:l1
        S2 = integrate(F, k1[j1]..k1[j1+1], k2[f2]..k2[f2+1], gl1, gl2)
        for j2 in f2+1:l2
            S2 += integrate(F, k1[j1]..k1[j1+1], k2[j2]..k2[j2+1], gl1, gl2)
        end
        S1 += S2
    end
    return S1
end

function _f_b_int_R(func, i1, i2, i3, P1::AbstractBSplineSpace, P2::AbstractBSplineSpace, P3::AbstractBSplineSpace, gl1, gl2, gl3)
    k1, k2, k3 = knots(P1), knots(P2), knots(P3)
    p1, p2, p3 = degree(P1), degree(P2), degree(P3)
    F(t1, t2, t3) = bsplinebasis(P1, i1, t1) * bsplinebasis(P2, i2, t2) * bsplinebasis(P3, i3, t3) * func(t1, t2, t3)

    f1, l1 = i1, i1+p1
    f2, l2 = i2, i2+p2
    f3, l3 = i3, i3+p3

    S3 = integrate(F, k1[f1]..k1[f1+1], k2[f2]..k2[f2+1], k3[f3]..k3[f3+1], gl1, gl2, gl3)
    for j3 in f3+1:l3
        S3 += integrate(F, k1[f1]..k1[f1+1], k2[f2]..k2[f2+1], k3[j3]..k3[j3+1], gl1, gl2, gl3)
    end
    S2 = S3
    for j2 in f2+1:l2
        S3 = integrate(F, k1[f1]..k1[f1+1], k2[j2]..k2[j2+1], k3[f3]..k3[f3+1], gl1, gl2, gl3)
        for j3 in f3+1:l3
            S3 += integrate(F, k1[f1]..k1[f1+1], k2[j2]..k2[j2+1], k3[j3]..k3[j3+1], gl1, gl2, gl3)
        end
        S2 += S3
    end
    S1 = S2
    for j1 in f1+1:l1
        S3 = integrate(F, k1[j1]..k1[j1+1], k2[f2]..k2[f2+1], k3[f3]..k3[f3+1], gl1, gl2, gl3)
        for j3 in f3+1:l3
            S3 += integrate(F, k1[j1]..k1[j1+1], k2[f2]..k2[f2+1], k3[j3]..k3[j3+1], gl1, gl2, gl3)
        end
        S2 = S3
        for j2 in f2+1:l2
            S3 = integrate(F, k1[j1]..k1[j1+1], k2[j2]..k2[j2+1], k3[f3]..k3[f3+1], gl1, gl2, gl3)
            for j3 in f3+1:l3
                S3 += integrate(F, k1[j1]..k1[j1+1], k2[j2]..k2[j2+1], k3[j3]..k3[j3+1], gl1, gl2, gl3)
            end
            S2 += S3
        end
        S1 += S2
    end
    return S1
end

function _f_b_int_I(func, i1, i2, i3, P1::AbstractBSplineSpace, P2::AbstractBSplineSpace, P3::AbstractBSplineSpace, gl1, gl2, gl3)
    k1, k2, k3 = knots(P1), knots(P2), knots(P3)
    m1, m2, m3 = length(k1), length(k2), length(k3)
    p1, p2, p3 = degree(P1), degree(P2), degree(P3)
    F(t1, t2, t3) = bsplinebasis(P1, i1, t1) * bsplinebasis(P2, i2, t2) * bsplinebasis(P3, i3, t3) * func(t1, t2, t3)

    f1, l1 = max(i1, p1+1), min(i1+p1, m1-p1-1)
    f2, l2 = max(i2, p2+1), min(i2+p2, m2-p2-1)
    f3, l3 = max(i3, p3+1), min(i3+p3, m3-p3-1)

    S3 = integrate(F, k1[f1]..k1[f1+1], k2[f2]..k2[f2+1], k3[f3]..k3[f3+1], gl1, gl2, gl3)
    for j3 in f3+1:l3
        S3 += integrate(F, k1[f1]..k1[f1+1], k2[f2]..k2[f2+1], k3[j3]..k3[j3+1], gl1, gl2, gl3)
    end
    S2 = S3
    for j2 in f2+1:l2
        S3 = integrate(F, k1[f1]..k1[f1+1], k2[j2]..k2[j2+1], k3[f3]..k3[f3+1], gl1, gl2, gl3)
        for j3 in f3+1:l3
            S3 += integrate(F, k1[f1]..k1[f1+1], k2[j2]..k2[j2+1], k3[j3]..k3[j3+1], gl1, gl2, gl3)
        end
        S2 += S3
    end
    S1 = S2
    for j1 in f1+1:l1
        S3 = integrate(F, k1[j1]..k1[j1+1], k2[f2]..k2[f2+1], k3[f3]..k3[f3+1], gl1, gl2, gl3)
        for j3 in f3+1:l3
            S3 += integrate(F, k1[j1]..k1[j1+1], k2[f2]..k2[f2+1], k3[j3]..k3[j3+1], gl1, gl2, gl3)
        end
        S2 = S3
        for j2 in f2+1:l2
            S3 = integrate(F, k1[j1]..k1[j1+1], k2[j2]..k2[j2+1], k3[f3]..k3[f3+1], gl1, gl2, gl3)
            for j3 in f3+1:l3
                S3 += integrate(F, k1[j1]..k1[j1+1], k2[j2]..k2[j2+1], k3[j3]..k3[j3+1], gl1, gl2, gl3)
            end
            S2 += S3
        end
        S1 += S2
    end
    return S1
end

function innerproduct_R(func, P1::AbstractBSplineSpace)
    p1 = degree(P1)
    n1 = dim(P1)
    nip1 = p1 + 1
    gl1 = GaussLegendre(nip1)
    b = [_f_b_int_R(func, i1, P1, gl1) for i1 in 1:n1]
    return b
end

function innerproduct_I(func, P1::AbstractBSplineSpace)
    p1 = degree(P1)
    n1 = dim(P1)
    nip1 = p1 + 1
    gl1 = GaussLegendre(nip1)
    b = [_f_b_int_I(func, i1, P1, gl1) for i1 in 1:n1]
    return b
end

function innerproduct_R(func, P1::AbstractBSplineSpace, P2::AbstractBSplineSpace)
    p1, p2 = degree(P1), degree(P2)
    n1, n2 = dim(P1), dim(P2)
    nip1 = p1 + 1
    nip2 = p2 + 1
    gl1 = GaussLegendre(nip1)
    gl2 = GaussLegendre(nip2)
    b = [_f_b_int_R(func, i1, i2, P1, P2, gl1, gl2) for i1 in 1:n1, i2 in 1:n2]
    return b
end

function innerproduct_I(func, P1::AbstractBSplineSpace, P2::AbstractBSplineSpace)
    p1, p2 = degree(P1), degree(P2)
    n1, n2 = dim(P1), dim(P2)
    nip1 = p1 + 1
    nip2 = p2 + 1
    gl1 = GaussLegendre(nip1)
    gl2 = GaussLegendre(nip2)
    b = [_f_b_int_I(func, i1, i2, P1, P2, gl1, gl2) for i1 in 1:n1, i2 in 1:n2]
    return b
end

function innerproduct_R(func, P1::AbstractBSplineSpace, P2::AbstractBSplineSpace, P3::AbstractBSplineSpace)
    p1, p2, p3 = degree(P1), degree(P2), degree(P3)
    n1, n2, n3 = dim(P1), dim(P2), dim(P3)
    nip1 = p1 + 1
    nip2 = p2 + 1
    nip3 = p3 + 1
    gl1 = GaussLegendre(nip1)
    gl2 = GaussLegendre(nip2)
    gl3 = GaussLegendre(nip3)
    b = [_f_b_int_R(func, i1, i2, i3, P1, P2, P3, gl1, gl2, gl3) for i1 in 1:n1, i2 in 1:n2, i3 in 1:n3]
    return b
end

function innerproduct_I(func, P1::AbstractBSplineSpace, P2::AbstractBSplineSpace, P3::AbstractBSplineSpace)
    p1, p2, p3 = degree(P1), degree(P2), degree(P3)
    n1, n2, n3 = dim(P1), dim(P2), dim(P3)
    nip1 = p1 + 1
    nip2 = p2 + 1
    nip3 = p3 + 1
    gl1 = GaussLegendre(nip1)
    gl2 = GaussLegendre(nip2)
    gl3 = GaussLegendre(nip3)
    b = [_f_b_int_I(func, i1, i2, i3, P1, P2, P3, gl1, gl2, gl3) for i1 in 1:n1, i2 in 1:n2, i3 in 1:n3]
    return b
end

"""
* func: Real -> ℝ-vector space
"""
function fittingcontrolpoints(func, P1::AbstractBSplineSpace; domain=:I)
    if domain == :I
        b = innerproduct_I(func, P1)
        A = innerproduct_I(P1)
    elseif domain == :R
        b = innerproduct_R(func, P1)
        A = innerproduct_R(P1)
    end
    return inv(A) * b
end

"""
* func: (Real,Real) -> ℝ-vector space
"""
function fittingcontrolpoints(func, P1::AbstractBSplineSpace, P2::AbstractBSplineSpace; domain=:I)
    n1, n2 = dim(P1), dim(P2)
    if domain == :I
        A1, A2 = innerproduct_I(P1), innerproduct_I(P2)
        b = innerproduct_I(func, P1, P2)
    elseif domain == :R
        A1, A2 = innerproduct_R(P1), innerproduct_R(P2)
        b = innerproduct_R(func, P1, P2)
    end
    A = [A1[i1, j1] * A2[i2, j2] for i1 in 1:n1, i2 in 1:n2, j1 in 1:n1, j2 in 1:n2]
    _A = reshape(A, n1 * n2, n1 * n2)
    _b = reshape(b, n1 * n2)
    return reshape(inv(_A) * _b, n1, n2)
end

"""
* func: (Real,Real,Real) -> ℝ-vector space
"""
function fittingcontrolpoints(func, P1::AbstractBSplineSpace, P2::AbstractBSplineSpace, P3::AbstractBSplineSpace; domain=:I)
    n1, n2, n3 = dim(P1), dim(P2), dim(P3)
    if domain == :I
        A1, A2, A3 = innerproduct_I(P1), innerproduct_I(P2), innerproduct_I(P3)
        b = innerproduct_I(func, P1, P2, P3)
    elseif domain == :R
        A1, A2, A3 = innerproduct_R(P1), innerproduct_R(P2), innerproduct_R(P3)
        b = innerproduct_R(func, P1, P2, P3)
    end
    A = [A1[i1, j1] * A2[i2, j2] * A3[i3, j3] for i1 in 1:n1, i2 in 1:n2, i3 in 1:n3, j1 in 1:n1, j2 in 1:n2, j3 in 1:n3]
    _A = reshape(A, n1 * n2 * n3, n1 * n2 * n3)
    _b = reshape(b, n1 * n2 * n3)
    return reshape(inv(_A) * _b, n1, n2, n3)
end

"""
Approximate given function by linear combination of B-spline functions.
This function returns its control points.
* func: Vector{<:Real} -> ℝ-vector space
"""
function fittingcontrolpoints(func, Ps::AbstractVector{<:AbstractBSplineSpace}; domain=:I)
    # TODO: currently, this function only supports for 1-dim, 2-dim and 3-dim B-spline manifold.
    _func(t...) = func([t...])
    return fittingcontrolpoints(_func, Ps..., domain=domain)
end


using MDCT, Test

# O(N^2) implementations straight from the definitions

function slow_mdct(x::AbstractVector{T}) where {T<:Number}
    n = length(x)
    m = div(n,2)
    y = Array{T}(undef, m)
    for j = 1:m
        y[j] = sum(x .* cos.(2*pi/n * (j-0.5) * ((1:n) .- 0.5 .+ n/4)))
    end
    return y
end

function slow_imdct(y::AbstractVector{T}) where {T<:Number}
    m = length(y)
    n = m*2
    x = Array{T}(undef, n)
    for k = 1:n
        x[k] = 2/n * sum(y .* cos.(2*pi/n * ((1:m).-0.5) * (k - 0.5 + n/4)))
    end
    return x
end

for N=100:4:108
    X = rand(N)
    Y = mdct(X)
    Z = imdct(Y)
    Yp = plan_mdct(X)*X
    Zp = plan_imdct(Y)*Y
    Ys = slow_mdct(X)
    Zs = slow_imdct(Y)
    @test Y ≈ Ys rtol=1e-13
    @test Z ≈ Zs rtol=1e-13
    @test Yp ≈ Ys rtol=1e-13
    @test Zp ≈ Zs rtol=1e-13
end


using LinearAlgebra

# order 2
Qp = [
  -1//4 5//4 -1//2 0 0 0
  -1//4 -5//4 2 -1//2 0 0
  0 0 -3//2 2 -1//2 0
  0 0 0 -3//2 2 -1//2
  0 0 0 0 -5//4 5//4
  0 0 0 0 -1//4 -1//4
]
Qm = Matrix(-Qp')
B = zero(Qp); B[1,1] = -1; B[end,end] = 1
M = Diagonal([1//4, 5//4, 1, 1, 5//4, 1//4])
Dp = M \ (Qp + B / 2)
Dm = M \ (Qm + B / 2)

# order 3
Qp = [
  -1//12 3//4 -1//6 0 0 0
  -5//12 -5//12 1 -1//6 0 0
  0 -1//3 -1//2 1 -1//6 0
  0 0 -1//3 -1//2 1 -1//6
  0 0 0 -1//3 -5//12 3//4
  0 0 0 0 -5//12 -1//12
]
Qm = Matrix(-Qp')
B = zero(Qp); B[1,1] = -1; B[end,end] = 1
M = Diagonal([5//12, 13//12, 1, 1, 13//12, 5//12])
Dp = M \ (Qp + B / 2)
Dm = M \ (Qm + B / 2)

# order 4
Qp = [
    -1//48  205//288 -29//144   1//96   0      0      0     0     0 0 0
  -169//288 -11//48   33//32  -43//144  1//12  0      0     0     0 0 0
    11//144 -13//32  -29//48  389//288 -1//2   1//12  0     0     0 0 0
     1//32  -11//144 -65//288 -13//16  3//2   -1//2   1//12 0     0 0 0
     0        0        0       -1//4   -5//6   3//2  -1//2  1//12 0 0 0
     0        0        0        0      -1//4  -5//6   3//2 -1//2  1//12 0 0
  0 0 0 0 0 -1//4 -5//6 3//2 -1//2 1//12 0
  0 0 0 0 0 0 -1//4 -13//16 389//288 -43//144 1//96
  0 0 0 0 0 0 0 -65//288 -29//48 33//32 -29//144
  0 0 0 0 0 0 0 -11//144 -13//32 -11//48 205//288
  0 0 0 0 0 0 0 1//32 11//144 -169//288 -1//48
]
Qm = Matrix(-Qp')
B = zero(Qp); B[1,1] = -1; B[end,end] = 1
m = [49//144, 61//48, 41//48, 149//144]
M = Diagonal(vcat(m, ones(Int, size(Qp,1)-2*length(m)), m[end:-1:1]))
Dp = M \ (Qp + B / 2)
Dm = M \ (Qm + B / 2)

# order 5
Qp = [
    -1//120 941//1440 -47//360 -7//480    0      0      0     0     0 0 0
  -869//1440 -11//120 25//32  -43//360    1//30  0      0     0     0 0 0
    29//360 -17//32  -29//120 1309//1440 -1//4 1//30  0     0     0 0 0
     1//32  -11//360 -661//1440 -13//40   1     -1//4  1//30 0     0 0 0
     0        0        1//20   -1//2     -1//3   1     -1//4  1//30 0 0 0
     0        0        0        1//20    -1//2  -1//3   1     -1//4  1//30 0 0
  0 0 0 0 1//20   -1//2     -1//3   1     -1//4  1//30 0
  0 0 0 0 0 1//20 -1//2 -13//40 1309//1440 -43//360 -7//480
  0 0 0 0 0 0 1//20 -661//1440 -29//120 25//32 -47//360
  0 0 0 0 0 0 0 -11//360 -17//32 -11//120 941//1440
  0 0 0 0 0 0 0 1//32 29//360 -869//1440 -1//120
]
Qm = Matrix(-Qp')
B = zero(Qp); B[1,1] = -1; B[end,end] = 1
# m = [251//720, 299//240, 41//48, 149//144]
m = [251//720, 299//240, 211//240, 739//720]
M = Diagonal(vcat(m, ones(Int, size(Qp,1)-2*length(m)), m[end:-1:1]))
Dp = M \ (Qp + B / 2)
Dm = M \ (Qm + B / 2)

# order 6
Qptmp = [
  -265//128688 1146190567//1737288000 -1596619//18384000 -55265831//579096000 26269819//3474576000 2464501//144774000 0 0 0 0 0 0
  -1116490567//1737288000 -8839//214480 190538869//347457600 102705469//694915200 413741//9651600 -191689861//3474576000 0 0 0 0 0 0
  1096619//18384000 -135385429//347457600 -61067//321720 45137333//57909600 -253641811//694915200 70665929//579096000 -1//60 0 0 0 0 0
  66965831//579096000 -208765789//694915200 -17623253//57909600 -18269//45960 410905829//347457600 -477953317//1158192000 2//15 -1//60 0 0 0 0
  -49219819//3474576000 293299//9651600 26422771//694915200 -141938309//347457600 -346583//643440 2217185207//1737288000 -1//2 2//15 -1//60 0 0 0
  -2374501//144774000 142906261//3474576000 -3137129//579096000 -29884283//1158192000 -630168407//1737288000 -3559//6128 4//3 -1//2 2//15 -1//60 0 0
  0 0 0 0 1//30 -2//5 -7//12 4//3 -1//2 2//15 -1//60 0
  0 0 0 0 0 1//30 -2//5 -7//12 4//3 -1//2 2//15 -1//60
]
bnd = size(Qptmp, 1)-2
Qp = zeros(eltype(Qptmp), 2*bnd+2, 2*bnd+2)
Qp[1:size(Qptmp,1), 1:size(Qptmp,2)] = Qptmp
for i in 1:bnd
  Qp[end+1-i, end+1-size(Qptmp,1):end] = Qptmp[end:-1:1, i]
end
Qm = Matrix(-Qp')
B = zero(Qp); B[1,1] = -1; B[end,end] = 1
m = [13613//43200, 12049//8640, 535//864, 1079//864, 7841//8640, 43837//43200]
M = Diagonal(vcat(m, ones(Int, size(Qp,1)-2*length(m)), m[end:-1:1]))
Dp = M \ (Qp + B / 2)
Dm = M \ (Qm + B / 2)

# order 7
Qptmp = Rational{Int128}[
  -265//300272 1587945773//2432203200 -1926361//25737600 -84398989//810734400 48781961//4864406400 3429119//202683600 0 0 0 0 0 0 0
  -1570125773//2432203200 -26517//1501360 240029831//486440640 202934303//972881280 118207//13512240 -231357719//4864406400 0 0 0 0 0 0 0
  1626361//25737600 -206937767//486440640 -61067//750680 49602727//81073440 -43783933//194576256 51815011//810734400 -1//140 0 0 0 0 0 0
  91418989//810734400 -53314099//194576256 -33094279//81073440 -18269//107240 440626231//486440640 -365711063//1621468800 1//15 -1//140 0 0 0 0 0
  -62551961//4864406400 799//35280 82588241//972881280 -279245719//486440640 -346583//1501360 2312302333//2432203200 -3//10 1//15 -1//140 0 0 0 0
  -3375119//202683600 202087559//4864406400 -11297731//810734400 61008503//1621468800 -1360092253//2432203200 -10677//42896 1 -3//10 1//15 -1//140 0 0 0
  0 0 0 -1//105 1//10 -3//5 -1//4 1 -3//10 1//15 -1//140 0 0
  0 0 0 0 -1//105 1//10 -3//5 -1//4 1 -3//10 1//15 -1//140 0
  0 0 0 0 0 -1//105 1//10 -3//5 -1//4 1 -3//10 1//15 -1//140
]
bnd = size(Qptmp, 1)-3
Qp = zeros(eltype(Qptmp), 2*bnd+3, 2*bnd+3)
Qp[1:size(Qptmp,1), 1:size(Qptmp,2)] = Qptmp
for i in 1:bnd
  Qp[end+1-i, end+1-size(Qptmp,1):end] = Qptmp[end:-1:1, i]
end
Qm = Matrix(-Qp')
B = zero(Qp); B[1,1] = -1; B[end,end] = 1
m = [19087//60480, 84199//60480, 18869//30240, 37621//30240, 55031//60480, 61343//60480]
M = Diagonal(vcat(m, ones(Int, size(Qp,1)-2*length(m)), m[end:-1:1]))
Dp = M \ (Qp + B / 2)
Dm = M \ (Qm + B / 2)


# using Pkg; Pkg.add("SineFit")

begin
    using Printf
    using DSP, SineFit
    using Plots, StatsPlots
end

struct SinregResult
    period::Float64
    params::SineFit.WaveFitParams
    plot
end

"""Sinusoidal regression function.

Fits a sinusoid to data provided as as `x` and `y` and returns a named tuple
containing both the fitted curve and the parameters of that curve.
Used to produce initial values for modeling data windows and later?|
to find peaks in modeled julian day values to align the result to
a cumulative age timeline.

[x] Vector of `x` values of input data

[y] Vector of `y` values of input data

[fixed_period] Optional variable for fixing the period of the sinusoid
in the depth domain. Defaults to `NA`, period is not fixed. Supply a
single value to fix the period.

[plot] Should the fitting result be plotted? `TRUE/FALSE`

RETURNS: A list containing a vector of parameters of the fitted sinusoid
and the fitted values belonging to each `x` value.
Fitting parameters:

`I` | sineparams.vertoffst = the mean annual value of the sinusoid (height)
`A` | sineparams.amplitude = the amplitude of the sinusoid
`Dper` | sineparams.frequency = the period of the sinusoid in `x` domain
`peak` | sineparams.phaseshft = the location of the peak in the sinusoid
`R2adj` = the adjusted `R^2` value of the fit
`p` = the p-value of the fit

R Examples
```
# Create dummy data
x <- seq(1000, 11000, 1000)
y <- sin((2 * pi * (seq(1, 11, 1) - 8 + 7 / 4)) / 7)
sinlist <- sinreg(x, y, plot = FALSE) # Run the function
@export
```
"""
function sinreg(
        x::Union{AbstractArray{Number, 1}, AbstractVector},
        y::Union{AbstractArray{Number, 1}, AbstractVector};
        fixed_period=nothing, plot::Bool=false, verbose::Bool=false
    )

    # Determine sine period
    if fixed_period == nothing
        # Get periodogram from FFT
        ssp = periodogram(y)
        # Get frequency of maximum density
        per = 1 / ssp.freq[ssp.power .== maximum(ssp.power)]
        println(per)
        dper = per * diff([extrema(x)...])
    else
        dper = fixed_period
    end

    # SineFit
    sineparams = calculate_wave_shape(x, y)
    sinpred = SineFit.sin(x, sineparams)
    fstring = @sprintf(
                "y = %.2f sin(%.2f x + %.2f) + %.2f",
                sineparams.amplitude, sineparams.frequency,
                sineparams.phaseshft, sineparams.vertoffst
                )

    # Print results
    if verbose
        print(fstring)
    end

    # Draw plot
    if plot
        # Plot results
        scatter(x, y, label="Data")
        plot = plot!(x, sinpred, lw=2, label="SinFit")
        annotate!(plot, [(-12, -0.8, (fstring, font("serif", 14, :red, :left,))),])
    else
        plot = nothing
    end

    # Gather results
    results = SinregResult(dper, sineparams, plot)

    # return results
    return results
end

x =  collect(-10π:0.01:10π)
y =  Base.sin.(x .* 2.1).+ rand(length(x))

result = sinreg(x, y, plot=true)

result.plot


#=
  Demos for gradient descent glm algos
=#

using glmSolverjl
using Random: seed!

seed!(0);
gammaLogX, gammaLogY = simulateData(Float64, GammaDistribution(), LogLink(), 30, 100_000);
gammaLogBlockX = matrixToBlock(gammaLogX, 100);
gammaLogBlockY = matrixToBlock(gammaLogY, 100);

function gdDataDemo(niter::Int64 = 20, learningRate::Float64 = 2E-8)

  #= Number of parameters =#
  p = size(gammaLogX)[2]

  gammaModel = glm(Block1DParallel(), gammaLogBlockX,
        gammaLogBlockY, GammaDistribution(), LogLink(),
        #= solver =# GESVSolver(), inverse = GETRIInverse());
  println("Full GLM Solve\n", gammaModel.coefficients)

  println("The outputs for all these models should be the same.");

  gammaModel = glm(RegularData(), gammaLogX,
        gammaLogY, GammaDistribution(), LogLink(),
        #= solver =# GradientDescentSolver(learningRate), inverse = GETRIInverse(),
        control = Control{Float64}(maxit = niter),
        calculateCovariance = true,
        doStepControl = false);
  println("Gradient Descent With Regular Data\n", gammaModel.coefficients)
  #===================================================================#
  #= Gradient Descent Block Model =#
  gammaModel = glm(Block1D(), gammaLogBlockX,
        gammaLogBlockY, GammaDistribution(), LogLink(),
        #= solver =# GradientDescentSolver(learningRate),
        inverse = GETRIInverse(), control = Control{Float64}(maxit = niter),
        calculateCovariance = true,
        doStepControl = false);
  println("Gradient Descent With Block Data \n", gammaModel.coefficients)

  #= Gradient Descent Block1DParallel Model =#
  gammaModel = glm(Block1DParallel(), gammaLogBlockX,
        gammaLogBlockY, GammaDistribution(), LogLink(),
        #= solver =# GradientDescentSolver(learningRate),
        inverse = GETRIInverse(), control = Control{Float64}(maxit = niter),
        calculateCovariance = true,
        doStepControl = false);
  println("Gradient Descent With Parallel Block Data \n", gammaModel.coefficients)

  return
end

function momentumDataDemo(niter::Int64 = 20, learningRate::Float64 = 2E-8, momentum::Float64 = 0.7)

  #= Number of parameters =#
  p = size(gammaLogX)[2]

  gammaModel = glm(Block1DParallel(), gammaLogBlockX,
      gammaLogBlockY, GammaDistribution(), LogLink(),
      #= solver =# GESVSolver(), inverse = GETRIInverse());
  println("Full GLM Solve\n", gammaModel.coefficients)

  println("The outputs for all these models should be the same.");

  gammaModel = glm(RegularData(), gammaLogX,
      gammaLogY, GammaDistribution(), LogLink(),
      #= solver =# MomentumSolver(learningRate, momentum, p),
      inverse = GETRIInverse(),
      control = Control{Float64}(maxit = niter),
      calculateCovariance = true,
      doStepControl = false);
  println("Gradient Descent With Regular Data\n", gammaModel.coefficients)
  #===================================================================#
  #= Gradient Descent Block Model =#
  gammaModel = glm(Block1D(), gammaLogBlockX,
      gammaLogBlockY, GammaDistribution(), LogLink(),
      #= solver =# MomentumSolver(learningRate, momentum, p),
      inverse = GETRIInverse(),
      control = Control{Float64}(maxit = niter),
      calculateCovariance = true,
      doStepControl = false);
  println("Gradient Descent With Block Data \n", gammaModel.coefficients)

  #= Gradient Descent Block1DParallel Model =#
  gammaModel = glm(Block1DParallel(), gammaLogBlockX,
      gammaLogBlockY, GammaDistribution(), LogLink(),
      #= solver =# MomentumSolver(learningRate, momentum, p),
      inverse = GETRIInverse(), control = Control{Float64}(maxit = niter),
      calculateCovariance = true,
      doStepControl = false);
  println("Gradient Descent With Parallel Block Data \n", gammaModel.coefficients)

  return
end

function nesterovDataDemo(niter::Int64 = 30, learningRate::Float64 = 2E-8, momentum::Float64 = 0.70)

  #= Number of parameters =#
  p = size(gammaLogX)[2]

  gammaModel = glm(Block1DParallel(), gammaLogBlockX,
      gammaLogBlockY, GammaDistribution(), LogLink(),
      #= solver =# GESVSolver(), inverse = GETRIInverse());
  println("Full GLM Solve\n", gammaModel.coefficients)

  println("The outputs for all these models should be the same.");

  gammaModel = glm(RegularData(), gammaLogX,
      gammaLogY, GammaDistribution(), LogLink(),
      #= solver =# NesterovSolver(learningRate, momentum, p),
      inverse = GETRIInverse(),
      control = Control{Float64}(maxit = niter),
      calculateCovariance = true,
      doStepControl = false);
  println("Gradient Descent With Regular Data\n", gammaModel.coefficients)
  #===================================================================#
  #= Gradient Descent Block Model =#
  gammaModel = glm(Block1D(), gammaLogBlockX,
      gammaLogBlockY, GammaDistribution(), LogLink(),
      #= solver =# NesterovSolver(learningRate, momentum, p),
      inverse = GETRIInverse(),
      control = Control{Float64}(maxit = niter),
      calculateCovariance = true,
      doStepControl = false);
  println("Gradient Descent With Block Data \n", gammaModel.coefficients)

  #= Gradient Descent Block1DParallel Model =#
  gammaModel = glm(Block1DParallel(), gammaLogBlockX,
      gammaLogBlockY, GammaDistribution(), LogLink(),
      #= solver =# NesterovSolver(learningRate, momentum, p),
      inverse = GETRIInverse(), control = Control{Float64}(maxit = niter),
      calculateCovariance = true,
      doStepControl = false);
  println("Gradient Descent With Parallel Block Data \n", gammaModel.coefficients)

  return
end

function adagradDataDemo(niter::Int64 = 10, learningRate::Float64 = 1E-10, epsilon::Float64 = 1E-8)

  #= Number of parameters =#
  p = size(gammaLogX)[2]

  gammaModel = glm(Block1DParallel(), gammaLogBlockX,
      gammaLogBlockY, GammaDistribution(), LogLink(),
      #= solver =# GESVSolver(), inverse = GETRIInverse());
  println("Full GLM Solve\n", gammaModel.coefficients)

  println("The outputs for all these models should be the same.");

  gammaModel = glm(RegularData(), gammaLogX,
      gammaLogY, GammaDistribution(), LogLink(),
      #= solver =# AdagradSolver(learningRate, p, epsilon),
      inverse = GETRIInverse(),
      control = Control{Float64}(maxit = niter),
      calculateCovariance = true,
      doStepControl = false);
  println("Gradient Descent With Regular Data\n", gammaModel.coefficients)
  #===================================================================#
  #= Gradient Descent Block Model =#
  gammaModel = glm(Block1D(), gammaLogBlockX,
      gammaLogBlockY, GammaDistribution(), LogLink(),
      #= solver =# AdagradSolver(learningRate, p, epsilon),
      inverse = GETRIInverse(),
      control = Control{Float64}(maxit = niter),
      calculateCovariance = true,
      doStepControl = false);
  println("Gradient Descent With Block Data \n", gammaModel.coefficients)

  #= Gradient Descent Block1DParallel Model =#
  gammaModel = glm(Block1DParallel(), gammaLogBlockX,
      gammaLogBlockY, GammaDistribution(), LogLink(),
      #= solver =# AdagradSolver(learningRate, p, epsilon),
      inverse = GETRIInverse(), control = Control{Float64}(maxit = niter),
      calculateCovariance = true,
      doStepControl = false);
  println("Gradient Descent With Parallel Block Data \n", gammaModel.coefficients)

  return
end

#= Poor convergence performance for this one =#
function adadeltaDataDemo(niter::Int64 = 10, momentum::Float64 = 0.9, epsilon::Float64 = 1E-8)

  #= Number of parameters =#
  p = size(gammaLogX)[2]

  gammaModel = glm(Block1DParallel(), gammaLogBlockX,
      gammaLogBlockY, GammaDistribution(), LogLink(),
      #= solver =# GESVSolver(), inverse = GETRIInverse());
  println("Full GLM Solve\n", gammaModel.coefficients)

  println("The outputs for all these models should be the same.");

  gammaModel = glm(RegularData(), gammaLogX,
      gammaLogY, GammaDistribution(), LogLink(),
      #= solver =# AdadeltaSolver(momentum, p, epsilon),
      inverse = GETRIInverse(),
      control = Control{Float64}(maxit = niter),
      calculateCovariance = true,
      doStepControl = false);
  println("Gradient Descent With Regular Data\n", gammaModel.coefficients)
  #===================================================================#
  #= Gradient Descent Block Model =#
  gammaModel = glm(Block1D(), gammaLogBlockX,
      gammaLogBlockY, GammaDistribution(), LogLink(),
      #= solver =# AdadeltaSolver(momentum, p, epsilon),
      inverse = GETRIInverse(),
      control = Control{Float64}(maxit = niter),
      calculateCovariance = true,
      doStepControl = false);
  println("Gradient Descent With Block Data \n", gammaModel.coefficients)

  #= Gradient Descent Block1DParallel Model =#
  gammaModel = glm(Block1DParallel(), gammaLogBlockX,
      gammaLogBlockY, GammaDistribution(), LogLink(),
      #= solver =# AdadeltaSolver(momentum, p, epsilon),
      inverse = GETRIInverse(), control = Control{Float64}(maxit = niter),
      calculateCovariance = true,
      doStepControl = false);
  println("Gradient Descent With Parallel Block Data \n", gammaModel.coefficients)

  return
end

function RMSPropDataDemo(niter::Int64 = 10, learningRate::Float64 = 1E-2, epsilon::Float64 = 1E-8)

  #= Number of parameters =#
  p = size(gammaLogX)[2]

  gammaModel = glm(Block1DParallel(), gammaLogBlockX,
        gammaLogBlockY, GammaDistribution(), LogLink(),
        #= solver =# GESVSolver(), inverse = GETRIInverse());
  println("Full GLM Solve\n", gammaModel.coefficients)

  println("The outputs for all these models should be the same.");

  gammaModel = glm(RegularData(), gammaLogX,
        gammaLogY, GammaDistribution(), LogLink(),
        #= solver =# RMSpropSolver(learningRate, p, epsilon),
        inverse = GETRIInverse(),
        control = Control{Float64}(maxit = niter),
        calculateCovariance = true,
        doStepControl = false);
  println("Gradient Descent With Regular Data\n", gammaModel.coefficients)
  #===================================================================#
  #= Gradient Descent Block Model =#
  gammaModel = glm(Block1D(), gammaLogBlockX,
        gammaLogBlockY, GammaDistribution(), LogLink(),
        #= solver =# RMSpropSolver(learningRate, p, epsilon),
        inverse = GETRIInverse(), control = Control{Float64}(maxit = niter),
        calculateCovariance = true,
        doStepControl = false);
  println("Gradient Descent With Block Data \n", gammaModel.coefficients)

  #= Gradient Descent Block1DParallel Model =#
  gammaModel = glm(Block1DParallel(), gammaLogBlockX,
        gammaLogBlockY, GammaDistribution(), LogLink(),
        #= solver =# RMSpropSolver(learningRate, p, epsilon),
        inverse = GETRIInverse(),
        control = Control{Float64}(maxit = niter),
        calculateCovariance = true,
        doStepControl = false);
  println("Gradient Descent With Parallel Block Data \n", gammaModel.coefficients)

  return
end

function adamDataDemo(niter::Int64 = 50, learningRate::Float64 = 1E-3, b1::Float64 = 0.9, b2::Float64 = 0.999, epsilon::Float64 = 1E-8)

  #= Number of parameters =#
  p = size(gammaLogX)[2]

  gammaModel = glm(Block1DParallel(), gammaLogBlockX,
        gammaLogBlockY, GammaDistribution(), LogLink(),
        #= solver =# GESVSolver(), inverse = GETRIInverse());
  println("Full GLM Solve\n", gammaModel.coefficients)

  println("The outputs for all these models should be the same.");

  gammaModel = glm(RegularData(), gammaLogX,
        gammaLogY, GammaDistribution(), LogLink(),
        #= solver =# AdamSolver(learningRate, b1, b2, p, epsilon),
        inverse = GETRIInverse(),
        control = Control{Float64}(maxit = niter),
        calculateCovariance = true,
        doStepControl = false);
  println("Gradient Descent With Regular Data\n", gammaModel.coefficients)
  #===================================================================#
  #= Gradient Descent Block Model =#
  gammaModel = glm(Block1D(), gammaLogBlockX,
        gammaLogBlockY, GammaDistribution(), LogLink(),
        #= solver =# AdamSolver(learningRate, b1, b2, p, epsilon),
        inverse = GETRIInverse(), control = Control{Float64}(maxit = niter),
        calculateCovariance = true,
        doStepControl = false);
  println("Gradient Descent With Block Data \n", gammaModel.coefficients)

  #= Gradient Descent Block1DParallel Model =#
  gammaModel = glm(Block1DParallel(), gammaLogBlockX,
        gammaLogBlockY, GammaDistribution(), LogLink(),
        #= solver =# AdamSolver(learningRate, b1, b2, p, epsilon),
        inverse = GETRIInverse(),
        control = Control{Float64}(maxit = niter),
        calculateCovariance = true,
        doStepControl = false);
  println("Gradient Descent With Parallel Block Data \n", gammaModel.coefficients)

  return
end

function adaMaxDataDemo(niter::Int64 = 10, learningRate::Float64 = 1E-1, b1::Float64 = 0.9, b2::Float64 = 0.999)

  #= Number of parameters =#
  p = size(gammaLogX)[2]

  gammaModel = glm(Block1DParallel(), gammaLogBlockX,
        gammaLogBlockY, GammaDistribution(), LogLink(),
        #= solver =# GESVSolver(), inverse = GETRIInverse());
  println("Full GLM Solve\n", gammaModel.coefficients)

  println("The outputs for all these models should be the same.");

  gammaModel = glm(RegularData(), gammaLogX,
        gammaLogY, GammaDistribution(), LogLink(),
        #= solver =# AdaMaxSolver(learningRate, b1, b2, p),
        inverse = GETRIInverse(),
        control = Control{Float64}(maxit = niter),
        calculateCovariance = true,
        doStepControl = false);
  println("Gradient Descent With Regular Data\n", gammaModel.coefficients)
  #===================================================================#
  #= Gradient Descent Block Model =#
  gammaModel = glm(Block1D(), gammaLogBlockX,
        gammaLogBlockY, GammaDistribution(), LogLink(),
        #= solver =# AdaMaxSolver(learningRate, b1, b2, p),
        inverse = GETRIInverse(),
        control = Control{Float64}(maxit = niter),
        calculateCovariance = true,
        doStepControl = false);
  println("Gradient Descent With Block Data \n", gammaModel.coefficients)

  #= Gradient Descent Block1DParallel Model =#
  gammaModel = glm(Block1DParallel(), gammaLogBlockX,
        gammaLogBlockY, GammaDistribution(), LogLink(),
        #= solver =# AdaMaxSolver(learningRate, b1, b2, p),
        inverse = GETRIInverse(),
        control = Control{Float64}(maxit = niter),
        calculateCovariance = true,
        doStepControl = false);
  println("Gradient Descent With Parallel Block Data \n", gammaModel.coefficients)

  return
end

function nadamDataDemo(niter::Int64 = 30, learningRate::Float64 = 3E-3, b1::Float64 = 0.9, b2::Float64 = 0.999, epsilon::Float64 = 1E-8)

  #= Number of parameters =#
  p = size(gammaLogX)[2]

  gammaModel = glm(Block1DParallel(), gammaLogBlockX,
        gammaLogBlockY, GammaDistribution(), LogLink(),
        #= solver =# GESVSolver(), inverse = GETRIInverse());
  println("Full GLM Solve\n", gammaModel.coefficients)

  println("The outputs for all these models should be the same.");

  gammaModel = glm(RegularData(), gammaLogX,
        gammaLogY, GammaDistribution(), LogLink(),
        #= solver =# NAdamSolver(learningRate, b1, b2, p, epsilon),
        inverse = GETRIInverse(),
        control = Control{Float64}(maxit = niter),
        calculateCovariance = true,
        doStepControl = false);
  println("Gradient Descent With Regular Data\n", gammaModel.coefficients)
  #===================================================================#
  #= Gradient Descent Block Model =#
  gammaModel = glm(Block1D(), gammaLogBlockX,
        gammaLogBlockY, GammaDistribution(), LogLink(),
        #= solver =# NAdamSolver(learningRate, b1, b2, p, epsilon),
        inverse = GETRIInverse(), control = Control{Float64}(maxit = niter),
        calculateCovariance = true,
        doStepControl = false);
  println("Gradient Descent With Block Data \n", gammaModel.coefficients)

  #= Gradient Descent Block1DParallel Model =#
  gammaModel = glm(Block1DParallel(), gammaLogBlockX,
        gammaLogBlockY, GammaDistribution(), LogLink(),
        #= solver =# NAdamSolver(learningRate, b1, b2, p, epsilon),
        inverse = GETRIInverse(),
        control = Control{Float64}(maxit = niter),
        calculateCovariance = true,
        doStepControl = false);
  println("Gradient Descent With Parallel Block Data \n", gammaModel.coefficients)

  return
end

function amsGradDataDemo(niter::Int64 = 50, learningRate::Float64 = 9E-3, b1::Float64 = 0.9, b2::Float64 = 0.999, epsilon::Float64 = 1E-8)

  #= Number of parameters =#
  p = size(gammaLogX)[2]

  gammaModel = glm(Block1DParallel(), gammaLogBlockX,
        gammaLogBlockY, GammaDistribution(), LogLink(),
        #= solver =# GESVSolver(), inverse = GETRIInverse());
  println("Full GLM Solve\n", gammaModel.coefficients)

  println("The outputs for all these models should be the same.");

  gammaModel = glm(RegularData(), gammaLogX,
        gammaLogY, GammaDistribution(), LogLink(),
        #= solver =# AMSGradSolver(learningRate, b1, b2, p, epsilon),
        inverse = GETRIInverse(),
        control = Control{Float64}(maxit = niter),
        calculateCovariance = true,
        doStepControl = false);
  println("Gradient Descent With Regular Data\n", gammaModel.coefficients)
  #===================================================================#
  #= Gradient Descent Block Model =#
  gammaModel = glm(Block1D(), gammaLogBlockX,
        gammaLogBlockY, GammaDistribution(), LogLink(),
        #= solver =# AMSGradSolver(learningRate, b1, b2, p, epsilon),
        inverse = GETRIInverse(), control = Control{Float64}(maxit = niter),
        calculateCovariance = true,
        doStepControl = false);
  println("Gradient Descent With Block Data \n", gammaModel.coefficients)

  #= Gradient Descent Block1DParallel Model =#
  gammaModel = glm(Block1DParallel(), gammaLogBlockX,
        gammaLogBlockY, GammaDistribution(), LogLink(),
        #= solver =# AMSGradSolver(learningRate, b1, b2, p, epsilon),
        inverse = GETRIInverse(),
        control = Control{Float64}(maxit = niter),
        calculateCovariance = true,
        doStepControl = false);
  println("Gradient Descent With Parallel Block Data \n", gammaModel.coefficients)

  return
end

#===================================================================#

# Simple Gradient Descent
gdDataDemo(30, 2E-8)
# Gradient Descent with Momentum
momentumDataDemo(20, 2E-8, 0.7)
# Nesterov
nesterovDataDemo(30, 2E-8, 0.7)
# Adagrad
adagradDataDemo(10, 1E-10, 1E-8)
# Adadelta (Poor performance)
adadeltaDataDemo(10, 0.9, 1E-8)
# RMSProp
RMSPropDataDemo(10, 1E-2, 1E-8)
# Adam
adamDataDemo(50, 1E-3, 0.9, 0.999, 1E-8)
# AdaMax
adaMaxDataDemo(10, 1E-1, 0.9, 0.999)
# NAdam
nadamDataDemo(30, 3E-3, 0.9, 0.999, 1E-8)
# AMSGrad
amsGradDataDemo(40, 5E-4, 0.9, 0.999, 1E-8)


# ---------------------------------------------------------
# Simple harmonic oscillator without forcing term
# ---------------------------------------------------------
function harmonic_oscillator_free()
    k  = 2 ; m  = .5 ;  c = .1 ;
    u0 = 1 ; v0 = 0 ;

    M = m * ones(1, 1)
    C = c * ones(1, 1)
    K = k * ones(1, 1)
    R = zeros(1)

    sys = SecondOrderAffineContinuousSystem(M, C, K, R)

    U₀ = u0 * ones(1)
    V₀ = v0 * ones(1)

    return InitialValueProblem(sys, (U₀, V₀))
end

# ---------------------------------------------------------
# Simple harmonic oscillator with forcing term
# ---------------------------------------------------------
function harmonic_oscillator_forced()
    k  = 2 ; m  = .5 ;  c = .1 ;
    u0 = 1 ; v0 = 0 ;

    M = m * ones(1, 1)
    C = c * ones(1, 1)
    K = k * ones(1, 1)

    NSTEPS = 100
    Δt = 0.1
    ωf = k/(2m)
    R = [[0.1 * sin(ωf * Δt * (i-1))] for i in 1:NSTEPS+1]

    X = nothing # Universe(1) ignores state constraints
    B = ones(1, 1)
    sys = SecondOrderConstrainedLinearControlContinuousSystem(M, C, K, B, X, R)

    U₀ = u0 * ones(1)
    V₀ = v0 * ones(1)

    return InitialValueProblem(sys, (U₀, V₀)), NSTEPS, Δt
end

# ---------------------------------------------------------
# This example can be found in
# [Chapter 9, in Finite Element Procedures, K-J Bathe].
# ---------------------------------------------------------
function example_9_1_Bathe(; U₀ = zeros(2),   # initial displacements
                             U₀′ = zeros(2)) # initial velocities

    # analytic solution (cf. Example 9.7)
    A = [1/√3  (1/2)*√(2/3);
         1/√3      -√(2/3)]
    x₁(t) = (5 / √3) * (1 - cos(t*√2))
    x₂(t) = (2 * √(2/3)) * (-1 + cos(t*√5))
    U(t) = A * [x₁(t), x₂(t)]

    U12 = U(12*0.28) # ≈ [1.157, 2.489]

    # problem formulation
    M = [2 0; 0 1.]
    K = [6 -2; -2 4.]
    C = zeros(2, 2)
    R = [0, 10.]

    sys = SecondOrderAffineContinuousSystem(M, C, K, R)
    return InitialValueProblem(sys, (U₀, U₀′)), U
end

# ---------------------------------------------------------
# Heat transfer problem
# ---------------------------------------------------------
function heat_transfer(; nelem=3)
    rho = 1.
    csh = 1.
    kco = 4.
    lelem = 1.0 / nelem
    Area  = .25
    C = rho * csh * lelem * Area * ( [0.5+0.5+0.0 0; 0  0.0+0.5+0.5] )
    K = kco *       lelem * Area * ( [1.0+1.0 -1.; -1. 1.0+1.0] * 1.0 / lelem^2 )
    M = zeros(2, 2)
    R = zeros(2)
    sys = SecondOrderAffineContinuousSystem(M, C, K, R)

    # build initial state
    U₀ = zeros(2)
    for j = 1:(nelem+1-2)
        U₀[j] =  sin( pi*lelem*j ) + 0.5 * sin( 3.0*pi*lelem*j )
    end

    # analytic solution (node 1)
    α = kco / ( rho * csh )
    j = 1
    U(t) = exp(-pi^2 * α * 12*t ) * sin( pi*lelem*j ) + 0.5 * exp(-(3*pi)^2 * α * 12*t ) * sin( 3.0*pi*lelem*j )

    # here the velocity condition U₀′ is ignored
    return InitialValueProblem(sys, (U₀, U₀)), U
end

# ---------------------------------------------------------
# Non-linear pendulum model
# ---------------------------------------------------------
function pendulum_nonlinear()
    # ========================
    # Model definition
    # ========================

    # bar length
    L = 2.0 # [m]

    # cross section area
    ϕ = 0.01 # [m]
    A = (π * ϕ^2/4) # [m^2]

    # Young's modulus (Steel)
    E = 210e9 # [Pa]

    # mass (lumped)
    m = 214 # [kg]

    # damping
    c = 0 # [kg/s]

    # gravity
    g = 9.81; # [m/s^2]

    # ============================
    # Finite element model
    # ============================

    # u = [u_1, u_2]^T is the position of the mass, with the origin as indicated:
    #                     _______
    #                        |
    #                        |
    #                        |
    #                        o    << origin

    # internal forces field
    daux(u) = u[1]^2 - 2*L*u[2] + u[2]^2
    Fint(u) = E*A*L*daux(u)/2/L^4*[u[1], -(L-u[2])]

    # external forces field
    Fext(t) = [0, -m*g] # [N]

    # lumped mass matrix
    M = [m 0; 0. m]

    # damping matrix
    C = [c 0; 0. c]

    # ===================================
    # Initial conditions for simulation
    # ===================================

    u0 = [L, L]
    v0 = [0, 0.]

    sys = SecondOrderContinuousSystem(M, C, Fint, Fext)
    prob = InitialValueProblem(sys, (u0, v0))

    return prob
end

# ---------------------------------------------------------
# Von Mises truss (cercha de Von Mises)
# ---------------------------------------------------------
function von_mises_truss()
    # structural parameters
    ρ = 7850               # [kg/m^3] (steel)
    Lx = .374/2
    Lz = sqrt(.205^2 - Lx^2) # [m]
    l0 = sqrt(Lx^2 + Lz^2) # [m]
    Lc = .240              # [m]
    Ic = .0254*.0032^3/12  # [m^4]
    Ac = .0254*.0032       # [m^2]
    E  = 200e9             # [Pa] (steel)
    kc = 3*E*Ic/Lc^3       # [N/m]
    mb = l0*Ac*ρ           # [kg]
    m  = 1.4               # kg (pandeo incipiente en 1.4)
    c  = 2                 # [kg/s] (amortiguamiento por friccion juntas y arrastre pesa)
    g  = 9.81              # [m/s^2]

    # internal forces vector
    Fint(u) = E*Ac*l0*(u[1]^2+u[2]^2-2*Lx*u[1]+2*Lz*u[2])/2/l0^4*[-Lx+u[1], Lz+u[2]] + [kc*u[1], 0]

    # external forces vector
    Fext(t) = [0, -(m+mb)/2*g] # [N]

    # mass matrix (lumped)
    M = [mb 0; 0 (mb+m)/2]

    # damping matrix
    C = [c/10 0; 0 c]

    # initial conditions for simulation
    u0 = [0, 0.]
    v0 = [0, 0.]

    sys = SecondOrderContinuousSystem(M, C, Fint, Fext)
    prob = InitialValueProblem(sys, (u0, v0))
end


include("MatrixMarket.jl")
using MatrixMarket
using LinearLeastSquares

# read in the data
A = full(MatrixMarket.mmread("largeCorpus.mtx"));

# extract the classes of each document
classes = A[:,1];
# TODO: modify classes so that 4 5 6 are 1 2 3
classes[classes .> 3] = classes[classes .> 3] - 3;
A = A[:, 2:end];

# split into train/test
numData = size(A, 1);
data = randperm(numData);
ind = floor(numData*0.7);
training = data[1:ind];
test = data[ind+1:end];
trainDocuments = A[training,:];
trainClasses = classes[training,:];
testDocuments = A[test,:];
testClasses = classes[test,:];

# change all other than sports to -1 (sports is 1)
holdClass = 1;
trainClasses[trainClasses .!= holdClass] = -1;
trainClasses[trainClasses .== holdClass] = 1;
testClasses[testClasses .!= holdClass] = -1;
testClasses[testClasses .== holdClass] = 1;

# build the problem and solve with LLS
lambda = 100;
w = Variable(size(A, 2));
v = Variable();
objective = sum_squares(trainDocuments * w + v - trainClasses) + lambda * sum_squares(w);
optval = minimize!(objective);

# print out the 5 words most indicative of sports and nonsports
words = String[];
f = open("largeCorpusfeatures.txt");
for i = 1:length(evaluate(w))
  push!(words, readline(f))
end
indices = sortperm(vec(evaluate(w)));
for i = 1:5
  print(words[indices[i]])
end
for i = 0:4
  print(words[indices[length(words) - i]])
end

# calculate training error
yhat = sign(trainDocuments * evaluate(w) + evaluate(v));
trainCE =  1/size(trainClasses,1)*sum(trainClasses .!= yhat)

# calculate performance of our classifier on the test set
yhat2 = sign(testDocuments * evaluate(w) + evaluate(v));
testCE = 1/size(testClasses,1)*sum(testClasses .!= yhat2)


using StanModels, CSV

df = CSV.read(joinpath(@__DIR__, "..", "..", "data", "chimpanzees.csv"), delim=';');

# Define the Stan language model

m10_2s = "
data{
    int N;
    int pulled_left[N];
    int prosoc_left[N];
}
parameters{
    real a;
    real bp;
}
model{
    vector[N] p;
    bp ~ normal( 0 , 10 );
    a ~ normal( 0 , 10 );
    for ( i in 1:N ) {
        p[i] = a + bp * prosoc_left[i];
        p[i] = inv_logit(p[i]);
    }
    pulled_left ~ binomial( 1 , p );
}
";

# Define the Stanmodel and set the output format to :mcmcchains.

sm = SampleModel("m10.2s", m10_2s);

# Input data for cmdstan

m10_2_data = Dict("N" => size(df, 1),
"pulled_left" => df[!, :pulled_left], "prosoc_left" => df[!, :prosoc_left]);

# Sample using cmdstan

(sample_file, log_file)= stan_sample(sm, data=m10_2_data);

# Result rethinking

rethinking = "
   mean   sd  5.5% 94.5% n_eff Rhat
a  0.04 0.12 -0.16  0.21   180 1.00
bp 0.57 0.19  0.30  0.87   183 1.01
";

# Describe the draws

if !(sample_file == nothing)
  chn = read_samples(sm)
  describe(chn)
end


using Parameters

export ϑ, ω, hamiltonian

function ϑ(t::Number, q::AbstractVector, Θ::AbstractVector)
    Θ[1] = ϑ₁(t,q)
    Θ[2] = ϑ₂(t,q)
    nothing
end

function ϑ(t::Number, q::AbstractVector)
    [ϑ₁(t,q), ϑ₂(t,q)]
end

function ϑ(t::Number, q::AbstractVector, k::Int)
    if k == 1
        ϑ₁(t, q)
    elseif k == 2
        ϑ₂(t, q)
    else
        throw(BoundsError(ϑ,k))
    end
end

ϑ(t::Number, q::AbstractVector, params::NamedTuple, k::Int) = ϑ(t,q,k)

function lotka_volterra_2d_pᵢ(qᵢ, tᵢ=0)
    pᵢ = zero(qᵢ)

    if ndims(qᵢ) == 1
        ϑ(tᵢ, qᵢ, pᵢ)
    else
        for i in 1:size(qᵢ,2)
            ϑ(tᵢ, (@view qᵢ[:,i]), (@view pᵢ[:,i]))
        end
    end
    pᵢ
end

function ω(t, q, Ω)
    Ω[1,1] = 0
    Ω[1,2] = dϑ₁dx₂(t,q) - dϑ₂dx₁(t,q)

    Ω[2,1] = dϑ₂dx₁(t,q) - dϑ₁dx₂(t,q)
    Ω[2,2] = 0

    nothing
end

function hamiltonian(t, q, params)
    @unpack a₁, a₂, b₁, b₂ = params
    a₁*q[1] + a₂*q[2] + b₁*log(q[1]) + b₂*log(q[2])
end

hamiltonian_iode(t, q, params) = hamiltonian(t, q, params) # This is a workaround. It should be removed asap.
hamiltonian_iode(t, q, v, params) = hamiltonian(t, q, params)

hamiltonian_pode(t, q, p, params) = hamiltonian(t, q, params)

function lagrangian(t, q, v, params)
    ϑ₁(t,q) * v[1] + ϑ₂(t,q) * v[2] - hamiltonian(t, q, params)
end

function dHd₁(t, q, params)
    @unpack a₁, b₁ = params
    a₁ + b₁ / q[1]
end

function dHd₂(t, q, params)
    @unpack a₂, b₂ = params
    a₂ + b₂ / q[2]
end

function v₁(t, q, params)
    @unpack a₁, a₂, b₁, b₂ = params
    + q[1] * (a₂*q[2] + b₂)
end

function v₂(t, q, params)
    @unpack a₁, a₂, b₁, b₂ = params
    - q[2] * (a₁*q[1] + b₁)
end

f₁(t, q, v) = dϑ₁dx₁(t,q) * v[1] + dϑ₂dx₁(t,q) * v[2]
f₂(t, q, v) = dϑ₁dx₂(t,q) * v[1] + dϑ₂dx₂(t,q) * v[2]

g₁(t, q, v) = dϑ₁dx₁(t,q) * v[1] + dϑ₁dx₂(t,q) * v[2]
g₂(t, q, v) = dϑ₂dx₁(t,q) * v[1] + dϑ₂dx₂(t,q) * v[2]

lotka_volterra_2d_ϑ(t, q, Θ, params) = ϑ(t, q, Θ)
lotka_volterra_2d_ϑ(t, q, v, Θ, params) = ϑ(t, q, Θ)
lotka_volterra_2d_ω(t, q, Ω, params) = ω(t, q, Ω)
lotka_volterra_2d_ω(t, q, v, Ω, params) = ω(t, q, Ω)

function lotka_volterra_2d_dH(t, q, dH, params)
    dH[1] = dHd₁(t, q, params)
    dH[2] = dHd₂(t, q, params)
    nothing
end

function lotka_volterra_2d_v(t, q, v, params)
    v[1] = v₁(t, q, params)
    v[2] = v₂(t, q, params)
    nothing
end

function lotka_volterra_2d_v(t, q, p, v, params)
    lotka_volterra_2d_v(t, q, v, params)
end

function lotka_volterra_2d_v_ham(t, q, p, v, params)
    v .= 0
    nothing
end

function lotka_volterra_2d_v_dae(t, q, v, params)
    v[1] = v[3]
    v[2] = v[4]
    v[3] = 0
    v[4] = 0
    nothing
end

function lotka_volterra_2d_f(t::Real, q::Vector, v::Vector, f::Vector, params)
    f[1] = f₁(t,q,v) - dHd₁(t, q, params)
    f[2] = f₂(t,q,v) - dHd₂(t, q, params)
    nothing
end

function lotka_volterra_2d_f_ham(t::Real, q::Vector, f::Vector, params)
    f[1] = - dHd₁(t, q, params)
    f[2] = - dHd₂(t, q, params)
    nothing
end

function lotka_volterra_2d_f_ham(t::Real, q::Vector, v::Vector, f::Vector, params)
    lotka_volterra_2d_f_ham(t, q, f, params)
end

function lotka_volterra_2d_g(t::Real, q::Vector, v::Vector, g::Vector, params)
    g[1] = f₁(t,q,v)
    g[2] = f₂(t,q,v)
    nothing
end

function lotka_volterra_2d_g(t::Real, q::Vector, p::Vector, v::Vector, g::Vector, params)
    lotka_volterra_2d_g(t, q, v, g, params)
end

function lotka_volterra_2d_g̅(t::Real, q::Vector, v::Vector, g::Vector, params)
    g[1] = g₁(t,q,v)
    g[2] = g₂(t,q,v)
    nothing
end

function lotka_volterra_2d_g̅(t::Real, q::Vector, p::Vector, v::Vector, g::Vector, params)
    lotka_volterra_2d_g̅(t, q, v, g, params)
end

function lotka_volterra_2d_u_dae(t, q, λ, u, params)
    u[1] = 0
    u[2] = 0
    u[3] = λ[1]
    u[4] = λ[2]
    nothing
end

function lotka_volterra_2d_u(t, q, v, u, params)
    u .= v
    nothing
end

function lotka_volterra_2d_u(t, q, p, v, u, params)
    lotka_volterra_2d_u(t, q, v, u, params)
end

function lotka_volterra_2d_u̅(t, q, v, u, params)
    u[1] = v[1]
    u[2] = v[2]
    nothing
end

function lotka_volterra_2d_u̅(t, q, p, v, u, params)
    lotka_volterra_2d_u̅(t, q, v, u, params)
end

function lotka_volterra_2d_ϕ_dae(t, q, ϕ, params)
    ϕ[1] = q[3] - v₁(t,q,params)
    ϕ[2] = q[4] - v₂(t,q,params)
    nothing
end

function lotka_volterra_2d_ϕ(t, q, p, ϕ, params)
    ϕ[1] = p[1] - ϑ₁(t,q)
    ϕ[2] = p[2] - ϑ₂(t,q)
    nothing
end

function lotka_volterra_2d_ψ(t, q, p, v, f, ψ, params)
    ψ[1] = f[1] - g₁(t,q,v)
    ψ[2] = f[2] - g₂(t,q,v)
    nothing
end

function lotka_volterra_2d_ψ_lode(t, q, p, v, f, ψ, params)
    ψ[1] = f₁(t,q,v) - g₁(t,q,v) - dHd₁(t, q, params)
    ψ[2] = f₂(t,q,v) - g₂(t,q,v) - dHd₂(t, q, params)
    nothing
end


"""
    number_of_dimensions(all_vars...)

Return the number of unique dimensions for the given variables `all_vars`.
"""
function number_of_dimensions(all_vars::Vararg{Unitful.Dimensions})
    return length(unique_dims(all_vars...))
end

number_of_dimensions(all_vars::Vararg{Pair{<:AbstractString,<:QuantityOrUnitlike}}) =
number_of_dimensions([var.second for var in all_vars]...)

number_of_dimensions(all_vars::Vararg{<:QuantityOrUnits}) =
number_of_dimensions(dimension.(all_vars)...)

"""
    number_of_dimensionless(all_vars...)

Return the number of dimensionless numbers that can be constructed for a problem characterized by the variables `all_vars`.
"""
function number_of_dimensionless(all_vars::Vararg{<:QuantityOrUnitlike})
    return length(all_vars) - number_of_dimensions(all_vars...)
end

number_of_dimensionless(all_vars::Vararg{Pair{<:AbstractString,<:QuantityOrUnitlike}}) =
number_of_dimensionless([var.second for var in all_vars]...)

"""
    dimensionless(quantity, basis)

Make a `quantity` dimensionless using a dimensional `basis`.
"""
function dimensionless(quantity::Unitful.AbstractQuantity, basis::QuantityDimBasis)
    dim_mat = dim_matrix(basis.basis_dims, quantity)
    basis_facs = prod(basis.basis_vectors .^ (basis.dim_mat \ dim_mat))
    return ustrip(uconvert(Unitful.NoUnits, quantity / basis_facs))
end

"""
    dimensionful(value, unit, basis)

Restore the `unit`s of a dimensionless `value` using a dimensional `basis`.
"""
function dimensionful(value::Number, unit::Unitful.Units, basis::QuantityDimBasis)
    dim_vec = dim_matrix(basis.basis_dims, dimension(unit))
    return uconvert(unit, value * prod(basis.basis_vectors .^ (basis.dim_mat \ dim_vec)))
end

"""
    current_to_new_fac(dims, basis, new_basis)

Return the factor that is needed to transform specified dimensions `dims` from a current `basis` to a `new_basis`.
"""
function current_to_new_fac(dims::Unitful.Dimensions, basis::QuantityDimBasis, new_basis::QuantityDimBasis)
    dim_mat = dim_matrix(basis.basis_dims, dims)
    new_dim_mat = dim_matrix(new_basis.basis_dims, dims)
    basis_fac = prod(basis.basis_vectors .^ (basis.dim_mat \ dim_mat))
    new_basis_multipler = prod(new_basis.basis_vectors .^ (new_basis.dim_mat \ new_dim_mat))
    current_to_new_fac = new_basis_multipler / basis_fac
    return ustrip(uconvert(Unitful.NoUnits, current_to_new_fac))
end

"""
    change_basis(var, basis, new_basis)

Transform the specified quantity or unit `var` from a current `basis` to a `new_basis`.
"""
change_basis(var::Unitful.AbstractQuantity, basis::QuantityDimBasis, new_basis::QuantityDimBasis) =
var * current_to_new_fac(dimension(var), basis, new_basis)

change_basis(var::Unitful.Units, basis::QuantityDimBasis, new_basis::QuantityDimBasis) =
current_to_new_fac(dimension(var), basis, new_basis)


# Laplacian Eigenmaps
# -------------------
# Laplacian Eigenmaps for Dimensionality Reduction and Data Representation,
# M. Belkin, P. Niyogi, Neural Computation, June 2003; 15 (6):1373-1396

#### LEM type
immutable LEM{T <: Real} <: SpectralResult
    k::Int
    λ::AbstractVector{T}
    t::T
    proj::Projection{T}
    component::Vector{Int}

    LEM{T}(k::Int, λ::AbstractVector{T}, t::T, proj::Projection{T}) = new(k, λ, t, proj)
    LEM{T}(k::Int, λ::AbstractVector{T}, t::Float64, proj::Projection{T}, cc::Vector{Int}) = new(k, λ, t, proj, cc)
end

## properties
outdim(M::LEM) = size(M.proj, 1)
projection(M::LEM) = M.proj

eigvals(M::LEM) = M.λ
neighbors(M::LEM) = M.k
ccomponent(M::LEM) = M.component

## show & dump
function show(io::IO, M::LEM)
    print(io, "Laplacian Eigenmaps(outdim = $(outdim(M)), neighbors = $(neighbors(M)), t = $(M.t))")
end

function dump(io::IO, M::Isomap)
    show(io, M)
    println(io, "eigenvalues: ")
    Base.showarray(io, eigvals(M)', header=false, repr=false)
    println(io)
    println(io, "projection:")
    Base.showarray(io, projection(M), header=false, repr=false)
end

## interface functions
function transform{T<:Real}(::Type{LEM}, X::DenseMatrix{T}; d::Int=2, k::Int=12, t::T=1.0)
    n = size(X, 2)

    # Construct NN graph
    D, E = find_nn(X, k)

    W = zeros(T,n,n)
    for i = 1 : n
        jj = E[:, i]
        W[i,jj] = D[:, i]
    end
    W .^= 2
    W ./= maximum(W)

    # Select largest connected component
    CC = components(E)
    C = length(CC) == 1 ? CC[1] : CC[indmax(map(size, CC))]

    # Compute weights
    W[W .> eps(T)] = exp(-W[W .> eps(T)] ./ t)
    D = diagm(sum(W,2)[:])
    L = D - W

    λ, V = decompose(L, D, d)
    return LEM{T}(k, λ, t, V', C)
end


mutable struct Sin <: DiffableFunction
    grad_field::GradField
end

mutable struct Cos <: DiffableFunction
    grad_field::GradField
end

mutable struct Tan <: DiffableFunction
    grad_field::GradField
end

mutable struct Log <: DiffableFunction
    grad_field::GradField
end

mutable struct Exp <: DiffableFunction
    grad_field ::GradField
end

@inline forward(::Sin, x) = sin(x)
@inline forward(::Cos, x) = cos(x)
@inline forward(::Tan, x) = tan(x)
@inline forward(::Log, x) = log(x)
@inline forward(::Exp, x) = exp(x)

function backward(f::Sin, gy::Variable{T}) where {T <: Real}
    x = f.grad_field.inputs[1]
    return cos(x) * gy
end

function backward(f::Sin, gy::Variable{T}) where {T <: AbstractArray}
    x = f.grad_field.inputs[1]
    @. return cos(x) * gy
end

function backward(f::Cos, gy::Variable{T}) where {T <: Real}
    x = f.grad_field.inputs[1]
    return -sin(x) * gy
end

function backward(f::Cos,  gy::Variable{T}) where {T <: AbstractArray}
    x = f.grad_field.inputs[1]
    @. return -sin(x) * gy
end

function backward(f::Tan, gy::Variable{T}) where {T <: Real}
    x = f.grad_field.inputs[1]
    return (1 / (cos(x)^2)) * gy
end

function backward(f::Tan,  gy::Variable{T}) where {T <: AbstractArray}
    x = f.grad_field.inputs[1]
    @. return (1 / (cos(x)^2)) * gy
end

function backward(f::Log, gy::Variable{T}) where {T <: Real}
    x = f.grad_field.inputs[1]
    return gy / x
end

function backward(f::Log,  gy::Variable{T}) where {T <: AbstractArray}
    x = f.grad_field.inputs[1]
    @. return gy / x
end

function backward(f::Exp, gy::Variable{T}) where {T <: Real}
    x = f.grad_field.inputs[1]
    return exp(x) * gy
end

function backward(f::Exp,  gy::Variable{T}) where {T <: AbstractArray}
    x = f.grad_field.inputs[1]
    @. return exp(x) * gy
end

const math_functions = Dict(
    :(Base.sin) => Sin,
    :(Base.cos) => Cos,
    :(Base.tan) => Tan,
    :(Base.log) => Log,
    :(Base.exp) => Exp
)

for (func, jt_func) in math_functions
    @eval $func(x::Variable) = $jt_func(GradField())(x)
    @eval is_support(::typeof($func)) = true
    @eval is_support_broadcast(::typeof($func)) = true
    @eval get_jt_struct(::typeof($func)) = $jt_func
end


mutable struct parameters
      H::Float64
     Lϕ::Float64
   ϕmin::Float64
   TwoΩ::Float64
      g::Float64
      a::Float64
     Lj::Float64
     Uj::Float64
     Nϕ::Int64
     dk::Float64
     kₘ::Float64
     Fr::Float64
     Ro::Float64
end

function parameters(;
    H  = 500,
    Lϕ = π/3,
  ϕmin = π/6,
  TwoΩ = 1e-4,
    g  = 9.81,
    a  = 1,
    Lj = 1e4,
    Uj = 0.1,
    Nϕ = 100,
    dk = 5e-6,
    kₘ = 2e-4,
    Fr = 0.0,
    Ro = 0.0
    )
    return parameters(H, Lϕ, ϕmin, TwoΩ, g, a, Lj, Uj, Nϕ, dk, kₘ, Fr, Ro)
end