using Test

@testset "UWDDynam" begin
  include("uwd_dynam.jl")
end

@testset "DWDDynam" begin
  include("dwd_dynam.jl")
end

@testset "CPGDynam" begin
  include("cpg_dynam.jl")
end

@testset "Trajectories and Solutions" begin
  include("trajectories.jl")
end

@testset "Tonti Diagrams" begin
  include("TontiDiagrams.jl")
end


using SpecialFunctions: gamma
export c2, c3

function c2(psi)
    eps = 1.0
    if psi > eps
        res = (1 - cos(sqrt(psi))) / psi
    elseif psi < -eps
        res = (cosh(sqrt(-psi)) - 1) / (-psi)
    else
        res = 1.0 / 2.0
        delta = (-psi) / gamma(2 + 2 + 1)
        k = 1
        while res + delta != res
            res += delta
            k += 1
            delta = (-psi)^k / gamma(2*k + 2 + 1)
        end
    end
    return res
end

function c3(psi)
    eps = 1.0
    if psi > eps
        res = (sqrt(psi) - sin(sqrt(psi))) / (psi * sqrt(psi))
    elseif psi < -eps
        res = (sinh(sqrt(-psi)) - sqrt(-psi)) / (-psi * sqrt(-psi))
    else
        res = 1.0 / 6.0
        delta = (-psi) / gamma(2 + 3 + 1)
        k = 1
        while res + delta != res
            res += delta
            k += 1
            delta = (-psi)^k / gamma(2*k + 3 + 1)
        end
    end
    return res
end


#=
The first known prime found to exceed one million digits was discovered in 1999, and is a Mersenne prime of the form 26972593−1; it contains exactly 2,098,960 digits. Subsequently other Mersenne primes, of the form 2p−1, have been found which contain more digits.

However, in 2004 there was found a massive non-Mersenne prime which contains 2,357,207 digits: 28433×2^7830457+1.

Find the last ten digits of this prime number.
=#
function calc()
  x = BigInt(28433) * (BigInt(2)^7830457) + 1
  "$x"[end-9:end]
end
@time println(calc())


@testset "Tensor-CUR" begin
Random.seed!(12345)

@testset "Small case" begin
    T = rand_kruskal3(2, (20, 40, 60), false)
    @testset "slab axis: $i" for i in 1:3
        @time cur3 = tensorcur3(T, 3, 15, i)
        @test sum(cur3.Cweight) == 3
        @test sum(cur3.Rweight) == 15
        @test all(err -> err < 1e-5, cur3.error)
    end
end

@testset "Large case without reconstruction" begin
    T = rand_kruskal3(2, (100, 120, 60), false)
    @time factors = tensorcur3(T, 10, 200, compute_u=false)
    @test sum(factors.Cweight) == 10
    @test sum(factors.Rweight) == 200
    @test size(factors.U) == (0, 0)
    @test size(factors.error) == (0,)
end

end


using ARFFFiles, DataFrames
using Random

"""
    Normalize(vector)
Devuelve un vector de datos normalizados
"""
function Normalize(vector)
    max = maximum(vector)
    min = minimum(vector)
    diff = (max-min)
    reescale = diff ≈ 0 ? 1 : diff

    return map( x -> (x - min)/reescale, vector)
end
"""
    DataLabel(file)
Dado un fichero .arff devuleve:
1. matriz de datos normalizados, cada columna una categoría
2. Las etiquetas

Se supone que:
1. Class es la última categoría y contiene las categorías.
2. Que  el resto son numéricas
"""
function DataLabelArff(file, class_name)
    Random.seed!(0)
    df = ARFFFiles.load(DataFrame, file)
    labels = df[!, "$class_name"]
    attributes  = names(df)
    pop!(attributes) # Elimina la clase
    data = [
            Normalize(df[!,"$name"])
            for name in attributes
    ]
    data = reduce(hcat, data)

    # desordenamos los datos
    index = shuffle(1:length(labels))
    data = data[index, :]
    labels = labels[index]

    return data, labels
end


using ADCME
using ADSeismic
using PyPlot
using DelimitedFiles
using JLD2

scale = 201

param = AcousticPropagatorParams(NX=scale, NY=scale,
    NSTEP=1000, DELTAT=1e-4,  DELTAX=1.0, DELTAY=1.0,
    PropagatorKernel = 0, Rcoef = 1e-8)

rc = Ricker(param, 30.0, 200.0, 1e6)
srci = [div(param.NX,2)]
srcj = [div(param.NY,5)]
srcv = reshape(rc, :, 1)
src = AcousticSource(srci, srcj, srcv)

C = Variable(3300*ones(param.NX+2, param.NY+2))
model = AcousticPropagatorSolver(param, src, C)
@load "data.jld2" u

U = model.u
loss = sum((u[:, :, 40] - U[:,:,40])^2)
g = gradients(loss, C)

sess = Session(); init(sess)
G = run(sess, g)

close("all")
G[:,1:49] .= NaN
pcolormesh(G')
gca().invert_yaxis()
savefig("gradient.png")


using Statistics
import MLJFlux
import Flux

@testset  "Penalizer" begin
    A = [-1 2; -3 4]
    lambda = 1

    # 100% L2:
    alpha = 0
    penalty = MLJFlux.Penalizer(lambda, alpha)
    @test penalty(A) ≈ 1 + 4 + 9 + 16

    # 100% L1:
    alpha = 1
    penalty = MLJFlux.Penalizer(lambda, alpha)
    @test penalty(A) ≈ 1 + 2 + 3 + 4

    # no strength:
    lambda = 0
    alpha = 42.324
    penalty = MLJFlux.Penalizer(lambda, alpha)
    @test penalty(A) == 0
end

@testset "Penalty" begin
    model = MLJFlux.NeuralNetworkRegressor(lambda=1, alpha=1, loss=Flux.mae)
    chain = Flux.Dense(3, 1, identity)
    w = Flux.params(chain)
    p = MLJFlux.Penalty(model)

    # compare loss by hand and with penalized loss function:
    penalty = (sum(abs.(chain.weight)) + abs(chain.bias[1]))
    @test p(w) ≈ penalty
end


function gen_timeevoarray(hamiltonian,finalTime,samples)
  # Generate the time evolution array for a time independent Hamiltonian
  time_vec=((1:samples)-1)*finalTime/samples
  time_evo_array = Array(typeof(hamiltonian),samples)

  for i=1:samples
    time_evo_array[i] = expm(-1im .* hamiltonian .* time_vec[i] ./ h_bar)
  end

  return time_vec, time_evo_array
end


using JuMP, Base.Test, AmplNLWriter

## Solve test problem with sind and cosd functions
 #
 #  min   (7 - (3*cosd(x1) + 5*cosd(x2)))^2 + (0 - (3*sind(x1) + 5*sind(x2)))^2
 #  s.t.  x1, x2 continuous
 #
 #  The optimal objective value is 0
 ##

# solver = AmplNLSolver(Ipopt.amplexe, ["print_level=0"])

@testset "example: jump_nltrig" begin
    m = Model(solver=solver)
    @variable(m, x[1:2])

    @NLobjective(m, Min, (7 - (3*cosd(x[1]) + 5*cosd(x[2])))^2 + (0 - (3*sind(x[1]) + 5*sind(x[2])))^2)

    setvalue(x[1], 30)
    setvalue(x[2], -50)
    @test solve(m) == :Optimal
    @test isapprox(getvalue(x)[:], [38.21321, -21.78678], atol=1e-5)
    @test isapprox(getobjectivevalue(m), 0.0, atol=1e-5)
    # Now try from the other side
    setvalue(x[1], -30)
    setvalue(x[2], 50)
    @test solve(m) == :Optimal
    @test isapprox(getvalue(x)[:], [-38.21321, 21.78678], atol=1e-5)
    @test isapprox(getobjectivevalue(m), 0.0, atol=1e-5)
end


using Base.Test
using OptimTools

if !isdefined(:probs) || isempty(probs)
	println("getting test problems")
	include("getTestFunctions.jl")
end

his = []; flag = []; x0 = [];

firstOrderMethods = (sd,bfgs,nlcg,sr1)
# try stopping based on atol
for method in firstOrderMethods
	println("testing $(string(method))")
	for k=1:length(probs)
		prob = probs[k]

		println("\t\t$(prob[1])")

		fk   = prob[2]
		dfk  = prob[3]
		xsol = float(prob[5])
		x0   = float(xsol) + .01*randn(size(xsol))*norm(xsol)
		atol = 1e-4

		xsd,flag,his = method(fk,dfk, x0,maxIter=1000000,atol=atol,out=1*(k==2))

		@test all(diff(his[:,1]).<=0)
		@test norm(dfk(xsd)) .< atol
	end
end


function mean!(μ::VecIO, r::VecI, X::MatI)
    N = sum(r)
    if iszero(N)
        @simd for m in eachindex(μ)
            @inbounds μ[m] = 0.0
        end
    else
        BLAS.gemv!('T', 1.0, X, r, 0.0, μ)
        @simd for m in eachindex(μ)
            @inbounds μ[m] /= N
        end
    end
    return nothing
end

function mean!(μ::MatIO, R::MatI, X::MatI, N::VecI; init::Bool=false)
    if init
        @inbounds for i in eachindex(N)
            N[i] = sum(view(R,i,:))
        end
    end
    BLAS.gemm!('N', 'N', 1.0, R, X, 0.0, μ)
    for k in axes(μ, 1)
        @inbounds Nk = N[k]
        if iszero(Nk)
            @simd for m in axes(μ, 2)
                @inbounds μ[k,m] = 0.0
            end
        else
            @simd for m in axes(μ, 2)
                @inbounds μ[k,m] /= Nk
            end
        end
    end
    return nothing
end

function deviation!(Δ::MatIO, X::MatI, μ::VecI)
    n, m = size(X)
    for j in eachindex(1:m)
        @inbounds μj = μ[j]
        @simd for i in eachindex(1:n)
            @inbounds Δ[i,j] = X[i,j] - μj
        end
    end
    return nothing
end

function covariance!(C::MatIO, X::MatI, μ::VecI, r::VecI, Δ::MatB)
    deviation!(Δ, X, μ)
    for n in eachindex(r)
        @inbounds rn = sqrt(r[n])
        @simd for m in eachindex(μ)
            @inbounds Δ[n,m] *= rn
        end
    end
    BLAS.syrk!('U', 'T', 1.0, Δ, 0.0, C)
    @inbounds for j in eachindex(μ), i in 1:j-1
        C[j,i] = C[i,j]
    end
    return nothing
end


if !(VERSION.major==0&&VERSION.minor<=6)
	using Test
else
	using Base.Test
end
using FinancialToolbox

print_colored("Starting Complex Number Test\n",:green)

#Test Parameters
testToll=1e-14;
spot=10;K=10;r=0.02;T=2.0;sigma=0.2;d=0.01;
assert_(value,toll)=@test abs(value)<toll

#EuropeanCall Option
PriceCall=blsprice(spot,K,r,T,sigma,d);
DeltaCall=blsdelta(spot,K,r,T,sigma,d);
ThetaCall=blstheta(spot,K,r,T,sigma,d);
LambdaCall=blslambda(spot,K,r,T,sigma,d);
RhoCall=blsrho(spot,K,r,T,sigma,d);

#EuropeanPut Option
PricePut=blsprice(spot,K,r,T,sigma,d,false);
DeltaPut=blsdelta(spot,K,r,T,sigma,d,false);
ThetaPut=blstheta(spot,K,r,T,sigma,d,false);
RhoPut=blsrho(spot,K,r,T,sigma,d,false);
LambdaPut=blslambda(spot,K,r,T,sigma,d,false);

#Equals for both Options
Gamma=blsgamma(spot,K,r,T,sigma,d);
Vega=blsvega(spot,K,r,T,sigma,d);

## Complex Test with Complex Step Approximation for European Call
#Test parameters
DerToll=1e-13;
di=1e-15;
df(f,x)=f(x+1im*di)/di;
#Function definition
Fcall1(spot)=blsprice(spot,K,r,T,sigma,d);
Gcall1(r)=blsprice(spot,K,r,T,sigma,d);
Hcall1(T)=blsprice(spot,K,r,T,sigma,d);
Lcall1(sigma)=blsprice(spot,K,r,T,sigma,d);
Pcall1(spot)=blsprice(spot,K,r,T,sigma,d)*spot.re/blsprice(spot.re,K,r,T,sigma,d);
#TEST
print_colored("--- European Call Sensitivities: Complex Step Approximation\n",:yellow)
print_colored("-----Testing Delta\n",:blue);
assert_(df(Fcall1,spot).im-DeltaCall,DerToll)
print_colored("-----Testing Rho\n",:blue);
assert_(df(Gcall1,r).im-RhoCall,DerToll)
print_colored("-----Testing Theta\n",:blue);
assert_(-df(Hcall1,T).im-ThetaCall,DerToll)
print_colored("-----Testing Lambda\n",:blue);
assert_(df(Pcall1,spot).im-LambdaCall,DerToll)

## Complex Test with Complex Step Approximation for European Put
#Function definition
Fput1(spot)=blsprice(spot,K,r,T,sigma,d,false);
Gput1(r)=blsprice(spot,K,r,T,sigma,d,false);
Hput1(T)=blsprice(spot,K,r,T,sigma,d,false);
Pput1(spot)=blsprice(spot,K,r,T,sigma,d,false)*spot.re/blsprice(spot.re,K,r,T,sigma,d,false);
#TEST
print_colored("--- European Put Sensitivities: Complex Step Approximation\n",:yellow)
print_colored("-----Testing Delta\n",:blue);
assert_(df(Fput1,spot).im-DeltaPut,DerToll)
print_colored("-----Testing Rho\n",:blue);
assert_(df(Gput1,r).im-RhoPut,DerToll)
print_colored("-----Testing Theta\n",:blue);
assert_(-df(Hput1,T).im-ThetaPut,DerToll)
print_colored("-----Testing Lambda\n",:blue);
assert_(df(Pput1,spot).im-LambdaPut,DerToll)

print_colored("-----Testing Vega\n",:blue);
assert_(df(Lcall1,sigma).im-Vega,DerToll)
print_colored("Complex Number Test Passed\n",:green)
println("")

#TEST OF INPUT VALIDATION
print_colored("Starting Input Validation Test Complex\n",:magenta)
print_colored("----Testing Negative  Spot Price \n",:cyan)
@test_throws(ErrorException, blsprice(-spot+1im,K,r,T,sigma,d))
@test_throws(ErrorException, blkprice(-spot+1im,K,r,T,sigma))
@test_throws(ErrorException, blsdelta(-spot+1im,K,r,T,sigma,d))
@test_throws(ErrorException, blsgamma(-spot+1im,K,r,T,sigma,d))
@test_throws(ErrorException, blstheta(-spot+1im,K,r,T,sigma,d))
@test_throws(ErrorException, blsrho(-spot+1im,K,r,T,sigma,d))
@test_throws(ErrorException, blspsi(-spot+1im,K,r,T,sigma,d))
@test_throws(ErrorException, blslambda(-spot+1im,K,r,T,sigma,d))
@test_throws(ErrorException, blsvanna(-spot+1im,K,r,T,sigma,d))
@test_throws(ErrorException, blsvega(-spot+1im,K,r,T,sigma,d))

print_colored("----Testing Negative  Strike Price \n",:cyan)
@test_throws(ErrorException, blsprice(spot,-K+1im,r,T,sigma,d))
@test_throws(ErrorException, blkprice(spot,-K+1im,r,T,sigma))
@test_throws(ErrorException, blsdelta(spot,-K+1im,r,T,sigma,d))
@test_throws(ErrorException, blsgamma(spot,-K+1im,r,T,sigma,d))
@test_throws(ErrorException, blstheta(spot,-K+1im,r,T,sigma,d))
@test_throws(ErrorException, blsrho(spot,-K+1im,r,T,sigma,d))
@test_throws(ErrorException, blspsi(spot,-K+1im,r,T,sigma,d))
@test_throws(ErrorException, blslambda(spot,-K+1im,r,T,sigma,d))
@test_throws(ErrorException, blsvanna(spot,-K+1im,r,T,sigma,d))
@test_throws(ErrorException, blsvega(spot,-K+1im,r,T,sigma,d))

print_colored("----Testing Negative  Time to Maturity \n",:cyan)
@test_throws(ErrorException, blsprice(spot,K,r,-T+1im,sigma,d))
@test_throws(ErrorException, blkprice(spot,K,r,-T+1im,sigma))
@test_throws(ErrorException, blsdelta(spot,K,r,-T+1im,sigma,d))
@test_throws(ErrorException, blsgamma(spot,K,r,-T+1im,sigma,d))
@test_throws(ErrorException, blstheta(spot,K,r,-T+1im,sigma,d))
@test_throws(ErrorException, blsrho(spot,K,r,-T+1im,sigma,d))
@test_throws(ErrorException, blspsi(spot,K,r,-T+1im,sigma,d))
@test_throws(ErrorException, blslambda(spot,K,r,-T+1im,sigma,d))
@test_throws(ErrorException, blsvanna(spot,K,r,-T+1im,sigma,d))
@test_throws(ErrorException, blsvega(spot,K,r,-T+1im,sigma,d))

print_colored("----Testing Negative  Volatility \n",:cyan)
@test_throws(ErrorException, blsprice(spot,K,r,T,-sigma+1im,d))
@test_throws(ErrorException, blkprice(spot,K,r,T,-sigma+1im))
@test_throws(ErrorException, blsdelta(spot,K,r,T,-sigma+1im,d))
@test_throws(ErrorException, blsgamma(spot,K,r,T,-sigma+1im,d))
@test_throws(ErrorException, blstheta(spot,K,r,T,-sigma+1im,d))
@test_throws(ErrorException, blsrho(spot,K,r,T,-sigma+1im,d))
@test_throws(ErrorException, blspsi(spot,K,r,T,-sigma+1im,d))
@test_throws(ErrorException, blslambda(spot,K,r,T,-sigma+1im,d))
@test_throws(ErrorException, blsvanna(spot,K,r,T,-sigma+1im,d))
@test_throws(ErrorException, blsvega(spot,K,r,T,-sigma+1im,d))

print_colored("Complex Input Validation Test Passed\n",:magenta)


using QBase: base_n_val

# """
# strategies(num_inputs, num_outputs):
#   Constructs the set of unique strategies which maps M unique inputs to K
#   unique outputs.
#
# Input:
#   num_inputs: Integer, the number of distinct inputs, M
#   num_outputs: Integer, the number of distinct outputs, K
#
# Output:
#   strategies: Array of (M x K) matrices which deterministically and uniquely
#               map an input to an output.
# """
function strategies(num_inputs, num_outputs)

    inputs = Matrix(1I, (num_inputs, num_inputs))
    outputs = Matrix(1I, (num_outputs, num_outputs))

    strategies = []
    if num_outputs > 1
        # loop through the number of ways to select 10 elements from the output set
        # Incrementing a base (num_outputs) number with num_inputs digit will achieve
        # all such combinations.
        for i in 0:(num_outputs^num_inputs - 1)
            base_n_array = digits(i, base = num_outputs, pad = num_inputs )

            # use the base (num_outputs) number to construct the strategy matrix
            output_index_array = base_n_array .+ 1

            push!(strategies,outputs[:,output_index_array])
        end
    else
        strat = fill(1,(1,num_inputs))
        push!(strategies, strat)
    end

    strategies
end

# """
# communication_protocols(num_inputs,num_outputs):
#   Partitions the m input set into k non-empty output sets. For m >= k,
#   the number of partitions are exactly described by stirling's number
#   of the second kind. For m <= k, the number of protocols is given by (k choose m).
#
# Inputs:
#   num_inputs: Integer, the number of distinct inputs, m.
#   num_outputs: Integer, the number of distinct outputs, k.
#
# Output:
#   protocols: Array, where elements are deterministic maps from input states
#              to output states.
# """
function communication_protocols(num_inputs,num_outputs)
    outputs = Matrix(1I, (num_outputs,num_outputs))
    protocols = []

    if num_inputs < num_outputs

        num_base = max(num_outputs,2)

        inputs = Matrix(1I, (num_inputs, num_inputs))

        min_n_array = collect(0:(num_inputs-1))
        min_num = base_n_val(min_n_array, num_base)

        base_n_array = reverse(digits(min_num, base = num_base, pad = num_inputs))
        id_array = base_n_array .+ 1

        push!(protocols, outputs[:,id_array])
    else
        num_base = max(num_outputs,2)

        max_n_array = cat(
            0:(num_outputs-1),
            fill(num_outputs-1,(num_inputs - num_outputs)),
            dims=1
        )
        max_num = base_n_val(max_n_array, num_base)

        min_n_array = cat(
            fill(0, (num_inputs - num_outputs)),
            0:(num_outputs - 1),
            dims = 1
        )
        min_num = base_n_val(min_n_array, num_base)

        partitions = []
        for i in min_num:max_num
            base_n_array = digits(i, base = num_base, pad = num_inputs)
            # subsets are required to be non-empty thus every digit should be represented
            if issetequal( unique(base_n_array), 0:(num_outputs - 1))
                partition = fill([],num_outputs)
                for j in 1:size(base_n_array)[1]
                    digit = base_n_array[j]
                    partition[digit + 1] = cat(partition[digit + 1], j, dims = 1)
                end

                # ordering of sets does not matter
                # [[1],[2],[3,4]] is same as [[2],[1],[4,3]]
                dupe_count = 0
                for set in partitions
                    if issetequal(partition,set[2])
                        dupe_count += 1
                    end
                end

                # only consider partition sets that are unique
                if dupe_count == 0
                    push!(partitions, (base_n_array, partition) )
                end
            end
        end

        # construct protocols from partitions
        for partition in partitions
            output_index_array = reverse(partition[1]) .+ 1
            push!(protocols, outputs[:,output_index_array])
        end
    end

    protocols
end

# """
# input_set(num_inputs):
#   Constructs a set of orthonormal vectors which represent distinct inputs to
#   the bell test.
#
# Input:
#   num_inputs: Integer, the number of distinct inputs
#
# Output:
#   input_set: Array, contains input vectors of dimension num_inputs
# """
function input_set(num_inputs)
    input_set = []
    for i in 1:num_inputs
        input = zeros(Int64, (num_inputs,1))
        input[i] = 1
        push!(input_set, input)
    end

    input_set
end

# """
# is_no_signaling_duplicate(strategy, num_inputs, num_rx_inputs):
#   The set of strategies that have an rx_input contains strategies that which
#   do not use the rx_input to map the output. These strategies yield redundant
#   vertices in a quantity equivalent to Stirling's number of the 2nd kind to
#   those generated by the non-signaling constraint. This function checks if a
#   strategy is a duplicate.
#
# Inputs:
#   strategy: Matrix, map from (num_inputs)⊗(num_rx_inputs) -> num_outputs
#   num_inputs: Integer, number of inputs supplied to expt (M)
#   num_rx_inputs: Integer, number of inputs supplied by communication (2^bits)
#
# Outputs:
#   is_duplicate: Boolean, true if strategy is a non-signaling duplicate
# """
function is_no_signaling_duplicate(strategy, num_inputs, num_rx_inputs)
    # count the times that changing the rx bit changes the output
    is_duplicate = true
    for i in 0:(num_inputs - 1)
        output = strategy[:,(i*num_rx_inputs + 1)]
        for j in 2:num_rx_inputs
            if output != strategy[:,(i*num_rx_inputs + j)]
                is_duplicate = false
                break
            end
        end
        if !is_duplicate
            break
        end
    end

    is_duplicate
end

# """
# compute_no_signaling_behavior(alice_strategy, bob_strategy; subspace="non-signaling"):
#   Given deterministic strategies for alice and bob, computes the non-signaling
#   behavior of the joint system.
#
# Inputs:
#   alice_strategy: Tuple, (strategy map, set of input, input dimension, output dimension)
#   bob_strategy: Tuple, (strategy map, set of input, input dimension, output dimension)
#   subspace: String, "non-signaling" is optimal representation, "normalization" is general representation
#
# Output:
#   behavior: Array, behavior vector in the specified subspace
# """
function compute_no_signaling_behavior(alice_strategy, bob_strategy; subspace="non-signaling")

    (α, α_in_set, α_in, α_out) = alice_strategy
    (β, β_in_set, β_in, β_out) = bob_strategy

    if subspace == "non-signaling"
        # Alice is assumed to be indpendent of Bob
        α_behavior = zeros(Int64, α_in*(α_out - 1))
        for x in α_in_set
            a = α[1:(end-1),:]*x
            α_behavior += kron(a,x)
        end

        β_behavior = zeros(Int64, β_in*(β_out - 1))
        for y in β_in_set
            b = β[1:(end-1),:]*y
            β_behavior += kron(b,y)
        end

        joint_behavior = zeros(Int64, α_in*(α_out - 1)*β_in*(β_out - 1))
        for x in α_in_set
            for y in β_in_set
                a = α[1:(end-1),:]*x
                b = β[1:(end-1),:]*y

                joint_behavior += kron(a, kron(b, kron(x, y)))
            end
        end

        behavior = cat(α_behavior, β_behavior, joint_behavior, dims = 1)
    elseif subspace == "fixed-direction"
        α_behavior = zeros(Int64, α_in*(α_out - 1))
        αβ_behavior = zeros(Int64, α_in*α_out*β_in*(β_out - 1))
        α_complement = zeros(Int64, α_in)
        for x in α_in_set

            # Alice output
            a = α*x

            α_behavior += kron(a[1:(end-1),:], x)

            for y in β_in_set

                b = β[1:(end-1),:]*y
                αβ_behavior += kron(a,kron(b,kron(x,y)))
            end
        end

        behavior = cat(
            α_behavior,
            αβ_behavior,
            dims = 1
        )

    elseif subspace == "normalized"
        behavior_dim = (α_in * α_out)*(β_in * β_out) - (α_in * β_in)
        behavior = zeros(Int64, behavior_dim)
        joint_strategy = kron(α, β)

        for x in α_in_set
            for y in β_in_set
                xy = kron(x,y)

                ab = joint_strategy[1:(end-1),:]*xy

                behavior += kron(ab,xy)
            end
        end
    else
        behavior_dim = (α_in * α_out)*(β_in * β_out)
        behavior = zeros(Int64, behavior_dim)
        joint_strategy = kron(α, β)
        for x in α_in_set
            for y in β_in_set
                xy = kron(x,y)

                ab = joint_strategy*xy

                behavior += kron(ab,xy)
            end
        end
    end

    behavior
end

# """
# compute_signaling_behavior(alice_strategy, bob_strategy; direction="fixed"):
#   Computes the behavior associated wiht the provided strategies and communication
#   protocols. Supports fixed-direction (Assume Alice -> Bob) and one-way (Alice
#   -> Bob or Alice <- Bob).
#
# Inputs
#   alice_strategy: Tuple, (strategy map, input set, num inputs, num outputs, communication protocol map)
#   bob_strategy: Tuple, (strategy map, input set, num inputs, num outputs, communication protocol map)
#   direction: String, "fixed", "<-", or "->"
#
# Output:
#   behavior: Vector, behavior represented in either the fixed-direction
#             represention or normalization representation
# """
function compute_signaling_behavior(alice_strategy, bob_strategy; direction="fixed", subspace="fixed-direction")
    (α, α_in_set, α_in, α_out, ρ) = alice_strategy
    (β, β_in_set, β_in, β_out, σ) = bob_strategy

    if (direction == "fixed") & (subspace != "fixed-direction")
        direction = "->"
    end

    behavior_dim = 0
    if subspace == "normalized"
        behavior_dim = (α_in * α_out)*(β_in * β_out) - α_in*β_in
    elseif subspace == "generalized"
        behavior_dim = (α_in*α_out)*(β_in*β_out)
    elseif subspace == "fixed-direction"
        behavior_dim = α_in*(α_out - 1) + α_in*(α_out)*β_in*(β_out - 1)
    end

    if direction == "fixed"
        # assume alice only signals to bob
        behavior_dim = α_in*(α_out - 1) + α_in*(α_out)*β_in*(β_out - 1)

        α_behavior = zeros(Int64, α_in*(α_out - 1))
        αβ_behavior = zeros(Int64,  α_in*(α_out)*β_in*(β_out - 1))

        behavior = zeros(Int64, behavior_dim)
        for x in α_in_set

            # Alice output
            a = α*x

            α_behavior += kron(a[1:(end-1),:], x)
            for y in β_in_set

                α_tx = ρ*x
                β_input = kron(y, α_tx)

                b = β[1:(end-1),:]*β_input
                αβ_behavior += kron(a, kron(b, kron(x,y)))
            end
        end

        behavior = cat(
            α_behavior,
            αβ_behavior,
            dims = 1
        )
    elseif direction == "->"
        # TODO: compute generalized behavior and normalized behavior based off subpsace
        behavior = zeros(Int64, behavior_dim)
        for x in α_in_set
            for y in β_in_set

                rx_input = ρ*x
                β_input = kron(y, rx_input)

                joint_input = kron(x, β_input)
                joint_strategy = kron(α,β)
                if subspace == "normalized"
                    ab = joint_strategy[1:(end-1),:]*joint_input
                    behavior += kron(ab,kron(x,y))
                elseif subspace == "generalized"
                    ab = joint_strategy*joint_input
                    behavior += kron(ab,kron(x,y))
                end
            end
        end
    elseif direction == "<-"

        behavior = zeros(Int64, behavior_dim)
        for x in α_in_set
            for y in β_in_set

                rx_input = σ*y
                α_input = kron(x, rx_input)

                joint_input = kron(α_input, y)
                joint_strategy = kron(α,β)

                if subspace == "normalized"
                    ab = joint_strategy[1:(end-1),:]*joint_input
                    behavior += kron(ab,kron(x,y))
                elseif subspace == "generalized"
                    ab = joint_strategy*joint_input
                    behavior += kron(ab,kron(x,y))
                end
            end
        end
    end

    behavior
end

# """
# vertices(alice_expt, bob_expt; bits=0, fixed_direction=true):
#   Enumerates the vertices of the local polytope corresponding to the supplied
#   bipartite bell experiment. Supports non-signaling (bits = 0), fixed direction
#   signaling, and one-way signaling scenarios.
#
# Input:
#   alice_expt:      Tuple, (num_inputs, num_outputs)
#   bob_expt:        Tuple, (num_inputs, num_outputs)
#   bits:            Integer, the number of bits used in communication (default = 0)
#   dits:            Integer, the number of distinct communication states (default = 0)
#   fixed_direction: Boolean, true if fixed-direction signaling is used (default = true)
#
# Output:
#   vertices: Array containing local polytope vertices represented in the optimal
#             subspace.
# """
function vertices(alice_expt, bob_expt; bits=0, dits=0, fixed_direction=true, rep="")

    (α_in, α_out) = alice_expt
    (β_in, β_out) = bob_expt

    comm_dits = 0
    if bits > 0
        comm_dits = 2^bits
    elseif dits > 0
        comm_dits = dits
    end

    α_in_set = input_set(α_in)
    β_in_set = input_set(β_in)

    α_set = strategies(α_in, α_out)
    β_set = strategies( ((comm_dits == 0) ? 1 : comm_dits)*β_in, β_out)

    α2_set = []
    β2_set = []
    ρ_set = []
    σ_set = []

    if !in(rep,("normalized","generalized","fixed-direction","non-signaling"))
        if comm_dits == 0
            rep = "non-signaling"
        elseif (comm_dits > 0) & fixed_direction
            rep = "fixed-direction"
        elseif (comm_dits > 0) & !fixed_direction
            rep = "normalized"
        else
            rep = "generalized"
        end
    end

    if (α_out < 2) & (β_out < 2)
        rep = "generalized"
    elseif (α_out < 2) | (β_out < 2)
        if rep != "generalized"
            rep = "normalized"
        end
    end

    if comm_dits > 0
        β2_set = strategies( β_in, β_out )
        ρ_set = communication_protocols(α_in, comm_dits)
        if !(fixed_direction)
            α2_set = strategies( (comm_dits)*α_in, α_out )
            σ_set = communication_protocols(β_in, comm_dits)
        end
    end

    vertices = []

    if comm_dits == 0
        for α in α_set
            for β in β_set
                vertex = compute_no_signaling_behavior(
                    (α, α_in_set, α_in, α_out),
                    (β, β_in_set, β_in, β_out),
                    subspace = rep
                )
                push!(vertices,vertex)
            end
        end
    elseif fixed_direction
        for α in α_set
            for β2 in β2_set
                vertex = compute_no_signaling_behavior(
                    (α, α_in_set, α_in, α_out),
                    (β2, β_in_set, β_in, β_out),
                    subspace = rep
                )
                push!(vertices, vertex)
            end

            for β in β_set
                if !( is_no_signaling_duplicate(β, β_in, comm_dits) ) & !(α_in < 2)
                    for ρ in ρ_set
                        vertex = compute_signaling_behavior(
                            (α, α_in_set, α_in, α_out, ρ),
                            (β, β_in_set, β_in, β_out, nothing),
                            direction = "fixed",
                            subspace = rep
                        )
                        push!(vertices, vertex)
                    end
                end
            end
        end
    else
        for α in α_set
            for β2 in β2_set
                no_signaling_vertex = compute_no_signaling_behavior(
                    (α, α_in_set, α_in, α_out),
                    (β2, β_in_set, β_in, β_out),
                    subspace = rep
                )
                push!(vertices, no_signaling_vertex)
            end
        end

        for β in β_set
            if !( is_no_signaling_duplicate(β, β_in, comm_dits) ) & !(α_in < 2)
                for α in α_set
                    for ρ in ρ_set
                        ab_vertex = compute_signaling_behavior(
                            (α, α_in_set, α_in, α_out, ρ),
                            (β, β_in_set, β_in, β_out, nothing),
                            direction = "->",
                            subspace = rep
                        )
                        push!(vertices, ab_vertex)
                    end
                end
            end
        end

        for α in α2_set
            if !( is_no_signaling_duplicate(α, α_in, comm_dits) ) & !(β_in < 2)
                for β in β2_set
                    for σ in σ_set
                        ba_vertex = compute_signaling_behavior(
                            (α, α_in_set, α_in, α_out, nothing),
                            (β, β_in_set, β_in, β_out, σ),
                            direction = "<-",
                            subspace = rep
                        )

                        push!(vertices, ba_vertex)
                    end
                end
            end
        end
    end

    vertices
end

# """
# strategy_to_behavior(strategy):
#
#     Converts a strategy matrix to its corresponding behavior representation.
#
# Input:
#     strategy: Matrix, dimensions do not matter, it should represent a valid strategy
#
# Output:
#     behavior: Col Vector, corresponding behavior representation for the strategy
# """
function strategy_to_behavior(strategy; rep="generalized", constant=true)

    if rep == "normalized"
        strategy = strategy[1:(end-1),:]
    end

    (num_out, num_in) = size(strategy)

    behavior_dim = num_out*num_in

    behavior = zeros(behavior_dim, 1) + collect(Iterators.flatten(map(i->strategy[i,:], 1:num_out)))

    if constant
        behavior = cat([1],behavior,dims=1)
    end

    behavior
end

# """
# behavior_to_strategy(num_in, num_out, gen_behavior):
#
#     converts a behavior in the generalized representation into it's correpsonding
#     strategy matrix.
#
# Inputs:
#     num_in/out: Integer, bell scenario parameters (dimensions of strategies)
#     gen_behavior: Col Vector, a valid behavior in the generalized representation
#
# Output:
#     strategy: Matrix, the strategy isomorphic to the provided behavior.
# """
function behavior_to_strategy(num_in, num_out, gen_behavior)
    behavior = []
    if length(gen_behavior) == num_out*num_in
        behavior = gen_behavior
    elseif length(gen_behavior) - 1 == num_out*num_in
        behavior = gen_behavior[2:end]
    else
        throw(DomainError(gen_behavior, "./src/BellComm/LocalPolytope.jl: generalized behavior does not have proper dimensions"))
    end

    strategy = reshape(behavior, num_in, num_out)'

    strategy
end

"""
    facet_to_matrix(num_in, num_out, gen_facet)

transforms the vector form of a facet to the minimal matrix with non-negative
elements.
"""
function facet_to_matrix(num_in, num_out, gen_facet)

    bound = -1*gen_facet[1]

    matrix = behavior_to_strategy(num_in, num_out, gen_facet[2:end])

    new_matrix = zeros(num_out,num_in)
    for col_id in 1:num_in
        col = matrix[:,col_id]

        col_min = min(col...)

        new_col = col
        if col_min != 0
            new_col = -1*col_min .+ col

            bound = bound + -1*col_min
        end

        new_matrix[:,col_id] = new_col
    end

    (bound, new_matrix)
end


abstract type AbstractTwoDimCA end
abstract type AbstractGameOfLife <: AbstractTwoDimCA end

struct GameOfLife{T<:Bool} <: AbstractGameOfLife
    generations::Int
    all_runs::AbstractArray{T}
end

"""
    function GameOfLife(initial_state::AbstractArray{T}, generations::Int)

Given a starting matrix return the evolution for given generations of Conway's Game of Life, as described in [1].

[1] Gardner, Martin. “Mathematical games: The fantastic combinations of John Conway’s new solitaire game “life”.” Scientific American 223.4 (1970): 120-123.
"""
function GameOfLife(initial_state::AbstractArray{T},
        generations::Int) where T<: Bool

    height, width = size(initial_state)

    all_runs = Array{T}(undef, height, width, generations)
    all_runs[:, :, 1] = initial_state

    for g = 2:generations
        for i = 1:height, j = 1:width
            live_neighbours = 0

            for ud = (i-1):(i+1), lr = (j-1):(j+1)
                #pbc
                if ud < 1; ud = height-ud; end
                if ud > height; ud = ud-height; end
                if lr < 1; lr = width-lr; end
                if lr > width; lr = lr-width; end

                if all_runs[ud, lr, g-1] == 1; live_neighbours+=1; end
            end

            all_runs[i, j, g] = apply_standard_rules(all_runs[i, j, g-1], live_neighbours)
        end
    end
    return GameOfLife{T}(generations, all_runs)
end

function apply_standard_rules(state::T,
        live_neighbours::Int) where T <: Bool

    if state == 1
        if live_neighbours == 3 || live_neighbours == 4
            return 1
        else
            return 0
        end
    elseif state == 0
        if live_neighbours == 3
            return 1
        else
            return 0
        end
    end
end


func(x) = sin(2pi * x)

fs, mins, maxs = univariatematrix(func, -.1, .25, 0., .1)

linprog(fs, [], '<', [], mins, maxs)


export Poisson_Regression, predict, fit

type poissonRegression
    #array of weights
    w
    #w0 constnat
    w_0::Float64
    #learning rate
    eta::Float64
    #decay rate for the learning rate
    alpha::Float64
    #number of samples parsed through (will be incremnted automatically)
    n::Int64
    #number of features
    d::Int64
end

#constructor for convenience
function Poisson_Regression(;eta::Float64=0.0001, alpha::Float64=1.00, d::Int64=0)
    if d != 0
        model = poissonRegression(zeros(d), 0.0, eta, alpha, 0, d)
    else
        model = poissonRegression(nothing, 0.0, eta, alpha, 0, d)
    end

    #return the model
    model
end

#prediction using n examles (nxd matrix)
function predict(model::poissonRegression, x::Array{Float64,2})
    linear_prediction = x*model.w+model.w_0
    prediction = exp(linear_prediction)
end

#prediction using 1xd array
function predict(model::poissonRegression, x::Array{Float64,1})
    linear_prediction = x*model.w+model.w_0
    prediction = exp(linear_prediction)
end

#takes in nxd batch data as an input, conducts stochastic gradient descent
function fit(model::poissonRegression, y::Array{Float64, 1}, x::Array{Float64, 2})
    #checking if y and x match in size
    if length(y)!=size(x)[1]
        return nothing
    end

    #initiating weight array if necessary
    if model.d == 0
        model.d = size(x)[2]
        model.w = zeros(model.d)
    end

    #updating info (right now its just the number of examples parsed)
    model.n += length(y)
    num_data = length(y)

    #making prediction
    prediction = predict(model, x)

    #updating w_0
    model.w_0 = model.w_0 + model.eta*(sum(y-prediction, 1)[1]*1.0/num_data)

    #updating w
    model.w = model.w + model.eta*((transpose(x)*(y-prediction))/num_data)

    model
end


# This file includes to blocks used in a digital communication system.

export Detector, isml

"""
    $TYPEDEF

Baseband optimal detector.

!!! note
    If elements of `probs` are equal to each other, than detector detects symbols in the maximum likelihood (ML)
    mode. Otherwise, it detects in the maximum a posterior probabilities (MAP) mode.

# Fields

    $TYPEDFIELDS
"""
struct Detector{T}
    "Basis reference of the detector"
    refs::T
    "A priori probabilities of the message symbols"
    probs::Vector{Float64}
    "2 times the power spectral density of channel noise"
    N0::Float64
end
Detector(refs) = Detector(refs, fill(1 / length(refs), length(refs)), NaN)

"""
    $SIGNATURES

Returns true if `detector` is a maximum likelihood detector, i.e, a priori probabilities of its reference refs are
equal.
"""
isml(detector::Detector) = (probs = detector.probs; all(probs .≈ probs[1]))

(detector::Detector)(rx) = isml(detector) ? mldetect(detector, rx) : mapdetect(detector, rx)

function mapdetect(detector::Detector, rx)
    Es = 1 / 2 * energy.(detector.refs)
    Pm = detector.N0 * log.(detector.probs)
    map(rx) do ri
        argmax(map(s -> real(ri ⋅ s), detector.refs) - Es + Pm)
    end
end

function mldetect(detector::Detector, rx)
    Es = 1 / 2 * energy.(detector.refs)
    map(rx) do ri
        argmax(map(s -> real(ri ⋅ s), detector.refs) - Es)
    end
end


@testset "Tropical Curve" begin
    @polyvar x y

    g = Tropical(0) + x^3 + y^3 + log(2)*x*y
    t = TropicalCurve(g)
    n3 = [0.0, -log(2)]
    n2 = [-log(2), 0.0]
    n1 = [log(2), log(2)]
    @test vertices(t)[1] ≈ n1
    @test vertices(t)[2] ≈ n3
    @test vertices(t)[3] ≈ n2

    @test segments(t)[1][1] ≈ n1
    @test segments(t)[1][2] ≈ n3
    @test segments(t)[2][1] ≈ n1
    @test segments(t)[2][2] ≈ n2
    @test segments(t)[3][1] ≈ n3
    @test segments(t)[3][2] ≈ n2

    @test length(halfrays(t)) == 3

    # c = ArchTrop(1 + x + y + x * y + y^2 + x^2 * y).curve
    # @test length(halfrays(c)) == 4
    # @test length(vertices(c)) == 1
end


mutable struct _Linear <: DiffableFunction
    grad_field :: GradField
end

function forward(::_Linear, x, W, b)
    return x * W .+ b
end

function backward(f::_Linear, gy)
    x, W, b = f.grad_field.inputs
    gx = matmul(gy, transpose(W))
    gW = matmul(transpose(x), gy)
    gb = sum_to(gy, size(b.values))
    return gx, gW, gb
end

linear(x, W, b) = _Linear(GradField())(x, W, b)


@doc raw"""
    CholeskySpace{N} <: AbstractManifold{ℝ}

The manifold of lower triangular matrices with positive diagonal and
a metric based on the cholesky decomposition. The formulae for this manifold
are for example summarized in Table 1 of [^Lin2019].

# Constructor

    CholeskySpace(n)

Generate the manifold of $n× n$ lower triangular matrices with positive diagonal.

[^Lin2019]:
    > Lin, Zenhua: "Riemannian Geometry of Symmetric Positive Definite Matrices via
    > Cholesky Decomposition", arXiv: [1908.09326](https://arxiv.org/abs/1908.09326).
"""
struct CholeskySpace{N} <: AbstractManifold{ℝ} end

CholeskySpace(n::Int) = CholeskySpace{n}()

@doc raw"""
    check_point(M::CholeskySpace, p; kwargs...)

Check whether the matrix `p` lies on the [`CholeskySpace`](@ref) `M`, i.e.
it's size fits the manifold, it is a lower triangular matrix and has positive
entries on the diagonal.
The tolerance for the tests can be set using the `kwargs...`.
"""
function check_point(M::CholeskySpace, p; kwargs...)
    if size(p) != representation_size(M)
        return DomainError(
            size(p),
            "The point $(p) does not lie on $(M), since its size is not $(representation_size(M)).",
        )
    end
    if !isapprox(norm(strictlyUpperTriangular(p)), 0.0; kwargs...)
        return DomainError(
            norm(UpperTriangular(p) - Diagonal(p)),
            "The point $(p) does not lie on $(M), since it strictly upper triangular nonzero entries",
        )
    end
    if any(diag(p) .<= 0)
        return DomainError(
            min(diag(p)...),
            "The point $(p) does not lie on $(M), since it hast nonpositive entries on the diagonal",
        )
    end
    return nothing
end

"""
    check_vector(M::CholeskySpace, p, X; kwargs... )

Check whether `v` is a tangent vector to `p` on the [`CholeskySpace`](@ref) `M`, i.e.
after [`check_point`](@ref)`(M,p)`, `X` has to have the same dimension as `p`
and a symmetric matrix.
The tolerance for the tests can be set using the `kwargs...`.
"""
function check_vector(M::CholeskySpace, p, X; kwargs...)
    if size(X) != representation_size(M)
        return DomainError(
            size(X),
            "The vector $(X) is not a tangent to a point on $(M) since its size does not match $(representation_size(M)).",
        )
    end
    if !isapprox(norm(strictlyUpperTriangular(X)), 0.0; kwargs...)
        return DomainError(
            norm(UpperTriangular(X) - Diagonal(X)),
            "The matrix $(X) is not a tangent vector at $(p) (represented as an element of the Lie algebra) since it is not lower triangular.",
        )
    end
    return nothing
end

@doc raw"""
    distance(M::CholeskySpace, p, q)

Compute the Riemannian distance on the [`CholeskySpace`](@ref) `M` between two
matrices `p`, `q` that are lower triangular with positive diagonal. The formula
reads

````math
d_{\mathcal M}(p,q) = \sqrt{\sum_{i>j} (p_{ij}-q_{ij})^2 +
\sum_{j=1}^m (\log p_{jj} - \log q_{jj})^2
}
````
"""
function distance(::CholeskySpace, p, q)
    return sqrt(
        sum((strictlyLowerTriangular(p) - strictlyLowerTriangular(q)) .^ 2) +
        sum((log.(diag(p)) - log.(diag(q))) .^ 2),
    )
end

@doc raw"""
    exp(M::CholeskySpace, p, X)

Compute the exponential map on the [`CholeskySpace`](@ref) `M` emanating from the lower
triangular matrix with positive diagonal `p` towards the lower triangular matrix `X`
The formula reads

````math
\exp_p X = ⌊ p ⌋ + ⌊ X ⌋ + \operatorname{diag}(p)
\operatorname{diag}(p)\exp\bigl( \operatorname{diag}(X)\operatorname{diag}(p)^{-1}\bigr),
````

where $⌊\cdot⌋$ denotes the strictly lower triangular matrix,
and $\operatorname{diag}$ extracts the diagonal matrix.
"""
exp(::CholeskySpace, ::Any...)

function exp!(::CholeskySpace, q, p, X)
    q .= (
        strictlyLowerTriangular(p) +
        strictlyLowerTriangular(X) +
        Diagonal(diag(p)) * Diagonal(exp.(diag(X) ./ diag(p)))
    )
    return q
end

@doc raw"""
    inner(M::CholeskySpace, p, X, Y)

Compute the inner product on the [`CholeskySpace`](@ref) `M` at the
lower triangular matric with positive diagonal `p` and the two tangent vectors
`X`,`Y`, i.e they are both lower triangular matrices with arbitrary diagonal.
The formula reads

````math
g_p(X,Y) = \sum_{i>j} X_{ij}Y_{ij} + \sum_{j=1}^m X_{ii}Y_{ii}p_{ii}^{-2}
````
"""
function inner(::CholeskySpace, p, X, Y)
    return (
        sum(strictlyLowerTriangular(X) .* strictlyLowerTriangular(Y)) +
        sum(diag(X) .* diag(Y) ./ (diag(p) .^ 2))
    )
end

@doc raw"""
    log(M::CholeskySpace, X, p, q)

Compute the logarithmic map on the [`CholeskySpace`](@ref) `M` for the geodesic emanating
from the lower triangular matrix with positive diagonal `p` towards `q`.
The formula reads

````math
\log_p q = ⌊ p ⌋ - ⌊ q ⌋ + \operatorname{diag}(p)\log\bigl(\operatorname{diag}(q)\operatorname{diag}(p)^{-1}\bigr),
````

where $⌊\cdot⌋$ denotes the strictly lower triangular matrix,
and $\operatorname{diag}$ extracts the diagonal matrix.
"""
log(::Cholesky, ::Any...)

function log!(::CholeskySpace, X, p, q)
    return copyto!(
        X,
        strictlyLowerTriangular(q) - strictlyLowerTriangular(p) +
        Diagonal(diag(p) .* log.(diag(q) ./ diag(p))),
    )
end

@doc raw"""
    manifold_dimension(M::CholeskySpace)

Return the manifold dimension for the [`CholeskySpace`](@ref) `M`, i.e.

````math
    \dim(\mathcal M) = \frac{N(N+1)}{2}.
````
"""
@generated manifold_dimension(::CholeskySpace{N}) where {N} = div(N * (N + 1), 2)

@doc raw"""
    reporesentation_size(M::CholeskySpace)

Return the representation size for the [`CholeskySpace`](@ref)`{N}` `M`, i.e. `(N,N)`.
"""
@generated representation_size(::CholeskySpace{N}) where {N} = (N, N)

Base.show(io::IO, ::CholeskySpace{N}) where {N} = print(io, "CholeskySpace($(N))")

# two small helpers for strictly lower and upper triangulars
strictlyLowerTriangular(p) = LowerTriangular(p) - Diagonal(diag(p))

strictlyUpperTriangular(p) = UpperTriangular(p) - Diagonal(diag(p))

@doc raw"""
    vector_transport_to(M::CholeskySpace, p, X, q, ::ParallelTransport)

Parallely transport the tangent vector `X` at `p` along the geodesic to `q`
on the [`CholeskySpace`](@ref) manifold `M`. The formula reads

````math
\mathcal P_{q←p}(X) = ⌊ X ⌋
+ \operatorname{diag}(q)\operatorname{diag}(p)^{-1}\operatorname{diag}(X),
````

where $⌊\cdot⌋$ denotes the strictly lower triangular matrix,
and $\operatorname{diag}$ extracts the diagonal matrix.
"""
vector_transport_to(::CholeskySpace, ::Any, ::Any, ::Any, ::ParallelTransport)

function vector_transport_to!(::CholeskySpace, Y, p, X, q, ::ParallelTransport)
    return copyto!(Y, strictlyLowerTriangular(p) + Diagonal(diag(q) .* diag(X) ./ diag(p)))
end

@doc raw"""
    zero_vector(M::CholeskySpace, p)

Return the zero tangent vector on the [`CholeskySpace`](@ref) `M` at `p`.
"""
zero_vector(::CholeskySpace, ::Any...)

zero_vector!(M::CholeskySpace, X, p) = fill!(X, 0)


using Distributions
using Gadfly
using RDatasets

function spiral(radius, theta, sequence)
    rs = map(radius, sequence)
    ts = map(theta, sequence)
    xs = map((r, t) -> r * sin(t), rs, ts)
    ys = map((r, t) -> r * cos(t), rs, ts)

    return [xs ys]
end

typealias Weight Float64
typealias Value Float64
typealias Point Vector{Value}
typealias Class Matrix{Value}
typealias DataSet Vector{Class}

sigmoid = n -> 1 / (1 + exp(-n))

unvec = (x) -> reshape(x, 1, length(x))

function computeOutput(weights, input)
    output = Matrix{Value}[input]
    for w in weights
        push!(output, map(sigmoid, last(output) * w))
    end
    return output
end

function classify(weights, input)
    output = input
    for w in weights
        output = map(sigmoid, output * w)
    end
    return indmax(output)
end

function mse(target, output)
    diff = target - output
    return sum(diff * diff) / 2
end

function learn(weights, input, target, learningRate = 0.8)
    n = size(weights, 1)
    outputs = computeOutput(weights, input)
    deltas = fill(Matrix{Value}(1, 1), n)

    # tabulate the output layer first
    o = last(outputs)
    deltas[n] = (target - o) .* o .* (1 - o)

    # tabulate from output to input
    for i in (n-1):-1:1
        o = outputs[i+1]
        w = weights[i+1]
        d = deltas[i+1]
        sums = (w * d')'
        deltas[i] = sums .* o .* (1 - o)
    end

    for i in 1:n
        delta = [d * o for o = outputs[i], d = deltas[i]]
        if (all(abs(delta) .< 0.0001))
            weights[i] = weights[i] + delta * learningRate * 1000
        else
            weights[i] = weights[i] + delta * learningRate
        end
    end

    return Dict(
        "weights" => weights,
        "error" => sum(map(mse, target, last(outputs)))
    )
end

function train(weights, dataset, iteration = 1000)
    n = size(dataset, 1)
    totalSize = sum([size(class, 1) for class = dataset])
    errors = []
    for i in 1:iteration
        accumulatedError = 0
        for (j, class) in enumerate(dataset)    # for each class
            target = zeros((1, n))
            target[j] = 1
            m = size(class, 1)
            for k in 1:m
                result = learn(weights, [class[k,:] 1], target)    # augmented
                weights = result["weights"]
                accumulatedError = accumulatedError + result["error"]
            end
        end
        push!(errors, accumulatedError)
        if (accumulatedError < 0.005 * totalSize)
            break
        end
        if (i % 100 == 0)
            @show (i, last(errors))
        end
    end
    return Dict(
        "weights" => weights,
        "error" => errors
    )
end

function generateRandomWeights(layers)
    n = size(layers, 1)
    w = Array{Weight}[]
    for i in 1:(n-1)
        s = layers[i] * layers[i+1]
        push!(w, reshape(rand(s), layers[i], layers[i+1]))
    end
    return w
end

function generateDataSet(dataset)
    result = DataFrame(x = [], y = [], class = [])
    for (i, class) in enumerate(dataset)
        result = [result; DataFrame(x = class[:,1], y = class[:,2], class = i)]
    end
    return result
end

function generateGrid(seq, weights)
    n = length(seq)
    xs = repeat(seq, inner=[n])
    ys = repeat(seq, outer=[n])
    rs = map((x, y) -> classify(weights, [x y 1]), xs, ys)
    return DataFrame(x = xs, y = ys, class = rs)
end

function drawPoint(data)
    return layer(data, x = "x", y = "y", color = "class", Geom.point,
        Theme(default_point_size = 3px, highlight_width = 1.5px))
end

function drawGrid(data)
    return layer(data, x = "x", y = "y", color = "class", Geom.rectbin)
end

function drawError(data)
    x = collect(1:length(data))
    return layer(x = x, y = data, Geom.line)
end

################################################################################
#   2
################################################################################

# PDF
spiralPDFS = Cairo.CairoPDFSurface("spiral.pdf", 600, 600)
spiralPDFC = Cairo.CairoContext(spiralPDFS)
spiralPDF = Compose.CAIROSURFACE(spiralPDFS)

# data points
spiral0 = spiral(i -> 6.5 * (104 - i) / 104,
    i -> pi * i / 16,
    40:96)

spiral1 = spiral(i -> 6.5 * (104 - i) / -104,
    i -> pi * i / 16,
    40:96)
spiralDataPoints = Class[spiral0, spiral1]

# weights
spiralWeights = generateRandomWeights([3, 80, 20, 10, 2])
spiralResult = train(spiralWeights, spiralDataPoints, 10000)
spiralWeights = spiralResult["weights"]
spiralErrors = spiralResult["error"]

# plot spirals
spiralDataSet = generateDataSet(spiralDataPoints)
spiralDataSetLayer = drawPoint(spiralDataSet)

# plot the grid
spiralGrid = generateGrid(collect(-8:0.1:8), spiralWeights)
spiralGridLayer = drawGrid(spiralGrid)

spiralPlot = plot(spiralDataSetLayer, spiralGridLayer,
    Coord.cartesian(fixed = true),
    Guide.title("The Two Spirals Problem"),
    Scale.color_discrete_manual(colorant"coral", colorant"cornflowerblue"))

spiralErrorsLayer = drawError(spiralErrors)
spiralErrorsPlot = plot(spiralErrorsLayer,
    Coord.Cartesian(ymin = 0),
    Guide.ylabel("Error"),
    Guide.xlabel("Iteration"))

draw(spiralPDF, spiralPlot)
Cairo.show_page(spiralPDFC)
draw(spiralPDF, spiralErrorsPlot)
Cairo.finish(spiralPDFS)

################################################################################
#   2
################################################################################

moonsPDFS = Cairo.CairoPDFSurface("moons.pdf", 600, 600)
moonsPDFC = Cairo.CairoContext(moonsPDFS)
moonsPDF = Compose.CAIROSURFACE(moonsPDFS)

# the double moon problem
n = 200
thetas = linspace(-180, 180, n)*pi/360;
r = 8
x0 = -5 + r * sin(thetas)' + randn(1, n)
y0 =      r * cos(thetas)' + randn(1, n)
x1 =  5 + r * sin(thetas)' + randn(1, n)
y1 =     -r * cos(thetas)' + randn(1, n)
moons = Class[[vec(x0) vec(y0)], [vec(x1) vec(y1)]]

#
moonsWeights = generateRandomWeights([3, 8, 8, 2])
moonsResult = train(moonsWeights, moons, 10000)
moonsWeights = moonsResult["weights"]
moonsErrors = moonsResult["error"]

# plot moons
moonsDS = generateDataSet(moons)
moonsDSLayer = drawPoint(moonsDS)

# plot the grid
moonsGrid = generateGrid(collect(-18:0.5:18), moonsWeights)
moonsGridLayer = drawGrid(moonsGrid)

moonsPlot = plot(moonsDSLayer, moonsGridLayer,
    Coord.cartesian(fixed = true),
    Guide.title("The Double Moons Problem"),
    Scale.color_discrete_manual(colorant"coral", colorant"cornflowerblue"))

moonsErrorsLayer = drawError(moonsResult["error"])
moonsErrorsPlot = plot(moonsErrorsLayer,
    Guide.ylabel("Error"),
    Guide.xlabel("Iteration"))

draw(moonsPDF, moonsPlot)
Cairo.show_page(moonsPDFC)
draw(moonsPDF, moonsErrorsPlot)
Cairo.finish(moonsPDFS)

################################################################################
#   3
################################################################################

# PDF
mvrPDFS = Cairo.CairoPDFSurface("mvrs.pdf", 600, 600)
mvrPDFC = Cairo.CairoContext(mvrPDFS)
mvrPDF = Compose.CAIROSURFACE(mvrPDFS)

mvr0 = rand(MvNormal([0.0, 0.0], [1.0 0.0; 0.0 1.0]), 100)'
mvr1 = rand(MvNormal([10.0, 0.0], [1.0 0.0; 0.0 4.0]), 100)'
mvr2 = rand(MvNormal([5.0, 10.0], [4.0 0.0; 0.0 1.0]), 100)'
mvr3 = rand(MvNormal([5.0, 5.0], [1.0 0.0; 0.0 1.0]), 100)'
mvrDataPoints = Class[mvr0, mvr1, mvr2, mvr3]

# weights
mvrWeights = generateRandomWeights([3, 10, 10, 4])
mvrResult = train(mvrWeights, mvrDataPoints, 10000)
mvrWeights = mvrResult["weights"]
mvrErrors = mvrResult["error"]

# plot distributions
mvrDataSet = generateDataSet(mvrDataPoints)
mvrDataSetLayer = drawPoint(mvrDataSet)

# plot the grid
mvrGrid = generateGrid(collect(-8:0.5:18), mvrWeights)
mvrGridLayer = drawGrid(mvrGrid)

mvrPlot = plot(mvrDataSetLayer, mvrGridLayer,
    Coord.cartesian(fixed = true),
    Guide.title("4 Multivariate Distributions"))
    # Scale.color_discrete_manual(colorant"coral", colorant"cornflowerblue"))

mvrErrorsLayer = drawError(mvrErrors)
mvrErrorsPlot = plot(mvrErrorsLayer,
    Coord.Cartesian(ymin = 0),
    Guide.ylabel("Error"),
    Guide.xlabel("Iteration"))

draw(mvrPDF, mvrPlot)
Cairo.show_page(mvrPDFC)
draw(mvrPDF, mvrErrorsPlot)
Cairo.finish(mvrPDFS)


__precompile__()

module PolyChaosODE

using BenchmarkTools,
    DifferentialEquations,
    Distributions,
    FastGaussQuadrature,
    LaTeXStrings,
    LinearAlgebra,
    Parameters,
    PolyChaos,
    Plots,
    Statistics

include("grids.jl")
include("orthogonal_polynomial_basis.jl")
include("quadrature.jl")
include("stochastic_galerkin_ode.jl")
include("monte_carlo_gsa.jl")
include("utils.jl")

export TensorIntegrationGrid
export OrthogonalPolyBasis, LegendrePolyBasis, HermitePolyBasis
export StochGalerkinODE
export compute_expectation_and_diag_variance,
    mc_gsa_estimate_total_indices,
    plot_with_plus_minus_std,
    compute_total_order_sobol_indices

end


##################################
# Operador de cruce  por media ponderada
##############################
using Random

"""
    MediaPonderada((C1, C2)
Operador de cruce por ponderación
Genera dos hijos
"""
function MediaPonderada(C1, C2)
    a1 = rand() # valor entre cero y 1
    a2 = rand() # valor entre cero y 1
    return [a1*C1 + (1-a1)*C2, a2*C1 + (1-a2)*C2]
end


module Stepwise

using Statistics, LinearAlgebra
using Parameters
using Base: ==

export AbstractDirection, Bidirection, Unidirection, ForwardBackward, BackwardForward, Forward, Backward,
    AbstractCriterion, AIC, BIC,
    AbstractAlgorithm, StepwiseFeatureSelection,
    AbstractAlgResult, SFSResult,
    Model, Null, Full, null_or_full, ==, update!,
    stepwise

abstract type AbstractDirection end
abstract type Bidirection <: AbstractDirection end
abstract type Unidirection <: AbstractDirection end
struct ForwardBackward <: Bidirection end
struct BackwardForward <: Bidirection end
struct Forward <: Unidirection end
struct Backward <: Unidirection end

abstract type AbstractCriterion end
struct AIC <: AbstractCriterion end
struct BIC <: AbstractCriterion end
(::AIC)(𝐗, 𝐲, 𝚺⁻¹) = 𝐲' * (𝚺⁻¹ - 𝚺⁻¹ * 𝐗 * pinv(𝐗' * 𝚺⁻¹ * 𝐗) * 𝐗' * 𝚺⁻¹) * 𝐲 + 2 * size(𝐗, 2)
(::BIC)(𝐗, 𝐲, 𝚺⁻¹) = 𝐲' * (𝚺⁻¹ - 𝚺⁻¹ * 𝐗 * pinv(𝐗' * 𝚺⁻¹ * 𝐗) * 𝐗' * 𝚺⁻¹) * 𝐲 + log(length(𝐲)) * size(𝐗, 2)

abstract type AbstractAlgorithm end
@with_kw struct StepwiseFeatureSelection{S <: AbstractDirection,T <: AbstractCriterion,U <: Integer} <: AbstractAlgorithm
    direction::S = ForwardBackward()
    criterion::T = AIC()
    max_step::U = typemax(Int)
end

# define "model"
struct Null end; struct Full end
null_or_full(::Forward) = Null(); null_or_full(::ForwardBackward) = Null()
null_or_full(::Backward) = Full(); null_or_full(::BackwardForward) = Full()
struct Model{T <: Integer}
    M::Vector{T} # the set of features in this model
    Mᶜ::Vector{T} # the set of features not included in this model
    Model{T}(::Null, dims::T) where {T} = new{T}(T[], collect(T, 1:dims))
    Model{T}(::Full, dims::T) where {T} = new{T}(collect(T, 1:dims), T[])
end
Base.:(==)(m::Model, m′::Model) = issetequal(m.M, m′.M) && issetequal(m.Mᶜ, m′.Mᶜ)
Model(direction::AbstractDirection, dims::T) where {T} = Model{T}(null_or_full(direction), dims)
## update model
function update!(model::Model, idx)
    @unpack M, Mᶜ = model
    if idx > 0
        @assert idx ∈ Mᶜ
        push!(M, idx); setdiff!(Mᶜ, idx)
    else
        @assert -idx ∈ M
        setdiff!(M, -idx); push!(Mᶜ, -idx)
    end
end

abstract type AbstractAlgResult end
struct SFSResult{S,T} <: AbstractAlgResult
    model::Model{S}
    history::T
end
function Base.show(io::IO, ::MIME"text/plain", result::SFSResult{S,T}) where {S,T}
    println(io, "SFSResult{$S,$T}")
    println(io, " selected features: ", result.model.M)
    println(io, " history of this algorithm: ", result.history)
end

# stepwise feature selection
function stepwise(𝐗, 𝐲, 𝚺, direction::Bidirection, criterion::AbstractCriterion; max_step=typemax(Int))
    𝚺⁻¹ = I / 𝚺
    d = size(𝐗, 2)
    model = Model(direction, d)
    history = typeof(d)[]
    L_prev = Inf

    for i = 1:max_step
        @unpack M, Mᶜ = model
        minL = Inf
        idx = 0
        # delete
        for j ∈ M
            L = criterion(𝐗[:, setdiff(M, j)], 𝐲, 𝚺⁻¹)
            L < minL && (minL = L; idx = -j)
        end
        # add
        for j′ ∈ Mᶜ
            L = criterion(𝐗[:, union(M, j′)], 𝐲, 𝚺⁻¹)
            L < minL && (minL = L; idx = +j′)
        end

        # if the criterion is not improved, then finish this algorithm
        L_prev ≤ minL && (return SFSResult(model, history))

        # update
        L_prev = minL
        push!(history, idx)
        update!(model, idx)
    end
    return SFSResult(model, history)
end
stepwise(𝐗, 𝐲_obs, 𝚺) = stepwise(𝐗, 𝐲_obs, 𝚺, ForwardBackward(), AIC()) # default
function stepwise(𝐗, 𝐲, 𝚺, direction::Forward, criterion::AbstractCriterion; max_step=typemax(Int))
    𝚺⁻¹ = I / 𝚺
    d = size(𝐗, 2)
    model = Model(direction, d)
    history = typeof(d)[]
    L_prev = Inf

    for i = 1:max_step
        @unpack M, Mᶜ = model
        minL = Inf
        idx = 0
        # add
        for j′ ∈ Mᶜ
            L = criterion(𝐗[:, union(M, j′)], 𝐲, 𝚺⁻¹)
            L < minL && (minL = L; idx = +j′)
        end

        # if the criterion is not improved, then finish this algorithm
        L_prev ≤ minL && (return SFSResult(model, history))

        # update
        L_prev = minL
        push!(history, idx)
        update!(model, idx)
    end
    return SFSResult(model, history)
end
function stepwise(𝐗, 𝐲, 𝚺, direction::Backward, criterion::AbstractCriterion; max_step=typemax(Int))
    𝚺⁻¹ = I / 𝚺
    d = size(𝐗, 2)
    model = Model(direction, d)
    history = typeof(d)[]
    L_prev = Inf

    for i = 1:max_step
        M = model.M
        minL = Inf
        idx = 0
        # delete
        for j ∈ M
            L = criterion(𝐗[:, setdiff(M, j)], 𝐲, 𝚺⁻¹)
            L < minL && (minL = L; idx = -j)
        end

        # if the criterion is not improved, then finish this algorithm
        L_prev ≤ minL && (return SFSResult(model, history))

        # update
        L_prev = minL
        push!(history, idx)
        update!(model, idx)
    end
    return SFSResult(model, history)
end
function stepwise(𝐗, 𝐲, 𝚺, alg::StepwiseFeatureSelection)
    @unpack direction, criterion, max_step = alg
    stepwise(𝐗, 𝐲, 𝚺, direction, criterion; max_step=max_step)
end # wrapper of stepwise

end # module Stepwise


# four1 test

Fs = 1000;            # Sampling frequency
T = 1/Fs;             # Sampling period

function prepare_input(input_size)
	t = (0:input_size-1)*T;        # Time vector
	S = zeros(2^ceil(log2(input_size)))
	S[1:input_size] = 0.7*sin.(2pi*50t) + sin.(2pi*120t) + 2*randn(size(t));
	return S
end

function test_function(input_data)
	isign = 1
	nn = length(input_data)
	nnlog = log2(nn)
	if ceil(nnlog) != nnlog
		error("Length of input_data vector must be a power of 2.")
	end
	j::Int64 = 1
	res::Array{Complex{Float64},1} = copy(input_data)

	for i = 1:nn
		if j > i
			res[j], res[i] = input_data[i], input_data[j]
		end
		m = nn/2
		while m >= 2 && j > m
			j -= m
			m = m/2
		end
		j += m
	end

	mmax = 1;
	while nn > mmax
		istep = 2*mmax
		theta = isign*(pi/mmax)
		wp = -2*sin(theta/2)^2 + sin(theta)*im
		w = 1 + 0im

		for m = 1:mmax
			for i = m:istep:nn
				j = i + mmax
				temp = w*res[j]
				res[j] = res[i] - temp
				res[i] += temp
			end
			w += w*wp
		end
		mmax = istep
	end

	return res
end


using VortexDistributions
using Test, Plots

    # simple test field
    Nx = 400; Ny = 400
    Lx = 200; Ly = Lx
    x = LinRange(-Lx/2,Lx/2, Nx+1)[1:end-1]; y = LinRange(-Ly/2,Ly/2, Ny+1)[1:end-1];
    psi0 = one.(x*y') |> complex
    psi = Torus(copy(psi0),x,y)
    xp = 1.133
    yp = 2.787
    vp = PointVortex(xp,yp,1)
    sp = ScalarVortex(vp)

## distant dipole 1
    vp = PointVortex(-60,-60,1)
    vn = PointVortex(-60,60,-1)
    psi2 = Torus(copy(psi0),x,y)
    dip2 = ScalarVortex([vp,vn])
    periodic_dipole!(psi2,dip2)
    heatmap(x,y,abs2.(transpose(psi2.ψ)))
    heatmap(x,y,angle.(transpose(psi2.ψ)))

    vort = findvortices(psi2)

## distant dipole 2
    vp = PointVortex(-60.1,-60,1)
    vn = PointVortex(60,-60,-1)
    psi2 = Torus(copy(psi0),x,y)
    dip2 = ScalarVortex([vp,vn])
    periodic_dipole!(psi2,dip2)
    heatmap(x,y,abs2.(transpose(psi2.ψ)))
    heatmap(x,y,angle.(transpose(psi2.ψ)))

## distant dipole 3
    vp = PointVortex(-61,-60,1)
    vn = PointVortex(60,60,-1)
    psi2 = Torus(copy(psi0),x,y)
    dip2 = ScalarVortex([vp,vn])
    periodic_dipole!(psi2,dip2)
    heatmap(x,y,abs2.(transpose(psi2.ψ)))
    heatmap(x,y,angle.(transpose(psi2.ψ)))

## close dipole (works!)
    vp = PointVortex(20,10,1)
    vn = PointVortex(-10,.7,-1)
    dip = Dipole(vp,vn)
    @test typeof(dip) <: VortexGroup
    # simple test field
    Nx = 400; Ny = 400
    Lx = 200; Ly = Lx
    x = LinRange(-Lx/2,Lx/2, Nx+1)[1:end-1];
    y = LinRange(-Ly/2,Ly/2, Ny+1)[1:end-1];
    psi0 = one.(x*y') |> complex
    psi = Torus(copy(psi0),x,y)
    dip2 = ScalarVortex([vp,vn])
    periodic_dipole!(psi,dip2)
    heatmap(x,y,abs2.(psi.ψ))
    heatmap(x,y,angle.(transpose(psi.ψ)))

## close dipole
    vp = PointVortex(-6.1,-6,1)
    vn = PointVortex(6,6,-1)
    psi2 = Torus(copy(psi0),x,y)
    dip2 = ScalarVortex([vp,vn])
    periodic_dipole!(psi2,dip2)
    heatmap(x,y,abs2.(psi2.ψ))
    heatmap(x,y,angle.(transpose(psi2.ψ)))

## large dipole. branch cut?
    vp = PointVortex(80,70,1)
    vn = PointVortex(-40,-30,-1)
    dip = Dipole(vp,vn)
    @test typeof(dip) <: VortexGroup
    # simple test field
    Nx = 400; Ny = 400
    Lx = 200; Ly = Lx
    x = LinRange(-Lx/2,Lx/2, Nx+1)[1:end-1]
    y = LinRange(-Ly/2,Ly/2, Ny+1)[1:end-1]
    psi0 = one.(x*y') |> complex
    psi = Torus(copy(psi0),x,y)
    dip2 = ScalarVortex([vp,vn])
    periodic_dipole!(psi,dip2)
    heatmap(x,y,abs2.(transpose(psi.ψ)))
    heatmap(x,y,angle.(transpose(psi.ψ)))

## summary:
# typo in Eq


#!/usr/bin/env julia

#load path to qjulia home directory
push!(LOAD_PATH, joinpath(@__DIR__, "..", "matrix"))

using MatrixBase
using MatrixMarket
using LinearAlgebra
using IterativeSolvers
using LinearMaps
using Random

#matrix_path = "/home/astrel/data/nasa2146/nasa2146.mtx"
#matrix_path = "/home/astrel/data/nasa2910/nasa2910.mtx"
matrix_path = "/home/astrel/data/nasa4704/nasa4704.mtx"
#matrix_path = "/home/astrel/data/smt/smt.mtx"

Random.seed!(2018)

csrM = MatrixBase.CSRMat{Float64}(matrix_path)

MatrixBase.print_CSRMat_info(csrM)

data_type = typeof(csrM.csrVals[1])

b = Vector{data_type}(undef, csrM.Dims[2])
x = Vector{data_type}(undef, csrM.Dims[2])
z = Vector{data_type}(undef, csrM.Dims[2])

x .= 0.0
z .= 0.0
b .= rand(data_type, csrM.Dims[2])

M(v::Vector{data_type}) = MatrixBase.csrmv(csrM, v)

A = LinearMaps.LinearMap{data_type}(M, nothing, csrM.Dims[1], csrM.Dims[2]; issymmetric=true )
#or just:
#M=MatrixMarket.mmread(matrix_path)

@time x, ch = IterativeSolvers.cg!(x, A, b; maxiter=30000,verbose=true, log=true)

println(ch)

#compute residual:
z=M(x)
z .= b - z

println("Solution norm: ", sqrt(dot(b,b)), ", absolute residual: ", sqrt(dot(z,z)))


"""
This method is inspired by (but not an implementation of) the method found
in

Nelder-mead enhanced extreme learning machine.

Philip Reiner and Bogdan M Wilamowski.

2013 IEEE 17th International Conference on Intelligent Engineering Systems
(INES), 2013 pp. 225-230.

http://ieeexplore.ieee.org/lpdocs/epic03/wrapper.htm?arnumber=6632816

"""
mutable struct OptimELM{TV,TN} <: AbstractSLFN where TV <: AbstractArray{Float64} where TN <: RBF
    p::Int  # Number of training points
    q::Int  # Dimensionality of function domain
    s::Int  # number of neurons
    ϵ::Float64  # expected learning accuracy

    # internal parameters
    Wt::Matrix{Float64}  # transpose of W matrix
    d::Vector{Float64}
    neuron_type::TN
    v::TV
    E::TV  # running error

    function OptimELM(p::Int, q::Int, s::Int, ϵ::Float64, neuron_type::TN) where TN
        Wt = 2*rand(q, s) - 1
        d = rand(s)
        v = TV(s)
        new{typeof(v), TN}(p, q, s, ϵ, Wt, d, neuron_type, v)
    end
end

function OptimELM(x::AbstractArray, y::AbstractArray;
                  s::Int=size(y, 1), ϵ::Float64=1e-6,
                  neuron_type::TN=RBF(Gaussian())) where TN <: RBF
    q = size(x, 2)  # dimensionality of function domain
    p = size(y, 1)  # number of training points
    s = min(p, s)   # Can't have more neurons than training points
    out = OptimELM(p, q, s, ϵ, neuron_type)
    out.E = copy(y)
    fit!(out, x, y)
end

function fit!(elm::OptimELM, x::AbstractArray, y::AbstractArray)
    # ------- Step 1: initialize
    n = 0
    err = Inf
    xt = x'
    E = elm.E

    # ------- Step 2: Learning
    while n < elm.s && err > elm.ϵ
        # --- Step 2.a: Increase n
        n += 1

        # --- Step 2.b: find index of max error
        j = argmax(abs.(E))

        # --- Step 2.c: Assign new center to be input x
        cn = xt[:, j]
        elm.Wt[:, n] = cn

        # --- Step 2.d: Initialze β = Eⱼ
        βn = E[j]

        # --- Step 2.e: define objective function
        function obj(σ)
            sse = 0.0
            for i in 1:elm.p
                gi = hidden_out(elm.neuron_type, view(xt, :, i), cn, σ)
                sse += (E[i] - βn * gi)^2
            end
            sse
        end

        # Step 2.f: do some iterations of an optimization algorithm
        res = Optim.optimize(obj, 1e-5, 1e2)
        elm.d[n] = Optim.minimizer(res)

        # Step 2.g: re-calculate vn
        num = 0.0
        denom = 0.0
        for i in 1:elm.p
            gp = hidden_out(elm.neuron_type, view(xt, :, i), cn, elm.d[n])
            num += E[i] * gp
            denom += gp*gp
        end
        elm.v[n] = num/denom

        # Step 2.h: update error trace
        for i in 1:elm.p
            E[i] -= elm.v[n] * hidden_out(elm.neuron_type, view(xt, :, i), cn, elm.d[n])
        end

        # Step 2.i: update err
        err = maximum(abs.(E))

    end

    # trim in case we achieved desired accuracy before adding elm.s neurons
    if n < elm.s
        elm.v = elm.v[1:n]
        elm.d = elm.d[1:n]
        elm.Wt = elm.Wt[:, 1:n]
    end

    return elm

end

function Base.show(io::IO, elm::OptimELM{TA,TN}) where TA where TN
    m = match(r"SLFN.RBF\{SLFN\.(.+),Distances\.(.+)\}", string(TN))
    ta, td = m[1], m[2]
    n = size(elm.v, 1)
    s = """
    OptimELM with
      - $(elm.q) input dimension(s)
      - $n RBF{$ta,$td} neuron$(n>1 ? "s" : "")
      - $(elm.p) training point(s)
    """
    print(io, s)
end


module Quadhps
using FastGaussQuadrature, FastClosures, StaticArrays

struct _Helper{T}
  abstol::T
  reltol::T
  max_depth::Int
  split::Int
  function _Helper(abstol::T, reltol::T, max_depth::Int, split::Int
      ) where {T<:Real}
    @assert split > 0 && max_depth >= 0 "$split, $max_depth"
    return new{T}(abstol, reltol, max_depth, split)
  end
end

#const memo = Dict{Int64, Tuple{Array{Float64,1},Array{Float64,1},
#                               Array{Float64,1}}}()
const memo = Dict()

@inline function memoisedgausslengendre(n::Int)
  if !haskey(memo, n)
    x, w = FastGaussQuadrature.gausslegendre(n)
    memo[n] = (x, w)
  end
  return memo[n]
end

@inline function calculate_node_weight(h::_Helper, n::Int,
    a, b)
  x, w = memoisedgausslengendre(n)
  halfdifference = @. (b - a) / 2
  halfsum = @. (b + a) / 2
  y = @. x * halfdifference + halfsum
  return y, w, halfdifference
end

@inline function simdmapreduce(f::T, op, iter) where {T<:Function}
  @inbounds output = f(iter[1])
  @inbounds @simd for i ∈ 2:length(iter)
    output += f(iter[i])
  end
  return output
end

@inline function applyquadrature(f::T, op, x, w) where {T<:Function}
  @inbounds output = f(x[1]) * w[1]
  @inbounds @simd for i ∈ 2:length(w)
    output += f(x[i]) * w[i]
  end
  return output
end

@inline function quad(f::T, h::_Helper, a, b, n::Int
    ) where {T<:Function}
  x, w, normalisation = calculate_node_weight(h, n, a, b)
  return applyquadrature(f, +, x, w) * normalisation
end

@inline function quad(f::T, h::_Helper, a, b, c, n::Int
    ) where {T<:Function}
  @assert a < b < c "$a, $b, $c"
  U = promote_type(typeof(a), typeof(b), typeof(c))
  @assert isapprox(b .- a, c .- b, rtol=sqrt(eps(U)), atol=0) "$a, $b, $c"
  x, w, normalisation = calculate_node_weight(h, n, a, b)
  b_a = b .- a
  reductionop = @closure x -> (f(x) .+ f(x .+ b_a))
  return applyquadrature(reductionop, +, x, w) * normalisation
end

@inline function hasconverged(a, b, h::_Helper, l::Int,
    nrm::T) where {T<:Function}
  all(@. nrm(a - b) < h.split^l * h.reltol * nrm(a + b) / 2) && return true
  all(@. nrm(a + b) / 2 <= h.abstol) && return true
  l > h.max_depth && return true
  return false
end

function quadhp(f::T, h::_Helper, a, b,
    n::Int, l::Int, nrm::U) where {T<:Function, U<:Function}
  @assert -Inf < a < b < Inf "a = $a, b = $b"
  @assert ispow2(n) "n is $n"
  qp = quad(f, h, a, b, n)
  qh = quad(f, h, a, (b .+ a) / 2, b, Int(n / 2))
  hasconverged(qp, qh, h, l, nrm) && return @SVector [(qp + qh) / 2, @. nrm(qp .- qh)]
  reductionop = @closure i -> quadhp(f, h, a .+ i * (b .- a) / h.split,
    a .+ (i + 1) * (b .- a) / h.split, 2n, l + 1, nrm)
  return simdmapreduce(reductionop, +, 0:(h.split - 1))
end

const default_order = 32
const default_split = 2
const default_max_depth = 8

function quadhp(f::T, a, b; norm::U=abs,
                rtol::Real=eps(), atol::Real=0.0,
                order::Int=default_order, max_depth::Int=default_max_depth,
                split::Int=default_split) where {T<:Function, U<:Function}
  return quadhp(f, _Helper(atol, rtol, max_depth, split), a, b, order, 0, norm)
end

function quadhp(f::T, ab...; norm::U=abs,
                rtol::Real=eps(), atol::Real=0.0,
                order::Int=default_order, max_depth::Int=default_max_depth,
                split::Int=default_split) where {T<:Function, U<:Function}
  reductionop = @closure i -> quadhp(f, ab[i], ab[i+1], norm=norm, rtol=rtol,
    atol=atol, order=order, max_depth=max_depth, split=split)
  return simdmapreduce(reductionop, +, 1:(length(ab)-1))
end

end # module


using ComplexSystems
using Plots
gr()

fl65 = Dict{String,String}("F" => "F[-F]F[+F]F")
HW = mult_evol_lsystem("F",fl65,3)
splitted = split(HW,"");
plot();
let
    global x = 0.
    global y = 0.
    global a = 0.
    global p
    global variables=[(x,y,a)]
    Δ = 25
    global da = Δ/180*π
    for s in splitted
        if s == "F" || s == "G"
            p,x,y=turtle_step(x,y,s,a)
        elseif s == "+" || s == "-"
            a = update_angle(a,da,1,s)
        elseif s == "["
            push!(variables,(x,y,a))
            x,y,a = variables[end]
        elseif s == "]"
            x,y,a = variables[end]
            pop!(variables)
        end
        p
    end
end

mf = Dict{String,String}("F" => "FF[+F][-F]","+" =>
"+F[+2F][-2F]","-"=>"-F[-3F][+3F]")
HW2 = mult_evol_lsystem("F",mf,4);
splitted = split(HW2,"");

plot();
let
    global x = 0.
    global y = 0.
    global a = 0.
    global p
    global variables=[(x,y,a)]
    Δ = 45
    global da = Δ/180*π
    for s in splitted
        if s == "F" || s == "G"
            p,x,y=turtle_step(x,y,s,a)
        elseif s == "+" || s == "-"
            a = update_angle(a,da,1,s)
        elseif s == "["
            push!(variables,(x,y,a))
            x,y,a = variables[end]
        elseif s == "]"
            x,y,a = variables[end]
            pop!(variables)
        end
        p
    end
end


using StatsBase
using UnPack

function rand_sample(P, q)
    L = Vector{Int64}(undef, q)
    for i in 1:q
        L[i] = sample(1:length(P), Weights(P))
        # L[i] = sample(1:sizeof(P), weights(P), 1, replace=true)
    end
    return L
end

function not_in_search_space(position, lb, ub)
    return any(position .<= lb) || any(position .>= ub)
end

function CCE(complex_pops, cce_params)
    # Step 1. Initialize
    @unpack q, alpha, beta, lb, ub, obj_func = cce_params
    n_pop = length(complex_pops)

    # Step 2. Assign weights
    P = [2*(n_pop+1-i) / (n_pop*(n_pop+1)) for i in 1:n_pop]

    # Calculate Population Range (Smallest Hypercube)
    new_lb = complex_pops[1].position
    new_ub = complex_pops[1].position
    for i in 2:n_pop
        new_lb = min.(new_lb, complex_pops[i].position)
        new_ub = max.(new_ub, complex_pops[i].position)
    end

    # CCE main loop
    for it in 1:beta
        # Step 3. Select parents
        L = rand_sample(P, q)
        B = complex_pops[L]

        # Step 4. Generate Offspring
        for k in 1:alpha
            # a) Sort population
            sorted_indexs = sortperm(B)
            sort!(B)
            L[:] = L[sorted_indexs]

            # Calculate the centroid
            g = zeros(length(lb))
            for i in 1:q-1
                g .= g .+ B[i].position
            end
            g .= g ./ (q-1)

            # b) Reflection step
            reflection = deepcopy(B[end])
            reflection.position = 2 .* g .- B[end].position # newly generated point using reflection
            if not_in_search_space(reflection.position, lb, ub)
                reflection.position = uniform_rand(new_lb, new_ub)
            end
            reflection.cost = obj_func(reflection.position)

            if reflection.cost < B[end].cost
                B[end] = deepcopy(reflection)
            else # Contraction
                contraction = deepcopy(B[end])
                contraction.position = (g .+ B[end].position) ./ 2
                contraction.cost = obj_func(contraction.position)

                if contraction.cost < B[end].cost
                    B[end] = deepcopy(contraction)
                else
                    B[end].position = uniform_rand(new_lb, new_ub)
                    B[end].cost = obj_func(B[end].position)
                end
            end

        end

        complex_pops[L] = B
    end
    return complex_pops
end


module VisualTests

using Gadfly, Interpolations
const nx = 10
xg = 1:nx
xf = -1.2nx:.1:3.3nx

f1(x) = sin(2pi/nx * (x-1))

y1 = f1.(xg)

p = plot()

if true

Btypes = (Periodic, Flat, Line, Free, Reflect)
Gtypes = (OnCell, OnGrid)
degrees = (
    Constant(), Linear(),
    [Quadratic(T(G())) for T in Btypes, G in Gtypes]...,
    [Cubic(T(G())) for T in Btypes[1:end-1], G in Gtypes]...,  # no Reflect for Cubic
)
Etypes = (Flat, Line, Reflect, Periodic)

for deg in degrees, ET in Etypes
    itp = extrapolate(interpolate(y1, BSpline(deg)), ET())

    stuff = Any[]

    push!(stuff, layer(x=xg,y=y1,Geom.point,Theme(default_color=colorant"green")))
    push!(stuff, layer(x=xf,y=[itp[x] for x in xf],Geom.path,Theme(point_size=2px)))
    title = "$deg, $(nameof(ET))"
    push!(stuff, Guide.title(title))
    display(plot(stuff...))
end

nx = 10
xg = 1:nx
ny = 6
yg = 1:ny

f2(x,y) = sin(2pi/nx * (x-1)) * cos(2pi/ny * (y-1))
zg = f2.(xg, yg')

xf = -1:.1:nx+1
yf = -1:.1:ny+1

itp2 = extrapolate(interpolate(zg, BSpline(Quadratic(Flat(OnCell())))), Line())

display(plot(
    layer(x=xf,y=yf,z=[itp2[x,y] for x in xf, y in yf], Geom.contour),
    Guide.title("Quadratic(Flat(OnCell)), Line")
))

end

end


##
"""
CELL5M is a 5-minute arc resolution view of the -180,-90
lat-long system. This corresponds to a cell-size of
0.08333333333. The following code converts between cell5m
values and latlong.
"""
function cell5m_to_latlong(cell::Int)
  nrows = 4320
  ncols = 2160
  col, row = divrem(cell, nrows)
  x = row*0.0833333333333333333-(180-0.0833333333333333333/2)
  y = (90-0.0833333333333333333/2) - col*0.0833333333333333333
  return x,y
end
cell5m_to_latlong(1300706)
## Test data (from spam2010V1r1_global_H_TA)

cell5m_test = [1300706
1300707
1300708
1305026
1305027
1305028
1348258
1348259
1348260
1352579
1352580
1473490]
cell5m_ref_x = [-147.7916666668
-147.7083333335
-147.6250000001
-147.7916666668
-147.7083333335
-147.6250000001
-145.1250000001
-145.0416666668
-144.9583333335
-145.0416666668
-144.9583333335
-149.1250000001]
cell5m_ref_y = [ 64.8749999994
 64.8749999994
 64.8749999994
 64.791666666
 64.791666666
 64.791666666
 63.9583333327
 63.9583333327
 63.9583333327
 63.8749999994
 63.8749999994
 61.5416666661]
for i in 1:length(cell5m_test)
  x,y = cell5m_to_latlong(cell5m_test[i])
  println(cell5m_test[i], " ",
          x, " ", cell5m_ref_x[i], " ",
          y, " ", cell5m_ref_y[i])
  @assert (x ≈ cell5m_ref_x[i])
  @assert (y ≈ cell5m_ref_y[i])
end
##


function linsolve(operator, b, x₀, alg::CG, a₀::Real = 0, a₁::Real = 1)
    # Initial function operation and division defines number type
    y₀ = apply(operator, x₀)
    T = typeof(dot(b, y₀)/norm(b)*one(a₀)*one(a₁))
    α₀ = convert(T, a₀)
    α₁ = convert(T, a₁)
    # Continue computing r = b - a₀ * x₀ - a₁ * operator(x₀)
    r = copyto!(similar(b, T), b)
    r = α₀ == 0 ? r : axpy!(-α₀, x₀, r)
    r = axpy!(-α₁, y₀, r)
    x = copyto!(similar(r), x₀)
    normr = norm(r)
    S = typeof(normr)

    # Algorithm parameters
    maxiter = alg.maxiter
    tol::S = alg.tol
    numops = 1 # operator has been applied once to determine r
    numiter = 0

    # Check for early return
    normr < tol && return (x, ConvergenceInfo(1, r, normr, numiter, numops))

    # First iteration
    ρ = normr^2
    p = copyto!(similar(r), r)
    q = apply(operator, p)
    if α₀ != zero(α₀) || α₁ != one(α₁)
        axpby!(α₀, p, α₁, q)
    end
    α = ρ / dot(p,q)
    axpy!(+α, p, x)
    axpy!(-α, q, r)
    normr = norm(r)
    ρold = ρ
    ρ = normr^2
    β = ρ/ρold
    numops += 1
    numiter += 1
    if alg.verbosity > 1
        msg = "CG linsolve in iter $numiter: "
        msg *= "normres = "
        msg *= @sprintf("%.12e", normr)
        @info msg
    end

    # Check for early return
    normr < tol && return (x, ConvergenceInfo(1, r, normr, numiter, numops))

    while numiter < maxiter
        axpby!(1, r, β, p)
        q = apply(operator, p)
        if α₀ != zero(α₀) || α₁ != one(α₁)
            axpby!(α₀, p, α₁, q)
        end
        α = ρ / dot(p,q)
        axpy!(+α, p, x)
        axpy!(-α, q, r)
        normr = norm(r)
        if normr < tol # recompute to account for buildup of floating point errors
            r = copyto!(similar(b, T), b)
            r = α₀ == 0 ? r : axpy!(-α₀, x, r)
            r = axpy!(-α₁, apply(operator, x), r)
            normr = norm(r)
            ρ = normr^2
            β = 0 # restart CG
        else
            ρold = ρ
            ρ = normr^2
            β = ρ/ρold
        end
        if normr < tol
            if alg.verbosity > 0
                @info """CG linsolve converged at iteration $numiter:
                 *  norm of residual = $normr
                 *  number of operations = $numops"""
            end
            return (x, ConvergenceInfo(1, r, normr, numiter, numops))
        end
        numops += 1
        numiter += 1
        if alg.verbosity > 1
            msg = "CG linsolve in iter $numiter: "
            msg *= "normres = "
            msg *= @sprintf("%.12e", normr)
            @info msg
        end
    end
    if alg.verbosity > 0
        @warn """CG linsolve finished without converging after $numiter iterations:
         *  norm of residual = $normr
         *  number of operations = $numops"""
    end
    return (x, ConvergenceInfo(0, r, normr, numiter, numops))
end


function poissonpvec(μ::Float64, n::Int)
    # Poisson probabilities, from 0 to n
    pv = Vector{Float64}(undef, n+1)
    @inbounds pv[1] = p = exp(-μ)
    for i = 1:n
        @inbounds pv[i+1] = (p *= (μ / i))
    end
    return pv
end

# Naive sampler by counting exp variables
#
# Suitable for small μ
#
struct PoissonCountSampler{T<:Real} <: Sampleable{Univariate,Discrete}
    μ::T
end

PoissonCountSampler(d::Poisson) = PoissonCountSampler(rate(d))

function rand(rng::AbstractRNG, s::PoissonCountSampler)
    μ = s.μ
    T = typeof(μ)
    n = 0
    c = randexp(rng, T)
    while c < μ
        n += 1
        c += randexp(rng, T)
    end
    return n
end

# Algorithm from:
#
#   J.H. Ahrens, U. Dieter (1982)
#   "Computer Generation of Poisson Deviates from Modified Normal Distributions"
#   ACM Transactions on Mathematical Software, 8(2):163-179
#
#   For μ sufficiently large, (i.e. >= 10.0)
#
struct PoissonADSampler{T<:Real} <: Sampleable{Univariate,Discrete}
    μ::T
    s::T
    d::T
    L::Int
end

PoissonADSampler(d::Poisson) = PoissonADSampler(rate(d))

function PoissonADSampler(μ::Real)
    s = sqrt(μ)
    d = 6 * μ^2
    L = floor(Int, μ - 1.1484)

    PoissonADSampler(promote(μ, s, d)..., L)
end

function rand(rng::AbstractRNG, sampler::PoissonADSampler)
    μ = sampler.μ
    s = sampler.s
    d = sampler.d
    L = sampler.L
    μType = typeof(μ)

    # Step N
    G = μ + s * randn(rng, μType)

    if G >= zero(G)
        K = floor(Int, G)
        # Step I
        if K >= L
            return K
        end

        # Step S
        U = rand(rng, μType)
        if d * U >= (μ - K)^3
            return K
        end

        # Step P
        px, py, fx, fy = procf(μ, K, s)

        # Step Q
        if fy * (1 - U) <= py * exp(px - fx)
            return K
        end
    end

    while true
        # Step E
        E = randexp(rng, μType)
        U = 2 * rand(rng, μType) - one(μType)
        T = 1.8 + copysign(E, U)
        if T <= -0.6744
            continue
        end

        K = floor(Int, μ + s * T)
        px, py, fx, fy = procf(μ, K, s)
        c = 0.1069 / μ

        # Step H
        if c*abs(U) <= py*exp(px + E) - fy*exp(fx + E)
            return K
        end
    end
end

# Procedure F
function procf(μ, K::Int, s)
    # can be pre-computed, but does not seem to affect performance
    ω = 0.3989422804014327/s
    b1 = 0.041666666666666664/μ
    b2 = 0.3*b1*b1
    c3 = 0.14285714285714285*b1*b2
    c2 = b2 - 15 * c3
    c1 = b1 - 6 * b2 + 45 * c3
    c0 = 1 - b1 + 3 * b2 - 15 * c3

    if K < 10
        px = -μ
        py = μ^K/factorial(K)
    else
        δ = 0.08333333333333333/K
        δ -= 4.8*δ^3
        V = (μ-K)/K
        px = K*log1pmx(V) - δ # avoids need for table
        py = 0.3989422804014327/sqrt(K)

    end
    X = (K-μ+0.5)/s
    X2 = X^2
    fx = -X2 / 2 # missing negation in pseudo-algorithm, but appears in fortran code.
    fy = ω*(((c3*X2+c2)*X2+c1)*X2+c0)
    return px,py,fx,fy
end


t1 = 1
t2 = 2
sum = 2

while t1 + t2 < 4_000_000

    if (t1 + t2) % 2 == 0
        global sum += (t1+t2)
    else
    end
    global t1, t2 = t2, t1 + t2
end

println(sum)


using Random
using LinearAlgebra

@testset "LTMADS" begin
    all_in_range(l, arr, rng) = mapreduce(x -> x in rng, (x,y) -> x && y, arr)
    T = Float64
    #= Audet & Dennis 2006 pg. 203 generation of vector b(ℓ) =#
    @testset "b_l_generation" begin
        b = Dict(0.0 => [0.0, 1.0, 0.0])
        i = Dict(0.0 => 2)
        N = 3
        l = 0
        @test DS.b_l_generation(b,i,l,N) == (b[0.0],i[0.0])

        l = 1
        nb, ni = DS.b_l_generation(b,i,l,N)
        @test nb[ni] == 2.0 || nb[ni] == -2.0
        @test all_in_range(1.0, nb, -2^l:2^l)
    end

    #= Audet & Dennis 2006 pg. 203 generation of matrix L =#
    @testset "L_generation" begin
        l = 2
        N = 3
        mat = DS.L_generation(N, l)
        #Verify determinant
        @test abs(det(mat)) == 2^(l*(N-1))
        #Upper triangle is zero
        @test all([mat[i,j] == 0 for i=1:N-1, j=1:N-1 if i<j])
        #Lower triangle has all value in range -2ˡ+1:2ˡ-1
        @test all_in_range(l, [mat[i,j] for i=1:N-1, j=1:N-1 if i>j], -2^l+1:2^l-1)
        #Diagonal has all value equal to ±2ˡ
        @test all_in_range(l, [mat[i,j] for i=1:N-1, j=1:N-1 if i==j], [-2^l,2^l])
    end

    #= Audet & Dennis 2006 pg. 203 generation of matrix B =#
    @testset "B_generation" begin
        l = 3; N = 4
        L = DS.L_generation(N, l)
        b, I = DS.b_l_generation(Dict{T,Vector{T}}(), Dict{T,Int64}(), l, N)
        p = shuffle!(setdiff(1:N, I))

        B = DS.B_generation(N, I, b, L, perm=p)
        @test all([B[p[i],1:N-1] == L[i,:] for i=1:N-1])
        @test all(B[I, 1:N-1] .== 0 )
        @test B[:,N] == b
        @test abs(det(B)) == 2^(l * N)
    end

    @testset "Match_Example" begin
        b = [-3, 2, 4, -1, 0]
        I = 3
        L = [-4.0  0.0  0.0 0.0;
              3.0  4.0  0.0 0.0;
             -1.0  2.0 -4.0 0.0;
              1.0 -2.0  0.0 4.0]
        p = [4,1,2,5]
        q = [5,1,3,2,4]
        N = 5

        B = [ 3.0  4.0  0.0 0.0 -3.0;
             -1.0  2.0 -4.0 0.0  2.0;
              0.0  0.0  0.0 0.0  4.0;
             -4.0  0.0  0.0 0.0 -1.0;
              1.0 -2.0  0.0 4.0  0.0]
        _B = DS.B_generation(N, I, b, L, perm=p)
        @test _B == B

        B′ = [ 4.0 0.0  0.0 -3.0  3.0;
              2.0 0.0 -4.0  2.0 -1.0;
              0.0 0.0  0.0  4.0  0.0;
              0.0 0.0  0.0 -1.0 -4.0;
             -2.0 4.0  0.0  0.0  1.0]
        _B′ = DS.B′_generation(B, N, perm=q)
        @test _B′ == B′

        Dₖmin = [ 4.0 0.0  0.0 -3.0  3.0 -4.0;
                  2.0 0.0 -4.0  2.0 -1.0  1.0;
                  0.0 0.0  0.0  4.0  0.0 -4.0;
                  0.0 0.0  0.0 -1.0 -4.0  5.0;
                 -2.0 4.0  0.0  0.0  1.0 -3.0]
        @test Dₖmin == DS.form_basis_matrix(N, B′, false)

        Dₖmax = [ 4.0 0.0  0.0 -3.0  3.0 -4.0 -0.0 -0.0  3.0 -3.0;
                  2.0 0.0 -4.0  2.0 -1.0 -2.0 -0.0  4.0 -2.0  1.0;
                  0.0 0.0  0.0  4.0  0.0 -0.0 -0.0 -0.0 -4.0 -0.0;
                  0.0 0.0  0.0 -1.0 -4.0 -0.0 -0.0 -0.0  1.0  4.0;
                 -2.0 4.0  0.0  0.0  1.0  2.0 -4.0 -0.0 -0.0 -1.0]
        @test Dₖmax == DS.form_basis_matrix(N, B′, true)
    end

    @testset "LTMADS" begin
        p = DS.DSProblem{T}(3, poll=LTMADS{T}())
        @test !isdefined(p, :objective)
        @test isdefined(p, :constraints)
        @test p.sense == DS.Min
        @test p.N == 3
        @test p.status.optimization_status_string == "Unoptimized"
        @test p.status.optimization_status == DS.Unoptimized
    end

    @testset "MeshUpdate" begin
        p = DS.DSProblem{T}(3, poll=LTMADS{T}())
        @test p.config.mesh.Δᵐ == 1
        DS.MeshUpdate!(p, DS.Unsuccessful)
        @test p.config.mesh.Δᵐ == 1/4
        DS.MeshUpdate!(p, DS.Dominating)
        @test p.config.mesh.Δᵐ == 1
        DS.MeshUpdate!(p, DS.Improving)
        @test p.config.mesh.Δᵐ == 1
        DS.MeshUpdate!(p, DS.Dominating)
        @test p.config.mesh.Δᵐ == 1
        DS.MeshUpdate!(p, DS.Unsuccessful)
        DS.MeshUpdate!(p, DS.Unsuccessful)
        @test p.config.mesh.Δᵐ == 1/16
        DS.MeshUpdate!(p, DS.Improving)
        @test p.config.mesh.Δᵐ == 1/16
        DS.MeshUpdate!(p, DS.Dominating)
        @test p.config.mesh.Δᵐ == 1/4
    end
end


include("../src/geom_module.jl")

module dtest
using ..geom
using PyPlot
PyPlot.pygui(true)

# x_ = vec([0 1 0.3 1.5 1.7 -0.9 -1])
# y_ = vec([ 0 0.2 1 1.5 3 2.55 0.65])

x_ = [0.0, 1, 2, 3, 2, 1, 0]
y_ = [0.0, 0, 1, 1, 3, 1, 1]

point_array = map( (x, y) -> geom.MutablePoint(x, y), x_, y_ )

delmesh = geom.build_delaunay_mesh!(point_array)

triidx = geom.triangle_indices(delmesh)

PyPlot.figure()
PyPlot.grid()
geom.plot_triangulation(delmesh; init_triangle = false)

end


# Let p = 89, q = 107, n = pq, a = 3, and b = 5. Find x in Zn such that
# a = x (mod p) and b = x (mod q).

p=89
q=107
n=p*q
a=3
b=5

for x in 1:n
    if a == x % p && b == x % q
        println("x=", x)
        exit(0)
    end
end

println("No x found.")


module Statifiers

using StatsBase: std, skewness, kurtosis, variation, sem, mad
using StatsBase: entropy, summarystats, autocor, pacf, rle, quantile
using Dates
using DataFrames
using Statistics

export fit!,transform!

export Statifier,tsmlfullstat

using AutoMLPipeline.AbsTypes
using AutoMLPipeline.Utils
import AutoMLPipeline.AbsTypes: fit!, transform!

# Transforms instances with nominal features into one-hot form
# and coerces the instance matrix to be of element type Float64.

"""
    Statifier(Dict(
       :processmissing => true
    ))

Outputs summary statistics such as mean, median, quartile, entropy, kurtosis, skewness, etc.
with parameter:

- `:processmissing` => `boolean` to indicate whether to include `missing` data stats.

Example:

    dt=[missing;rand(1:10,3);missing;missing;missing;rand(1:5,3)]
    dat = DataFrame(Date= DateTime(2017,12,31,1):Dates.Hour(1):DateTime(2017,12,31,10) |> collect,
                    Value = dt)

    statfier = Statifier(Dict(:processmissing=>false))

    fit!(statfier,dat)
    results=transform!(statfier,dat)

Implements: `fit!`, `transform!`
"""
mutable struct Statifier <: Transformer
  model
  args

  function Statifier(args=Dict())
    default_args = Dict(
      :processmissing => true
    )
    new(nothing, mergedict(default_args, args))
  end
end

"""
    fit!(st::Statifier, features::T=[], labels::Vector=[]) where {T<:Union{Vector,Matrix,DataFrame}}

Validate argument to make sure it's a 2-column format.
"""
function fit!(st::Statifier, features::DataFrame=DataFrame(), labels::Vector=[])
  typeof(features) <: DataFrame || error("Statifier.fit!: data should be a dataframe: Date,Val ")
  ncol(features) == 2 || error("dataframe must have 2 columns: Date, Val")
  st.model = st.args
end

"""
    transform!(st::Statifier, features::T=[]) where {T<:Union{Vector,Matrix,DataFrame}}

Compute statistics.
"""
function transform!(st::Statifier, features::DataFrame=DataFrame())
  features != [] || return DataFrame()
  typeof(features) <: DataFrame || error("Statifier.fit!: data should be a dataframe: Date,Val ")
  ncol(features) == 2 || error("dataframe must have 2 columns: Date, Val")
  sum(names(features) .== (:Date,:Value))  == 2 || error("wrong column names")
  fstat = tsmlfullstat(features.Value)
  timestat = timevalstat(features)
  if st.args[:processmissing] == true
    # full namedtuple: stat1,stat2,bstat
    hcat(timestat,fstat...)
  else
    hcat(timestat,fstat.stat1,fstat.stat2)
  end
end

function timevalstat(features::DataFrame)
  ldates=features.Date |> skipmissing |> collect
  totalhours = sum(diff(ldates)).value/1000/3600 # to hours
  lcount = length(ldates)
  timestart=first(features.Date)
  timeend=last(features.Date)
  sfreq = totalhours/lcount
  dftime = DataFrame(tstart=timestart,tend=timeend,sfreq=sfreq)
end

function tsmlfullstat(dat::Vector)
  data = skipmissing(dat) |> collect
  lcount = length(data)
  lmax = maximum(data)
  lmin = minimum(data)
  lsm = summarystats(data)
  q1 = quantile(data,0.1)
  q2 = quantile(data,0.2)
  q8 = quantile(data,0.8)
  q9 = quantile(data,0.9)
  lks = kurtosis(data)
  lsk = skewness(data)
  lvar = variation(data)
  lsem = sem(data)
  lentropy = entropy(data)
  # assume 24-hour period
  _autolags = 1:minimum([24,length(data)-1])
  _pacflags = 1:minimum([24,div(length(data),2)-1])
  lautocor = autocor(data,_autolags) .|> abs2 |> sum |> sqrt
  lpacf = pacf(data,_pacflags;method=:yulewalker) .|> abs2 |> sum |> sqrt
  lbrle = rlestatmissingblocks(dat)
  df1=DataFrame(count=lcount,max=lmax,min=lmin,median=lsm.median,
                mean=lsm.mean,q1=q1,q2=q2,q25=lsm.q25,
                q75=lsm.q75,q8=q8,q9=q9)
  df2=DataFrame(kurtosis=lks,skewness=lsk,variation=lvar,entropy=lentropy,
            autocor=lautocor,pacf=lpacf)
  df3=DataFrame(bmedian=lbrle.bmedian,bmean=lbrle.bmean,bq25=lbrle.bq25,bq75=lbrle.bq75,
                bmin=lbrle.bmin,bmax=lbrle.bmax)
  return (stat1=df1,stat2=df2,bstat=df3)
end

function rlestatmissingblocks(dat::Vector{T}) where {T<:Union{AbstractFloat,Integer,Missing}}
  data = deepcopy(dat)
  # replace missing with dummy value for rle processing
  indxmissing=findall(x->ismissing(x),data)
  dummy = -99999
  if eltype(data) <: Union{AbstractFloat,Missing}
    data[indxmissing] .= dummy
  elseif eltype(data) <: Union{Integer,Missing}
    data[indxmissing] .= dummy
  else
    error("not a valid type")
  end
  _rle = rle(data)
  indx=findall(x->x == dummy,_rle[1])
  blocks = _rle[2][indx]
  _sm = summarystats(blocks)
  return (bq25=_sm.q25,bmean=_sm.mean,bmedian=_sm.median,bq75=_sm.q75,bmax=_sm.max,bmin=_sm.min,bmiss=_sm.nmiss)
end

end


#############################################################################
# sum.jl
# Handles sums of expressions.
# All expressions and atoms are subtpyes of AbstractExpr.
# Please read expressions.jl first.
#############################################################################

import Base.sum
export sum

### Sum Atom
struct SumAtom <: AbstractExpr
    head::Symbol
    id_hash::UInt64
    children::Tuple{AbstractExpr}
    size::Tuple{Int, Int}

    function SumAtom(x::AbstractExpr)
        children = (x,)
        return new(:sum, hash(children), children, (1, 1))
    end
end

function sign(x::SumAtom)
    return sign(x.children[1])
end

# The monotonicity
function monotonicity(x::SumAtom)
    return (Nondecreasing(),)
end

# If we have h(x) = f o g(x), the chain rule says h''(x) = g'(x)^T f''(g(x))g'(x) + f'(g(x))g''(x);
# this represents the first term
function curvature(x::SumAtom)
    return ConstVexity()
end

function evaluate(x::SumAtom)
    return sum(evaluate(x.children[1]))
end

# Suppose x was of the form
# x = Ay where A was a coefficient. Then sum(x) can also be considered
# sum(A, 1) * y
function conic_form!(x::SumAtom, unique_conic_forms::UniqueConicForms)
    if !has_conic_form(unique_conic_forms, x)
        objective = conic_form!(x.children[1], unique_conic_forms)
        new_obj = copy(objective)
        for var in keys(new_obj)
            re = sum(new_obj[var][1], dims=1)
            im = sum(new_obj[var][2], dims=1)
            new_obj[var] = (re,im)
        end
        cache_conic_form!(unique_conic_forms, x, new_obj)
    end
    return get_conic_form(unique_conic_forms, x)
end

# Dispatch to an internal helper function that handles the dimension argument in
# the same manner as Base, with dims=: denoting a regular sum
sum(x::AbstractExpr; dims=:) = _sum(x, dims)

_sum(x::AbstractExpr, ::Colon) = SumAtom(x)

function _sum(x::AbstractExpr, dimension::Integer)
    if dimension == 1
        return Constant(ones(1, x.size[1]), Positive()) * x
    elseif dimension == 2
        return x * Constant(ones(x.size[2], 1), Positive())
    else
        error("Sum not implemented for dimension $dimension")
    end
end

Base.@deprecate sum(x::AbstractExpr, dim::Int) sum(x, dims=dim)


#using Pkg
#Pkg.add("Flux")
using Flux

using Flux.Tracker

# executable math
f(x) = x^2+1

# f'(x) = 2x
df(x) = gradient(f,x,nest=true)[1] # df is a tuple, [1] gets the first coordinate

df(4)

# f''(x) = 2
ddf(x) = gradient(df,x,nest=true)[1]
ddf(0)

h(x) = -cos(x)^cos(x)

# h'(x) = tan(x)cos(x)^(cos(x)+1)(log(cos(x))+1) obviously
dh(x)=gradient(h,x)[1]
dh(pi/4)

f(x,y,z) = x^2 + y^2 + z^2

#grad(f) = (2x,2y,2z)
gradient(f,1,2,3)

# Quick Example to introduce Params(): Linear Regression

# random initial parameters
W = rand(5,10)
b = rand(5)

fhat(x) = W*x + b

function loss(x,y)
    yhat = fhat(x) # our prediction for y
    return sum((y-yhat).^2)
end

x = rand(10)
y = rand(5)

loss(x,y) # big loss with random parameters
# I have 50+ paramters, how do I pass them all at once?

W = param(W)
b = param(b)

grads = gradient(() -> loss(x, y), Params([W, b]))

# gradient descent

alpha = 0.01 # learning rate, step size, etc
gW = grads[W]
gb = grads[b]

Tracker.update!(W,-alpha*gW) # essentially W = W - alpha * gW. It does something else I don't understand
Tracker.update!(b,-alpha*gb);

loss(x,y)

function sigmoid(x)
    return 1/(1+exp(-x))
end

W1 = param(rand(7, 10))
b1 = param(rand(7))
layer1(x) = W1 * x .+ b1

W2 = param(rand(5, 7))
b2 = param(rand(5))
layer2(x) = W2 * x .+ b2

model(x) = layer2(sigmoid.(layer1(x)))

layer1(x) = Dense(10,7,sigmoid)
layer2(x) = Dense(7,5)

model(x) = layer2(layer1(x))

# or equivalently
model2(x) = Chain(layer1,layer2)

# cool thing about Chain is that it supports indexing
model2(x)[1]

# To train a model call somthing like

train!(objective, parameters, data, optimizer, cb = () -> println("still training..."))

# cb stands for callback. Its useful to updating you about training (e.g. what the loss is currently)
# By default, it is called after every batch. Use Flux.throttle() to change this

@epoch 5 train!(...)