@testset "minimal_is_empty_test" begin
    @test isempty(emptyinterval())
    @test !isempty(interval(-Inf,+Inf))
    @test !isempty(interval(1.0,2.0))
    @test !isempty(interval(-1.0,2.0))
    @test !isempty(interval(-3.0,-2.0))
    @test !isempty(interval(-Inf,2.0))
    @test !isempty(interval(-Inf,0.0))
    @test !isempty(interval(-Inf,-0.0))
    @test !isempty(interval(0.0,Inf))
    @test !isempty(interval(-0.0,Inf))
    @test !isempty(interval(-0.0,0.0))
    @test !isempty(interval(0.0,-0.0))
    @test !isempty(interval(0.0,0.0))
    @test !isempty(interval(-0.0,-0.0))
end

@testset "minimal_is_entire_test" begin
    @test !isentire(emptyinterval())
    @test isentire(interval(-Inf,+Inf))
    @test !isentire(interval(1.0,2.0))
    @test !isentire(interval(-1.0,2.0))
    @test !isentire(interval(-3.0,-2.0))
    @test !isentire(interval(-Inf,2.0))
    @test !isentire(interval(-Inf,0.0))
    @test !isentire(interval(-Inf,-0.0))
    @test !isentire(interval(0.0,Inf))
    @test !isentire(interval(-0.0,Inf))
    @test !isentire(interval(-0.0,0.0))
    @test !isentire(interval(0.0,-0.0))
    @test !isentire(interval(0.0,0.0))
    @test !isentire(interval(-0.0,-0.0))
end

@testset "minimal_equal_test" begin
    @test ==(interval(1.0,2.0), interval(1.0,2.0))
    @test !==(interval(1.0,2.1), interval(1.0,2.0))
    @test ==(emptyinterval(), emptyinterval())
    @test !==(emptyinterval(), interval(1.0,2.0))
    @test ==(interval(-Inf,+Inf), interval(-Inf,+Inf))
    @test !==(interval(1.0,2.4), interval(-Inf,+Inf))
    @test ==(interval(1.0,Inf), interval(1.0,Inf))
    @test !==(interval(1.0,2.4), interval(1.0,Inf))
    @test ==(interval(-Inf,2.0), interval(-Inf,2.0))
    @test !==(interval(-Inf,2.4), interval(-Inf,2.0))
    @test ==(interval(-2.0,0.0), interval(-2.0,0.0))
    @test ==(interval(-0.0,2.0), interval(0.0,2.0))
    @test ==(interval(-0.0,-0.0), interval(0.0,0.0))
    @test ==(interval(-0.0,0.0), interval(0.0,0.0))
    @test ==(interval(0.0,-0.0), interval(0.0,0.0))
end

@testset "minimal_subset_test" begin
    @test issubset(emptyinterval(), emptyinterval())
    @test issubset(emptyinterval(), interval(0.0,4.0))
    @test issubset(emptyinterval(), interval(-0.0,4.0))
    @test issubset(emptyinterval(), interval(-0.1,1.0))
    @test issubset(emptyinterval(), interval(-0.1,0.0))
    @test issubset(emptyinterval(), interval(-0.1,-0.0))
    @test issubset(emptyinterval(), interval(-Inf,+Inf))
    @test !issubset(interval(0.0,4.0), emptyinterval())
    @test !issubset(interval(-0.0,4.0), emptyinterval())
    @test !issubset(interval(-0.1,1.0), emptyinterval())
    @test !issubset(interval(-Inf,+Inf), emptyinterval())
    @test issubset(interval(0.0,4.0), interval(-Inf,+Inf))
    @test issubset(interval(-0.0,4.0), interval(-Inf,+Inf))
    @test issubset(interval(-0.1,1.0), interval(-Inf,+Inf))
    @test issubset(interval(-Inf,+Inf), interval(-Inf,+Inf))
    @test issubset(interval(1.0,2.0), interval(1.0,2.0))
    @test issubset(interval(1.0,2.0), interval(0.0,4.0))
    @test issubset(interval(1.0,2.0), interval(-0.0,4.0))
    @test issubset(interval(0.1,0.2), interval(0.0,4.0))
    @test issubset(interval(0.1,0.2), interval(-0.0,4.0))
    @test issubset(interval(-0.1,-0.1), interval(-4.0, 3.4))
    @test issubset(interval(0.0,0.0), interval(-0.0,-0.0))
    @test issubset(interval(-0.0,-0.0), interval(0.0,0.0))
    @test issubset(interval(-0.0,0.0), interval(0.0,0.0))
    @test issubset(interval(-0.0,0.0), interval(0.0,-0.0))
    @test issubset(interval(0.0,-0.0), interval(0.0,0.0))
    @test issubset(interval(0.0,-0.0), interval(-0.0,0.0))
end

@testset "minimal_less_test" begin
    @test ≤(emptyinterval(), emptyinterval())
    @test !≤(interval(1.0,2.0), emptyinterval())
    @test !≤(emptyinterval(), interval(1.0,2.0))
    @test ≤(interval(-Inf,+Inf), interval(-Inf,+Inf))
    @test !≤(interval(1.0,2.0), interval(-Inf,+Inf))
    @test !≤(interval(0.0,2.0), interval(-Inf,+Inf))
    @test !≤(interval(-0.0,2.0), interval(-Inf,+Inf))
    @test !≤(interval(-Inf,+Inf), interval(1.0,2.0))
    @test !≤(interval(-Inf,+Inf), interval(0.0,2.0))
    @test !≤(interval(-Inf,+Inf), interval(-0.0,2.0))
    @test ≤(interval(0.0,2.0), interval(0.0,2.0))
    @test ≤(interval(0.0,2.0), interval(-0.0,2.0))
    @test ≤(interval(0.0,2.0), interval(1.0,2.0))
    @test ≤(interval(-0.0,2.0), interval(1.0,2.0))
    @test ≤(interval(1.0,2.0), interval(1.0,2.0))
    @test ≤(interval(1.0,2.0), interval(3.0,4.0))
    @test ≤(interval(1.0,3.5), interval(3.0,4.0))
    @test ≤(interval(1.0,4.0), interval(3.0,4.0))
    @test ≤(interval(-2.0,-1.0), interval(-2.0,-1.0))
    @test ≤(interval(-3.0,-1.5), interval(-2.0,-1.0))
    @test ≤(interval(0.0,0.0), interval(-0.0,-0.0))
    @test ≤(interval(-0.0,-0.0), interval(0.0,0.0))
    @test ≤(interval(-0.0,0.0), interval(0.0,0.0))
    @test ≤(interval(-0.0,0.0), interval(0.0,-0.0))
    @test ≤(interval(0.0,-0.0), interval(0.0,0.0))
    @test ≤(interval(0.0,-0.0), interval(-0.0,0.0))
end

@testset "minimal_precedes_test" begin
    @test precedes(emptyinterval(), interval(3.0,4.0))
    @test precedes(interval(3.0,4.0), emptyinterval())
    @test precedes(emptyinterval(), emptyinterval())
    @test !precedes(interval(1.0,2.0), interval(-Inf,+Inf))
    @test !precedes(interval(0.0,2.0), interval(-Inf,+Inf))
    @test !precedes(interval(-0.0,2.0), interval(-Inf,+Inf))
    @test !precedes(interval(-Inf,+Inf), interval(1.0,2.0))
    @test !precedes(interval(-Inf,+Inf), interval(-Inf,+Inf))
    @test precedes(interval(1.0,2.0), interval(3.0,4.0))
    @test precedes(interval(1.0,3.0), interval(3.0,4.0))
    @test precedes(interval(-3.0, -1.0), interval(-1.0,0.0))
    @test precedes(interval(-3.0, -1.0), interval(-1.0,-0.0))
    @test !precedes(interval(1.0,3.5), interval(3.0,4.0))
    @test !precedes(interval(1.0,4.0), interval(3.0,4.0))
    @test !precedes(interval(-3.0, -0.1), interval(-1.0,0.0))
    @test precedes(interval(0.0,0.0), interval(-0.0,-0.0))
    @test precedes(interval(-0.0,-0.0), interval(0.0,0.0))
    @test precedes(interval(-0.0,0.0), interval(0.0,0.0))
    @test precedes(interval(-0.0,0.0), interval(0.0,-0.0))
    @test precedes(interval(0.0,-0.0), interval(0.0,0.0))
    @test precedes(interval(0.0,-0.0), interval(-0.0,0.0))
end

@testset "minimal_interior_test" begin
    @test isinterior(emptyinterval(), emptyinterval())
    @test isinterior(emptyinterval(), interval(0.0,4.0))
    @test !isinterior(interval(0.0,4.0), emptyinterval())
    @test isinterior(interval(-Inf,+Inf), interval(-Inf,+Inf))
    @test isinterior(interval(0.0,4.0), interval(-Inf,+Inf))
    @test isinterior(emptyinterval(), interval(-Inf,+Inf))
    @test !isinterior(interval(-Inf,+Inf), interval(0.0,4.0))
    @test !isinterior(interval(0.0,4.0), interval(0.0,4.0))
    @test isinterior(interval(1.0,2.0), interval(0.0,4.0))
    @test !isinterior(interval(-2.0,2.0), interval(-2.0,4.0))
    @test isinterior(interval(-0.0,-0.0), interval(-2.0,4.0))
    @test isinterior(interval(0.0,0.0), interval(-2.0,4.0))
    @test !isinterior(interval(0.0,0.0), interval(-0.0,-0.0))
    @test !isinterior(interval(0.0,4.4), interval(0.0,4.0))
    @test !isinterior(interval(-1.0,-1.0), interval(0.0,4.0))
    @test !isinterior(interval(2.0,2.0), interval(-2.0,-1.0))
end

@testset "minimal_strictly_less_test" begin
    @test <(emptyinterval(), emptyinterval())
    @test !<(interval(1.0,2.0), emptyinterval())
    @test !<(emptyinterval(), interval(1.0,2.0))
    @test <(interval(-Inf,+Inf), interval(-Inf,+Inf))
    @test !<(interval(1.0,2.0), interval(-Inf,+Inf))
    @test !<(interval(-Inf,+Inf), interval(1.0,2.0))
    @test !<(interval(1.0,2.0), interval(1.0,2.0))
    @test <(interval(1.0,2.0), interval(3.0,4.0))
    @test <(interval(1.0,3.5), interval(3.0,4.0))
    @test !<(interval(1.0,4.0), interval(3.0,4.0))
    @test !<(interval(0.0,4.0), interval(0.0,4.0))
    @test !<(interval(-0.0,4.0), interval(0.0,4.0))
    @test !<(interval(-2.0,-1.0), interval(-2.0,-1.0))
    @test <(interval(-3.0,-1.5), interval(-2.0,-1.0))
end

@testset "minimal_strictly_precedes_test" begin
    @test strictprecedes(emptyinterval(), interval(3.0,4.0))
    @test strictprecedes(interval(3.0,4.0), emptyinterval())
    @test strictprecedes(emptyinterval(), emptyinterval())
    @test !strictprecedes(interval(1.0,2.0), interval(-Inf,+Inf))
    @test !strictprecedes(interval(-Inf,+Inf), interval(1.0,2.0))
    @test !strictprecedes(interval(-Inf,+Inf), interval(-Inf,+Inf))
    @test strictprecedes(interval(1.0,2.0), interval(3.0,4.0))
    @test !strictprecedes(interval(1.0,3.0), interval(3.0,4.0))
    @test !strictprecedes(interval(-3.0,-1.0), interval(-1.0,0.0))
    @test !strictprecedes(interval(-3.0,-0.0), interval(0.0,1.0))
    @test !strictprecedes(interval(-3.0,0.0), interval(-0.0,1.0))
    @test !strictprecedes(interval(1.0,3.5), interval(3.0,4.0))
    @test !strictprecedes(interval(1.0,4.0), interval(3.0,4.0))
    @test !strictprecedes(interval(-3.0,-0.1), interval(-1.0,0.0))
end

@testset "minimal_disjoint_test" begin
    @test isdisjoint(emptyinterval(), interval(3.0,4.0))
    @test isdisjoint(interval(3.0,4.0), emptyinterval())
    @test isdisjoint(emptyinterval(), emptyinterval())
    @test isdisjoint(interval(3.0,4.0), interval(1.0,2.0))
    @test !isdisjoint(interval(0.0,0.0), interval(-0.0,-0.0))
    @test !isdisjoint(interval(0.0,-0.0), interval(-0.0,0.0))
    @test !isdisjoint(interval(3.0,4.0), interval(1.0,7.0))
    @test !isdisjoint(interval(3.0,4.0), interval(-Inf,+Inf))
    @test !isdisjoint(interval(-Inf,+Inf), interval(1.0,7.0))
    @test !isdisjoint(interval(-Inf,+Inf), interval(-Inf,+Inf))
end


# this file contains code for the simulated annealing TSP heuristic

"""
  simulated_annealing(distmat; ...)

Use a simulated annealing strategy to return a closed tour. The temperature
decays exponentially from `init_temp` to `final_temp`. Return a tuple `(path, cost)`.

### Optional keyword arguments:
- `steps::Int = 50n^2`: number of steps to take; defaults to 50n^2 where n is number of cities
- `num_starts::Int = 1`: number of times to run the simulated annealing algorithm, each time
    starting with a random path, or `init_path` if non-null. Defaults to 1.
- `init_temp::Real = exp(8)`: initial temperature which controls initial chance of accepting an
    inferior tour.
- `final_temp::Real = exp(-6.5)` final temperature which controls final chance of accepting an
    inferior tour; lower values roughly correspond to a longer period of 2-opt.
- `init_path::Union{Vector{Int}, Nothing} = nothing`: path to start the annealing from.
    Either a `Vector{Int}` or `nothing`. If a `Vector{Int}` is given, it will be used as the initial path;
    otherwise a random initial path is used.
"""
function simulated_annealing(distmat::Matrix{T} where {T<:Real};
                             steps = 50*length(distmat),
                             num_starts = 1,
                             init_temp = exp(8), final_temp = exp(-6.5),
                             init_path::Union{Vector{Int}, Nothing} = nothing)

    # check inputs
    n = check_square(distmat, "Must pass a square distance matrix to simulated_annealing.")

    # cooling rate: we multiply by a constant mult each step
    cool_rate = (final_temp / init_temp)^(1 / (steps - 1))

    # do SA with a single starting path
    function sahelper!(path)
        temp = init_temp / cool_rate # divide by cool_rate so when we first multiply we get init_temp
        n = size(distmat, 1)
        cost_cur = pathcost(distmat, path)

        for i in 1:steps
            temp *= cool_rate

            # take a step
            # keep first and last cities fixed
            first, last = rand(2:n), rand(2:n)
            if first > last
                first, last = last, first
            end
            cost_other = pathcost_rev(distmat, path, first, last)
            @fastmath accept = cost_other < cost_cur ? true : rand() < exp((cost_cur - cost_other) / temp)
            # should we accept?
            if accept
                reverse!(path, first, last)
                cost_cur = cost_other
            end
        end

        return path, cost_cur
    end

    # unpack the initial path
    if init_path == nothing
        randstart = true
        path = randpath(n)
    else
        if !legal_circuit(init_path)
            error("The init_path passed to simulated_annealing must be a legal circuit.")
        end
        randstart = false
        path = init_path
    end
    cost = pathcost(distmat, path)
    for _ in 1:num_starts
        path_this_start = randstart ? randpath(n) : deepcopy(init_path)
        otherpath, othercost = sahelper!(path_this_start)
        if othercost < cost
            cost = othercost
            path = otherpath
        end
    end

    return path, cost
end

function randpath(n)
    path = 1:n |> collect |> shuffle
    push!(path, path[1]) # loop
    return path
end


#=
  Beta comparison

  From infer.net src/tests/Test/BlogTests.cs
  """
  a = Beta(11,500)[mean=0.02153], b = Beta(16,695)[mean=0.0225]
  aGreaterThanConstant = Bernoulli(0.9849)
  P(A > B) =  Bernoulli(0.4467)
  """

  (Yet another A/B test.)

  Summary Statistics
         parameters      mean       std   naive_se      mcse         ess      rhat   ess_per_sec
             Symbol   Float64   Float64    Float64   Float64     Float64   Float64       Float64

              aRate    0.0215    0.0065     0.0001    0.0002   1208.1553    1.0015       35.7887
              bRate    0.0238    0.0056     0.0001    0.0002   1433.9797    1.0000       42.4782
               aGtB    0.3834    0.4862     0.0049    0.0174    709.5007    0.9999       21.0173
            aGr0_01    0.9853    0.1204     0.0012    0.0031   1829.2015    1.0001       54.1857
  aSuccessCountPost   10.7714    4.5923     0.0459    0.0939   2137.7235    1.0006       63.3249
  bSuccessCountPost   16.5616    5.5983     0.0560    0.1184   2440.4334    1.0001       72.2920

  Cf ~/webppl/beta_comparison.wppl

=#

using Turing
include("jl_utils.jl")

@model function beta_comparison(aTrialCount=400,aSuccessCount=10,bTrialCount=700,bSuccessCount=15)
    aRate ~ Beta(1,10)
    bRate ~ Beta(1,10)

    aSuccessCount ~ Binomial(aTrialCount,aRate)
    bSuccessCount ~ Binomial(bTrialCount,bRate)

    aGtB ~ Dirac(aRate > bRate)
    aGt0_01 ~ Dirac(aRate > 0.01)

    aSuccessCountPost ~ Binomial(aTrialCount,aRate)
    bSuccessCountPost ~ Binomial(bTrialCount,bRate)
end

aTrialCount=500
aSuccessCount=10
bTrialCount=700
bSuccessCount=16
model = beta_comparison(aTrialCount,aSuccessCount,bTrialCount,bSuccessCount)

num_chns = 4
# chs = sample(model, Prior(), 1000)
# chs = sample(model, MH(), 10_000)
# chs = sample(model, PG(15), 10_000)
# chs = sample(model, IS(), 10_000)
chs = sample(model, SMC(), 10_000)
# chs = sample(model, SMC(), MCMCThreads(), 10_000, num_chs)

# chs = sample(model, SGLD(), 10_000)
# chs = sample(model,NUTS(), 10_000)
# chs = sample(model,HMC(0.01,5), 10_000)

display(chs)
# display(plot(chs))


#############################################################################
#  Copyright 2016, Iain Dunning, Joey Huchette, Miles Lubin, and contributors
#  This Source Code Form is subject to the terms of the Mozilla Public
#  License, v. 2.0. If a copy of the MPL was not distributed with this
#  file, You can obtain one at http://mozilla.org/MPL/2.0/.
#
# JuMP implementation of the cannery problem
# Dantzig, Linear Programming and Extensions,
# Princeton University Press, Princeton, NJ, 1963.
#
# Author: Louis Luangkesorn
# Date January 30, 2015
#############################################################################

using JuMP

function PrintSolution(status, plants, markets, ship)
    println("RESULTS:")
    if status == :Optimal
      for i = 1:length(plants)
        for j in 1:length(markets)
          println("  $(plants[i]) $(markets[j]) = $(getvalue(ship[i,j]))")
        end
      end
    else
        println("  No solution")
    end
    println("")
end

function solveCannery(plants, markets, capacity, demand, distance, freight)
  numplants = length(plants)
  nummarkets = length(markets)
  cannery = Model()

  @variable(cannery, ship[1:numplants, 1:nummarkets] >= 0)

  # Ship no more than plant capacity
  @constraint(cannery, xyconstr[i=1:numplants],
                   sum{ship[i,j], j=1:nummarkets}<=capacity[i])

  # Ship at least market demand
  @constraint(cannery, xyconstr[j=1:nummarkets],
               sum{ship[i,j], i=1:numplants}>=demand[j])

  # Minimize transporatation cost
  @objective(cannery, Min,
              sum{distance[i,j]* freight * ship[i,j], i=1:numplants, j=1:nummarkets})

  solution = solve(cannery)
  if solution == :Optimal
    result = ship
    PrintSolution(solution, plants, markets, ship)
  else
    result = solution
    print("No solution")
  end
  result
end

# PARAMETERS

plants = ["Seattle", "San-Diego"]
markets = ["New-York", "Chicago", "Topeka"]

# capacity and demand in cases
capacitycases = [350, 600]
demandcases = [300, 300, 300]

# distance in thousand miles
distanceKmiles = [2.5 1.7 1.8;
            2.5 1.8 1.4]

# cost per case per thousand miles
freightcost = 90

solveCannery(plants, markets, capacitycases, demandcases, distanceKmiles, freightcost)


if Pkg.installed("Ledger") == nothing
    Pkg.clone("https://github.com/ozanarkancan/Ledger.jl.git")
end

for p in ("Knet","ArgParse")
    Pkg.installed(p) == nothing && Pkg.add(p)
end

using Knet, Ledger

include(Pkg.dir("Knet","data","housing.jl"))

predict(w,x)=(w[1]*x.+w[2])

loss(w,x,y)=(sum(abs2,y-predict(w,x)) / size(x,2))

lossgradient = grad(loss)

function train(w, x, y; lr=.1, epochs=20)
    for epoch=1:epochs
        g = lossgradient(w, x, y)
        update!(w, g; lr=lr)
    end
    return w
end

function experimentf(;seed=-1, epochs=20, lr=0.1, test=0.0, winit=0.1)
    srand(seed)
    w = map(Array{Float32}, [winit*randn(1,13), winit*randn(1,1) ])
    (xtrn,ytrn,xtst,ytst) = map(Array{Float32}, Main.housing(test))
    experiment_log = []

    push!(experiment_log, Dict("epoch" => 0, "train-mse" => loss(w, xtrn, ytrn), "test-mse" => loss(w, xtst, ytst)))
    println(experiment_log[end])

    for i=1:epochs
        train(w, xtrn, ytrn; lr=lr, epochs=1)
        push!(experiment_log, Dict("epoch" => i, "train-mse" => loss(w, xtrn, ytrn), "test-mse" => loss(w, xtst, ytst)))
        println(experiment_log[end])
        sleep(0.5)
    end
    return experiment_log, Dict("test-mse" => loss(w, xtst, ytst))
end

function main()
    engine = LedgerEngine("mongodb://localhost:27017")
    configf = Dict(:seed=>123, :epochs=>100, :lr=>0.1, :test=>0.0, :winit=>0.1)
    Ledger.run(engine, "housing", "linear-regression", experimentf, configf, ["epoch", "train-mse", "test-mse"])
end

PROGRAM_FILE=="knet_linreg.jl" && main()


"""
    crossTable(x, y; row=true)

Make cross-tabulation table between arrays `x` and `y`.

Optional `row` report row percentage where `row=false` report column percentage, `digits` specifies the digits use in `round` function.

Return table of complete cases only, i.e., missing values are removed. This table can be used as the input to `chisquaretest`, `oddsratio`, and `riskratio` functions.

## Examples
```julia
julia> using Epi

julia> gender = ["Female", "Male"][rand(1:2, 100)];

julia> age = ["0-5", "6-10", "11-15"][rand(1:3, 100)];

julia> tm_ = crossTable(age, gender);
Frequency (row percentage)
4×3 Named Array{Text{String},2}
A ╲ B │      Male     Female      Total
──────┼────────────────────────────────
11-15 │ 16 (0.53)  14 (0.47)   30 (1.0)
0-5   │ 15 (0.43)  20 (0.57)   35 (1.0)
6-10  │ 18 (0.51)  17 (0.49)   35 (1.0)
Total │ 49 (0.49)  51 (0.51)  100 (1.0)

julia> tm_ = crossTable(age, gender, row=false);
Frequency (column percentage)
4×3 Named Array{Text{String},2}
A ╲ B │      Male     Female      Total
──────┼────────────────────────────────
11-15 │ 16 (0.33)  14 (0.27)   30 (0.3)
0-5   │ 15 (0.31)  20 (0.39)  35 (0.35)
6-10  │ 18 (0.37)  17 (0.33)  35 (0.35)
Total │  49 (1.0)   51 (1.0)  100 (1.0)
```
"""
function crossTable(x::AbstractArray, y::AbstractArray; row=true, digits=2)
	if length(x)!=length(y) error("Different length") end
	data = removeMissing(hcat(x, y))
	xx, yy = (unique(data[:, 1]), unique(data[:, 2]))
	nr, nc = (length(xx), length(yy))
	if nc | nr > 7 print("Consider grouping the variables") end
	A = NamedArray{Int}(nr, nc)
	for i in 1:nr, j in 1:nc
		A[i, j] = crossCount(xx[i], yy[j], data)
	end
 	A = hcat(A, sum(A, dims=2))
 	A = vcat(A, sum(A, dims=1))
	setnames!(A, push!(string.(xx), string("Total")), 1)
	setnames!(A, push!(string.(yy), string("Total")), 2)
	if row
		pc = round.(A ./ A[:, end], digits=digits)
		println("Frequency (row percentage)")
		println(NamedArray(Text.(string.(Array(A), " (", pc, ")")),
		                   (names(A, 1), names(A, 2))))
	else
		println("Frequency (column percentage)")
		pr = round.(A ./ A[end, :]', digits=digits)
		println(NamedArray(Text.(string.(Array(A), " (", pr, ")")),
		                   (names(A, 1), names(A, 2))))
	end
	A # return Frequency table, no %
end


using Pkg
Pkg.activate(".")
Pkg.instantiate()
# using Revise
using LazySets, Optim, Plots
import LinearAlgebra: I

# STRUCTURE AND CONSTRUCTORS:

struct NormalForm
    nPlayers::Int64
    nMoves::Tuple{Vararg{Int64}}
    payoffMatList::Tuple{ Vararg{ Array{<:Real} } }

    function NormalForm(players, nmoves, payofflist)
        if size(payofflist[1])==nmoves && players==ndims(payofflist[1])
            new(players, nmoves, payofflist)
        else
            error("Violates dimension requirements for NormalForm.")
        end
    end

end

function NormalForm(normalFormTable::Array{<:Tuple})
    nplayers=ndims(normalFormTable)
    nmoves=size(normalFormTable)

    payoffmatlist=ntuple(
        i -> [normalTable[k][i] for k in CartesianIndices(normalFormTable)],
        nplayers
    )

    return NormalForm(nplayers, nmoves, payoffmatlist)
end

#CALCULATING VALUES

function pureMinMax2(normalform::NormalForm)
    pure_mm1= minimum(
        maximum(
            normalform.payoffMatList[1];
            dims=1
        ),
        dims=2
    )
    pure_mm2= minimum(
        maximum(
            normalform.payoffMatList[2];
            dims=2
        ),
        dims=1
    )
    return [pure_mm1[1], pure_mm2[1]]
end

function brCorrelatMix(normalform::NormalForm,player::Int64,corrMix::Vector)
    payoff=normalform.payoffMatList[player]
    permutation=Tuple(
        unique( [player; 1:normalform.nPlayers] )
    )
    permutedPayoff=permutedims(payoff,permutation)
    permutedPayoff= reshape(permutedPayoff, ( size(permutedPayoff,1) , :))
    return maximum(permutedPayoff*corrMix)
end

function miniMax_i(normalform::NormalForm, player::Int64)
    uBR_i(μ) = brCorrelatMix(normalform, player, μ)
    parametrize_mix(z)=exp.(z)/ sum(exp.(z))
    obj(z) = uBR_i(parametrize_mix(z))

    z0=zeros( prod(normalform.nMoves) ÷ normalform.nMoves[player])
    solution=optimize(obj, z0)
    return solution.minimum
end

function miniMaxProfile(normalform::NormalForm)
    return [miniMax_i(normalform, i) for i=1:normalform.nPlayers]
end

function randomNormalForm(nMovesList::Tuple{Vararg{<:Real}})
    nplayers=length(nMovesList)
    payoffmatlist= Tuple(rand(Float64, nMovesList) for i in 1:nplayers)
    NormalForm(nplayers, nMovesList, payoffmatlist)
end

function greaterThanSet(minPoint::Vector{<:Real})::HPolyhedron
    HPolyhedron(
        -Matrix{Float64}(I,length(minPoint), length(minPoint)),
        -minPoint
    )
end

#PLOTTING

function plotFeasible2!(p,normalform::NormalForm)
    payoff1=normalform.payoffMatList[1]
    payoff2=normalform.payoffMatList[2]

    payoff_points=[
        Vector{Float64}( [payoff1[i], payoff2[i]] )
        for i in eachindex(payoff1)
    ]
    feasible = VPolygon(payoff_points)
    Plots.plot!(p,
        feasible,
        xlabel="Payoff player 1",
        ylabel="Payoff player 2",
        label="Feasible set"
    );

    Plots.scatter!(p,
        [payoff_points[i][1] for i in eachindex(payoff_points)],
        [payoff_points[i][2] for i in eachindex(payoff_points)],
        label="Pure payoffs"
    );
end

function plotFeasible2(normalform::NormalForm)
    p=Plots.plot(title="Normal Form Payoffs")
    plotFeasible2!(p,normalform)
    display(p)
    return p
end

function plotIR_Set2!(p::Plots.Plot, normalform::NormalForm)
    plot!(
        p,
        greaterThanSet( miniMaxProfile(normalform) ),
        label="IR Set",
        legend=:bottomleft
    )
end

function plotMinimax2!(p::Plots.Plot, normalform::NormalForm)
    mm=miniMaxProfile(normalform)
    Plots.scatter!(p,
        [mm[1]],
        [mm[2]],
        label="Minimax"
    );
end

function plotall(normalform::NormalForm)
    p=plot(
        title="Normal form payoffs"
        )
    plotFeasible2!(p,normalform)
    plotIR_Set2!(p,normalform)
    plotMinimax2!(p,normalform)
    display(p)
    return p
end

#= TO DO:
- generate functions to plot 3D
=#

#= CODE TO PLOT 3D FEASIBLE SET
Using LazySets, GLMakie, Polyhedra

test=CH(
    Singleton([1.,1.,3.]),
    Singleton([2.,1.,4.])
)

test=CH(test,
    Singleton([2., 1.5, 3.])
)

figureplot3d(
    overapproximate(test)
) =#


# plot residual oil saturation versus contact angle
using Plots
sor_ww=0.15
sor_ow=0.2
sor_mw=0.08
teta_ow=π
teta_ww=0.0
teta_mw=π/2+0.2 # just to shift it a bit
a=(sor_mw-0.5*(sor_ww-sor_ow)*cos(teta_mw)-0.5*(sor_ww+sor_ow))/(cos(teta_mw)^2-1)
b=0.5*(sor_ww-sor_ow)
c=0.5*(sor_ww+sor_ow)-a

teta=linspace(0, π, 100)
sor=a*cos(teta).^2+b*cos(teta)+c

plot(
plot(teta, sor, xlabel="contact angle [rad]", ylabel="Sor"),
plot(cos(teta), sor, xlabel="contact angle [rad]", ylabel="Sor")
)


using MultilayerPerceptron

using MLDatasets: MNIST

N_CLASSES = 10

function preprocess(x, y=nothing)
    n = size(x)[end]
    mat_x = reshape(x, :, n)

    if isnothing(y)
        return mat_x
    end

    mat_y = one_hot(y .+ 1, N_CLASSES)  # digits to 1-based labels
    mat_x, mat_y
end

function mnist_example()
    train_x, train_y = preprocess(MNIST.traindata(Float64)...)

    N_EPOCHS = 10
    n_pixels = size(train_x, 1)
    hidden_layer = Layer((32, n_pixels); activation_fn=relu, grad_fn=∇relu)
    output_layer = Layer((N_CLASSES, 32))

    mlp = MLP([hidden_layer, output_layer])
    fit!(mlp, train_x, train_y;
        loss_fn=logit_cross_entropy,
        learning_rate=5.0, batch_size=64, n_epochs=N_EPOCHS)

    test_x, test_y = begin
        test_x, test_y = MNIST.testdata(Float64)
        test_x = preprocess(test_x)
        test_y .+= 1
        test_x, test_y
    end
    n_test = length(test_y)
    n_correct = 0
    for i in 1:n_test
        x = test_x[:, [i]]
        y = test_y[i]
        pred_label = argmax(vec(predict(mlp, x)))
        if pred_label == y
            n_correct += 1
        end
    end
    println("Accuracy = $(n_correct / n_test)")

    # The current configuration achieves comparable accuracy with its Flux counterpart in
    # https://github.com/FluxML/model-zoo (> 95%).
end

mnist_example()


using Spectra
using Test

# testing long correction function
x_for_long = [20.,21.,22.,23.,24.,25.]
y_for_long = [1.0,1.0,1.0,1.0,1.0,1.0]
h_plank = 6.626070040e-34 #Plank constant
k_bolt = 1.38064852e-23  #Boltzman constant
c_light = 299792458. #speed of light in vaccum
nu0 = 1.0./(514.532).*1e9 #laser wavenumber at 514.532
nu = 100.0.*x_for_long # cm-1 to m-1
T = 23.0+273.15 # the temperature in K

x_long,long_res,eselong = tlcorrection([x_for_long y_for_long],23.0,514.532) # using the function
t0 = nu0.^3.0 .*nu./((nu0.-nu).^4)
t1= 1 .- exp.(-h_plank.*c_light.*nu./(k_bolt.*T)) # c in m/s  : t1 dimensionless
long_calc= y_for_long.*t0.*t1 # pour les y
long_calc = long_calc./trapz(x_for_long,long_calc) # area normalisation

@test isapprox(long_res,long_calc)
@test isapprox(x_for_long,x_long)


#Signal Processing Functions

#Consts
AF_CONV_EXPAND = icxx"AF_CONV_EXPAND;"

# Fourier Transforms
import Base: fft, ifft
fft{T}(a::AFVector{T}) = AFArray{Complex{T}}(af_fft(a))
fft{T}(a::AFMatrix{T}) = AFArray{Complex{T}}(af_fft2(a))
fft{T}(a::AFAbstractArray{T,3}) = AFArray{Complex{T}}(af_fft3(a))

fft{T<:Complex}(a::AFVector{T}) = AFArray{T}(af_fft(a))
fft{T<:Complex}(a::AFMatrix{T}) = AFArray{T}(af_fft2(a))
fft{T<:Complex}(a::AFAbstractArray{T,3}) = AFArray{T}(af_fft3(a))

ifft{T}(a::AFVector{T}) = AFArray{Complex{T}}(af_ifft(a))
ifft{T}(a::AFMatrix{T}) = AFArray{Complex{T}}(af_ifft2(a))
ifft{T}(a::AFAbstractArray{T,3}) = AFArray{Complex{T}}(af_ifft3(a))

ifft{T<:Complex}(a::AFVector{T}) = AFArray{T}(af_ifft(a))
ifft{T<:Complex}(a::AFMatrix{T}) = AFArray{T}(af_ifft2(a))
ifft{T<:Complex}(a::AFAbstractArray{T,3}) = AFArray{T}(af_ifft3(a))

#FIR Filter
fir{T}(a::AFAbstractArray{T}, x::AFAbstractArray{T}) = AFArray{T}(af_fir(a,x))

#IIR Filter
iir{T}(ff::AFAbstractArray{T}, fb::AFAbstractArray{T}, a::AFAbstractArray{T}) = AFArray{T}(af_iir(ff, fb, a))

#Convolutions

import Base: conv, conv2

conv{T}(a::AFAbstractArray{T}, b::AFAbstractArray{T}) = AFArray{T}(af_fftConvolve(a, b, AF_CONV_EXPAND))
conv2{T}(a::AFAbstractArray{T}, b::AFAbstractArray{T}) = AFArray{T}(af_fftConvolve2(a,b, AF_CONV_EXPAND))


# Rank-based fitness assignment
# sp - selective linear presure in [1.0, 2.0]
function ranklinear(sp::Float64)
  @assert 1.0 <= sp <= 2.0 "Selective pressure has to be in range [1.0, 2.0]."
  function rank(fitness::Vector{Float64}, N::Int)
      λ = float(length(fitness))
      idx = sortperm(fitness)
      ranks = zeros(Integer(λ))
      for i in 1:Integer(λ)
          ranks[i] = ( 2.0- sp + 2.0*(sp - 1.0)*(idx[i] - 1.0) / (λ - 1.0) ) /λ
      end
      return sample(1:Integer(λ),weights(ranks),N,replace = false)
  end
  return rank
end

# Tournament selection
function tournament(groupSize::Integer)
  groupSize <= 0 && error("Group size needs to be positive")
  function tournamentN(fitness::Vector{Float64},N::Integer)
    selection = Array(Int, N)
    nF = length(fitness)

    for i in 1:N
      contenders = sample(1:nF,groupSize,replace = true)
      winner = first(contenders)
      wf = fitness[winner]
      for k in 2:groupSize
        contender = contenders[k]
        if fitness[contender] > wf
          winner = contender
          wf = fitness[contender]
        end
      end
      selection[i] = winner
    end
    return selection
  end
  return tournamentN
end

# Roulette wheel (proportionate selection) selection
function roulette(fitness::Vector{Float64}, N::Integer)
  n = length(fitness)
  wv = weights(fitness)
  sample(1:n,wv,N,replace = true)
end

# Stochastic universal sampling (SUS)
function sus(fitness::Vector{Float64}, N::Integer)
  n = length(fitness)
  @assert N <= n "Number of selections (N) must be less than numeber of elements"
  wv = weights(fitness)
  sample(1:n,wv,N,replace = false)
end


using NCDatasets, ColorSchemes, RPRMakie
using ImageShow

# Taken from https://lazarusa.github.io/BeautifulMakie/GeoPlots/topography/
cmap = dataset = Dataset(joinpath(@__DIR__, "ETOPO1_halfdegree.nc"))
lon = dataset["lon"][:]
lat = dataset["lat"][:]

data = Float32.(dataset["ETOPO1avg"][:, :])
function toCartesian(lon, lat; r=1, cxyz=(0, 0, 0))
    lat, lon = lat * π / 180, lon * π / 180
    x = cxyz[1] + (r + 80_000) * cos(lat) * cos(lon)
    y = cxyz[2] + (r + 80_000) * cos(lat) * sin(lon)
    z = cxyz[3] + (r + 80_000) * sin(lat)
    return (x, y, z) ./ 80_000
end

function lonlat3D(lon, lat, data; cxyz=(0, 0, 0))
    xyzw = zeros(size(data)..., 3)
    for (i, lon) in enumerate(lon), (j, lat) in enumerate(lat)
        x, y, z = toCartesian(lon, lat; r=data[i, j], cxyz=cxyz)
        xyzw[i, j, 1] = x
        xyzw[i, j, 2] = y
        xyzw[i, j, 3] = z
    end
    return xyzw[:, :, 1], xyzw[:, :, 2], xyzw[:, :, 3]
end
# this is needed in order to have a closed surface
lonext = cat(collect(lon), lon[1]; dims=1)
dataext = begin
    tmpdata = zeros(size(lon)[1] + 1, size(lat)[1])
    tmpdata[1:size(lon)[1], :] = data
    tmpdata[size(lon)[1] + 1, :] = data[1, :]
    tmpdata
end
#now let's plot that surface!
xetopo, yetopo, zetopo = lonlat3D(lonext, lat, dataext)

begin
    r = 30
    lights = [PointLight(Vec3f(2, 1, 3), RGBf(r, r, r))]
    fig = Figure(; resolution=(1200, 1200), backgroundcolor=:black)
    ax = LScene(fig[1, 1]; show_axis=false)#, scenekw=(lights=lights,))
    pltobj = surface!(ax, xetopo, yetopo, zetopo; color=dataext, colormap=:hot, colorrange=(-6000, 5000))
    cam = cameracontrols(ax.scene)
    cam.fov[] = 10
    cam.eyeposition[] = Vec3f(3, 1, 1)
    cam.lookat[] = Vec3f(0)
    cam.upvector[] = Vec3f(0, 0, 1)
    Makie.update_cam!(ax.scene, cam)
    RPRMakie.activate!(; iterations=100)
    display(ax.scene)
end


function fact(n::Integer)
    n < 0 && return zero(n)
    f = one(n)
    for i in 2:n
        f *= i
    end
    return f
end

for i in 10:20
	println("$i -> ", fact(i))
end


import MitosisStochasticDiffEq as MSDE
using Mitosis
using Statistics
using LinearAlgebra
using StochasticDiffEq

# Estimate the parameters of a Lorenz SDE

u0 = [1.0,1.0]
tspan = (0.0,10.0)
function multiplicative_noise!(du,u,p,t)
  x,y = u
  du[1] = p[5]*x
  du[2] = p[6]*y
end
p = ptrue = [1.5,1.0,3.0,1.0,0.1,0.1]

function lotka_volterra!(du,u,p,t)
  x,y = u
  α,β,γ,δ = p
  du[1] = dx = α*x - β*x*y
  du[2] = dy = δ*x*y - γ*y
end

# define log.(u) via Ito

ξ0 = log.(u0)
function g!(du,u,p,t)
  x,y = u
  du[1] = p[5]
  du[2] = p[6]
  nothing
end

function f!(du,u,p,t)
  x,y = u
  α,β,γ,δ = p
  du[1] = dx = α - β*exp(y)
  du[2] = dy = δ*exp(x) - γ
  nothing
end
function f(u, p, t)
  du = zeros(2)
  f!(du, u, p, t)
  du
end

# f is linear in the parameters with Jacobian
function f_jac(J,u,p,t)
  x,y = u
  J .= false
  J[1,1] = 1.0
  J[1,2] = -exp(y)
  J[2,3] = -1.0
  J[2,4] = exp(x)
  nothing
end
# and intercept
function ϕ0(du,u,p,t)
  du .= false
  nothing
end

using Plots

prob_sde = SDEProblem(lotka_volterra!,multiplicative_noise!,u0,tspan,p)
ensembleprob = EnsembleProblem(prob_sde)
@time data = solve(ensembleprob,SOSRI(),saveat=0.1,trajectories=1000)
plot(EnsembleSummary(data))

prob_sde2 = SDEProblem(f!,g!,ξ0,tspan,p)
ensembleprob2 = EnsembleProblem(prob_sde2)
data2 = solve(ensembleprob2,SOSRI(),saveat=0.1,trajectories=1000)

@time data2exp = solve(ensembleprob2,SOSRI(),saveat=0.1,trajectories=1000)
for u in data2exp.u
  u.u .= [exp.(x) for x in u.u]
end

plot(EnsembleSummary(data2exp))

quvar(u) = sum((du - f(u, p, t)*dt).^2 for (t,u, dt, du) in zip(u.t, u.u, diff(u.t), diff(u.u)))

p = (0.5 .+ 1.5rand(length(ptrue))).*ptrue
ps = [copy(p)]
for k in 1:5
  global p
  # Solve the regression problem

  sdekernel = MSDE.SDEKernel(f!,g!,tspan,p)

  ϕprototype = zeros((2,4)) # prototypes for vectors
  yprototype = zeros((2,))
  R = MSDE.Regression!(sdekernel,yprototype,paramjac_prototype=ϕprototype,paramjac=f_jac,intercept=ϕ0)

  G = MSDE.conjugate(R, data2, 0.1I(4))

  # Estimate
  p̂ = mean(G)
  se = sqrt.(diag(cov(G)))
  display(map((p̂, se, p) -> "$(round(p̂, digits=2)) ± $(round(se, digits=2)) (true: $p)", p̂, se, ptrue))

  p[1:4] = rand(convert(Gaussian{(:μ, :Σ)}, G))

  # Estimate covariance
  T = sum(u.t[end] - u.t[1] for u in data2)

  Q = sum(quvar(u) for u in data2)

  p[end-1:end] = sqrt.(Q/T)
  push!(ps, copy(p))
end
ps
its = eachindex(ps)
cols = [k for j in its, k in eachindex(ptrue)]
pl = scatter(reduce(hcat, ps)', c=cols, labels=["α" "β" "γ" "δ" "s1" "s2"])
plot!(reduce(hcat, fill(ptrue, length(ps)))', c=cols, labels=["αtrue" "βtrue" "γtrue" "δtrue" "s1true" "s2true"])


points = [
    3im
    1 + im
    0im
    -1 - im
    0 - 2im
    1.5 + 0.5im
    2im
    -1.5 + 0.5im
    0 - 2im
    1 - im
    0im
    -1 + im
]

function draw_curve_and_tangents(points)
    newdraw()
    draw_curve(points, color = :black, line = 3)

    S = Spline(points, :closed)
    n = npatches(S)

    d = 1 / 3
    T = 1:d:n+1-d

    for t in T
        tt = t + d / 3
        v = S'(tt)
        v /= 2 * abs(v)
        draw_vector(v, S(tt), color = :green)
    end

    finish()
end

function tangent_plot(points)
    S = Spline(points, :closed)
    f(t) = angle(S'(t)) / (2pi)
    plot(f, 0.99, npatches(S) + 1.01)
    plot!(legend = false)
end


#=
Modia library with 1D heat transfer component models (inspired from Modelica Standard Library).

Developer: Martin Otter, DLR-SR
Copyright 2021, DLR Institute of System Dynamics and Control
License: MIT (expat)
=#

using ModiaLang
using Unitful

HeatPort = Model( T = potential,   # Absolute temperature
                  Q_flow = flow )  # Heat flow into the component

FixedTemperature = Model(
    T    = 293.15u"K",
    port = HeatPort,
    equations = :[port.T = T]
)

FixedHeatFlow = Model(
    Q_flow = 0.0u"W",   # Fixed heat flow into the connected component
    port   = HeatPort,
    equations = :[port.Q_flow = -Q_flow]
)

HeatCapacitor = Model(
    C = 1.0u"J/K",
    port = HeatPort,
    T = Var(init = 293.15u"K"),
    equations = :[T = port.T,
                  der(T) = port.Q_flow/C]
)

ThermalConductor = Model(
    G = 1.0u"W/K",
    port_a = HeatPort,
    port_b = HeatPort,
    equations = :[dT = port_a.T - port_b.T
                  0  = port_a.Q_flow + port_b.Q_flow
                  port_a.Q_flow = G*dT]
)

# Fully insulated rod with 1D heat transfer and port_a/port_b on left/right side

T_grad1(T,Ta,dx,i) = i == 1         ? (Ta - T[1])/(dx/2) : (T[i-1] - T[i]  )/dx
T_grad2(T,Tb,dx,i) = i == length(T) ? (T[i] - Tb)/(dx/2) : (T[i]   - T[i+1])/dx

InsulatedRod = Model(
    L       = 1.0u"m",            # Length of rod
    A       = 0.0004u"m^2",       # Rod area
    rho     = 7500.0u"kg/m^3",    # Density of rod material
    lambda  = 74.0u"W/(m*K)",     # Thermal conductivity of rod material
    c       = 450.0u"J/(kg*K)",   # Specific heat capacity of rod material
    port_a  = HeatPort,           # Heat port on left side
    port_b  = HeatPort,           # Heat port on right side
    T       = Map(init = fill(293.15u"K", 1)),  # Initial temperature and number of nodes
    equations = :[
        n   = length(T)
        dx  = L/n
        Ce  = c*rho*A*dx
        k   = lambda*A/Ce
        der(T) = k*[T_grad1(T,port_a.T,dx,i) - T_grad2(T,port_b.T,dx,i) for i in eachindex(T)]
        port_a.Q_flow = lambda*A*(port_a.T - T[1])/(dx/2)
        port_b.Q_flow = lambda*A*(port_b.T - T[n])/(dx/2)
    ]
)


using Polynomials
using Plots
using PolynomialRoots
using ProgressMeter
using BenchmarkTools
using FileIO
using JLD2

gr()
#plotlyjs()
begin
	quot(v)=v[1]//v[2]
	vect(r)=[numerator(r),denominator(r)]
end

begin
	remainder(v) = v[1] % v[2]
	flr(v)=numerator(floor(v[1] // v[2]))
	newpair(v) = [v[2], remainder(v)]
	function continued_fraction(v)
		cf=[]
		while remainder(v) != 0
			push!(cf, flr(v))
			v=newpair(v)
		end
		push!(cf, flr(v))
		cf
	end
	function cf_value(cf)
		rev = reverse(cf)
		rat = popfirst!(rev)
		for j in rev
			rat = j+1//rat
		end
		[numerator(rat), denominator(rat)]
	end
end

function center_corner_index(v)
	v[1] == 1 && return [[0,1],[1,0],v[2]]
	center_cf = continued_fraction(v)[1:end-1]
	center = cf_value(center_cf)
	push!(center_cf, 1)
	k2 = cf_value(center_cf)
	corner = k2 - center
	ind = numerator((v[1] - corner[1]) // center[1])
	[center, corner, ind]
end

# testing updating q
function dQ(v)
		return Polynomial(push!(zeros(BigInt,v[2]), (-1)^v[1]))
end

function Q(v)
		v==[0,1] && return 1
		v==[1,0] && return 0
		v==[1,1] && return 1
		cci=center_corner_index(v)
		c,k,i=cci[1],cci[2],cci[3]
		return ([0 1; -dQ(c) Q(c)]^(i-1) * [Q(k); Q(k+c)])[2]
end

function bigroots(poly)
    PolynomialRoots.roots(convert(Vector{BigInt},coeffs(poly)))
end

function cleanpair(p)
    	r=p[1]//p[2]
    	return [numerator(r), denominator(r)]
end

function fractionline(n)  #fractions in (0,1/2) with denominator n.
    	top=ceil(n//2)
    	L=[cleanpair([j,n]) for j in 1:top-1 ]
    	return filter(rat-> rat[2]==n ,L)
end

function id(z)
	return z
end

function F64(z)
	return convert(Complex{Float64},z)
end

function generateDiscriminants(rational)
    q2 = Q(rational)^2
    dq  = dQ(rational)
    return (q2 - 4 * dq)
end

function generateQDict(max_denom)
	qDict = Dict{Vector{Int64}, Polynomial{BigInt}}([1,3] => Q([1,3]))

	for key in 4:max_denom
		rats = fractionline(key)
		for rat in rats
			qDict[rat] = Q(rat)
		end
	end
	return qDict
end

function generateDiscriminantsDict(qDict)
	discDict = Dict{Vector{Int64}, Polynomial{BigInt}}([1,3] => generateDiscriminants([1,3]))

	Threads.@threads for key in collect(keys(qDict))
		discDict[key] = generateDiscriminants(key)
	end
	return discDict
end

function generateRoots(dict)
	setprecision(BigFloat, 512) do
		n = length(dict)

		p = Progress(n, 1, "Generating roots...", 50)

		keysdict = collect(keys(dict))
		rootsDict = Dict(keysdict[1] => PolynomialRoots.roots(keysdict[1]))

		@Threads.threads for key in keysdict
			poly = coeffs(dict[key])
			rootsDict[key] = PolynomialRoots.roots(poly)
			next!(p)
		end
	return rootsDict
	end
end

function generateRootset(rootsdict)
	rootset = []

	denKey = collect(keys(rootsdict))
	n = length(denKey) +2

	p = Progress(n, 1, "Mapping unions...", 50)

	Threads.@threads for key in denKey
        	rootset=union(rootset, map(id,rootsdict[key]))
		next!(p)
	end
	return rootset
end

function runQPlot(rootset, discset, max_denom)

	c1=filter(z->abs2(z)<3, rootset);
	c2=filter(z->abs2(z)<3, discset);

	scatter([c1[j] for j in 1:length(c1)], markersize = 1,markerstrokewidth=0, c = :black, size = (5000,5000), label=false, aspect_ratio=1, framestyle= :none, background_color= :ivory)
	scatter!([c2[j] for j in 1:length(c1)], markersize = 1,markerstrokewidth=0, c = :red)

	savefig("plots/scatter_" * string(max_denom) * ".svg")
end

function main(max_denom)
	if isfile("data/roots_" * string(max_denom) * ".jld2")
		r = load("data/roots_" * string(max_denom) * ".jld2")
		rs = load("data/disc_roots_" * string(max_denom) * ".jld2")
		r = generateRootset(r)
		rs = generateRootset(rs)

		runQPlot(r,rs,max_denom)

	else
	q = generateQDict(max_denom)
	d = generateDiscriminantsDict(q)

	r = generateRoots(q)
	save(r, "data/roots_" * string(max_denom) * ".jld2")

	rs = generateRoots(d)
	save(rs, "data/disc_roots_" * string(max_denom) * ".jld2")

	r = generateRootset(r)
	rs = generateRootset(rs)

	runQPlot(r,rs,max_denom)
	end
end

main(200)


using ADCME
using AdFem
using LinearAlgebra
using MAT
using PyPlot; matplotlib.use("agg")
using SparseArrays

function k_exact(x,y)
    1 + x^2 + x / (1+y^2)
end

θ0 = [1.0; 1.0; 1.0]
θ = Variable([1.0; 1.0; 1.0])
paramA = θ[1]
paramB = θ[2]
paramC = θ[3]

# RBF2D

function k_func(x,y)
    1 + paramA * x^2 + paramB * x / (1+ paramC * y^2)
end

include("Boussinesq_common.jl")

m = 20
n = 20
h = 1/n

mesh = Mesh(m, n, h, degree=2)
nnode = mesh.nnode
nedge = mesh.nedge
ndof = mesh.ndof
nelem = mesh.nelem
ngauss = get_ngauss(mesh)

kgauss = eval_f_on_gauss_pts(k_func, mesh); kgauss=stack(kgauss)
LaplaceK = compute_fem_laplace_matrix1(kgauss, mesh)

S_computed = S[end, :]
S_data = matread("data.mat")["V"]

loss =  mean((S_computed .- S_data)^2)
loss = loss * 1e10

sess = Session(); init(sess)

# @info run(sess, loss, θ=>θ0)
# lineview(sess, θ, loss, θ0, zeros(3))
# savefig("lineview.png")
# gradview(sess, θ, loss, zeros(3))
# savefig("gradview.png")

# gradview(sess, θ, sum(kgauss), zeros(3))
# savefig("gradview_kgauss.png")

# gradview(sess, θ, sum(values(LaplaceK)^2), zeros(3))
# savefig("gradview_LaplaceK.png")
# @info S[2, :]

# gradview(sess, θ, sum(S[2, :]^2), zeros(3))
# savefig("gradview_S2.png")

gradview(sess, θ, sum(S[3, :]^2), zeros(3))
savefig("gradview_S3.png")

# gradview(sess, θ, sum(S_computed^2), zeros(3))
# savefig("gradview_S_computed.png")


"""
    BondPrice3(Y,cf,m)

Calculate bond price as sum of discounted cash flows.

# Input:
- Y:  scalar or K vector of interest rates
- cf: K vector of cash flows
- m:  K vector of times for the cash flows
"""
function BondPrice3(Y,cf,m)              #cf is a vector of all cash flows at times m
    (length(cf) != length(m)) && error("BondPrice3: cf and m must have the same lengths")
    cdisc = cf./((1.0.+Y).^m)            #cf1/(1+Y1)^m1, cf2/(1+Y2)^m2 + ...
    P     = sum(cdisc)                   #price
    return P
end


function SeisSmooth1(A;L=7)
	a = isodd(L) ? 0 : 1
	kernel = ones(eltype(A),L)/L;
	A = conv(A,kernel)
	A = A[floor(Int,L/2) + 1:end-floor(Int,L/2) + a]
	return A
end


#------------------------------------------------------------------------------------------------------------
#   Define functions needed to solve the covariance estimation problem:
#           min_{X \in \R^{d x r}}   F(X)   :=    E_{a,b} | <XX^T, aa^T> - b |
#   and it's elastic net variant with
#           F(X) :=  E_{a,b} | <XX^T, aa^T> - b | + (<XX^T, aa^T> - b )^2
#
#   Solve via two methods:
#           1) stochastic subgradient (Φ = 1/2 ||⋅||^2)
#           2) stochastic mirror descent (Φ = a polynomial)
#
#   Note that stochastic subgradients are
#           ∂ (  | <XX^T, a a^T> - b |  )   =    sign(res) * 2 aa^T X
#           ∂ (  | <XX^T, a a^T> - b |  )   =    (2 * res + sign(res) ) * 2 aa^T X
#   where (a, b) ∈ Rᵐ x R are chosen randomly, and
#           res = <XX^T, a a^T> - b = || aTX ||₂² - b  (residual)
#------------------------------------------------------------------------------------------------------------

using LinearAlgebra
using Polynomials
using Random

#------------------------------------------------------------------------------------------
# Draw stochastic a's (gaussian) and corresponding b's
# Returns: maxIter of them, in format:
#                       A = matrix with a's in columns
#                       B = vector with b's in entries
#------------------------------------------------------------------------------------------
function get_ab(XTtrue, stochErr::Float64, maxIter::Int64; noiseType::String="gaussian")
    (r,d) = size(XTtrue);
    A = randn(d, maxIter);  # gaussian entries in a's
    bError = stochErr * randn(1, maxIter);  # normal(0,stochErr²) errors
    if noiseType=="sparse"
        p = 0.25; # probability of corrupting a measurement
        noiseless_idxs = randsubseq(collect(1:maxIter), 1-p);
        bError[noiseless_idxs] = zeros(length(noiseless_idxs));
    end
    B = sum(abs2, XTtrue*A, dims=1) + bError;
    return(A,B)
end

#---------------------------------------------------------------------------------------------------
#   Compute empirical function values, based on (A,B) as above
#   Inputs:     X = point to evaluate function (d x r matrix)
#                   A = matrix with columns as the a's
#                   B = vector with b's as entries
#---------------------------------------------------------------------------------------------------
# vanilla covariance estimation function value
function compute_empirical_function(X, A, B)
    residuals = sum(abs2, X'*A, dims=1) - B;
    return sum(abs, residuals)
end

# elastic net empirical function value
function compute_enet_empirical_function(X, A, B)
    residuals = sum(abs2, X'*A, dims=1) - B;
    return (sum(abs, residuals) + sum(abs2, residuals)) / size(A,2)
end

#----------------------------------------------------------------------------
# Update the subgradient G, in place
#----------------------------------------------------------------------------
function subgrad!(G, XTa, a, b)
    res = sum(abs2, XTa) - b;
    lmul!(0.0,G)  # reset G to zero
    BLAS.ger!(2*sign(res), a, XTa, G);  # rank one update
end

function enet_subgrad!(G, XTa, a, b)
    res = sum(abs2, XTa) - b;
    lmul!(0.0,G)  # reset G to zero
    BLAS.ger!(2*sign.(res)+2*res, a, XTa, G);  # rank one update
end

#------------------------------------------------------------------------------------------------------------
# Find the smallest real, positive root of a polynomial
# input : coefficients, as a vector by increasing degree (e.g. [a0, a1, a2, a3])
#------------------------------------------------------------------------------------------------------------
function get_root(coeffs)
    all_roots = roots(Poly(coeffs));

    # reduce to positive real roots
    pos_real_idxs = map(x->(imag(x)==0.0 && real(x)>0), all_roots);
    pos_real_roots = real(all_roots[pos_real_idxs]);

    # return smallest positive real root
    return(minimum(pos_real_roots))
end


using TimeModels

println("Testing Kalman.jl")

srand(1)

F = diagm([1.0])
V = diagm([2.0])
G = reshape([1, 2, -0.5], 3, 1)
W = diagm([8.0, 2.5, 4.0])
x0 = randn(1)
P0 = diagm([1e7])
mod1 = StateSpaceModel(F, V, G, W, x0, P0)

# Test simulating
println("Simulating series")
x, y = simulate(mod1, 100)
@time x, y = simulate(mod1, 100)
t = [1:size(x, 1)]

# Test filtering
println("Filtering")
filt = kalman_filter(y, mod1)
@time filt = kalman_filter(y, mod1)
println("Log likelihood:")
println(filt.loglik)

# Test smoothing
println("Smoothing")
smooth = kalman_smooth(y, mod1)
@time smooth = kalman_smooth(y, mod1)

# Test model-fitting
function build(theta)
	F = diagm(theta[1])
	V = diagm(exp(theta[2]))
	G = reshape(theta[3:5], 3, 1)
	W = diagm(exp(theta[6:8]))
	x0 = [theta[9]]
	P0 = diagm(1e7)
	StateSpaceModel(F, V, G, W, x0, P0)
end
theta0 = zeros(9)
build(theta0)

fit(y, build, theta0)

# println("Plotting")
# p = FramedPlot()
# for i=1:3
# 	add(p, Curve(t, y[:, i], "color", "grey"))
# end
# add(p, Curve(t, filt.filtered, "type", "dashed"))
# add(p, Curve(t, smooth.smoothed, "color", "blue"))
# # add(p, Curve(t, x[:, 1], "color", "blue"))
# # add(p, Curve(t, filt.filtered[:, 1], "color", "green"))
# # add(p, Curve(t, smooth.smoothed[:, 1], "color", "red"))
# Winston.display(p)

println("\n\nPassed Kalman tests.")


# By default, Julia/LLVM does not use fused multiply-add operations (FMAs).
# Since these FMAs can increase the performance of many numerical algorithms,
# we need to opt-in explicitly.
# See https://ranocha.de/blog/Optimizing_EC_Trixi for further details.
@muladd begin

# mapping formula from a point (xi, eta) in reference space [-1,1]^2 to a point (x,y)
# in physical coordinate space for a quadrilateral element with straight sides
#     Alg. 95 from the blue book of Kopriva
function straight_side_quad_map(xi, eta, corner_points)

  x = (0.25 * (  corner_points[1,1] * (1.0 - xi) * (1.0 - eta)
               + corner_points[2,1] * (1.0 + xi) * (1.0 - eta)
               + corner_points[3,1] * (1.0 + xi) * (1.0 + eta)
               + corner_points[4,1] * (1.0 - xi) * (1.0 + eta)) )

  y = (0.25 * (  corner_points[1,2] * (1.0 - xi) * (1.0 - eta)
               + corner_points[2,2] * (1.0 + xi) * (1.0 - eta)
               + corner_points[3,2] * (1.0 + xi) * (1.0 + eta)
               + corner_points[4,2] * (1.0 - xi) * (1.0 + eta)) )

  return x, y
end

# Compute the metric terms for the straight sided quadrilateral mapping
#     Alg. 100 from the blue book of Kopriva
function straight_side_quad_map_metrics(xi, eta, corner_points)

  X_xi  = ( 0.25 * (  (1.0 - eta) * (corner_points[2,1] - corner_points[1,1])
                    + (1.0 + eta) * (corner_points[3,1] - corner_points[4,1])) )

  X_eta = ( 0.25 * (  (1.0 - xi) * (corner_points[4,1] - corner_points[1,1])
                    + (1.0 + xi) * (corner_points[3,1] - corner_points[2,1])) )

  Y_xi  = ( 0.25 * (  (1.0 - eta) * (corner_points[2,2] - corner_points[1,2])
                    + (1.0 + eta) * (corner_points[3,2] - corner_points[4,2])) )

  Y_eta = ( 0.25 * (  (1.0 - xi) * (corner_points[4,2] - corner_points[1,2])
                    + (1.0 + xi) * (corner_points[3,2] - corner_points[2,2])) )

  return X_xi, X_eta, Y_xi, Y_eta
end

# construct the (x,y) node coordinates in the volume of a straight sided element
function calc_node_coordinates!(node_coordinates::AbstractArray{<:Any, 4}, element, nodes, corners)

  for j in eachindex(nodes), i in eachindex(nodes)
    node_coordinates[:, i ,j ,element] .= straight_side_quad_map(nodes[i], nodes[j], corners)
  end

  return node_coordinates
end

# construct the metric terms for a straight sided element
function calc_metric_terms!(jacobian_matrix, element, nodes, corners)

  # storage format:
  #   jacobian_matrix[1,1,:,:,:] <- X_xi
  #   jacobian_matrix[1,2,:,:,:] <- X_eta
  #   jacobian_matrix[2,1,:,:,:] <- Y_xi
  #   jacobian_matrix[2,2,:,:,:] <- Y_eta
  for j in eachindex(nodes), i in eachindex(nodes)
    (jacobian_matrix[1, 1, i, j, element],
     jacobian_matrix[1, 2, i, j, element],
     jacobian_matrix[2, 1, i, j, element],
     jacobian_matrix[2, 2, i, j, element]) = straight_side_quad_map_metrics(nodes[i], nodes[j],
                                                                            corners)
  end

  return jacobian_matrix
end

# construct the normal direction vectors (but not actually normalized) for a straight sided element
# normalization occurs on the fly during the surface flux computation
function calc_normal_directions!(normal_directions, element, nodes, corners)

  # normal directions on the boundary for the left (local side 4) and right (local side 2)
  for j in eachindex(nodes)
    # side 2
    X_xi, X_eta, Y_xi, Y_eta = straight_side_quad_map_metrics(1.0, nodes[j], corners)
    Jtemp = X_xi * Y_eta - X_eta * Y_xi
    normal_directions[1, j, 2, element] = sign(Jtemp) * ( Y_eta )
    normal_directions[2, j, 2, element] = sign(Jtemp) * (-X_eta )

    # side 4
    X_xi, X_eta, Y_xi, Y_eta = straight_side_quad_map_metrics(-1.0, nodes[j], corners)
    Jtemp =  X_xi * Y_eta - X_eta * Y_xi
    normal_directions[1, j, 4, element] = -sign(Jtemp) * ( Y_eta )
    normal_directions[2, j, 4, element] = -sign(Jtemp) * (-X_eta )
  end

  # normal directions on the boundary for the top (local side 3) and bottom (local side 1)
  for i in eachindex(nodes)
    # side 1
    X_xi, X_eta, Y_xi, Y_eta = straight_side_quad_map_metrics(nodes[i], -1.0, corners)
    Jtemp =  X_xi * Y_eta - X_eta * Y_xi
    normal_directions[1, i, 1, element] = -sign(Jtemp) * (-Y_xi )
    normal_directions[2, i, 1, element] = -sign(Jtemp) * ( X_xi )

    # side 3
    X_xi, X_eta, Y_xi, Y_eta = straight_side_quad_map_metrics(nodes[i], 1.0, corners)
    Jtemp = X_xi * Y_eta - X_eta * Y_xi
    normal_directions[1, i, 3, element] = sign(Jtemp) * (-Y_xi )
    normal_directions[2, i, 3, element] = sign(Jtemp) * ( X_xi )
  end

  return normal_directions
end

end # @muladd


@DeepShiba_Func mutable struct Log <: Func end

@DeepShiba_Func mutable struct Sin <: Func end

@DeepShiba_Func mutable struct Cos <: Func end

@DeepShiba_Func mutable struct Tan <: Func end

@DeepShiba_Func mutable struct Tanh <: Func end

_Log(x::Variable) = Log([x, ], nothing, 0)(x)
_Sin(x) = Sin([x, ],   nothing, 0)(x)
_Cos(x) = Cos([x, ], nothing, 0)(x)
_Tan(x) = Tan([x, ], nothing, 0)(x)
_Tanh(x) = Tanh([x, ], nothing, 0)(x)

forward(f::Log, x) = log(x)
forward(f::Sin, x) = sin(x)
forward(f::Cos, x) = cos(x)
forward(f::Tan, x) = tan(x)
forward(f::Tanh, x) = tanh(x)

function backward(f::Log)::AbstractArray{Variable}
    gys = get_gys(f.outputs)
    x = f.inputs[1]
    return [1 / x,] .* gys
end

function backward(f::Sin)::AbstractArray{Variable}
    gys = get_gys(f.outputs)
    x, = f.inputs
    return [cos(x),] .* gys
end

function backward(f::Cos)::AbstractArray{Variable}
    gys = get_gys(f.outputs)
    x, = f.inputs
    return [-sin(x),] .* gys
end

function backward(f::Tan)::AbstractArray{Variable}
    gys = get_gys(f.outputs)
    x, = f.inputs
    return [1 / cos(x)^2,] .* gys
end

function backward(f::Tanh)::AbstractArray{Variable}
    gys = get_gys(f.outputs)
    y = f.outputs[1]
    return [1 - y^2,] .* gys
end


using OrthogonalPolynomialsQuasi, BandedMatrices, LazyArrays, Test

@testset "Ultraspherical" begin
    @testset "Transforms" begin
        U = Ultraspherical(1)
        x = axes(U,1)
        Un = U[:,Base.OneTo(5)]
        @test factorize(Un) isa ContinuumArrays.TransformFactorization
        @test (Un \ x) ≈ [0,0.5,0,0,0]
        @test (U * (U \ exp.(x)))[0.1] ≈ exp(0.1)
    end

    @testset "Operators" begin
        @testset "Lowering" begin
            λ = 1
            wC1 = WeightedUltraspherical(λ)
            wC2 = WeightedUltraspherical(λ+1)
            L = wC1 \ wC2
            @test L isa BandedMatrix
            @test bandwidths(L) == (2,0)
            @test L[1,1] ≈ 2λ*(2λ+1)/(4λ*(λ+1))
            @test L[3,1] ≈ -2/(4λ*(λ+1))
            u = [randn(10); zeros(∞)]
            @test wC1[0.1,:]'*(L*u) ≈ wC2[0.1,:]'*u
        end

        @testset "Weighted Derivative" begin
            T = Chebyshev()
            wC1 = WeightedUltraspherical(1)
            wC2 = WeightedUltraspherical(2)
            x = axes(wC2,1)
            D = Derivative(x)

            @test wC1 \ (D*wC2) isa BandedMatrix
            @test (wC1 \ (D*wC2))[1:5,1:5] == BandedMatrix(-1 => [-1.5,-4,-7.5,-12])

            u = wC2 * [randn(5); zeros(∞)]
            @test (D*u)[0.1] ≈ ((D*T) * (T\u))[0.1]
        end

        @testset "Interrelationships" begin
            @testset "Chebyshev–Ultrashperical" begin
                T = Chebyshev()
                U = ChebyshevU()
                C = Ultraspherical(2)
                D = Derivative(axes(T,1))

                @test C\C === pinv(C)*C === Eye(∞)
                D₀ = U\(D*T)
                D₁ = C\(D*U)
                @test D₁ isa BandedMatrix
                @test (D₁*D₀)[1:10,1:10] == diagm(2 => 4:2:18)
                @test D₁*D₀ isa MulMatrix
                @test bandwidths(D₁*D₀) == (-2,2)

                S₁ = (C\U)[1:10,1:10]
                @test S₁ isa BandedMatrix{Float64}
                @test S₁ == diagm(0 => 1 ./ (1:10), 2=> -(1 ./ (3:10)))
            end
            @testset "Legendre" begin
                @test Ultraspherical(0.5) \ (UltrasphericalWeight(0.0) .* Ultraspherical(0.5)) == Eye(∞)
                @test Legendre() \ (UltrasphericalWeight(0.0) .* Ultraspherical(0.5)) == Eye(∞)
            end
        end
    end

    @testset "test on functions" begin
        T = Chebyshev()
        U = Ultraspherical(1)
        D = Derivative(axes(T,1))
        f = T*Vcat(randn(10), Zeros(∞))
        @test (U*(U\f)).args[1] isa Ultraspherical
        @test (U*(U\f))[0.1] ≈ f[0.1]
        @test (D*f)[0.1] ≈ ForwardDiff.derivative(x -> (ChebyshevT{eltype(x)}()*f.args[2])[x],0.1)
    end

    @testset "Evaluation" begin
        C = Ultraspherical(2)
        @test @inferred(C[0.1,Base.OneTo(0)]) == Float64[]
        @test @inferred(C[0.1,Base.OneTo(1)]) == [1.0]
        @test @inferred(C[0.1,Base.OneTo(2)]) == [1.0,0.4]
        @test @inferred(C[0.1,Base.OneTo(3)]) == [1.0,0.4,-1.88]
    end

end


using Gadfly, Distributions, Random

set_default_plot_size(6inch, 16inch)

Random.seed!(1234)

x = rand(Normal(), 100)
y = rand(Normal(10), 100)

xd = Normal()
yd = Normal(10)

# two numeric vectors
pl1 = plot(x=x, y=y, Stat.qq, Geom.point)

# one numeric and one Distribution
pl2 = plot(x=x, y=yd, Stat.qq, Geom.point)
pl3 = plot(x=xd, y=y, Stat.qq, Geom.point)

# Apply different scales to x and y
pl4 = plot(x=x, y=exp.(y), Stat.qq, Geom.point, Scale.y_log10)
pl5= plot(x=exp.(x), y=y, Stat.qq, Geom.point, Scale.x_log10)

# Apply scales to Distributions
z = rand(Exponential(), 100)
pl5 = plot(x=z, y=Exponential(), Stat.qq, Geom.point)
pl6 = plot(x=log.(z), y=Exponential(), Stat.qq, Geom.point)
pl7 = plot(x=log.(z), y=Exponential(), Stat.qq, Geom.point, Scale.y_log)
pl8 = plot(x=z, y=Exponential(), Stat.qq, Geom.point, Scale.x_log, Scale.y_log)

# by analogy with Stat.func, computed (.y) aes should be scaled:
#plot(xmin=[1], xmax=[15], y=[exp], Stat.func, Geom.line)
# (y value is log(exp(x), or y=x)
#plot(xmin=[1], xmax=[15], y=[exp], Stat.func, Geom.line, Scale.y_log)

vstack(pl1, pl2, pl3, pl4, pl5, pl6, pl7, pl8)


push!(LOAD_PATH,"C:\\julia_code")

const Float = Float64;

using Distances
using GPbox

function sqvd2(ndim::Int64,x::Vector{T},y::Vector{T},len_fact::T) where T<:Float64
    # len_fact = 1/(2*len^2);
    s::T = 0.0;
    for i=1:ndim
        s+=(x[i] - y[i])^2;
    end
    # return s
    return exp(-s*len_fact)
end

function gen_cov(x::Matrix{T},y::Matrix{T},gpk::GPkernel) where T<:Float
    if(gpk.kernel_type=="squared exp")
        nx = size(x,2);
        ny = size(y,2);
        ndim::Int64 = size(x,1);
        lensq::T = (get(gpk.param,"len",1))^2;
        covs = Array{T}(nx,ny);
        for i=1:nx
            for j=1:ny
                # @inbounds covs[i,j] = sqvd(x[:,i],y[:,j]);
                @inbounds covs[i,j] = sqvd2(ndim,x[:,i],y[:,j],1/(2*lensq));
            end
        end
    end
    return covs
end

len = 0.3;
gpk = GPkernel("squared exp",Dict("len"=>len));
r1 = rand(10,5000);
r2 = rand(10,5000);

@time tt1 = gen_cov(r1,r2,gpk);
@time tt2 = genCovs(r1,r2,gpk);


# Factorize two different matrices (one real, one complex) and solve for multiple rhs and free memory
using MUMPSjInv
using Test

include("getDivGrad.jl")
A = getDivGrad(24, 23, 25)
A2 = getDivGrad(34, 32, 36)
n = size(A, 1)
n2 = size(A2, 1)
A = A + im * spdiagm(0 => rand(n))

nrhs = 10
rhs = randn(n, nrhs) + im*randn(n, nrhs)
rhs2 = randn(n2, nrhs)

println("Factorize complex matrix")
@time F1 = factorMUMPS(A,1)

println("Factorize real matrix")
@time F2 = factorMUMPS(A2,1)

println("Solve complex system")
@time x = applyMUMPS(F1,rhs)
err = zeros(nrhs)
for i=1:nrhs
    err[i] = norm(A * x[:, i] - rhs[:, i]) / norm(rhs[:, i])
end
@test maximum(err) < 1e-14

println("Solve real system")
@time x2 = applyMUMPS(F2, rhs2)
err = zeros(nrhs)
for i = 1:nrhs
    err[i] = norm(A2 * x2[:, i] - rhs2[:, i]) / norm(rhs2[:, i])
end
@test maximum(err) < 1e-14

println("Free memory")
destroyMUMPS(F1)
destroyMUMPS(F2)


using JuMP, Compat.Test, ReSHOP

## Solve test problem with non-linear binary variables
 #
 #  min   100 * (x2 - (0.5 + x1) ^ 2) ^ 2 + (1 - x1) ^ 2
 #  s.t.  x1, x2 binary
 #
 #  The solution is (0, 0).
 ##

if !isdef(:solver_sbb); solver_sbb = ReSHOP.ReSHOPSolver("sbb"); end

@testset "example: jump_nonlinearbinary" begin
    m = Model(solver=solver_sbb)
    @variable(m, x[1:2], Bin)

    # Set some non-binary bounds on x1 and x2. These should be ignored.
    # The optimal solution if x is Int is (1, 2) which is allowed by these bounds
    setupperbound(x[1], 2)
    setupperbound(x[2], 2)

    @NLobjective(m, Min, 100*(x[2] - (0.5 + x[1])^2)^2 + (1 - x[1])^2)

    @test solve(m) == :Optimal
    @test getvalue(x)[:] == [0.0, 0.0]
    @test getobjectivevalue(m) == 7.25
end


using DataStructures

# this is the main heurisctic used to find the test fragments
function generate_basis(eq, x; homotopy=true)
    # if homotopy return generate_homotopy(eq, x) end
    eq = expand(eq)
    S = 0 #Set{Any}()
    for t in terms(eq)
        q = equivalent(t, x)
        f = kernel(q)

        C₁ = closure(f, x)
        p = q * inverse(f)

        if homotopy
            if isdependent(p, x)
                C₂ = generate_homotopy(p, x)
            else
                C₂ = 1
            end
        else
            C₂ = find_candidates(p, x)
        end

        S += sum(c₁*c₂ for c₁ in C₁ for c₂ in C₂)
    end
    return unique([one(x); [equivalent(t,x) for t in terms(S)]])
end

function expand_basis(basis, x)
    b = sum(basis)
    eq = (1 + x) * (b + Differential(x)(b))
    eq = expand(expand_derivatives(eq))
    S = Set{Any}()
    enqueue_expr!(S, eq, x)
    return [one(x); [s for s in S]]
end

function closure(eq, x; max_terms=50)
    if !isdependent(eq, x) return [one(x)] end
    D = Differential(x)
    S = Set{Any}()
    q = Queue{Any}()
    enqueue_expr!(S, q, eq, x)

    while !isempty(q) && length(S) < max_terms
        y = dequeue!(q)
        enqueue_expr!(S, q, expand_derivatives(D(y)), x)
    end
    unique([one(x); [s for s in S]; [s*x for s in S]])
end

function find_candidates(eq, x)
    eq = apply_d_rules(eq)
    D = Differential(x)

    S = Set{Any}()
    q = Queue{Any}()
    enqueue_expr!(S, q, eq, x)

    for y in q
        ∂y = expand_derivatives(D(y))
        enqueue_expr!(S, ∂y, x)
    end

    return unique([one(x); [s for s in S]])
end

function candidate_pow_minus(p, k; abstol=1e-8)
    if isnan(poly_deg(p))
        return [p^k, p^(k+1), log(p)]
    end

    x = var(p)
    d = poly_deg(p)

    for j = 1:10  # will try 10 times to find the roots
        r, s = find_roots(p, x)
        if length(r) + length(s) >= d break end
    end
    s = s[1:2:end]
    r = nice_parameter.(r)
    s = nice_parameter.(s)

    # ∫ 1 / ((x-z₁)(x-z₂)) dx = ... + c₁ * log(x-z₁) + c₂ * log(x-z₂)
    q = Any[log(x - u) for u in r]
    for i in eachindex(s)
        β = s[i]
        if abs(imag(β)) > abstol
            push!(q, atan((x - real(β))/imag(β)))
            push!(q, (1+x)*log(x^2 - 2*real(β)*x + abs2(β)))
        else
            push!(q, log(x - real(β)))
        end
    end
    q = unique(q)

    # return [[p^k, p^(k+1)]; candidates(q₁, x)]
    if k ≈ -1
        return [[p^k]; q]
    else
        return [[p^k, p^(k+1)]; q]
    end
end

function candidate_sqrt(p, k)
    x = var(p)

    h = Any[p^k, p^(k+1)]

    if poly_deg(p) == 2
        r, s = find_roots(p, x)
        l = leading(p, x)
        if length(r) == 2
            if sum(r) ≈ 0
                r₁ = abs(r[1])
                if l > 0
                    push!(h, acosh(x/r₁))
                else
                    push!(h, asin(x/r₁))
                end
            end
        elseif real(s[1]) ≈ 0
            push!(h, asinh(x/imag.(s[1])))
        end
    end

    Δ = expand_derivatives(Differential(x)(p))
    push!(h, log(0.5*Δ + sqrt(p)))
    return h
end

###############################################################################

function enqueue_expr!(S, q, eq::SymbolicUtils.Add, x)
    for t in arguments(eq)
        enqueue_expr!(S, q, t, x)
    end
end

function enqueue_expr!(S, q, eq, x)
    y = eq / coef(eq, x)
    if y ∉ S && isdependent(y, x)
        enqueue!(q, y)
        push!(S, y)
    end
end

function enqueue_expr!(S, eq::SymbolicUtils.Add, x)
    for t in arguments(eq)
        enqueue_expr!(S, t, x)
    end
end

function enqueue_expr!(S, eq, x)
    y = eq / coef(eq, x)
    if y ∉ S && isdependent(y, x)
        push!(S, y)
    end
end


using Statistics, DataFrames, CSV
data = CSV.read("data/purchaseData.csv", DataFrame)

println(mean(data.Price),"\n")
println(mean(skipmissing(data.Price)),"\n")
println(coalesce.(data.Grade, "QQ")[1:4],"\n")
println(first(dropmissing(data,:Price), 4),"\n")
println(sum(ismissing.(data.Name)),"\n")
println(findall(completecases(data))[1:4])


using FractionalCalculus
using Test

@testset "Test Caputo Direct Fractional Derivative" begin
    @test isapprox(fracdiff(x->x, 0.5, 0, 1, 1e-8, CaputoDirect())[1], 2/sqrt(pi); atol = 1e-5)
    @test isapprox(fracdiff(x->x^5, 0.5, 0, 3.2, 1e-8, CaputoDirect())[1], 4.300306216488329e2; atol = 1e-5)
    @test isapprox(fracdiff(x->x^5, 0.25, 0, 3.2, 1e-8, CaputoDirect())[1], 382.1228113951680; atol = 1e-5)
    @test isapprox(fracdiff(x->(x-3)^3+(x+2)^2+5, 0.5, 0, 0.83, 1e-8, CaputoDirect())[1], 23.899930581118699; atol = 1e-5)
    @test isapprox(fracdiff(x->(x-3)^3+(x+2)^2+5, 0.25, 0, 0.83, 1e-8, CaputoDirect())[1], 22.96354626572943; atol = 1e-5)
    @test isapprox(fracdiff(x->cos(x), 0.5, 0, 0.34, 1e-8, CaputoDirect())[1], -0.147174774607683; atol = 1e-5)
    @test isapprox(fracdiff(x->cos(x), 0.25, 0, 0.34, 1e-8, CaputoDirect())[1], -0.093074344902431; atol = 1e-5)
    @test isapprox(fracdiff(x->exp(-x^2), 0.5, 0, 2.3, 1e-8, CaputoDirect())[1], -0.505891017154289; atol = 1e-5)
    @test isapprox(fracdiff(x->exp(-x^2), 0.25, 0, 2.3, 1e-8, CaputoDirect())[1], -0.762927553656252; atol = 1e-5)

    @test isapprox(fracdiff(x->x, 0.5, 0, [1, 2, 3], 0.0001, CaputoDirect()), [2/sqrt(pi), 1.5957691216057408, 1.9544100476116828]; atol=1e-3)
end

@testset "Test Caputo Piecewise Fractional Derivative" begin
    @test isapprox(fracdiff(x->x, 0.5, 1, 1e-4, CaputoPiecewise()), 2/sqrt(pi); atol = 1e-5)
    @test isapprox(fracdiff(x->x^5, 0.5, 3.2, 1e-4, CaputoPiecewise()), 4.300306216488329e2; atol = 1e-5)
    @test isapprox(fracdiff(cos, 0.5, 0.34, 1e-4, CaputoPiecewise()), -0.147174774607683; atol = 1e-5)
    @test isapprox(fracdiff(x->exp(-x^2), 0.25, 2.3, 1e-3, CaputoPiecewise()), -0.762927553656252; atol = 1e-5)
end

@testset "Test Caputo Piecewise Fractional Derivative" begin
    @test isapprox(fracdiff(x->x, 0.5, [1, 2, 3], 1e-4, CaputoPiecewise()), [1.1283791670557781, 1.5957691213491374, 1.9544100474864603])
end

@testset "Test Caputo Diethelm Fractional Derivative" begin
    @test isapprox(fracdiff(x->x, 0.5, 1, 1e-4, CaputoDiethelm()), 2/sqrt(pi); atol = 1e-5)
    @test isapprox(fracdiff(x->x^5, 0.5, 3.2, 1e-5, CaputoDiethelm()), 4.300306216488329e2; atol = 1e-5)
end

@testset "Test Caputo Diethelm Fractional Derivative" begin
    @test isapprox(fracdiff(x->x, 0.5, [1, 2, 3], 0.0001, CaputoDiethelm()), [1.1283791670955023, 1.5957691216057386, 1.954410047611745])
end

@testset "Test Caputo High Order Fractional Derivative" begin
    @test isapprox(fracdiff(x->x^8, 0.5, 1, 0.05, 3, CaputoHighOrder()), 2.8542562386883445; atol=1e-4)
end

@testset "Test Grunwald-Letnikov Fractional Derivative" begin
    @test isapprox(fracdiff(x->x, 0.5, 0, 0.5, GLDirect())[1], 0.7978845583186518; atol = 1e-5)
    @test isapprox(fracdiff(x->x^5, 0.5, 0, 3.2, GLDirect())[1], 4.300306216488329e2; atol = 1e-5)
    @test isapprox(fracdiff(x->x, 0.5, collect(0:0.01:1), 2, GLHighPrecision())[end], 2/sqrt(pi); atol = 1e-4)
end

@testset "Test GL_Multiplicative_Additive()" begin
    @test isapprox(fracdiff(x->x, 0.5, 1, 0.009, GLMultiplicativeAdditive()), 2/sqrt(pi); atol=1e-2)
end

@testset "Test GL_Lagrange_Three_Point_Interp" begin
    @test isapprox(fracdiff(x->x, 0.5, 1, 0.009, GLLagrangeThreePointInterp()), 2/sqrt(pi); atol=1e-2)
end

@testset "Test GL_Finite_Difference" begin
    @test isapprox(fracdiff(x->x, 0.5, 1, 0.009, GLFiniteDifference()), 2/sqrt(pi); atol=1e-2)
end

@testset "Test Grunwald Letnikov Lagrange_Three_Point_Interp" begin
    @test isapprox(fracdiff(x->x, 0.5, 1, 0.01, GLLagrangeThreePointInterp()), 2/sqrt(pi); atol=1e-4)
end

@testset "Test RLDiff Matrix Fractional Deriavtive" begin
    @test isapprox(fracdiff(x->x, 0.5, 1, 0.001, RLDiffMatrix())[end], 2/sqrt(pi); atol = 1e-3)
end

@testset "Test RLDiff Approx" begin
    @test isapprox(fracdiff(x->x, 0.5, 1, 0.001, RLDiffApprox()), 2/sqrt(pi); atol = 1e-4)

    # Test Array input end point
    result = fracdiff(x->x, 0.5, [1, 2, 3], 0.001, RLDiffApprox())
    @test isapprox(result[1], 2/sqrt(pi); atol=1e-4)
    @test isapprox(result[2], 1.5957691216057306; atol=1e-4)
    @test isapprox(result[3], 1.954410047611678; atol=1e-4)
end

@testset "Test RL Linear Spline Interpolation" begin
    @test isapprox(fracdiff(x->x, 0.5, 1, 0.0001, RLLinearSplineInterp()), 2/sqrt(pi); atol = 1e-4)

    # Test Array input end point
    result = fracdiff(x->x, 0.5, [1, 2, 3], 0.001, RLLinearSplineInterp())
    @test isapprox(result[1], 2/sqrt(pi); atol=1e-4)
    @test isapprox(result[2], 1.5957691216057306; atol=1e-4)
    @test isapprox(result[3], 1.954410047611678; atol=1e-4)
end

@testset "Test RL_G1 method" begin
    @test isapprox(fracdiff(x->x, 0.5, 0, 1, 0.006, RLG1()), 2/sqrt(pi); atol=1e-2)
end

@testset "Test RL_D method" begin
    @test isapprox(fracdiff(x->x, 0.5, 1, 0.001, RLD()), 2/sqrt(pi); atol=1e-4)
end

@testset "Test Hadamard Fractional Derivative" begin
    @test isapprox(fracdiff(log, 0.5, 1, 2, 1/4000000, HadamardLRect()), 0.9391460; atol=1e-4)
    @test isapprox(fracdiff(log, 0.3, 1, 2, 1/4000000, HadamardLRect()), 0.8514935; atol=1e-4)
    @test isapprox(fracdiff(log, 0.5, 1, 2, 1/4000000, HadamardRRect()), 0.9391460; atol=1e-4)
    @test isapprox(fracdiff(log, 0.3, 1, 2, 1/4000000, HadamardRRect()), 0.8514936; atol=1e-4)
    @test isapprox(fracdiff(log, 0.5, 1, 2, 1/4000000, HadamardTrap()), 0.9391460; atol=1e-4)
    @test isapprox(fracdiff(log, 0.3, 1, 2, 1/4000000, HadamardTrap()), 0.8514935; atol=1e-4)
end

@testset "Test Riesz Symmetric fractional derivative" begin
    @test isapprox(fracdiff(x->x, 0.5, 1, 0.5, RieszSymmetric()), [0.2651650429449553; 0.2651650429449553; -0.39774756441743303]; atol=1e-2)
end

@testset "Test macros" begin
    @test isapprox(@fracdiff(x->x, 0.5, 1), 1.1283791670955126; atol=1e-4)
    @test isapprox(@fracdiff(x->x^5, 0.5, 3.2), 4.300306216488329e2; atol=1e-2)
end

@testset "Test auxiliary functions" begin
    @test isapprox(RieszMatrix(0.5, 2, 0.1), [-1.58114 1.3835; 1.3835 -1.58114]; atol=1e-3)
    @test isapprox(omega(3, 2), [1, -2, 1, 0]; atol=1e-3)
    @test isapprox(B(2, 2), [1.0 0.0; -2.0 1.0]; atol=1e-3)
    @test isapprox(genfun(2), [1.5 -2 0.5]; atol=1e-3)
    @test isapprox(first_order(exp, 2, 1e-6), exp(2); atol=1e-5)
    @test isapprox(first_order(sin, 2, 1e-6), cos(2); atol=1e-5)
end


using Random: RandomDevice, randstring
using Nettle: hexdigest, gen_key32_iv16
using Primes: nextprime

const Bytes = Vector{UInt8}

abstract type RSAKey end

struct PublicRSAKey <: RSAKey
  e::BigInt
  n::BigInt
end

struct PrivateRSAKey <: RSAKey
  d::BigInt
  n::BigInt
end

"""
  `secure_rand(start, stop)`

Generates a relatively secure random number, based on the OS
"""
function secure_rand(start, stop)
  return rand(RandomDevice(), start:stop)
end

"""
  `secure_randstring(len)`

Generates a relatively secure random string, based on the OS
"""
function secure_randstring(len::Integer)::String
  return randstring(RandomDevice(), len)
end

"""
  `pack_int(num::Integer)::Bytes`

Packs an integer into an array of bytes
"""
function pack_int(num::Integer)::Bytes
  hex = string(num, base=16)
  pad = length(hex) % 2
  hex = ("0"^pad) * hex
  return hex2bytes(hex)
end

"""
  `unpack_int(b::Bytes)::BigInt`

Unpacks an array of bytes into a BigInt
"""
function unpack_int(b::Bytes)::BigInt
  hex = bytes2hex(b)
  return parse(BigInt, hex, base=16)
end

"""
  `gen_prime(num_bits::Integer)::BigInt`

Generates a random prime number with given number of bits
"""
function gen_prime(num_bits::Integer)::BigInt
  lower_bound = secure_rand(BigInt(2)^num_bits + 1, BigInt(2)^(num_bits+1))
  return nextprime(lower_bound)
end

"""
  `gen_RSAKeys(num_bits::Integer = 2048)::Tuple{PublicRSAKey, PrivateRSAKey}`

Generates Public and Private RSA Keys, given bit size of primes (default 2048)
"""
function gen_RSAKeys(num_bits::Integer = 2048)::Tuple{PublicRSAKey, PrivateRSAKey}
  size_offset = secure_rand(1, 5)

  # generate large primes p and q
  # make p and q differ by a few bits to make factoring harder
  p = gen_prime(num_bits)
  q = gen_prime(num_bits + size_offset)

  n = p*q # RSAKey modulus
  λ_n = lcm(p - 1, q - 1) # Carmichael's totient function for n

  # compute e (public encryption key)
  e = secure_rand(2^16 + 1, λ_n)
  while gcd(e, λ_n) != 1
    e = secure_rand(1, λ_n)
  end

  # compute d (private decryption key)
  # such that d*e ≡ 1 mod λ_n
  d = invmod(e, λ_n)

  public_key = PublicRSAKey(e, n)
  private_key = PrivateRSAKey(d, n)

  return public_key, private_key
end

"""
  `gen_salt()::Bytes`

Generates random 8-bit salt
"""
function gen_salt()::Bytes
  hex_val = hexdigest("sha256", secure_randstring(16))[1:16]
  return hex2bytes(hex_val)
end

"""
  `gen_key([passphrase::String])`

Generates 32 bit key, (can be influenced by a given passphrase, optional)
"""
function gen_key32()::Bytes
  passphrase = secure_randstring(16)
  salt = gen_salt()
  # generate key and initialization vector using passphrase and salt
  (key32, iv16) = gen_key32_iv16(Vector{UInt8}(passphrase), salt)
  return key32
end

function gen_key32(passphrase::String)::Bytes
  salt = gen_salt()
  # generate key and initialization vector using passphrase and salt
  (key32, iv16) = gen_key32_iv16(Vector{UInt8}(passphrase), salt)
  return key32
end

"""
  `to_str(bytes::Bytes)::String`

Converts binary to ascii string
"""
function to_str(bytes::Bytes)::String
  return join(Char(i) for i in bytes)
end

"""
  `to_bytes(str::String)::Bytes`

Converts ascii string to bytes (Vector{UInt8})
"""
function to_bytes(str::String)::Bytes
  return Bytes(str)
end


using LinearAlgebra, ForwardDiff

function row_col!(row,col,r,c)
    for cc in c
        for rr in r
            push!(row,convert(Int,rr))
            push!(col,convert(Int,cc))
        end
    end
    return row, col
end

v = [10.0; 1.0; 0.]
n = 0.0
μ = 0.5

P = [1. 0. 0.;
	 0. 1. 0.]

_norm(x) = sqrt(x'*x)
function vec_norm(x)
	if norm(x) == 0.0
		return ones(length(x))./norm(ones(length(x)))
	else
		x./_norm(x)
	end
end

function d_vec_norm(x)
	if norm(x) == 0.0
		y = 1.0*ones(length(x))#./norm(ones(length(x)))
		return (I - y*y'/(y'*y))/_norm(y)
	else
		(I - x*x'/(x'*x))/_norm(x)
	end
end
b = 1.0e-3*ones(2)
b = zeros(2)
d_vec_norm(b)
_norm(b)
vec_norm(b)
d_vec_norm(b)
norm(vec(d_vec_norm(b)) - vec(ForwardDiff.jacobian(vec_norm,norm(b) == 0.0 ? 1.0*ones(2) : b)))

ψ = 0.1
P*v + ψ*vec_norm(b)

##
include(joinpath(pwd(),"src/Ipopt.jl"))

nx = 4
nc = 2 + 1 + 1
function obj(x)
	b = x[1:2]
	ψ = x[3]
	s = x[4]

	return 100.0*s
end

function con!(c,x)
	b = x[1:2]
	ψ = x[3]
	s = x[4]

	c[1:2] = P*v + ψ*vec_norm(b)
	c[3] = μ*n - _norm(b)
	c[4] = s - min(ψ,(μ*n - _norm(b)))

	return nothing
end

function ∇con!(C,x)
	b = x[1:2]
	ψ = x[3]

	C[1:2,1:2] = ψ*d_vec_norm(b)
	C[1:2,3] = vec_norm(b)

	C[3,1:2] = -1.0*vec_norm(b)
	C[3,3] = 0.0

	if ψ < μ*n - _norm(b)
		C[4,3] = -1.0
	else
		C[4,1:2] = vec_norm(b)
	end
	C[4,4] = 1.0
	return nothing
end

_x = [1.0e-3; 1.0e-3; 1.0e-3; 1.0e-3]
obj(_x)
_c = zeros(nc)
con!(_c,_x)
_c
_C = zeros(nc,nx)
∇con!(_C,_x)

function eval_f(x::Vector{Float64})
  return obj(x)
end

function eval_g(x::Vector{Float64}, g::Vector{Float64})
	con!(g,x)

	return g
end

function eval_grad_f(x::Vector{Float64}, grad_f::Vector{Float64})
	grad_f .= ForwardDiff.gradient(obj,x)
end

function eval_jac_g(x::Vector{Float64}, mode, rows::Vector{Int32}, cols::Vector{Int32}, values::Vector{Float64})
	if mode == :Structure
		row = []
		col = []
		row_col!(row,col,(1:nc),(1:nx))
		rows .= vec(row)
		cols .= vec(col)
	else
		∇con!(reshape(values,nc,nx),x)
	end
end

function eval_h(x::Vector{Float64}, mode, rows::Vector{Int32}, cols::Vector{Int32}, obj_factor::Float64, lambda::Vector{Float64}, values::Vector{Float64})
	if mode == :Structure
		nothing
	else
		nothing
	end
end

x_L = [-1.0e8; -1.0e8; 0.0; 0.0]
x_U = 1.0e8*ones(nx)

g_L = zeros(nc)
g_U = [zeros(2);1.0e8;1.0e8]

prob = Ipopt.createProblem(nx, x_L, x_U, nc, g_L, g_U, nc*nx, nx*nx,
                     eval_f, eval_g, eval_grad_f, eval_jac_g, eval_h)
Ipopt.addOption(prob,"hessian_approximation","limited-memory")
# Ipopt.addOption(prob,"mu_strategy","monotone")

Ipopt.addOption(prob,"constr_viol_tol",1.0e-3)
Ipopt.addOption(prob,"tol",1.0e-3)

prob.x .= 1.0*rand(nx)
@time solvestat = Ipopt.solveProblem(prob)

b_sol = prob.x[1:2]
ψ_sol = prob.x[3]
s_sol = prob.x[4]

norm(v[1:2])
vec_norm(v[1:2])'*vec_norm(b_sol)

con!(_c,prob.x)
_c
_C = zeros(nc,nx)
∇con!(_C,prob.x)
_C


using MarketData, TradeModels, Quandl, Distributions
import Base.show, TimeSeries.findall

tinv(df::Int, pval::Float64) = quantile(TDist(df), pval) # t table
tinv(df::Int, pval::FloatRange{Float64}) = quantile(TDist(df), pval) # t table

include(Pkg.dir("FinancialSeries/src/dev/byvolume.jl"))

crude = quandl("CHRIS/CME_CL1", rows=1000)
myvol = 825399
c = byvolume(crude, myvol)

function findall(b::BitArray)
    rownums = zeros(Int, sum(b))
    j = 1
    for i in 1:length(b)
      if b[i]
        rownums[j] = i
        j+=1
      end
    end
    rownums
end

# Phi = phi_0 to phi_p or length of p + 1
# y = all the ys starting at the p index, because we need pre values
# [yp+1 yp+2 ... end]
# the number of rows in y == number of rows in X
# X is a big matrix with the first row of 1s and the number of columns after that equal to p for a total of p+1 columns
# the number of columns in X must match the number of rows in Phi, because that's how it lines up
# to estimate Phi  :
##     inv(X'X) * X' * y

ret = percentchange(cl, method="log")
ret = flipud(ret.values)

#myfit = ar(c["Settle"],6)


using MultiQuad, Test, Unitful

@testset "quad" begin
    rtol = 1e-10

    xmin = 0
    xmax = 4
    func1(x) = x^2 * exp(-x)
    res = 2 - 26 / exp(4)
    int, err = quad(func1, xmin, xmax, rtol = rtol)
    @test isapprox(int, res, atol = err)
    @test typeof(quad(
        func1,
        xmin,
        xmax,
        rtol = rtol,
        method = :vegas,
    )[1]) == Float64
    @test typeof(quad(
        func1,
        xmin,
        xmax,
        rtol = rtol,
        method = :suave,
    )[1]) == Float64

    xmin2 = 0 * u"m"
    xmax2 = 4 * u"m"
    func2(x) = x^2
    res2 = 64 / 3
    int2, err2 = quad(func2, xmin2, xmax2, rtol = rtol)
    @test isapprox(int2, res2 * u"m^3", atol = err2)
    @test typeof(quad(
        func2,
        xmin2,
        xmax2,
        rtol = rtol,
        method = :vegas,
    )[1]) == typeof(1.0 * u"m^3")
    @test typeof(quad(
        func2,
        xmin2,
        xmax2,
        rtol = rtol,
        method = :suave,
    )[1]) == typeof(1.0 * u"m^3")

    f(x) = x^4

    @test quad(f, -1, 1, method=:gausslegendre, order=100000)[1] ≈ 2/5

    @test quad(f, method=:gausshermite, order=10000)[1] ≈ 3(√π)/4

    @test quad(f, method=:gausslaguerre, order=10000)[1] ≈ 24
    @test quad(f, method=:gausslaguerre, order=3, α=1.0)[1] ≈ 120

    @test quad(f, -1, 1, method=:gausschebyshev, order=3)[1] ≈ 3π/8
    @test quad(f, -1, 1, method=:gausschebyshev, order=3, kind=1)[1] ≈ 3π/8
    @test quad(f, -1, 1, method=:gausschebyshev, order=3, kind=2)[1] ≈ π/16
    @test quad(f, -1, 1, method=:gausschebyshev, order=3, kind=3)[1] ≈ 3π/8
    @test quad(f, -1, 1, method=:gausschebyshev, order=3, kind=4)[1] ≈ 3π/8

    @test quad(f, -1, 1, method=:gaussradau, order=3)[1] ≈ 2/5
    @test quad(f, -1, 1, method=:gausslobatto, order=4)[1] ≈ 2/5

    @test_throws ErrorException quad(func1, 0, 4, method = :none)
end

@testset "dblquad" begin
    rtol = 1e-10

    xmin = 0
    xmax = 4
    ymin(x) = x^2
    ymax(x) = x^3

    func1(y, x) = y * x^3
    res1 = 241664 / 5
    int1, err1 = dblquad(
        func1,
        xmin,
        xmax,
        ymin,
        ymax,
        rtol = rtol,
        method = :cuhre,
    )
    int1b, err1b = dblquad(
        func1,
        xmin,
        xmax,
        ymin,
        ymax,
        rtol = rtol,
        method = :hcubature,
    )

    @test dblquad(
        (y,x)->x*y,
        0,
        2,
        0,
        3,
        rtol = rtol,
        method = :hcubature,
    )[1] ≈ 9

    @test isapprox(int1, res1, atol = err1)
    @test isapprox(int1b, res1, atol = err1b)
    @test typeof(dblquad(
        func1,
        xmin,
        xmax,
        ymin,
        ymax,
        rtol = rtol,
        method = :vegas,
    )[1]) == Float64
    @test typeof(dblquad(
        func1,
        xmin,
        xmax,
        ymin,
        ymax,
        rtol = rtol,
        method = :suave,
    )[1]) == Float64
    @test typeof(dblquad(
        func1,
        xmin,
        xmax,
        ymin,
        ymax,
        rtol = rtol,
        method = :divonne,
    )[1]) == Float64

    xmin2 = 0 * u"m"
    xmax2 = 4 * u"m"
    ymin2(x) = x^2 * u"m"
    ymax2(x) = x^3
    func2(y, x) = y * x^3

    int2, err2 = dblquad(func2, xmin2, xmax2, ymin2, ymax2, rtol = rtol)
    int2b, err2b = dblquad(
        func2,
        xmin2,
        xmax2,
        ymin2,
        ymax2,
        rtol = rtol,
        method = :hcubature,
    )

    @test isapprox(int2, res1 * u"m^10", atol = err2)
    @test isapprox(int2b, res1 * u"m^10", atol = err2b)
    @test typeof(dblquad(
        func2,
        xmin2,
        xmax2,
        ymin2,
        ymax2,
        rtol = rtol,
        method = :vegas,
    )[1]) == typeof(1.0 * u"m^10")
    @test typeof(dblquad(
        func2,
        xmin2,
        xmax2,
        ymin2,
        ymax2,
        rtol = rtol,
        method = :suave,
    )[1]) == typeof(1.0 * u"m^10")
    @test typeof(dblquad(
        func2,
        xmin2,
        xmax2,
        ymin2,
        ymax2,
        rtol = rtol,
        method = :divonne,
    )[1]) == typeof(1.0 * u"m^10")

    @test_throws ErrorException dblquad(
        func1,
        xmin,
        xmax,
        ymin,
        ymax,
        rtol = rtol,
        method = :none,
    )
end

@testset "tblquad" begin
    rtol = 1e-10

    xmin = 0
    xmax = 4
    ymin(x) = x^2
    ymax(x) = x^3
    zmin(x, y) = 3
    zmax(x, y) = 4

    func1(z, y, x) = y * x^3 * sin(z)
    res = 241664 / 5 * (cos(3) - cos(4))
    int1, err1 = tplquad(func1, xmin, xmax, ymin, ymax, zmin, zmax, rtol = rtol)
    int1b, err1b = tplquad(
        func1,
        xmin,
        xmax,
        ymin,
        ymax,
        zmin,
        zmax,
        rtol = rtol,
        method = :hcubature,
    )
    @test isapprox(int1, res, atol = err1)
    @test isapprox(int1b, res, atol = err1b)
    @test typeof(tplquad(
        func1,
        xmin,
        xmax,
        ymin,
        ymax,
        zmin,
        zmax,
        rtol = rtol,
        method = :vegas,
    )[1]) == Float64
    @test typeof(tplquad(
        func1,
        xmin,
        xmax,
        ymin,
        ymax,
        zmin,
        zmax,
        rtol = rtol,
        method = :suave,
    )[1]) == Float64

    @test tplquad(
        (z,y,x)->x*y*z,
        0,
        2,
        0,
        2,
        0,
        3,
        rtol = rtol,
        method = :hcubature,
    )[1] ≈ 18

    @test typeof(tplquad(
        func1,
        xmin,
        xmax,
        ymin,
        ymax,
        zmin,
        zmax,
        rtol = rtol,
        method = :divonne,
    )[1]) == Float64

    xmin2 = 0 * u"m"
    xmax2 = 4 * u"m"
    ymin2(x) = x^2 * u"m"
    ymax2(x) = x^3
    zmin2(x, y) = 3
    zmax2(x, y) = 4

    func2(z, y, x) = y * x^3 * sin(z)
    int2, err2 = tplquad(
        func2,
        xmin2,
        xmax2,
        ymin2,
        ymax2,
        zmin2,
        zmax2,
        rtol = rtol,
    )
    int2b, err2b = tplquad(
        func2,
        xmin2,
        xmax2,
        ymin2,
        ymax2,
        zmin2,
        zmax2,
        rtol = rtol,
        method = :hcubature,
    )
    @test isapprox(int2, res * u"m^10", atol = err2)
    @test isapprox(int2b, res * u"m^10", atol = err2b)
    @test typeof(tplquad(
        func2,
        xmin2,
        xmax2,
        ymin2,
        ymax2,
        zmin2,
        zmax2,
        rtol = rtol,
        method = :vegas,
    )[1]) == typeof(1.0 * u"m^10")
    @test typeof(tplquad(
        func2,
        xmin2,
        xmax2,
        ymin2,
        ymax2,
        zmin2,
        zmax2,
        rtol = rtol,
        method = :suave,
    )[1]) == typeof(1.0 * u"m^10")
    @test typeof(tplquad(
        func2,
        xmin2,
        xmax2,
        ymin2,
        ymax2,
        zmin2,
        zmax2,
        rtol = rtol,
        method = :divonne,
    )[1]) == typeof(1.0 * u"m^10")

    @test_throws ErrorException tplquad(
        func2,
        xmin2,
        xmax2,
        ymin2,
        ymax2,
        zmin2,
        zmax2,
        rtol = rtol,
        method = :none,
    )[1]
end


# SPDX-License-Identifier: MIT

using CUDA
using Documenter
using ExpFamilyDistributions
using LinearAlgebra
using SpecialFunctions: loggamma, digamma
using Test

CUDA.allowscalar(false)

const types = [Float32, Float64]
const arraytypes = [Array]

if CUDA.functional()
    @info "gpu available testing on CuArray"
    CUDA.versioninfo()
    push!(arraytypes, CuArray)
end

for T in types

    @testset "PD matrix operations ($T)" begin
        M = T[1 0.5; 0.5 1]
        @test all(inv(M) .≈ pdmat_inverse(M))
        @test all(logdet(M) .≈ pdmat_logdet(M))
    end

    for AT in arraytypes

        @testset "DefaultParameter ($AT | $T)" begin
            ξ = AT(T[1.0, 2.0, 3.0])
            p = DefaultParameter(ξ)

            @test all(naturalform(p) .≈ ξ)
            @test all(realform(p) .≈ ξ)
            @test all(Array(jacobian(p)) .≈ Matrix{T}(I, length(ξ), length(ξ)))
            p̄ = reallocate(p, Tuple)
            @test typeof(realform(p̄)) == Tuple{T, T, T}

            dict = todict(p)
            np = fromdict(typeof(p), dict)
            @test all(realform(p) .≈ realform(np))
            @test typeof(realform(p)) == typeof(realform(np))
        end

        @testset "Normal ($AT | $T)" begin
            n = Normal(AT(T[1, 2]), AT(T[2 0; 0 3]))
            @test typeof(n).mutable

            η = AT(T[1/2, 2/3, -1/4, -1/6, 0])
            @test all(naturalform(n.param) .≈ η)

            x = AT(T[2, 3])
            Tx = AT(T[2, 3, 4, 9, 6])
            @test all(stats(n, x) .≈ Tx)

            A = .5 * pdmat_logdet(n.Σ) + .5 * dot(n.μ, pdmat_inverse(n.Σ) * n.μ)
            @test typeof(lognorm(n)) == T
            @test lognorm(n) ≈ T(A)

            @test basemeasure(n, x) ≈ T(-log(2π))

            ETx = AT(T[1, 2, 3, 7, 2])
            @test all(gradlognorm(n) .≈ ETx)

            s1, s2, s3 = splitgrad(n, gradlognorm(n))
            @test size(s1) == (2,)
            @test size(s2) == (2,)
            @test size(s3) == (1,)

            n2 = Normal(AT(zeros(T, 2)), AT(Matrix{T}(I, 2, 2)))
            @test isapprox(kldiv(n, n), 0; atol = (T == Float32 ? 1e-6 : 1e-16))
            @test kldiv(n, n2) >= 0
            @test kldiv(n2, n) >= 0

            n2 = Normal(stdparam(n, naturalform(n.param))...)
            @test isapprox(kldiv(n, n2),  0; atol = (T == Float32 ? 1e-6 : 1e-16))

            @test size(sample(n2, 1)) == (2, 1)
            @test size(sample(n2, 10)) == (2, 10)
            @test eltype(sample(n2, 2)) == T
        end

        @testset "NormalDiag ($AT | $T)" begin
            n = NormalDiag(AT(T[1, 2]), AT(T[2, 2]))
            @test typeof(n).mutable

            η = AT(T[.5, 1, -.25, -.25])
            @test all(naturalform(n.param) .≈ η)

            x = AT(T[2, 3])
            Tx = AT(T[2, 3, 4, 9])
            @test all(stats(n, x) .≈ Tx)

            A = .5 * pdmat_logdet(n.Σ) + .5 * dot(n.μ, pdmat_inverse(n.Σ) * n.μ)
            @test typeof(lognorm(n)) == T
            @test lognorm(n) ≈ T(A)

            @test basemeasure(n, x) ≈ T(-log(2π))

            ETx = AT(T[1, 2, 3, 6])
            @test all(gradlognorm(n) .≈ ETx)

            s1, s2 = splitgrad(n, gradlognorm(n))
            @test size(s1) == (2,)
            @test size(s2) == (2,)

            n2 = NormalDiag(AT(zeros(T, 2)), AT(ones(T, 2)))
            @test kldiv(n, n) ≈  0
            @test kldiv(n, n2) >= 0
            @test kldiv(n2, n) >= 0

            n2 = NormalDiag(stdparam(n, naturalform(n.param))...)
            @test kldiv(n, n2) ≈  0

            @test size(sample(n, 10)) == (2, 10)
            @test eltype(sample(n, 2)) == T
       end

        @testset "Gamma ($AT | $T)" begin
            g = Gamma(AT(T[1, 2]), AT(T[2, 3]))
            @test typeof(g).mutable

            η = AT(T[-2, -3, 1, 2])
            @test all(naturalform(g.param) .≈ η)

            x = AT(T[1, 2])
            Tx = vcat(x, log.(x))
            @test all(stats(g, x) .≈ Tx)

            A = loggamma(1) + loggamma(2) - 1*log(2) - 2*log(3)
            @test lognorm(g) ≈ T(A)

            @test basemeasure(g, x) ≈ sum(-log.(x))

            ETx = AT(T[1/2, 2/3, digamma(1) - log(2), digamma(2) - log(3)])
            @test all(gradlognorm(g) .≈ ETx)

            s1, s2 = splitgrad(g, gradlognorm(g))
            @test length(s1) == 2
            @test length(s2) == 2

            g2 = Gamma(AT(T[1, 1]), AT(T[1, 1]))
            @test kldiv(g, g) ≈  0
            @test kldiv(g, g2) >= 0
            @test kldiv(g2, g) >= 0

            g2 = Gamma(stdparam(g, naturalform(g.param))...)
            @test kldiv(g, g2) ≈  0

            @test size(sample(g, 10)) == (2,10)
        end

        @testset "Dirichlet ($AT | $T)" begin
            d = Dirichlet(AT(T[1, 2, 3]))
            @test typeof(d).mutable

            η = AT(T[1, 2, 3])
            @test all(naturalform(d.param) .≈ η)

            x = AT(T[1, 2, 3])
            Tx = AT(T[log(1), log(2), log(3)])
            @test all(stats(d, x) .≈ Tx)

            A = sum(loggamma_dot(d.α)) - loggamma(sum(d.α))
            @test lognorm(d) ≈ T(A)

            @test basemeasure(d, x) ≈ -log.(x)

            ETx = digamma.(d.α) .- digamma(sum(d.α))
            @test all(gradlognorm(d) .≈ ETx)

            s = splitgrad(d, gradlognorm(d))
            @test size(s) == (3,)

            d2 = Dirichlet(AT(ones(T, 3)))
            @test kldiv(d, d) ≈  0
            @test kldiv(d, d2) >= 0
            @test kldiv(d2, d) >= 0

            d2 = Dirichlet(stdparam(d, naturalform(d.param))...)
            @test kldiv(d, d2) ≈  0

            @test size(sample(d, 10)) == (3, 10)
        end

        @testset "Wishart($AT | $T)" begin
            X = AT(AT(T[1 0.5; 0.5 2]))

            D = T(2)
            W = AT(Matrix{T}(I, 2, 2))
            v = T(2)
            w = Wishart(W, v)
            @test typeof(w).mutable

            invW = pdmat_inverse(W)
            η = vcat(-.5 * diag(invW), vec_tril(invW), v/2)
            @test eltype(naturalform(w.param)) == T
            @test all(naturalform(w.param) .≈ η)

            TX = vcat(diag(X), vec_tril(X), pdmat_logdet(X))
            @test all(stats(w, X) .≈ TX)

            A = .5*( v*pdmat_logdet(W) + v*D*log(2) ) + sum([loggamma((v+1-i)/2) for i in 1:D])
            @test lognorm(w) ≈ T(A)

            B = -.5*( (D-1)*pdmat_logdet(X) + .5*D*(D-1)*log(π) )
            @test basemeasure(w, X) ≈ B

            vW = v*W
            ETX = vcat(diag(vW), vec_tril(vW), sum([digamma(T(0.5)*(v+1-i)) for i in 1:D]) + D*T(log(2)) + pdmat_logdet(W))
            @test eltype(gradlognorm(w)) == T
            @test all(gradlognorm(w) .≈ ETX)

            s = splitgrad(w, gradlognorm(w))
            @test length(s) == 3
            @test all(s[1] .≈ diag(vW))
            @test all(s[2] .≈ vec_tril(vW))
            @test s[3] .≈ sum([digamma((v+1-i)/2) for i in 1:D]) + D*log(2) + pdmat_logdet(W)

            w2 = Wishart(AT(Matrix{T}(I, 2, 2)), 2)
            @test kldiv(w, w) ≈  0
            @test kldiv(w, w2) >= 0
            @test kldiv(w2, w) >= 0

            w2 = Wishart(stdparam(w, naturalform(w.param))...)
            @test kldiv(w, w2) ≈  0

            @test size(sample(w, 10)) == (2, 2, 10)
        end
    end
end

# Finally, test the documentation
DocMeta.setdocmeta!(ExpFamilyDistributions, :DocTestSetup,
                    :(using ExpFamilyDistributions), recursive = true)
doctest(ExpFamilyDistributions)


using Test, OrdinaryDiffEq, HyperbolicDiffEq

u₀func(x) = sinpi(x)

balance_law = Burgers()
@inferred UniformPeriodicMesh1D(0., 2., 50)
meshx = UniformPeriodicMesh1D(0., 2., 50)
@inferred LobattoLegendre(4)
basis = LobattoLegendre(4)
tspan = (0.0, 0.3)

@inferred UniformPeriodicFluxDiffDisc1D(balance_law, meshx, basis, EnergyConservativeFlux(), GodunovFlux())
semidisc = UniformPeriodicFluxDiffDisc1D(balance_law, meshx, basis, EnergyConservativeFlux(), GodunovFlux())
semidiscretise(semidisc, u₀func, tspan)
ode = semidiscretise(semidisc, u₀func, tspan)
sol = solve(ode, SSPRK104(), dt=max_dt(tspan[1], ode.u0, semidisc), save_everystep=false)
@inferred evaluate_coefficients(sol.u[end], semidisc)
x, u = evaluate_coefficients(sol.u[end], semidisc)

@inferred UniformPeriodicFluxDiffDisc1D(balance_law, meshx, basis, L2L4ConservativeFlux(), GodunovFlux())
semidisc = UniformPeriodicFluxDiffDisc1D(balance_law, meshx, basis, L2L4ConservativeFlux(), GodunovFlux())
ode = semidiscretise(semidisc, u₀func, tspan)
sol = solve(ode, SSPRK104(), dt=max_dt(tspan[1], ode.u0, semidisc), save_everystep=false)
@inferred evaluate_coefficients(sol.u[end], semidisc)
x, u = evaluate_coefficients(sol.u[end], semidisc)

@inferred UniformPeriodicFluxDiffDisc1D(balance_law, meshx, basis, L2L2sConservativeFlux(Val{3}()), GodunovFlux())
semidisc = UniformPeriodicFluxDiffDisc1D(balance_law, meshx, basis, L2L2sConservativeFlux(Val{3}()), GodunovFlux())
ode = semidiscretise(semidisc, u₀func, tspan)
sol = solve(ode, SSPRK104(), dt=max_dt(tspan[1], ode.u0, semidisc), save_everystep=false)
@inferred evaluate_coefficients(sol.u[end], semidisc)
x, u = evaluate_coefficients(sol.u[end], semidisc)

balance_law = HyperbolicDiffEq.Euler{Float64,2}(1.4)
ϱ₀(x,y)  = 1.
vx₀(x,y) = sin(x)*cos(y)
vy₀(x,y) = -cos(x)*sin(y)
p₀(x,y)  = 100/1.4 + (cos(2x)+cos(2y))*3/16
u₀(x,y) = conserved_variables(ϱ₀(x,y), vx₀(x,y), vy₀(x,y), p₀(x,y), balance_law)
xmin=0.; xmax=2π; ymin=0.; ymax=2π
fvol = KennedyGruberFlux()
fnum = fvol
p = 9
Nx = 5
Ny = 5
meshx = UniformPeriodicMesh1D(xmin, xmax, Nx)
meshy = UniformPeriodicMesh1D(ymin, ymax, Ny)
basis = LobattoLegendre(p)
tspan = (0., 14.)
semidisc = UniformPeriodicFluxDiffDisc2D(balance_law, meshx, meshy, basis, fvol, fnum, true)
ode = semidiscretise(semidisc, u₀, tspan)
tstops = range(tspan[1], tspan[end], length=100)
sol = solve(ode, SSPRK33(), dt=0.1/((2p+1)*max(Nx,Ny)), saveat=tstops, tstops=tstops)


function ldl_dense!(A::Symmetric{T, Matrix{T}}) where {T <: Real}
  # use symmetric lower only
  @assert A.uplo == 'L'
  ldl_lowtri!(A.data)
end

function ldl_lowtri!(A::Matrix)
  n = size(A, 1)
  for j = 1:n
    djj = A[j, j]
    for k = 1:(j - 1)
      djj -= A[j, k]^2 * A[k, k]
    end
    A[j, j] = djj
    for i = (j + 1):n
      lij = A[i, j] # we use the lower block
      for k = 1:(j - 1)
        lij -= A[i, k] * A[j, k] * A[k, k]
      end
      A[i, j] = lij / djj
    end
  end
end

function solve_lowertri!(L, b)
  n = length(b)
  @inbounds for j = 1:n
    @simd for i = (j + 1):n
      b[i] -= L[i, j] * b[j]
    end
  end
end

function solve_lowertri_transpose!(LT, b)
  n = length(b)
  @inbounds for i = n:-1:1
    @simd for j = (i - 1):-1:1
      b[j] -= LT[i, j] * b[i]
    end
  end
end

function solve_diag!(D, b)
  n = length(b)
  @inbounds for j = 1:n
    b[j] /= D[j, j]
  end
end

# function solve(L, D, b)
#   x = copy(b)
#   solve_lowertri!(L, x)
#   solve_diag!(D, x)
#   solve_lowertri_transpose!(L, x)
#   return x
# end

function ldiv_dense!(LD, b)
  solve_lowertri!(LD, b)
  @views solve_diag!(Diagonal(LD[diagind(LD)]), b)
  solve_lowertri_transpose!(LD, b)
  return b
end

function ldiv_dense(x, LD, b)
  x .= b
  ldiv_dense!(LD, x)
end


using Makie
using Colors

using IntervalConstraintProgramming, IntervalArithmetic

## Constraint programming
solid_torus = @constraint (3 - sqrt(x^2 + y^2))^2 + z^2 <= 1

half_space = @constraint (x + y) + z <= 1

x = interval(-5, 5)
Y = IntervalBox(x, 3)

@time paving = pave(solid_torus ∩ half_space, Y, 0.1);

inner = paving.inner
boundary = paving.boundary;

## Makie plotting set-up
positions = Point{3, Float32}[Point3(mid(x)...) for x in vcat(inner, boundary)]
scales = Vec3f0[Vec3f0([diam(x) for x in xx]) for xx in vcat(inner, boundary)]

zs = Float32[x[3] for x in positions]
minz = minimum(zs)
maxz = maximum(zs)

xs = Float32[x[1] for x in positions]
minx = minimum(xs)
maxx = maximum(xs)

colors1 = RGBA{Float32}[RGBA( (zs[i]-minz)/(maxz-minz), (xs[i]-minx)/(maxx-minx), 0f0, 0.1f0)
        for i in 1:length(inner)];
colors2 = RGBA{Float32}[RGBA( 0.5f0, 0.5f0, 0.5f0, 0.02f0) for x in boundary];
colors = vcat(colors1, colors2);

## Makie plotting:
cube = Rect{3, Float32}(Vec3f0(-0.5, -0.5, -0.5), Vec3f0(1, 1, 1))
# centre, widths

meshscatter(positions, marker=cube, scale=scales, color=colors, transparency=true)