using QuantLib

const numRows = 5
const numCols = 5
const swaptionVols = [0.1490, 0.1340, 0.1228, 0.1189, 0.1148, 0.1290, 0.1201, 0.1146, 0.1108, 0.1040, 0.1149, 0.1112, 0.1070, 0.1010, 0.0957, 0.1047, 0.1021, 0.0980, 0.0951, 0.1270, 0.1000, 0.0950, 0.0900, 0.1230, 0.1160]
const swaptionLengths = [Dates.Year(1), Dates.Year(2), Dates.Year(3), Dates.Year(4), Dates.Year(5)]

function generate_flatforward_ts{C <: QuantLib.Time.BusinessCalendar}(cal::C, settlementDate::Date)
  flat_rate = Quote(0.04875825)

  ffts = FlatForwardTermStructure(settlementDate, cal, flat_rate, QuantLib.Time.Actual365())

  return ffts
end

function calibrate_model{M <: ShortRateModel}(model::M, helpers::Vector{SwaptionHelper})
  om = QuantLib.Math.LevenbergMarquardt()
  calibrate!(model, helpers, om, QuantLib.Math.EndCriteria(400, 100, 1.0e-8, 1.0e-8, 1.0e-8))

  for i=1:numRows
    j = numCols - (i - 1)
    k = (i - 1) * numCols + j

    npv = model_value!(helpers[i])
    implied = implied_volatility!(helpers[i], npv, 1e-4, 1000, 0.05, 0.50)
    diff = implied - swaptionVols[k]

    println(@sprintf("%i x %i: model %.5f%%, market: %.5f%% (%.5f%%)", i, Int(swaptionLengths[j]), implied * 100, swaptionVols[k] * 100, diff * 100))
  end
end

function main()
  cal = QuantLib.Time.TargetCalendar()
  settlementDate = Date(2002, 2, 19)
  todays_date = Date(2002, 2, 15)
  set_eval_date!(settings, todays_date)

  const swaptionMats = [Dates.Year(1), Dates.Year(2), Dates.Year(3), Dates.Year(4), Dates.Year(5)]

  # flat yield term strucutre implying 1x5 swap at 5%
  rhTermStructure = generate_flatforward_ts(cal, settlementDate)

  # Define the ATM/OTM/ITM swaps
  fixedLegFrequency = QuantLib.Time.Annual()
  fixedLegConvention = QuantLib.Time.Unadjusted()
  floatingLegConvention = QuantLib.Time.ModifiedFollowing()
  fixedLegDayCounter = QuantLib.Time.EuroThirty360()
  floatingLegFrequency = QuantLib.Time.Semiannual()

  swapType = Payer()
  dummyFixedRate = 0.03
  indexSixMonths = euribor_index(QuantLib.Time.TenorPeriod(Dates.Month(6)), rhTermStructure)

  startDate = QuantLib.Time.advance(Dates.Year(1), cal, settlementDate, floatingLegConvention)
  maturity = QuantLib.Time.advance(Dates.Year(5), cal, startDate, floatingLegConvention)

  fixedSchedule = QuantLib.Time.Schedule(startDate, maturity, QuantLib.Time.TenorPeriod(fixedLegFrequency), fixedLegConvention, fixedLegConvention, QuantLib.Time.DateGenerationForwards(), false, cal)
  floatSchedule = QuantLib.Time.Schedule(startDate, maturity, QuantLib.Time.TenorPeriod(floatingLegFrequency), floatingLegConvention, floatingLegConvention, QuantLib.Time.DateGenerationForwards(), false, cal)

  swap = VanillaSwap(swapType, 1000.0, fixedSchedule, dummyFixedRate, fixedLegDayCounter, indexSixMonths, 0.0, floatSchedule, indexSixMonths.dc, DiscountingSwapEngine(rhTermStructure))

  fixedATMRate = fair_rate(swap)
  fixedOTMRate = fixedATMRate * 1.2
  fixedITMRate = fixedATMRate * 0.8

  atmSwap = VanillaSwap(swapType, 1000.0, fixedSchedule, fixedATMRate, fixedLegDayCounter, indexSixMonths, 0.0, floatSchedule, indexSixMonths.dc, DiscountingSwapEngine(rhTermStructure))
  otmSwap = VanillaSwap(swapType, 1000.0, fixedSchedule, fixedOTMRate, fixedLegDayCounter, indexSixMonths, 0.0, floatSchedule, indexSixMonths.dc, DiscountingSwapEngine(rhTermStructure))
  itmSwap = VanillaSwap(swapType, 1000.0, fixedSchedule, fixedITMRate, fixedLegDayCounter, indexSixMonths, 0.0, floatSchedule, indexSixMonths.dc, DiscountingSwapEngine(rhTermStructure))

  times = zeros(0)
  swaptions = Vector{SwaptionHelper}(numRows)

  # First Model
  modelG2 = G2(rhTermStructure)
  hullWhiteModel = HullWhite(rhTermStructure)
  hullWhiteModel2 = HullWhite(rhTermStructure)
  blackKarasinski = BlackKarasinski(rhTermStructure)

  for i = 1:numRows
    j = numCols - (i - 1)
    k = (i - 1) * numCols + j

    sh = SwaptionHelper(swaptionMats[i], swaptionLengths[j], Quote(swaptionVols[k]), indexSixMonths, indexSixMonths.tenor, indexSixMonths.dc, indexSixMonths.dc,
                        rhTermStructure, G2SwaptionEngine(modelG2, 6.0, 16))

    times = add_times_to!(sh, times)
    swaptions[i] = sh
  end

  tg = QuantLib.Time.TimeGrid(times, 30)

  # models
  # modelG2 = G2(rhTermStructure)
  # hullWhiteModel = HullWhite(rhTermStructure)
  # hullWhiteModel2 = HullWhite(rhTermStructure)
  # blackKarasinski = BlackKarasinski(rhTermStructure)
  #
  # for swaptionHelper in swaptions
  #   swaptionHelper.pricingEngine = G2SwaptionEngine(modelG2, 6.0, 16)
  # end

  println("G2 (analytic formulae) calibration")
  calibrate_model(modelG2, swaptions)
  println("calibrated to: ")
  println(@sprintf("a = %.6f, sigma = %.6f", get_params(modelG2)[1], get_params(modelG2)[2]))
  println(@sprintf("b = %.6f, eta = %.6f", get_params(modelG2)[3], get_params(modelG2)[4]))
  println(@sprintf("rho = %.6f", get_params(modelG2)[5]))

  # Hull White
  for i in eachindex(swaptions)
    swaptions[i] = update_pricing_engine(swaptions[i], JamshidianSwaptionEngine(hullWhiteModel))
  end

  println("")
  println("Hull-White (analytic formulae) calibration")
  calibrate_model(hullWhiteModel, swaptions)
  println("calibrated to: ")
  println(@sprintf("a = %.6f, sigma = %.6f", get_params(hullWhiteModel)[1], get_params(hullWhiteModel)[2]))

  # Hull White (numeric)
  for i in eachindex(swaptions)
    swaptions[i] = update_pricing_engine(swaptions[i], TreeSwaptionEngine(hullWhiteModel2, tg))
  end

  println("")
  println("Hull-White (numerical) calibration")
  calibrate_model(hullWhiteModel2, swaptions)
  println("calibrated to: ")
  println(@sprintf("a = %.6f, sigma = %.6f", get_params(hullWhiteModel2)[1], get_params(hullWhiteModel2)[2]))

  # Black Karasinski
  for i in eachindex(swaptions)
    swaptions[i] = update_pricing_engine(swaptions[i], TreeSwaptionEngine(blackKarasinski, tg))
  end

  println("")
  println("Black-Karasinski (numerical) calibration")
  calibrate_model(blackKarasinski, swaptions)
  println("calibrated to: ")
  println(@sprintf("a = %.6f, sigma = %.6f", get_params(blackKarasinski)[1], get_params(blackKarasinski)[2]))

  # ATM Bermudan swaption pricing
  println("")
  println(@sprintf("Payer bermudan swaption struck at %.6f %% (ATM)", fixedATMRate * 100.0))

  swapLeg = swap.legs[1] # Fixed Leg

  bermudanDates = Vector{Date}(length(swapLeg.coupons))
  for i=1:length(swapLeg.coupons)
    bermudanDates[i]  = accrual_start_date(swapLeg.coupons[i])
  end

  bermudanExercise = BermudanExercise(bermudanDates)

  bermudanSwaption = Swaption(atmSwap, bermudanExercise)

  bermudanSwaption = update_pricing_engine(bermudanSwaption, TreeSwaptionEngine(modelG2, 50))

  println(@sprintf("G2 (tree):       %.6f", npv(bermudanSwaption)))

  bermudanSwaption = update_pricing_engine(bermudanSwaption, FdG2SwaptionEngine(modelG2))

  println(@sprintf("G2 (fdm):       %.6f", npv(bermudanSwaption)))

  bermudanSwaption = update_pricing_engine(bermudanSwaption, TreeSwaptionEngine(hullWhiteModel, 50))

  println(@sprintf("HW (tree):       %.6f", npv(bermudanSwaption)))

  bermudanSwaption = update_pricing_engine(bermudanSwaption, FdHullWhiteSwaptionEngine(hullWhiteModel))

  println(@sprintf("HW (fdm):       %.6f", npv(bermudanSwaption)))

  bermudanSwaption = update_pricing_engine(bermudanSwaption, TreeSwaptionEngine(hullWhiteModel2, 50))

  println(@sprintf("HW (num, tree):       %.6f", npv(bermudanSwaption)))

  bermudanSwaption = update_pricing_engine(bermudanSwaption, FdHullWhiteSwaptionEngine(hullWhiteModel2))

  println(@sprintf("HW (num, fdm):       %.6f", npv(bermudanSwaption)))

  bermudanSwaption = update_pricing_engine(bermudanSwaption, TreeSwaptionEngine(blackKarasinski, 50))

  println(@sprintf("BK:       %.6f", npv(bermudanSwaption)))

  # # OTM Bermudan swaption pricing
  # println("")
  # println(@sprintf("Payer bermudan swaption struck at %.6f %% (OTM)", fixedOTMRate * 100.0))
  #
  # otmBermudanSwaption = Swaption(otmSwap, bermudanExercise)
  #
  # update_pricing_engine!(otmBermudanSwaption, TreeSwaptionEngine(modelG2, 300))
  #
  # println(@sprintf("G2 (tree):       %.6f", npv(otmBermudanSwaption)))
  #
  # update_pricing_engine!(otmBermudanSwaption, FdG2SwaptionEngine(modelG2))
  #
  # println(@sprintf("G2 (fdm):       %.6f", npv(otmBermudanSwaption)))
  #
  # update_pricing_engine!(otmBermudanSwaption, TreeSwaptionEngine(hullWhiteModel, 50))
  #
  # println(@sprintf("HW (tree):       %.6f", npv(otmBermudanSwaption)))
  #
  # update_pricing_engine!(otmBermudanSwaption, FdHullWhiteSwaptionEngine(hullWhiteModel))
  #
  # println(@sprintf("HW (fdm):       %.6f", npv(otmBermudanSwaption)))
  #
  # update_pricing_engine!(otmBermudanSwaption, TreeSwaptionEngine(hullWhiteModel2, 50))
  #
  # println(@sprintf("HW (num, tree):       %.6f", npv(otmBermudanSwaption)))
  #
  # update_pricing_engine!(otmBermudanSwaption, FdHullWhiteSwaptionEngine(hullWhiteModel2))
  #
  # println(@sprintf("HW (num, fdm):       %.6f", npv(otmBermudanSwaption)))
  #
  # update_pricing_engine!(otmBermudanSwaption, TreeSwaptionEngine(blackKarasinski, 50))
  #
  # println(@sprintf("BK:       %.6f", npv(otmBermudanSwaption)))
  #
  # # ITM Bermudan swaption pricing
  # println("")
  # println(@sprintf("Payer bermudan swaption struck at %.6f %% (OTM)", fixedITMRate * 100.0))
  #
  # itmBermudanSwaption = Swaption(itmSwap, bermudanExercise)
  #
  # update_pricing_engine!(itmBermudanSwaption, TreeSwaptionEngine(modelG2, 50))
  #
  # println(@sprintf("G2 (tree):       %.6f", npv(itmBermudanSwaption)))
  #
  # update_pricing_engine!(itmBermudanSwaption, FdG2SwaptionEngine(modelG2))
  #
  # println(@sprintf("G2 (fdm):       %.6f", npv(itmBermudanSwaption)))
  #
  # update_pricing_engine!(itmBermudanSwaption, TreeSwaptionEngine(hullWhiteModel, 50))
  #
  # println(@sprintf("HW (tree):       %.6f", npv(itmBermudanSwaption)))
  #
  # update_pricing_engine!(itmBermudanSwaption, FdHullWhiteSwaptionEngine(hullWhiteModel))
  #
  # println(@sprintf("HW (fdm):       %.6f", npv(itmBermudanSwaption)))
  #
  # update_pricing_engine!(itmBermudanSwaption, TreeSwaptionEngine(hullWhiteModel2, 50))
  #
  # println(@sprintf("HW (num, tree):       %.6f", npv(itmBermudanSwaption)))
  #
  # update_pricing_engine!(itmBermudanSwaption, FdHullWhiteSwaptionEngine(hullWhiteModel2))
  #
  # println(@sprintf("HW (num, fdm):       %.6f", npv(itmBermudanSwaption)))
  #
  # update_pricing_engine!(itmBermudanSwaption, TreeSwaptionEngine(blackKarasinski, 50))
  #
  # println(@sprintf("BK:       %.6f", npv(itmBermudanSwaption)))
end


import Base: +, -

struct Point
  x::Float64
  y::Float64
end

a::Point + b::Point = Point(a.x + b.x, a.y + b.y)
a::Point - b::Point = Point(a.x - b.x, a.y - b.y)
dist(p::Point) = sqrt(p.x^2 + p.y^2)

a=Point(2,7)


import StatsBase

"""
	mean(uv::AbstractUncertainValue, n::Int)

Compute the mean of an uncertain value over an `n`-draw sample of it.
"""
StatsBase.mean(x::AbstractUncertainValue, n::Int) = resample(StatsBase.mean, x, n)

"""
	mode(uv::AbstractUncertainValue, n::Int)

Compute the mode of an uncertain value over an `n`-draw sample of it.
"""
StatsBase.mode(x::AbstractUncertainValue, n::Int) = resample(StatsBase.mode, x, n)

"""
	quantile(uv::AbstractUncertainValue, q, n::Int)

Compute the quantile(s) `q` of an uncertain value over an `n`-draw sample of it.
"""
StatsBase.quantile(x::AbstractUncertainValue, q, n::Int) = resample(StatsBase.quantile, x, n, q)

"""
	iqr(uv::AbstractUncertainValue, n::Int)

Compute the interquartile range (IQR), i.e. the 75th percentile minus the 25th percentile,
over an `n`-draw sample of an uncertain value.
"""
StatsBase.iqr(x::AbstractUncertainValue, n::Int) = resample(StatsBase.iqr, x, n, q)

"""
	median(uv::AbstractUncertainValue, n::Int)

Compute the median of an uncertain value over an `n`-draw sample of it.
"""
StatsBase.median(x::AbstractUncertainValue, n::Int) = resample(StatsBase.median, x, n)

"""
	middle(uv::AbstractUncertainValue, n::Int)

Compute the middle of an uncertain value over an `n`-draw sample of it.
"""
StatsBase.middle(x::AbstractUncertainValue, n::Int) = resample(StatsBase.middle, x, n)

"""
	std(uv::AbstractUncertainValue, n::Int)

Compute the standard deviation of an uncertain value over an `n`-draw sample of it.
"""
StatsBase.std(x::AbstractUncertainValue, n::Int) = resample(StatsBase.std, x, n)

"""
	variance(uv::AbstractUncertainValue, n::Int)

Compute the variance of an uncertain value over an `n`-draw sample of it.
"""
StatsBase.var(x::AbstractUncertainValue, n::Int) = resample(StatsBase.var, x, n)

"""
	genmean(uv::AbstractUncertainValue, p, n::Int)

Compute the generalized/power mean with exponent `p` of an uncertain value over an
`n`-draw sample of it.
"""
StatsBase.genmean(x::AbstractUncertainValue, p, n::Int) = resample(StatsBase.genmean, x, n, p)

"""
	genvar(uv::AbstractUncertainValue, n::Int)

Compute the generalized sample variance of an uncertain value over an
`n`-draw sample of it.
"""
StatsBase.genvar(x::AbstractUncertainValue, n::Int) = resample(StatsBase.genvar, x, n)

"""
	harmmean(uv::AbstractUncertainValue, n::Int)

Compute the harmonic mean of an uncertain value over an `n`-draw sample of it.
"""
StatsBase.harmmean(x::AbstractUncertainValue, n::Int) = resample(StatsBase.harmmean, x, n)

"""
	geomean(uv::AbstractUncertainValue, n::Int)

Compute the geometric mean of an uncertain value over an `n`-draw sample of it.
"""
StatsBase.geomean(x::AbstractUncertainValue, n::Int) = resample(StatsBase.geomean, x, n)

"""
	kurtosis(uv::AbstractUncertainValue, n::Int, m = mean(uv, n))

Compute the excess kurtosis of an uncertain value over an `n`-draw sample of it,
optionally specifying a center `m`).
"""
StatsBase.kurtosis(x::AbstractUncertainValue, n::Int, m = mean(x, n)) = resample(StatsBase.kurtosis, x, n, m)

"""
	moment(x::AbstractUncertainValue, k, n::Int, m = mean(x, n))

Compute the `k`-th order central moment of an uncertain value over an
`n`-draw sample of it, optionally specifying a center `m`.
"""
StatsBase.moment(x::AbstractUncertainValue, k, n::Int, m = mean(x, n)) = resample(StatsBase.moment, x, n, k, m)

"""
	percentile(x::AbstractUncertainValue, p, n::Int)

Compute the percentile(s) `p` of an uncertain value over an `n`-draw sample of it.
"""
StatsBase.percentile(x::AbstractUncertainValue, p, n::Int) = resample(StatsBase.percentile, x, n, p)

"""
	renyientropy(uv::AbstractUncertainValue, α, n::Int)

Compute the Rényi (generalized) entropy of order `α` of an uncertain value over an
`n`-draw sample of it.
"""
StatsBase.renyientropy(x::AbstractUncertainValue, α, n::Int) = resample(StatsBase.quantile, x, n, α)

"""
	rle(x::AbstractUncertainValue, n::Int)

Compute the run-length encoding of an uncertain value over a `n`-draw
sample of it as a tuple. The first element of the tuple is a vector of
values of the input and the second is the number of consecutive occurrences of
each element.
"""
StatsBase.rle(x::AbstractUncertainValue, n::Int) = resample(StatsBase.rle, x, n)

"""
	sem(x::AbstractUncertainValue, n::Int)

Compute the standard error of the mean of an uncertain value over a `n`-draw
sample of it, optionally specifying a center `m`, i.e.
`sqrt(var(x_draw, corrected = true) / length(x_draw))`.
"""
StatsBase.sem(x::AbstractUncertainValue, n::Int) = resample(StatsBase.sem, x, n)

"""
	skewness(x::AbstractUncertainValue, n::Int, m = mean(x, n))

Compute the standardized skewness of an uncertain value over an `n`-draw sample of
it, optionally specifying a center `m`.
"""
StatsBase.skewness(x::AbstractUncertainValue, n::Int; m = mean(x, n)) = resample(StatsBase.skewness, x, n, m)

"""
	span(x::AbstractUncertainValue, n::Int)

Compute the span of a collection, i.e. the range `minimum(x):maximum(x)`, of an
uncertain value over an `n`-draw sample of it.  The minimum and
maximum of the draws of `x` are computed in one pass using extrema.
"""
StatsBase.span(x::AbstractUncertainValue, n::Int) = resample(StatsBase.span, x, n)

"""
	summarystats(uv::AbstractUncertainValue, n::Int)

Compute summary statistics of an uncertain value over an `n`-draw sample of it. Returns
a `SummaryStats` object containing the mean, minimum, 25th percentile, median,
75th percentile, and maximum.
"""
StatsBase.summarystats(x::AbstractUncertainValue, n::Int) = resample(StatsBase.summarystats, x, n)

"""
	totalvar(uv::AbstractUncertainValue, n::Int)

Compute the total sample variance of an uncertain value over an
`n`-draw sample of it. For a single uncertain value, this is
equivalent to the sample variance.
"""
StatsBase.totalvar(x::AbstractUncertainValue, n::Int) = resample(StatsBase.totalvar, x, n)


using ITensors
using ITensorMPOCompression

N=10; #10 sites
NNN=7; #Include up to 7th nearest neighbour interactions
sites = siteinds("S=1/2",N);
H=make_transIsing_MPO(sites,NNN);
is_lower_regular_form(H)==true
pprint(H[2])
orthogonalize!(H)
pprint(H[2])
get_Dw(H)
is_lower_regular_form(H)==true
is_orthogonal(H,left)==true


p1 = codom(Open([1], PetriNet(1), [1]))
p2 = codom(Open([1,2], PetriNet(2), [1,2]))
pc1 = id(p1)
pc2 = id(p2)

@test dom(pc1 ⊗ pc2) == dom(pc1) ⊗ dom(pc2)
@test codom(pc1 ⊗ pc2) == codom(pc1) ⊗ codom(pc2)
@test dom(braid(p1,p2)) == p1 ⊗ p2
@test codom(braid(p1,p2)) == p2 ⊗ p1

@test dom(mcopy(p1)) == p1
@test codom(mcopy(p1)) == p1 ⊗ p1
@test dom(delete(p1)) == p1
@test codom(delete(p1)) == munit(OpenPetriNetOb)


using LowRankModels
using Distributions

# This is just to create a low-rank representation of some count data, I don't care how accurate it is.
m,n = 100, 50;
k = 2;
A = rand(Poisson(),(m,n));
losses = convert(Array{Loss,1}, fill(PoissonLoss(10),n));
rx, ry = ZeroReg(), ZeroReg();
g_pre = GLRM(A, losses, rx, ry, k, scale=false, offset=false);
X_real, Y_real, ch = fit!(g_pre, params=Params(0.00001,min_stepsize=0.00000000001,max_iter=10000));

# Now we do the actual model using the perfect data and try to recapture it.
A_real = impute(losses, X_real'*Y_real);

g = GLRM(A_real, losses, rx, ry, k, scale=true, offset=true);
X, Y, ch = fit!(g, params=Params(0.00001,min_stepsize=0.00000000001,max_iter=10000));
U = X'*Y;
A_imputed = impute(losses, X'*Y);

error_metric(g)
errors(Domain[l.domain for l in losses], losses, U, A_real);


#Function for evaluating radial matrix elements
#23/07/2017

function radiel(atom,nn1,ll1,jj1,nn2,ll2,jj2)

  #Compute radial wavefunctions
    radial1, rscale1 = numerovfunc(atom,nn1,ll1,jj1)

    radial2, rscale2 = numerovfunc(atom,nn2,ll2,jj2)

  #Some preparation
  if nn1>=nn2
    Yk1 = radial1
    Yk2 = radial2
    rscale = rscale1
  else
    Yk1 = radial2
    Yk2 = radial1
    rscale = rscale2
  end

  #Resize smaller solution vector by attaching zeros to the end such that the two solution vectors are the same length
  if length(Yk1)≠length(Yk2)
    Yk2conc = append!(Yk2,zeros(length(Yk1)-length(Yk2)))
  else
    Yk2conc = Yk2
  end

  #Solve the matrix elements using method adapted from Zimmerman et al. (1979)
  deltar = rscale[2:end]-rscale[1:(end-1)]
  numervec = Yk1[2:end].*Yk2conc[2:end].*(rscale[2:end].^2).*deltar
  return matrixelement = sum(numervec)
end


using GLVisualize, GeometryTypes, GLAbstraction, Colors

if !isdefined(:runtests)
    window = glscreen()
    timesignal = bounce(linspace(0.0, 1.0, 360))
end
const N = 2048
function spiral(i, start_radius, offset)
	Point2f0(sin(i), cos(i)) * (start_radius + ((i/2pi)*offset))
end
# 2D particles
curve_data(i, N) = Point2f0[spiral(i+x/20f0, 1, (i/20)+1) for x=1:N]

t = const_lift(x-> (1f0-x)*100f0, timesignal)
color = map(RGBA{Float32}, colormap("Blues", N))
_view(visualize(const_lift(curve_data, t, N), :lines, color=color))

if !isdefined(:runtests)
    renderloop(window)
end


using Test
using Random
using Distributions
using MEstimation
using NLsolve

include("new_logistic_regression.jl")

Random.seed!(123);
n = 50;
p = 10;
x = rand(n, p);
x[:, 1] = fill(1.0, n);
true_betas = rand(p) * 0.02;
my_data = new_logistic_regression.simulate(true_betas, x, fill(1, n));

logistic_template = MEstimation.objective_function_template(
    new_logistic_regression.nobs,
    new_logistic_regression.observation,
    new_logistic_regression.loglik
)

logistic_obj_template = MEstimation.objective_function_template(new_logistic_regression.nobs,
                                                    new_logistic_regression.observation,
                                                    new_logistic_regression.loglik)

obj_br_implicit = MEstimation.fit(logistic_obj_template, my_data, true_betas, estimation_method="RBM")

obj_br_explicit = MEstimation.fit(logistic_obj_template, my_data, true_betas, estimation_method="RBM", br_method="explicit_trace")

@test isapprox(coef(obj_br_implicit), coef(obj_br_explicit))


#######################################################################################
#                                 Load Packages
#######################################################################################
# set the working directory to the directory in which this file is contained
cd(@__DIR__)
# load the package manager
using Pkg
# activate the project environment
Pkg.activate("../../")
using Revise, AdaptiveDesignOptimization, Random, Distributions
includet("Delay_Discounting.jl")
#######################################################################################
#                               Define Experiment Design
#######################################################################################
Random.seed!(120341)
prior = [Uniform(-5, 5), Uniform(-5, 50)]

model = Model(;prior, loglike)

parm_list = (κ = range(-5, 0, length=50) .|> x->10^x,
   τ = range(0, 5, length=11)[2:end])

# parm_list = (κ = [.1],
#    τ = [.2,.5])

design_list = (
    t_ss = [0.0],
    t_ll = [0.43, 0.714, 1, 2, 3,
        4.3, 6.44, 8.6, 10.8, 12.9,
        17.2, 21.5, 26, 52, 104,
        156, 260, 520],
    r_ss = 12.5:12.5:787.5,
    r_ll = [800.0]
)

# design_list = (
#     t_ss = [0.0],
#     t_ll = [5.0, 10.0],
#     r_ss = [12.0, 20.0],
#     r_ll = [80.0]
# )

data_list = (choice=[true, false],)
#######################################################################################
#                              Simulate Experiment
#######################################################################################
using DataFrames
true_parms = (κ=.12, τ=1.5)
n_trials = 100
optimizer = Optimizer(;design_list, parm_list, data_list, model);
design = optimizer.best_design
df = DataFrame(design=Symbol[], trial=Int[], mean_κ=Float64[], mean_τ=Float64[],
    std_κ=Float64[], std_τ=Float64[])
new_data = [:optimal, 0, mean_post(optimizer)..., std_post(optimizer)...]
push!(df, new_data)

for trial in 1:n_trials
    data = simulate(true_parms..., design...)
    design = update!(optimizer, data)
    new_data = [:optimal, trial, mean_post(optimizer)..., std_post(optimizer)...]
    push!(df, new_data)
end
#######################################################################################
#                              Random Experiment
#######################################################################################
randomizer = Optimizer(;design_list, parm_list, data_list, model, design_type=Randomize);
design = randomizer.best_design
new_data = [:random, 0, mean_post(randomizer)..., std_post(randomizer)...]
push!(df, new_data)

for trial in 1:n_trials
    data = simulate(true_parms..., design...)
    design = update!(randomizer, data)
    new_data = [:random, trial, mean_post(randomizer)..., std_post(randomizer)...]
    push!(df, new_data)
end
#######################################################################################
#                                 Plot Results
#######################################################################################
using StatsPlots
@df df plot(:trial, :mean_κ, xlabel="trial", ylabel="mean κ", group=:design, grid=false)
hline!([true_parms.κ], label="true")

@df df plot(:trial, :mean_τ, xlabel="trial", ylabel="mean τ", group=:design, grid=false)
hline!([true_parms.τ], label="true")

@df df plot(:trial, :std_κ, xlabel="trial", ylabel="σ of κ", grid=false, group=:design, ylims=(0,.3))

@df df plot(:trial, :std_τ, xlabel="trial", ylabel="σ of τ", grid=false, group=:design, ylims=(0,2))


using NCDatasets
using Formatting
using ArgParse
using Statistics
using JSON

include("constants.jl")

function mreplace(x)
    return replace(x, missing=>NaN)
end

function integrate(x, dydx)
    y = copy(dydx) * 0.0

    for i = 2:length(x)
        y[i] = y[i-1] + (dydx[i-1] + dydx[i]) * (x[i] - x[i-1]) / 2.0
    end

    return y
end

function parse_commandline()

    s = ArgParseSettings()
    @add_arg_table s begin

        "--data-file"
            help = "Atm data file."
            arg_type = String
            required = true

        "--output-file"
            help = "Output file."
            arg_type = String
            required = true

        "--domain-file"
            help = "Domain file."
            arg_type = String
            required = true

        "--beg-year"
            help = "Year of begin."
            arg_type = Int64
            required = true

        "--end-year"
            help = "Year of end."
            arg_type = Int64
            required = true

    end

    return parse_args(ARGS, s)
end

parsed = parse_commandline()
print(json(parsed, 4))

Dataset(parsed["domain-file"], "r") do ds

    global mask = ds["mask"][:] |> mreplace
    global lat = ds["yc"][1, :] |> mreplace

    global lat_weight = 2π * Re^2.0 * cos.(deg2rad.(lat))
    global ϕ          = deg2rad.(lat)

    global L_idx = (mask .== 1)
    global O_idx = (mask .== 0)

end

Dataset(parsed["data-file"], "r") do ds

    beg_t = (parsed["beg-year"] - 1) * 12 + 1
    end_t = (parsed["end-year"] - 1) * 12 + 12
    rng = (:,:,beg_t:end_t)

    global A_flx_2D = (
        FFLX_TOA = ( - mean(ds["FSNT"][rng...], dims=(3, ))  + mean(ds["FLNT"][rng...],  dims=(3, )) )[:, :, 1],
        FFLX_SFC = ( - mean(ds["FSNS"][rng...], dims=(3, ))  + mean(ds["FLNS"][rng...],  dims=(3, )) )[:, :, 1],
        HFLX_SFC = (   mean(ds["SHFLX"][rng...], dims=(3, )) + mean(ds["LHFLX"][rng...], dims=(3, )) )[:, :, 1],
    )

    global (Nx, Ny) = size(A_flx_2D.FFLX_TOA)

    O_flx_2D = (
        FFLX_SFC = zeros(Nx, Ny),
        HFLX_SFC = zeros(Nx, Ny),
    )

    L_flx_2D = (
        FFLX_SFC = zeros(Nx, Ny),
        HFLX_SFC = zeros(Nx, Ny),
    )

    O_flx_2D.FFLX_SFC[O_idx] .= A_flx_2D.FFLX_SFC[O_idx]
    O_flx_2D.HFLX_SFC[O_idx] .= A_flx_2D.HFLX_SFC[O_idx]

    L_flx_2D.FFLX_SFC[L_idx] .= A_flx_2D.FFLX_SFC[L_idx]
    L_flx_2D.HFLX_SFC[L_idx] .= A_flx_2D.HFLX_SFC[L_idx]

    # zonal average
    global A_flx = (
        FFLX_TOA = mean(A_flx_2D.FFLX_TOA, dims=(1,))[1,:] .* lat_weight,
        FFLX_SFC = mean(A_flx_2D.FFLX_SFC, dims=(1,))[1,:] .* lat_weight,
        HFLX_SFC = mean(A_flx_2D.HFLX_SFC, dims=(1,))[1,:] .* lat_weight,
    )

    global O_flx = (
        FFLX_SFC = mean(O_flx_2D.FFLX_SFC, dims=(1,))[1,:] .* lat_weight,
        HFLX_SFC = mean(O_flx_2D.HFLX_SFC, dims=(1,))[1,:] .* lat_weight,
    )

    global L_flx = (
        FFLX_SFC = mean(L_flx_2D.FFLX_SFC, dims=(1,))[1,:] .* lat_weight,
        HFLX_SFC = mean(L_flx_2D.HFLX_SFC, dims=(1,))[1,:] .* lat_weight,
    )

end

A_EFLX_CONV = - ( A_flx.FFLX_TOA - A_flx.FFLX_SFC - A_flx.HFLX_SFC )
O_EFLX_CONV = - ( O_flx.FFLX_SFC + O_flx.HFLX_SFC )
L_EFLX_CONV = - ( L_flx.FFLX_SFC + L_flx.HFLX_SFC )

A_IET = integrate(ϕ, A_EFLX_CONV)
O_IET = integrate(ϕ, O_EFLX_CONV)
L_IET = integrate(ϕ, L_EFLX_CONV)

Dataset(parsed["output-file"], "c") do ds

    defDim(ds, "Ny", Ny)

    for (varname, vardata, vardim, attrib) in [
        ("A_FFLX_TOA",  A_flx.FFLX_TOA,  ("Ny",), Dict()),
        ("A_FFLX_SFC",  A_flx.FFLX_SFC,  ("Ny",), Dict()),
        ("A_HFLX_SFC",  A_flx.HFLX_SFC,  ("Ny",), Dict()),

        ("O_FFLX_SFC",  O_flx.FFLX_SFC,  ("Ny",), Dict()),
        ("O_HFLX_SFC",  O_flx.HFLX_SFC,  ("Ny",), Dict()),

        ("L_FFLX_SFC",  L_flx.FFLX_SFC,  ("Ny",), Dict()),
        ("L_HFLX_SFC",  L_flx.HFLX_SFC,  ("Ny",), Dict()),

        ("A_IET",       A_IET,           ("Ny",), Dict()),
        ("O_IET",       O_IET,           ("Ny",), Dict()),
        ("L_IET",       L_IET,           ("Ny",), Dict()),
    ]

        println("Doing var: ", varname)

        var = defVar(ds, varname, Float64, vardim)
        var.attrib["_FillValue"] = 1e20

        var = ds[varname]

        for (k, v) in attrib
            var.attrib[k] = v
        end

        rng = []
        for i in 1:length(vardim)-1
            push!(rng, Colon())
        end
        push!(rng, 1:size(vardata)[end])
        var[rng...] = vardata

    end

end


using CellListMap, StaticArrays

function push_pair!(i,j,d2,matrix)
  d = sqrt(d2)
  matrix[i,j] = d
  return matrix
end

function push_pair2!(i,j,d2,matrix)
  d = sqrt(d2)
  matrix[j,i] = d
  return matrix
end

function find_min!(M,dist,nframe)
  min = +Inf
  for i in 1:length(M[1,:])
    a = -Inf
    data = M[:,i]
    for h in 1:length(data)
      if a < data[h] && data[h] != 0
        a = data[h]
      end
    end
    if a == -Inf
      a = 0.
    end
    dist[nframe,i] = a
  end
end

let

  N1=10
  N2=80
  sides = [250,250,250]
  cutoff = 10.
  box = Box(sides,cutoff)
  matrix = zeros(N1,N2)

  nframes   = 10
  Dist      = zeros(nframes,N2)

  for i in 1:nframes

    # Particle positions
    x = [ box.sides .* rand(SVector{3,Float64}) for i in 1:N1 ]
    y = [ box.sides .* rand(SVector{3,Float64}) for i in 1:N2 ]

    # Initialize auxiliary linked lists (largest set!)
    cl = CellList(x,y,box)

    matrix = map_pairwise!((x,y,i,j,d2,output) -> push_pair!(i,j,d2,output),matrix,box,cl)

    #println(matrix)
    find_min!(matrix, Dist, i)

  end

  println(Dist)

end


"""
$(SIGNATURES)

Compute the minimum sample size needed for the desired power, type I error rate and effect size

Arguments
---------
* `Test` -- The test for calculating the power
* `std` -- Standard deviation
* `power` -- Power
* `alpha` -- Desired type I error rate
* `side` -- Side of the test, "two" is two-sided, "one" is one-sided
"""
function sample_size{T <: TrialTest}(
    test::T;
    power::Union{Real, Void} = nothing,
    std::Union{Real, Tuple{Real, Real}, Void} = nothing,
    alpha::Real = 0.05,
    side::String = "two",
)
    # Transcode side
    side = lowercase(side)
    # Check arguments
    check_args_sample_size(test, power = power, std = std, alpha = alpha, side = side)
    # Calculate sample size
    gap(n::Real) = analytic_power(test, n = n, std = std, alpha = alpha, side = side) - power
    # Round to the minimum integer that is larger than the root
    # print(fzero(gap, 2, MAXSAMPLESIZE))
    return round(fzero(gap, 2, MAXSAMPLESIZE), RoundUp)

end


"""
    GeneralizedGamma(a,d,p)

The *GeneralizedGamma distribution* with parameter a, d and p has probability density
function

```math
f(x; a, d, p) = \\frac{p}{a^p\\Gamma(d/p)} x^{d-1} e^{-\\left(\\frac{x}{a}\\right)^p},
\\quad x > 0
```

```julia
GeneralizedGamma()          # GeneralizedGamma distribution GeneralizedGamma(1, 1, 1)
GeneralizedGamma(a)         # GeneralizedGamma distribution GeneralizedGamma(a, 1, 1)

params(d)        # Get the parameters, i.e. (a, d, p)

S = rand(GeneralizedGamma(2, 3, 4), 5000); ##Sample from GeneralizedGamma(2, 3, 4)
fit_mle(GeneralizedGamma, S)  #MLE
```

External links

* [Generalized gamma distribution on Wikipedia](https://en.wikipedia.org/wiki/Generalized_gamma_distribution)
"""

struct GeneralizedGamma{T<:Real} <: ContinuousUnivariateDistribution
    a::T
    d::T
    p::T
    GeneralizedGamma{T}(a, d, p) where {T} = new{T}(a, d, p)
end

function GeneralizedGamma(a::T, d::T, p::T; check_args = true) where {T<:Real}
    check_args &&
        @check_args(GeneralizedGamma, a > zero(a) && d > zero(d) && p > zero(p))
    return GeneralizedGamma{T}(a, d, p)
end

GeneralizedGamma(a::Real, d::Real, p::Real) =
    GeneralizedGamma(promote(a, d, p)...)
GeneralizedGamma(a::Integer, d::Integer, p::Integer) =
    GeneralizedGamma(float(a), float(d), float(p))
GeneralizedGamma(a::T) where {T<:Real} = GeneralizedGamma(a, one(T), one(T))
GeneralizedGamma() = GeneralizedGamma(1.0, 1.0, 1.0, check_args = false)

@distr_support GeneralizedGamma 0.0 Inf

#### Conversions
convert(::Type{GeneralizedGamma{T}}, a::S, d::S, p::S) where {T<:Real,S<:Real} =
    GeneralizedGamma(T(a), T(d), T(p))
convert(
    ::Type{GeneralizedGamma{T}},
    d::GeneralizedGamma{S},
) where {T<:Real,S<:Real} =
    GeneralizedGamma(T(d.a), T(d.d), T(d.p), check_args = false)

#### Parameters

#shape(d::GeneralizedGamma) = d.p ##or d.d??
#scale(d::GeneralizedGamma) = d.a
#rate(d::GeneralizedGamma) = 1 / d.a

params(d::GeneralizedGamma) = (d.a, d.d, d.p)
partype(::GeneralizedGamma{T}) where {T} = T

#### Statistics

function mean(d::GeneralizedGamma)
    ip = 1 / d.p
    d.a * gamma((d.d + 1) * ip) / gamma(d.d * ip)
end
function var(d::GeneralizedGamma)
    ip = 1 / d.p
    iΓdp = 1 / gamma(d.d * ip)
    Γd1p = gamma((d.d + 1) * ip)
    Γd2p = gamma((d.d + 2) * ip)
    d.a^2 * (Γd2p * iΓdp - (Γd1p * iΓdp)^2)
end

function skewness(d::GeneralizedGamma)
    (a, d, p) = params(d)
    ip = 1 / p
    iΓdp = 1 / gamma(d * ip)
    Γd1p = gamma((d + 1) * ip)
    Γd2p = gamma((d + 2) * ip)
    Γd3p = gamma((d + 3) * ip)

    μ = a * Γd1p * iΓdp
    σ2 = a^2 * (Γd2p * iΓdp - (Γd1p * iΓdp)^2)
    EX3 = a^3 * Γd3p * iΓdp

    (EX3 - 3 * μ * σ2 - μ^3) / sqrt(σ2)^3
end

function kurtosis(d::GeneralizedGamma)
    (a, d, p) = params(d)
    ip = 1 / p
    iΓdp = 1 / gamma(d * ip)
    Γd1p = gamma((d + 1) * ip)
    Γd2p = gamma((d + 2) * ip)
    Γd3p = gamma((d + 3) * ip)
    Γd4p = gamma((d + 4) * ip)

    μ = a * Γd1p * iΓdp
    EX2 = a^2 * Γd2p * iΓdp
    EX3 = a^3 * Γd3p * iΓdp
    EX4 = a^4 * Γd4p * iΓdp
    (EX4 - 4 * EX3 * μ + 6 * EX2 * μ^2 - 3 * μ^4) / (EX2 - μ^2)^2 - 3
end

function mode(d::GeneralizedGamma)
    (a, d, p) = params(d)
    ip = 1 / p
    d >= 1 ? a * ((d - 1) * ip)^ip :
    error("GeneralizedGamma has no mode when d < 1")
end

function entropy(d::GeneralizedGamma)
    (a, d, p) = params(d)
    ip = 1 / p
    log(a * ip) + loggamma(d * ip) + d * ip - (d - 1) * ip * digamma(d * ip)
end

function logpdf(d::GeneralizedGamma{T}, x::Real) where {T<:Real}
    if x >= 0
        (a, d, p) = params(d)
        return log(p) - d * log(a) + (d - 1) * log(x) - (x / a)^p -
               loggamma(d / p)
    else
        return -T(Inf)
    end
end

pdf(d::GeneralizedGamma{T}, x::Real) where {T<:Real} = exp(logpdf(d, x))

# function cdf(d::GeneralizedGamma{T}, x::Real) where {T<:Real}
#     if x >= 0
#         (a, d, p) = params(d)
#         Γinc = gamma_inc(d / p, (x / a)^p, 0)
#         return Γinc[1] / (Γinc[1] + Γinc[2])
#     else
#         return zero(T)
#     end
# end
cdf(d::GeneralizedGamma{T}, x::Real) where {T<:Real} =
    cdf(Gamma(d.d / d.p, 1), (x / d.a)^d.p)
ccdf(d::GeneralizedGamma{T}, x::Real) where {T<:Real} =
    ccdf(Gamma(d.d / d.p, 1), (x / d.a)^d.p)

function quantile(d::GeneralizedGamma{T}, q::Real) where {T<:Real}
    (a, d, p) = params(d)
    ip = 1 / p
    a * quantile(Gamma(d * ip, 1), q)^ip
end

#function mgf(d::GeneralizedGamma{T}, t::Real) where {T<:Real}
#        g(x) = exp(logpdf(d, x) + t*x)
#        return quadgk(g, 0.0, Inf)
#end

# cf(d::GeneralizedGamma, t::Real)

function gradlogpdf(d::GeneralizedGamma{T}, x::Real) where {T<:Real}
    (a, d, p) = params(d)
    insupport(GeneralizedGamma, x) ? (-x^p * a^(-p) * p + d - 1) / x : zero(T)
end

#### Sampling
function rand(rng::AbstractRNG, d::GeneralizedGamma)
    (a, d, p) = params(d)
    if p == 1
        return rand(rng, Gamma(d, a))
    end
    ip = 1 / p
    return a * rand(rng, Gamma(d * ip, 1))^ip
end

#### Fit model
function fit_mle(
    ::Type{<:GeneralizedGamma},
    x::AbstractArray{T};
    maxiter::Int = 5000,
    tol::Float64 = 1e-6,
) where {T<:Real}
    n = length(x)
    logx = zeros(T, n)
    Σlogx = 0.0
    for i = 1:n
        logx[i] = log(x[i])
        Σlogx += logx[i]
    end
    ##Initial values
    pu, pl = 0.0, 0.0
    for i = 1.0:2.0:5000.0
        MM = gengamma_MLE_update_function(i, n, x, logx, Σlogx)
        if MM[4] < 0
            pu = i
            pl = i - 2

            break
        end
    end
    if gengamma_MLE_update_function(pu * 1.05, n, x, logx, Σlogx)[5]
        return error("K is zero.") # For some extreme conditions.
    end
    #print((pl,pu))
    if pu == 0
        return error("p is very large.")
    end
    pl = max(pl, 1.0e-7)
    # if isnan(gengamma_MLE_update_function(pl, n, x, logx, Σlogx)[4])
    #     return error("p is very small.")
    # end
    #if isnan(gengamma_MLE_update_function(pu, n, x, logx, Σlogx)[4])
    #    return error("p is very large.")
    #end
    a = 0.0
    d = 0.0
    p = 0.0
    p0 = (pl + pu) * 0.5
    for i = 1:maxiter
        #println(p0)
        if p0 < pl || p0 > pu || isnan(p0)
            p0 = pl + (pu - pl) * rand()
        end
        a, d, p = gengamma_MLE_update_function(p0, n, x, logx, Σlogx)
        if abs(p0 - p) <= tol
            return GeneralizedGamma(a, d, p)
        end
        p0 = p
    end
    @warn("Maximum iteration reached.")
    return GeneralizedGamma(a, d, p)
end

function gengamma_MLE_update_function(p, n, x, logx, Σlogx)
    Σxp = 0.0
    ∂Σxp = 0.0
    ∂2Σxp = 0.0
    ∂3Σxp = 0.0
    ip = 1 / p
    for i = 1:n
        xp = x[i]^p
        log_x = logx[i]
        Σxp += xp
        ∂Σxp += xp * log_x
        ∂2Σxp += xp * log_x^2
        ∂3Σxp += xp * log_x^3
    end
    K1 = n * ∂Σxp - Σlogx * Σxp
    K = 1 / K1
    ap = n^2 * ip * K
    a = ap^(-ip) ##exp(-ip*log(ap))
    log_ap = log(ap)
    log_a = -ip * log_ap
    ia = 1 / a
    d = n * Σxp * K
    dip = d * ip
    pΣxp = p * Σxp
    loggamma_dip = loggamma(d * ip)
    #ld = -n * log(a) + Σlogx - n * digamma_dip * ip
    ##Derivatives
    ∂ip = -ip^2
    ∂K1 = n * ∂2Σxp - Σlogx * ∂Σxp
    ∂K = -∂K1 * K^2
    ∂ap = n^2 * (∂ip * K + ip * ∂K)
    ∂log_ap = ∂ap / ap

    a_temp = -(∂ip * log_ap + ip * ∂log_ap)
    ∂a = a_temp * a
    ∂log_a = ∂a * ia
    #∂ia = (∂ip * log(ap) + ip * ∂ap / ap) * ia
    ∂d = n * (∂Σxp * K + Σxp * ∂K)
    #∂pΣxp = Σxp + p * ∂Σxp
    ∂dip = (∂d * ip + ∂ip * d)
    digamma_dip = digamma(dip)

    ∂loggamma_dip = ∂dip * digamma_dip
    #∂digamma_dip = trigamma(d * ip) * (∂d * ip + ∂ip * d)
    #∂ld = -n * ∂a * ia - n * (∂digamma_dip * ip + digamma_dip * ∂ip)
    ##Second derivatives
    ∂2ip = -2 * ∂ip * ip
    ∂2K1 = n * ∂3Σxp - Σlogx * ∂2Σxp
    ∂2K = -∂2K1 * K^2 - 2 * K * ∂K * ∂K1
    ∂2ap = n^2 * ((∂2ip * K + ∂ip * ∂K) + (∂ip * ∂K + ip * ∂2K))
    ∂2log_ap = (∂2ap * ap - ∂ap^2) / ap^2
    ∂a_temp =
        -((∂2ip * log_ap + ∂ip * ∂log_ap) + (∂ip * ∂log_ap + ip * ∂2log_ap))
    ∂2a = ∂a_temp * a + a_temp * ∂a
    ∂2log_a = (∂2a * a - ∂a^2) * ia^2

    ∂2d = n * ((∂2Σxp * K + ∂Σxp * ∂K) + (∂Σxp * ∂K + Σxp * ∂2K))
    ∂digamma_dip = ∂dip * trigamma(dip)
    ∂2dip = (∂2d * ip + ∂d * ∂ip + ∂2ip * d + ∂ip * ∂d)
    ∂2loggamma_dip = ∂2dip * digamma_dip + ∂dip * ∂digamma_dip
    ### log likelihood as a function of p. (a and d have closed form depend on p)
    llike =
        n * log(p) - n * d * log_a + (d - 1) * Σlogx - Σxp * ap -
        n * loggamma_dip

    ∂llike =
        n * ip - n * (∂d * log_a + d * ∂log_a) + ∂d * Σlogx -
        (∂Σxp * ap + ∂ap * Σxp) - n * ∂loggamma_dip

    ∂2llike =
        n * ∂ip - n * (∂2d * log_a + 2 * ∂d * ∂log_a + d * ∂2log_a) +
        ∂2d * Σlogx - (∂2Σxp * ap + 2 * ∂Σxp * ∂ap + Σxp * ∂2ap) -
        n * ∂2loggamma_dip

    return a, d, p - ∂llike / ∂2llike, ∂llike, ∂K == 0, llike
end

# Functions h(α) and h'(α). We will solve h(α) = 0 using Newton's method.
# α = p
# function gengamma_h_dh(
#     α::Float64,
#     n::Int64,
#     X::AbstractArray{T},
#     logX::AbstractArray{T},
#     slogX::Float64,
# ) where {T<:Real}
#     iα1 = 1 / (α + 1)
#     Sx = 0.0
#     Sxlog = 0.0
#     dSxlog = 0.0
#     for i = 1:n
#         xα = X[i]^α
#         Sx += xα
#         Sxlog += xα * logX[i]
#         dSxlog += xα * logX[i]^2
#     end
#     k = (n - 1.5 - α * iα1) / α - log(α) * α * iα1^2
#     dk = (α - 1) * log(α) * iα1^3 - (n - 1.5) / α^2
#     dSx = Sxlog
#     A = k * Sx
#     B = n * Sxlog - Sx * slogX
#     ϕ = A / B
#     dA = dk * Sx + k * dSx
#     dB = n * dSxlog - dSx * slogX
#     dϕ = (dA * B - A * dB) / B^2
#
#     H = n * digamma(n * ϕ) + α * slogX - 1 / ϕ - n * digamma(ϕ) - n * log(Sx)
#
#     dH =
#         n^2 * dϕ * trigamma(n * ϕ) + slogX + dϕ / ϕ^2 - n * dϕ * trigamma(ϕ) -
#         n * dSx / Sx
#     return H, dH, ϕ, Sx
# end

# # function sumlogpdf(x, n, slogx, a, d, p)
# #     a1 = 1 / a
# #     n * log(p) + (d - 1) * slogx - sum((x * a1) .^ p) + n * d * log(a1) -
# #     n * log(gamma(d / p))
# # end


using PortHamiltonianBenchmarkSystems, MAT

for n_cells in [50, 500, 10_000]
  J, R, Q, B = gugercin_pH_msd_chain(n_cells = n_cells)
  filename = "GugercinN$(2n_cells)"
  writeHDF5(filename*".h5"; J, R, Q, B)
  writeMAT(filename*".mat"; J, R, Q, B)
end


include("../LF_common/m_LF1d.jl")
include("../LF_common/m_LF3d.jl")

include("init_pot_Hcoul.jl")
include("init_pot_harm_3d.jl")
include("../utils/ortho_gram_schmidt.jl")
include("../utils/orthonormalize.jl")
include("EnergiesT.jl")
include("op_nabla2.jl")
include("op_H.jl")
include("calc_Energies.jl")
include("calc_grad.jl")
include("Sch_solve_Emin_cg.jl")
include("Sch_solve_Emin_pcg.jl")
include("calc_evals.jl")
include("calc_rho.jl")
include("../LF_common/sparse_LF3d.jl")
include("../LF_common/prec_mkl_ilu0.jl")
include("../LF_common/apply_prec_ilu0.jl")
include("diag_lobpcg.jl")

function test_main( ; method = "Emin_cg" )
  # LF parameters
  NN = [25, 25, 25]
  hh = [0.3, 0.3, 0.3]

  Npoints = prod(NN)

  # Initialize LF
  LF = init_LF3d_sinc( NN, hh )
  ΔV = LF.LFx.h * LF.LFy.h * LF.LFz.h

  # Parameter for potential
  center = [0.0, 0.0, 0.0]

  ω = 2.0
  Vpot = init_pot_harm_3d( LF, ω, center )
  Ncols = 4

  #Vpot = init_pot_Hcoul( LF, center )
  #Ncols = 1

  if method == "Emin_cg_sparse"
    #
    ∇2 = get_Laplacian3d_kron(LF)
    prec = prec_mkl_ilu0( -0.5*∇2 + spdiagm(Vpot) )
    Energies, evecs = Sch_solve_Emin_pcg( LF, ∇2, prec, Vpot, Ncols, verbose=true )
    evals = calc_evals( LF, Vpot, evecs )
    #
  elseif method == "diag_lobpcg"
    #
    ∇2 = get_Laplacian3d_kron(LF)
    prec = prec_mkl_ilu0( -0.5*∇2 + spdiagm(Vpot) )
    srand(1234)
    v = rand( Npoints, Ncols )
    evals, evecs = diag_lobpcg( LF, ∇2, prec, Vpot, v, verbose=true)
    evecs = evecs/sqrt(ΔV) # normalize evecs
    Energies = calc_Energies( LF, ∇2, Vpot, evecs )
    #
  else
    #
    Energies, evecs = schsolve_Emin_cg( LF, Vpot, Ncols, verbose=true )
    evals = calc_evals( LF, Vpot, evecs )
  end

  @printf("Etot = %18.10f\n", Energies.Total)
  @printf("\nEigenvalues:\n")
  for i = 1:Ncols
    @printf("%8d %f\n", i, evals[i])
  end

  rho = get_rho( evecs )
  @printf( "\nintegRho = %lf\n", sum(rho)*ΔV )

end

@code_native test_main()
@time test_main(method="Emin_cg_sparse")
#@time test_main(method="diag_lobpcg")
#@time test_main(method="Emin_cg")


# force calculation using linked lists method

fvec(data) = [ Vector{Float64}(undef,2) for i in 1:data.N ]

function force!(box, DATA, frc, first_atom, next_atom, nc)
  ut = 0.

  for i in 1:length(frc)
    frc[i][1] = 0.
    frc[i][2] = 0.
  end

  for iat in 1:DATA.N
    icell = trunc(Int64,box[iat][1]/DATA.cutoff) + 1
    jcell = trunc(Int64,box[iat][2]/DATA.cutoff) + 1
    for i in icell-1:icell+1
      for j in jcell-1:jcell+1
        iw, jw = wrap_cell(nc, i, j)
        jat = first_atom[iw,jw]
        while jat > 0
          if jat > iat
            wrap!(box[jat],box[iat],DATA.side)
            rij = sqrt( (box[jat][1] - box[iat][1])^2 + (box[jat][2]-box[iat][2])^2)

            if rij <= DATA.cutoff
              up,fp = fpair(box[jat], box[iat], rij, DATA)
              frc[iat] .= frc[iat] .+ fp
              frc[jat] .= frc[jat] .- fp
              ut = ut + up
            end

          end
          jat = next_atom[jat]
        end
      end
    end
  end

  return ut

end

export fvec, force!


using Unmarked, DataFrames

λ_formula = @formula(λ~elev+forest);
p_formula = @formula(p~precip+wind);
β_truth = [0, -0.3, 0.7, -0.2, 0, 0.7];

umd = simulate(Nmix, λ_formula, p_formula, (1000, 5), β_truth);

fit = nmix(λ_formula, p_formula, umd);

hcat(coef(fit), β_truth)

#Prediction
pr_df = DataFrame(elev=[0.5, -0.3], forest=[1,-1]);

predict(abundance(fit), pr_df, interval=true)

#Goodness-of-fit
gof(fit)

#Fit all subsets of covariates
fit_all = nmix(allsub(λ_formula), allsub(p_formula), umd);

#Model selection table
fit_all

#Missing values
yna = Array{Union{Int,Missing}}(deepcopy(umd.y))
yna[1,:] = fill(missing, 5)
yna[2,1] = missing

inp2 = UmData(yna, umd.site_covs, umd.obs_covs);

fit2 = nmix(@formula(psi~elev+forest), @formula(p~precip+wind), inp2)

gof(fit2)


mutable struct SLOB
    num_paths::Int64
    T::Int64
    p₀::Float64
    M::Int64
    L::Float64
    D::Float64
    σ::Float64
    nu::Float64
    α::Float64
    source_term::SourceTerm
    x::Array{Float64, 1}
    Δx::Float64
    Δt::Float64
end
function SLOB(num_paths::Int64, T::Int64, p₀::Float64,
    M::Int64, L::Real, D::Float64, σ::Float64, nu::Float64,
    α::Float64, source_term::SourceTerm)

    x₀ = p₀ - 0.5*L
    x_m = p₀ + 0.5*L
    @assert x₀ >= 0
    x = collect(Float64, range(x₀, stop=x_m, length=M+1))
    Δx = L/M
    Δt = (Δx^2) / (2.0*D)
    return SLOB(num_paths, T, p₀, M, L, D, σ, nu, α, source_term, x, Δx, Δt)
end

SLOB(dict) = SLOB(
    dict["num_paths"], dict["T"], dict["p₀"], dict["M"],
    dict["L"], dict["D"], dict["σ"], dict["nu"], dict["α"],
    SourceTerm(dict["λ"], dict["μ"]))

SLOB(;num_paths::Int64=1, T::Int64=100, p₀::Real=100.0,
    M::Int64=100, L::Real=50.0, D::Real=4.0, σ::Real=1.0,
    nu::Real=0.1, α::Real=20.0, λ::Real=1.0, μ::Real=0.5) =
    SLOB(num_paths, T, p₀, M, L, D, σ, nu, α,
    SourceTerm(λ, μ))

function (slob::SLOB)(seed::Int=-1)
    if seed == -1
            seeds = Int.(rand(MersenneTwister(), UInt32, slob.num_paths))
        else
            seeds = Int.(rand(MersenneTwister(seed), UInt32, slob.num_paths))
    end

    time_steps = get_time_steps(slob.T, slob.Δt)

    mid_price_paths = ones(Float64, slob.T + 1, slob.num_paths)
    for path in 1:slob.num_paths
        Random.seed!(seeds[path])
        mid_price_paths[:, path] = dtrw_solver(slob)
    end

    return mid_price_paths
end

function (slob::SLOB)(debug::Bool, seed::Int=-1)
    logger = FileLogger(Dict(Logging.Info => "info.log", Logging.Error => "error.log"), append=false)
    global oldglobal = global_logger(logger)
    if seed == -1
            seeds = Int.(rand(MersenneTwister(), UInt32, slob.num_paths))
        else
            seeds = Int.(rand(MersenneTwister(seed), UInt32, slob.num_paths))
    end
    time_steps = get_time_steps(slob.T, slob.Δt)

    raw_price_paths = ones(Float64, time_steps + 1, slob.num_paths)

    sample_price_paths = ones(Float64, slob.T + 1, slob.num_paths)

    lob_densities = zeros(Float64, slob.M+1, time_steps + 1, slob.num_paths)
    P⁺s = ones(Float64, time_steps, slob.num_paths)
    P⁻s = ones(Float64, time_steps, slob.num_paths)
    Ps = ones(Float64, time_steps, slob.num_paths)
    for path in 1:slob.num_paths
        Random.seed!(seeds[path])
        @info "path $path with seed $(seeds[path])"
        lob_densities[:, :, path], raw_price_paths[:, path],
            sample_price_paths[:, path], P⁺s[:, path],
            P⁻s[:, path], Ps[:, path] = dtrw_solver(debug, slob)
    end

    return lob_densities, raw_price_paths, sample_price_paths, P⁺s, P⁻s, Ps
end

function get_time_steps(T::Int, Δt::Float64)
    time_steps =  T / Δt
    return floor(Int, time_steps + eps(time_steps))
end

function sample_mid_price_path(slob::SLOB, price_path)
    Δt = slob.Δt
    get_mid_price_inds(x) = get_time_steps(x, Δt)
    mid_price_inds = get_mid_price_inds.(0:slob.T) .+ 1
    mid_prices = price_path[mid_price_inds]
    return mid_prices
end


# get the baseline of an ECG using median filters

"""
    getECGBaseline(ecg::Vector{<:Real}, samplerate::Real)

Get the baseline of an ECG signal for baseline correction. Source: Advances in Cardiac Signal Processing - Acharya, U.R. and Suri, J. and Spaan, J.A.E. and Krishnan, S.M. and Technologies, B.
    - ISBN: 9783540366751 page: 58f. adaption by Jan F. Kraemer

# Args:

* ecg: ECG signal
* samplerate: Sampling rate
"""
function getECGBaseline(ecg::Vector{<:Real}, samplerate::Real)
    samplerate > 10 || throw(DomainError("Samplerate needs to be positive and larger than 10 Hz, is: $samplerate"))

    # Baseline drift via median filter
    # 1. get rid of QRS
    baseline = medfilt1(ecg, n = Int(ceil(0.25*samplerate)))
    # 2. get rid of P and T waves
    baseline = medfilt1(baseline, n = Int(ceil(0.75*samplerate)))

    # Lowpass filter to avoid sudden changes
    low = Lowpass(5, fs = samplerate)
    cheby = Chebyshev2(5, 10)
    lowChebyFilt = digitalfilter(low, cheby)
    baseline = filtfilt(lowChebyFilt, baseline)

    return baseline
end


function transform_to_steps!(A::AbstractMatrix; epsilon = 1e-7, degenerate_exeption = true)
   # @inbounds - заметного ускорения здесь это не даёт!!!
   for k ∈ 1:size(A, 1)
        absval, Δk = findmax(abs, @view(A[k:end,k]))
        (degenerate_exeption && absval <= epsilon) && throw("Вырожденая матрица")
        Δk > 1 && swap!(@view(A[k,k:end]), @view(A[k+Δk-1,k:end]))
        for i ∈ k+1:size(A,1)
            t = A[i,k]/A[k,k]
            @. @views A[i,k:end] = A[i,k:end] - t * A[k,k:end]
            # макрос @. заменяет все "сквозные точки"
            # макрос @views заменяет многократное применение макроса @view
        end
    end
    return A
end

function swap!(A,B)
    for i in eachindex(A)
        A[i], B[i] = B[i], A[i]
    end
end

function swap_2!(A, i, iMax, k)
    A[i, k], A[iMax, k] = A[iMax, k], A[i, k]
end

function transform_to_steps_2!(a::Matrix)
    countSwaps = 1
    c = a
    eps = 0.000001
    for i = 1:size(a, 1)
        iMax = i
        for j = i:size(a, 1)
            if abs(c[j, i]) > abs(c[iMax, i])
                iMax = j
            end
        end
        if abs(c[iMax, i]) >= eps
            for k = 1:size(a, 2)
                swap_2!(c, i, iMax, k)
            end
            countSwaps = -countSwaps * (i != iMax ? 1 : -1)

            for j = i+1:size(a, 1)
                q = -c[j, i] / c[i, i]
                for k = size(a, 2):-1:i
                    c[j, k] += q*c[i, k]
                    if abs(c[j, k]) < eps
                        c[j, k] = 0
                    end
                end
            end
        end
    end
    if countSwaps > 0
        c * -1
    end
    c
end

function transform_to_steps_edit!(A::AbstractMatrix; epsilon = 1e-7, degenerate_exeption = true)
    ans = []
    for k ∈ 1:size(A, 1)
        absval, Δk = findmax(abs, @view(A[k:end,k]))
        (degenerate_exeption && absval <= epsilon) && throw("Вырожденая матрица")
        Δk > 1 && swap!(@view(A[k,k:end]), @view(A[k+Δk-1,k:end]))
        for i ∈ k+1:size(A,1)
            t = A[i,k]/A[k,k]
            @. @views A[i,k:end] = A[i,k:end] - t * A[k,k:end]
        end
    end
    for i = 1:size(A,1)
        for j = i:size(A,2)
            if abs(A[i, j]) > epsilon
                push!(ans, j)
                break;
            end
        end
    end
    return ans
end

function rank_(B::AbstractMatrix; epsilon = 1e-7, degenerate_exeption = true)
    ans = []
    A = copy(B)
    for k ∈ 1:size(A, 1)
        absval, Δk = findmax(abs, @view(A[k:end,k]))
        (degenerate_exeption && absval <= epsilon) && throw("Вырожденая матрица")
        Δk > 1 && swap!(@view(A[k,k:end]), @view(A[k+Δk-1,k:end]))
        for i ∈ k+1:size(A,1)
            t = A[i,k]/A[k,k]
            @. @views A[i,k:end] = A[i,k:end] - t * A[k,k:end]
        end
    end
    for i = 1:size(A,1)
        for j = i:size(A,2)
            if abs(A[i, j]) > epsilon
                push!(ans, j)
                break;
            end
        end
    end
    return length(ans)
end

function det_(a)
    ans = 1
    epsilon = 1e-7
    s = 1
    c = copy(a)
    for k ∈ 1:size(c, 1)
         absval, Δk = findmax(abs, @view(c[k:end,k]))
         Δk > 1 && swap!(@view(c[k,k:end]), @view(c[k+Δk-1,k:end]))
         if Δk > 1
             s*=-1
         end
         for i ∈ k+1:size(c,1)
             t = c[i,k]/c[k,k]
             @. @views c[i,k:end] = c[i,k:end] - t * c[k,k:end]
         end
    end
    for i = 1:size(c, 1)
        ans*=c[i, i]
    end
    ans*s
end

function slau(A, b)
    countSwaps = 1
    a = [A b]
    c = copy(a)
    eps = 0.000001
    for i = 1:size(a, 1)
        iMax = i
        for j = i:size(a, 1)
            if abs(c[j, i]) > abs(c[iMax, i])
                iMax = j
            end
        end
        if abs(c[iMax, i]) >= eps
            for k = 1:size(a, 2)
                swap_2!(c, i, iMax, k)
            end
            countSwaps = -countSwaps * (i != iMax ? 1 : -1)

            for j = i+1:size(a, 1)
                q = -c[j, i] / c[i, i]
                for k = size(a, 2):-1:i
                    c[j, k] += q*c[i, k]
                    if abs(c[j, k]) < eps
                        c[j, k] = 0
                    end
                end
            end
        end
    end
    if countSwaps > 0
        c * -1
    end
    c[size(a, 1), size(a, 1)+1] /= c[size(a, 1), size(a, 1)]
    c[size(a, 1), size(a, 1)] = 1
    v = []
    for i = size(a, 1):-1:1
        t = c[i, size(a, 1)+1]
        for j = i+1:size(a, 1)
            t-=c[j]*c[i,j]
        end
        c[size(a, 1), i] = t
    end
    c[size(a,1),1:size(a,1)]
end

function inv_(a)
    c = copy(a)
    for i = 1:size(c, 1)
        for j = i+1:size(c, 1)
            c[i,j], c[j,i] = c[j,i], c[i,j]
        end
    end
    c
end
function transform_to_low_steps!(A::AbstractMatrix; epsilon = 1e-7, degenerate_exeption = true)
   # @inbounds - заметного ускорения здесь это не даёт!!!
   for k ∈ size(A, 1):-1:1
        absval, Δk = findmax(abs, @view(A[k:end,k]))
        (degenerate_exeption && absval <= epsilon) && throw("Вырожденая матрица")
        Δk > 1 && swap!(@view(A[k,k:end]), @view(A[k+Δk-1,k:end]))
        for i ∈ k-1:-1:1
            t = A[i,k]/A[k,k]
            @. @views A[i,k:end] = A[i,k:end] - t * A[k,k:end]
            # макрос @. заменяет все "сквоные точки"
            # макрос @views заменяет многократное применение макроса @view
        end
    end
    return A
end


@testset verbose=true "sYlm" begin

    @testset "Test NINJA expressions ($T)" for T in [Float64, Float32, BigFloat]
        ## This is just to test my implementation of the equations give in the paper.
        ## Note that this is a test of the testing code itself, not of the main code.
        tol = 2eps(T)
        @showprogress "Test NINJA expressions ($T)" for ι in βrange(T)
            for ϕ in αrange(T)
                @test NINJA.sYlm(-2, 2, 2, ι, ϕ) ≈ NINJA.m2Y22(ι, ϕ) atol=tol rtol=tol
                @test NINJA.sYlm(-2, 2, 1, ι, ϕ) ≈ NINJA.m2Y21(ι, ϕ) atol=tol rtol=tol
                @test NINJA.sYlm(-2, 2, 0, ι, ϕ) ≈ NINJA.m2Y20(ι, ϕ) atol=tol rtol=tol
                @test NINJA.sYlm(-2, 2, -1, ι, ϕ) ≈ NINJA.m2Y2m1(ι, ϕ) atol=tol rtol=tol
                @test NINJA.sYlm(-2, 2, -2, ι, ϕ) ≈ NINJA.m2Y2m2(ι, ϕ) atol=tol rtol=tol
            end
        end
    end

    @testset "Compare to NINJA expressions ($T)" for T in [Float64, Float32, BigFloat]
        ℓₘₐₓ = 8
        sₘₐₓ = 2
        ℓₘᵢₙ = 0
        tol = ℓₘₐₓ^2 * 2eps(T)  # Mostly because the NINJA.sYlm expressions are inaccurate
        Y, H_rec_coeffs, Hwedge, expimϕ = Yprep(ℓₘₐₓ, sₘₐₓ, T)

        let R = randn(Rotor{T})
            @test_throws ErrorException Y!(Y, R, ℓₘₐₓ+1, 0, H_rec_coeffs, Hwedge, expimϕ)
            ℓ = ℓₘₐₓ - 1
            i = WignerHsize(ℓ, abs(0)) - 1
            @test_throws ErrorException Y!(Y, R, ℓ, 0, H_rec_coeffs, Hwedge[1:i], expimϕ)
        end

        @showprogress "Compare to NINJA expressions ($T)" for spin in -sₘₐₓ:sₘₐₓ
            for ι in βrange(T)
                for ϕ in αrange(T)
                    R = from_spherical_coordinates(ι, ϕ)
                    Y!(Y, R, ℓₘₐₓ, spin, H_rec_coeffs, Hwedge, expimϕ)
                    i = 1
                    for ℓ in 0:abs(spin)-1
                        for m in -ℓ:ℓ
                            @test Y[i] == 0
                            i += 1
                        end
                    end
                    for ℓ in abs(spin):ℓₘₐₓ
                        for m in -ℓ:ℓ
                            sYlm1 = Y[i]
                            sYlm2 = NINJA.sYlm(spin, ℓ, m, ι, ϕ)
                            @test sYlm1 ≈ sYlm2 atol=tol rtol=tol
                            i += 1
                        end
                    end
                end
            end
        end
    end

    @testset "Spin property ($T)" for T in [Float64, Float32, BigFloat]
        # Test that ₛYₗₘ(R exp(γ*z/2)) = ₛYₗₘ(R) * exp(-im*s*γ)
        # See https://spherical.readthedocs.io/en/main/SWSHs/
        # for a more detailed explanation
        ℓₘₐₓ = 8
        sₘₐₓ = 2
        ℓₘᵢₙ = 0
        tol = 4ℓₘₐₓ * eps(T)
        Y1, H_rec_coeffs, Hwedge, expimϕ = Yprep(ℓₘₐₓ, sₘₐₓ, T)
        Y2 = Ystorage(ℓₘₐₓ, T)
        @showprogress "Spin property ($T)" for spin in -sₘₐₓ:sₘₐₓ
            for ι in βrange(T)
                for ϕ in αrange(T)
                    for γ in γrange(T)
                        R = from_spherical_coordinates(ι, ϕ)
                        Y!(Y1, R * exp(γ*𝐤/2), ℓₘₐₓ, spin, H_rec_coeffs, Hwedge, expimϕ)
                        Y!(Y2, R, ℓₘₐₓ, spin, H_rec_coeffs, Hwedge, expimϕ)
                        @test Y1 ≈ Y2 * cis(-spin*γ) atol=tol rtol=tol
                    end
                end
            end
        end
    end

    @testset "sYlm vs WignerD ($T)" for T in [Float64, Float32, BigFloat]
        # ₛYₗₘ(R) = (-1)ˢ √((2ℓ+1)/(4π)) 𝔇ˡₘ₋ₛ(R)
        #        = (-1)ˢ √((2ℓ+1)/(4π)) 𝔇̄ˡ₋ₛₘ(R̄)
        ℓₘₐₓ = 8
        sₘₐₓ = 2
        ℓₘᵢₙ = 0
        tol = 4ℓₘₐₓ * eps(T)
        𝔇, _, expimα, expimγ = Dprep(ℓₘₐₓ, T)
        Y, H_rec_coeffs, Hwedge, expimϕ = Yprep(ℓₘₐₓ, sₘₐₓ, T)
        @showprogress "sYlm vs WignerD ($T)" for s in -sₘₐₓ:sₘₐₓ
            for ι in βrange(T)
                for ϕ in αrange(T)
                    R = from_spherical_coordinates(ι, ϕ)
                    D!(𝔇, R, ℓₘₐₓ, H_rec_coeffs, expimα, expimγ)
                    Y!(Y, R, ℓₘₐₓ, s, H_rec_coeffs, Hwedge, expimϕ)
                    i = 1
                    for ℓ in 0:abs(s)-1
                        for m in -ℓ:ℓ
                            @test Y[i] == 0
                            i += 1
                        end
                    end
                    for ℓ in abs(s):ℓₘₐₓ
                        for m in -ℓ:ℓ
                            sYlm1 = Y[i]
                            sYlm2 = (-1)^s * √((2ℓ+1)/(4T(π))) * 𝔇[WignerDindex(ℓ, m, -s)]
                            if ≉(sYlm1, sYlm2, atol=tol, rtol=tol)
                                println("Unequal at i=$i (s,ℓ,m)=$((s,ℓ,m)): ")
                                @show ι ϕ
                                @show sYlm1 sYlm2
                            end
                            @test sYlm1 ≈ sYlm2 atol=tol rtol=tol
                            i += 1
                        end
                    end
                end
            end
        end
    end

    @testset "sYlm conjugation ($T)" for T in [Float64, Float32, BigFloat]
        # ₛȲₗₘ = (-1)ˢ⁺ᵐ ₋ₛYₗ₋ₘ
        ℓₘₐₓ = 8
        sₘₐₓ = 2
        ℓₘᵢₙ = 0
        tol = 4ℓₘₐₓ * eps(T)
        Y1, H_rec_coeffs, Hwedge, expimϕ = Yprep(ℓₘₐₓ, sₘₐₓ, T)
        Y2 = Ystorage(ℓₘₐₓ, T)
        @showprogress "sYlm conjugation ($T)" for ι in βrange(T)
            for ϕ in αrange(T)
                for γ in γrange(T)
                    for s in -sₘₐₓ:sₘₐₓ
                        R = from_spherical_coordinates(ι, ϕ)
                        Y!(Y1, R, ℓₘₐₓ, s, H_rec_coeffs, Hwedge, expimϕ)
                        Y!(Y2, R, ℓₘₐₓ, -s, H_rec_coeffs, Hwedge, expimϕ)
                        for ℓ in abs(s):ℓₘₐₓ
                            for m in -ℓ:ℓ
                                sYlm1 = conj(Y1[Yindex(ℓ, m)])
                                sYlm2 = (-1)^(s+m) * Y2[Yindex(ℓ, -m)]
                                @test sYlm1 ≈ sYlm2 atol=tol rtol=tol
                            end
                        end
                    end
                end
            end
        end
    end

end


using Stheno, Plots, Random
using Stheno: @model

###########################  Define and inspect our model  ###########################

#=
We wish to perform inference in some latent process `f`, but it is corrupted by both some
iid noise, and some non-iid noise. Specifically, ω is some periodic behaviour in which we
are uninterested. Similarly, g is also drawn from a GP with an EQ kernel, but with a much
shorter length-scale. We wish to use observations of
`f + ω + g + GP(ZeroMean{Float64}(), Noise(0.1))`
to infer `f`. In this context, `ω + g + GP(ZeroMean{Float64}(), Noise(0.1))` is
regarded as our "noise process".

The described model is unidentiable. Conditioning on an observation of the periodic process
at some point, however, allows us to infer a particular `f`. Use the `model_identifiable`
function rather than `model` to see the results of this. Note the example of sequential
conditioning.
=#
@model function model()
    f = GP(EQ())
    ω = GP(periodic(EQ(), 1))
    return f, ω, ω + f, ω + f + GP(Noise(0.1))
end
@model function model_identifiable()
    f = GP(EQ())
    t = GP(periodic(EQ(), 1))
    ω = t | (t([0])←[0.0])
    return f, ω, ω + f, ω + f + GP(Noise(0.001))
end

# Select some input locations and sample from the prior.
rng, N, Nplot, S = MersenneTwister(123456), 100, 500, 100;
X, Xp = sort(rand(rng, N) * 10), range(-2.5, stop=12.5, length=Nplot);
f, ω, fpω, y = model();
fs, ωs, ys, ŷ = rand(rng, [f(Xp), ω(Xp), fpω(Xp), y(X)]);

# Compute posterior distribution over f′.
f′ = f | (y(X) ← ŷ);

# Sample from the posterior and write to file.
f′Xp = rand(rng, f′(Xp), S);

# Get posterior mean and marginals f′ and y′ and write them for plotting.
μf′, σf′ = marginals(f′(Xp));

###########################  Plot results - USE ONLY Julia-0.6!  ###########################

posterior_plot = plot();

# Prior over `f`.
plot!(posterior_plot, Xp, ys;
    linewidth=1.0,
    linecolor=:red,
    label="y");
plot!(posterior_plot, Xp, fs;
    linewidth=2.0,
    linecolor=:black,
    label="f");

# Posterior over `f`.
plot!(posterior_plot, Xp, μf′;
    linecolor=:blue,
    linewidth=2.0,
    label="f1");
plot!(posterior_plot, Xp, [μf′ μf′];
    linewidth=0.0,
    fillrange=[μf′ .- 3 .* σf′, μf′ .+ 3 * σf′],
    fillalpha=0.3,
    fillcolor=:blue,
    label="");
scatter!(posterior_plot, X, ŷ;
    markercolor=:red,
    markershape=:circle,
    markerstrokewidth=0.0,
    markersize=4,
    markeralpha=0.7,
    label="");
plot!(posterior_plot, Xp, f′Xp,
    linecolor=:blue,
    linealpha=0.2,
    label="");

display(posterior_plot);


# TODO expand tests here
let
    f(x) = x[1]^4
    function g!(x, storage)
        storage[1] = 4 * x[1]^3
        return
    end

    d = DifferentiableFunction(f, g!)

    initial_x = [1.0]

    Optim.optimize(d, initial_x,
                   method = AcceleratedGradientDescent()
                   show_trace = true,
                   iterations = 10)
end


using Revise
using Koala
using KoalaRidge
using Base.Test
using DataFrames

## SYNTHETIC DATA TEST

# Define some linear, noise-free, synthetic data:
const bias = -42.0
const coefficients = Float64[1, 3, 7]
const A = randn(1000, 3)
const X = DataFrame(A)
const y = A*coefficients

# Train model on all data with no regularization and no
# standardization of target:
const ridge = RidgeRegressor(lambda=0.0)
ty = default_transformer_y(ridge)
ty.standardize = false
ridgeM = Machine(ridge, X, y, eachindex(y), transformer_y=ty)
fit!(ridgeM, eachindex(y))

# Training error:
E = err(ridgeM, eachindex(y))
@test E < 1e-10

# Get the true bias?
@test abs(ridgeM.predictor.bias) < 1e-10
@test norm(ridgeM.predictor.coefficients - coefficients) < 1e-10

# Train with standardization of target:
ridgeM1 = Machine(ridge, X, y, eachindex(y))
fit!(ridgeM1, eachindex(y))

# Get same predictions?
@test isapprox(predict(ridgeM, X), predict(ridgeM1, X))

## TEST OF OTHER METHODS ON REAL DATA

# Load some data and define train/test rows:
const X, y = load_ames();
y = log.(y) # log of the SalePrice
const train, test = split(eachindex(y), 0.7); # 70:30 split

# Instantiate a model:
ridge = RidgeRegressor(lambda=0.1)
ty = default_transformer_y(ridge)
ty.boxcox = false
showall(ty)
showall(ridge)

# Build a machine:
ridgeM = Machine(ridge, X, y, train, transformer_y=ty)

fit!(ridgeM, train)
showall(ridgeM)

# tune lambda using cross-validation
lambdas, rmserrors = @curve λ logspace(-3,1,100) begin
    ridge.lambda = λ
    mean(cv(ridgeM, train, n_folds=9, verbosity=0))
end

# set lambda to the optimal value and do final train:
ridge.lambda = lambdas[indmin(rmserrors)]
fit!(ridgeM, train)

score = err(ridgeM, test)
println("score = $score")
@test score > 0.12 && score < 0.14


module auxiliaryFunctions

import LinearAlgebra: zeros, Matrix, svd, pinv, transpose, det, norm, I, rank
import HomotopyContinuation: solve, randn, differentiate, solutions, real_solutions, System, @polyvar
import DynamicPolynomials: Polynomial, Term
import Combinatorics: binomial, powerset, multiexponents
import Suppressor: @suppress

export affineVeronese,
	projVeronese,
	addNoise,
	calculateMeanDistanceToVariety,
	findEqListOfDegrees,
	comparisonOfMethods,
	makeCombinations,
	fillUpWithZeros,
	sampsonDistance,
	weightedGradientDescent,
	cleanUp

function addNoise(points, variance)
	points = [[entry+randn(Float64) for entry in point] for point in points]
	return(points)
end

#Take the filledzero list of degrees and return, if the polynomial generators are minimal.
function cleanUp(intermediateValues, listOfDegrees, varlength)
	totalexponents = vcat(map(i -> collect(multiexponents(varlength,-i)), -listOfDegrees[end][1]:0)...)
	array=[]
	indecesOfBasis = []
	for q in 1:length(listOfDegrees)
		currentsum = listOfDegrees[q][1]==listOfDegrees[1][1] ? 0 : sum([en[2] for en in listOfDegrees[1:q-1]])
		for i in 1:listOfDegrees[q][2]
			push!(array,intermediateValues[i+currentsum])
			push!(indecesOfBasis, length(array))
		end
		currentexponents = vcat(map(i -> collect(multiexponents(varlength,-i)), -listOfDegrees[q][1]:0)...)
		for val in 1:sum(totalexponents[1])-sum(currentexponents[1])
			indexarray = [[findfirst(t->t==adder+entry, totalexponents) for entry in currentexponents] for adder in collect(multiexponents(varlength,val)) ]
			for i in 1:listOfDegrees[q][2]
				for j in 1:length(indexarray)
					helper = [0. for _ in 1:length(totalexponents)]
					for k in 1:length(indexarray[j])
						helper[indexarray[j][k]] = intermediateValues[currentsum+i][length(totalexponents)-length(collect(multiexponents(varlength,listOfDegrees[q][1])))+k-1]
					end
					push!(array,helper)
				end
			end
		end
	end
	#Return true if the rank does not drop (ideal is minimal) and return the reduced ideal generators. Else return false
	if rank(hcat(array...))==length(array)
		redarray = reduce([array[ind] for ind in indecesOfBasis], array)
		return(true, [ar./norm(ar) for ar in redarray])
	else
		return(false, [])
	end
end

#Calculate f mod I for f not in I
function reduce(truebasis, array)
	for i in 1:length(truebasis)
		element = Base.copy(truebasis[i])
		fil = sort(filter(t->element!=t, array), rev=true)
		indeces = [findfirst(t->norm(t)>1e-15, el) for el in fil]

		#Gauss elimination algorithm
		for j in 1:length(fil)
			filind = findfirst(t->norm(t)>1e-15, fil[j])
			if norm(element[filind])>1e-15
				c1 = fil[j][filind]
				c2 = element[filind]
				element = element - fil[j]./c1.*c2
			end
			truebasis[i] = element
		end
	end
	return(truebasis)
end

function jacobianProd(veronese, var)
	d = length(var)
	dMatrix = Array{Polynomial, 2}(undef, length(veronese), length(veronese))
	dArray = [collect(differentiate(poly, var) for poly in veronese)][1]
	for i in 1:size(dMatrix)[1]
		for j in 1:size(dMatrix)[2]
			dMatrix[i,j] = Polynomial(differentiate(veronese[i], var)'*differentiate(veronese[j], var))
		end
	end
	return(dMatrix)
end

function vandermonde(n, d, array, proj=false)
	exponents = vcat(map(i -> collect(multiexponents(n,-i)), -d:0)...)
	if (proj == true)
		exponents = []
		for k in n:-1:0
			exp = collect(multiexponents(length(array[1])-1,k))
			[append!(entry,n-k) for entry in exp]
			append!(exponents,exp)
		end
	end
	output = Array{Float64,2}(undef, length(array), length(exponents))
	for i in 1:length(array)
		for j in 1:length(exponents)
			prod = 1.0
			for k in 1:length(exponents[j])
				if(exponents[j][k]!=0)
					prod = prod*((array[i][k])^(exponents[j][k]))
				end
			end
			output[i,j]=prod
		end
	end
	return(output)
end

function evaluationOfMatrix(gamma, Z, var)
	output = zeros(size(gamma)[1], size(gamma)[2])
	for iter in 1:length(Z)
		helper = Array{Float64, 2}(undef, size(gamma)[1], size(gamma)[2])
		for i in 1:size(output)[1]
			for j in 1:size(output)[2]
				helper[i,j] = (gamma[i,j])(var=>Z[iter])
			end
		end
		output = output + helper
	end
	return(output./length(Z))
end

function affineVeronese(n, var)
	exponents = vcat(map(i -> collect(multiexponents(length(var),-i)), -n:0)...)
	output = [prod(var.^exp) for exp in exponents]
	return(output)
end

function projVeronese(n,var)
	exponents = vcat(map(i -> collect(multiexponents(length(var)-1,-i)), -n:0)...)
	output = [prod(var[1:length(var)-1].^exp)*var[length(var)]^(n-sum(exp)) for exp in exponents]
	return(output)
end

function calculateMeanDistanceToVariety(points, equations, var)
	#HomotopyContinuation: https://www.juliahomotopycontinuation.org/examples/critical-points/
	@polyvar u[1:length(var)]
	if (length(var)-length(equations) == 1)
		d = [differentiate(equation, var) for equation in equations]
		matrix = Array{Polynomial, 2}(undef, length(var), length(var))
		matrix[1,:] = var-u
		for i in 1:length(d)
			matrix[i+1,:] = d[i]
		end
		systemArray = [det(matrix)]
		append!(systemArray, equations)
		systemArray = filter!(t->t!=0,systemArray)
		F_u = System(systemArray, variables = var, parameters = u)
	elseif (length(var)-length(equations) > 1)
		d = [differentiate(equation, var) for equation in equations]
		matrix = Array{Polynomial, 2}(undef, length(equations)+1, length(var))
		matrix[1,:] = var-u
		for i in 1:length(d)
			matrix[i+1,:] = d[i]
		end
		binomialsets = filter(p->length(p)==length(equations)+1, collect(powerset(1:length(var))))
		saverMatrix=Array{Polynomial,2}(undef,length(equations)+1,length(equations)+1)
		systemArray=Array{Polynomial,1}(undef,0)
		for entry in binomialsets
			for i in 1:length(entry)
				saverMatrix[:,i] = matrix[:, entry[i]]
			end
			push!(systemArray,det(saverMatrix))
		end
		append!(systemArray, equations)
		systemArray = filter!(t->t!=0,systemArray)
		F_u = System(systemArray, variables = var, parameters = u)
	else
		throw(error("The method is not yet supported for non-complete intersections!"))
	end
	p = randn(ComplexF64, length(points[1]))
	#@suppress begin
		try
			result_p = solve(F_u, target_parameters = p)
			realSolutions = solve(
								F_u,
								solutions(result_p);
								start_parameters =  p,
								target_parameters = points,
								transform_result = (r,u) -> minimum([norm(rel-u) for rel in solutions(r)])
			)
			realSolutions = filter(p -> p!=Inf, realSolutions)
			if !isempty(realSolutions)
				return(sum(realSolutions)/length(realSolutions))
			else
				return("dim>0")
			end
		catch e
			return("dim>0")
		end
	#end
end

#=
 Output is a good estimate for the starting vector of our Least Square Iteration
 =#
function comparisonOfMethods(n,points,numEq,tau)
	timer = round(Int64, time() * 1000)
	@polyvar var[1:length(points[1])]
	projPoints = [vcat(point,[1]) for point in points]
	@polyvar projVar[1:length(projPoints[1])]

	veroneseProj = projVeronese(n,projVar)
	veroProdProj = sum([projVeronese(n,point)*projVeronese(n,point)' for point in projPoints])/length(projPoints)
	jacoProdProj = evaluationOfMatrix(jacobianProd(veroneseProj,projVar), projPoints, projVar)
	svdSingular = svd(pinv(jacoProdProj)*veroProdProj)
	firstS = [entry / maximum(svdSingular.S) for entry in svdSingular.S]
	smallestS = firstS[length(firstS)-numEq]*tau
	numberOfSmallSingularValues = length(filter(p-> p<=smallestS, firstS))
	firstV = [svdSingular.V[:,i] for i in (length(veroneseProj)-numberOfSmallSingularValues+1):length(veroneseProj)]
	timer2 = round(Int64, time() * 1000)
	try
		Vandermonde = vandermonde(length(var),n,points,false)
		svdVander = svd(Vandermonde)
		secondS = [entry / maximum(svdVander.S) for entry in svdVander.S]
		secondV = svdVander.V[:,(length(veroneseProj)-numberOfSmallSingularValues)+1:length(veroneseProj)]
		secondV = [secondV[:,i] for i in 1:size(secondV)[2]]
		return(firstV, secondV)
	catch e
		return(firstV, firstV)
		prinln("Error caught",e)
	end
end

function weightedGradientDescent(points, n, var, curw0, nEq, maxIter, saverArray, zeroEntries)

	w0Matrix = Array{Float64,2}(undef,length(curw0[1]),length(curw0))
	for i in 1:length(curw0)
		w0Matrix[:,i] = curw0[i]
	end
	function zeroMatrix(w0Matrix)
		mat = zeros(Float64,length(curw0[1]),length(curw0))
		for zero in zeroEntries
			mat[zero[2],zero[1]] = w0Matrix[zero[2],zero[1]]
		end
		return(mat)
	end

	lossFct = w0Matrix -> sum([(projVeronese(n,vcat(points[j],1))'*w0Matrix)*saverArray[j]*w0Matrix'*projVeronese(n,vcat(points[j],1)) for j in 1:length(points)])/length(points)+100*sum([entry.^2 for entry in zeroMatrix(w0Matrix)])
	curLoss = lossFct(w0Matrix)
	dLossFct = w0Matrix -> 2*sum([(projVeronese(n,vcat(points[j],1))*projVeronese(n,vcat(points[j],1))')*w0Matrix*saverArray[j] for j in 1:length(points)])./length(points)+200*zeroMatrix(w0Matrix)
	i = 1
	while  i < maxIter && norm(dLossFct(w0Matrix)) > 1e-2
		w0Matrix, curLoss = backtracking_line_search(w0Matrix, dLossFct, lossFct)
		i=i+1
	end
	return([w0Matrix[:,i]./norm(w0Matrix[:,i]) for i in 1:size(w0Matrix)[2]], curLoss)
end

function backtracking_line_search(w0Matrix, dLossFct, lossFct; r=1e-3, s=0.7)
	dLoss = dLossFct(w0Matrix)./norm(dLossFct(w0Matrix))
	α = 0; t = 0.1; β = 10000
	while norm(β-α)>1e-15
		if lossFct(w0Matrix)-lossFct(w0Matrix - t * dLoss) < r*t*norm(dLoss)^2
			β = t
			t = (α+β)/2
		elseif s*norm(dLoss'*dLoss)<norm(dLoss'*dLossFct(w0Matrix - t*dLoss)./norm(dLossFct(w0Matrix - t*dLoss)))
			α = t
			t = (α+β)/2
		else
			break
		end
	end
	return(w0Matrix - t * dLoss, lossFct(w0Matrix - t * dLoss))
end

function sampsonDistance(points, nEq, n, var, startValues)
	@polyvar zed[1:length(points[1])+1]
	veronese = projVeronese(n, zed)
	Qstart = [start'*veronese for start in startValues]
	matrix = Array{Polynomial,2}(undef, nEq, length(points[1]))
	for i in 1:nEq, j in 1:nEq
		matrix[i,j] = differentiate(Qstart[i],zed)'*differentiate(Qstart[j],zed)
	end
	#TODO Whatever needs to be done here.
	helper = Array{Float64,2}(undef, nEq, nEq)
	saverArray = []

	for point in points
		for i in 1:nEq
			for j in 1:nEq
				helper[i,j] = matrix[i,j](zed=>vcat(point,1))
			end
		end
		append!(saverArray,[pinv(helper)])
	end
	return(saverArray)
end

function makeCombinations(values)
	saver = []
	for entry in values
		combis = collect(powerset(entry[2],entry[1],entry[1]))
		append!(saver, [combis])
	end
	return(cartesianUnion(saver))
end

function cartesianUnion(combinations)

	product = 1
	for comb in combinations
		product = product*length(comb)
	end
	output = Array{Any, 1}(undef, product)
	for i in 1:length(output)
		output[i]=[]
	end
	number = [1 for i in 1:length(combinations)]
	max = [length(comb) for comb in combinations]
	a = []
	j = 1
	while number[1]<=length(combinations[1])
		a = []
		for i in 1:length(combinations)
			append!(a,combinations[i][number[i]])
		end
		output[j] = [element for element in a]
		number[length(number)] = number[length(number)]+1
		number = checkNumber(number, max)
		j = j+1
	end
	return(output)
end

function checkNumber(number, max)
	for i in length(number):-1:2
		if(number[i]>max[i])
			number[i-1]=number[i-1]+1
			number[i]=1
		end
	end
	return(number)
end

function fillUpWithZeros(combination, n, numEq,d)
	zeroEntries = []

	for i in 1:length(combination)
		output = Array{Float64, 1}(undef, length(combination[end]))
		output[1:length(output)-length(combination[i])] .= 0
		output[length(output)-length(combination[i])+1:end] = combination[i]
		append!(zeroEntries,[[i,j] for j in 1:length(output)-length(combination[i])])
		combination[i] = [out for out in output]
	end
	return(combination, zeroEntries)
end

function findEqListOfDegrees(listOfDegrees)
	sort!(listOfDegrees)
	degreeList = []
	for x in listOfDegrees
		index = 0
		for entry in degreeList
			if entry[1]==x
				entry[2]=entry[2]+1
				index = 1
				break
			end
		end
		if(index==0)
			append!(degreeList, [[x,1]])
		end
	end
	return(	degreeList)
end

function findAllCompositions(amount, number)
	saverList = []
	for j in 1:number
		if(number%j==0)
			append!(saverList,[[j]])
		end
	end

	product = 1
	for k in (amount-1):-1:1
		helperList = []
		for entry in saverList
			append!(helperList, findDivisors(number/prod(entry), entry))
		end
		saverList = helperList
	end

	saverList = filter(p-> prod(p)==number,saverList)
	for list in saverList
		sort!(list)
	end
	saverSet = Set(saverList)
	saverList = collect(saverSet)
	return(saverList)
end

function findDivisors(number, currentList)
	if(number <= 0)
		return([])
	end
	saverList = []
	for k in 1:number
		if number%k==0
			helper = [list for list in currentList]
			append!(helper,[k])
			append!(saverList,[helper])
		end
	end
	return(saverList)
end

end


"""
    minimum_weight_perfect_bmatching(g, b[, w]; cutoff=Inf)

Given a graph `g` and an edgemap `w` containing the weights associated to each edge,
returns the perfect `b`-matching with the minimum total weight.

A perfect `b`-matching `M` is a collection of edges in `g` such that every vertex
has exactly `b` incident edges in `M` .

If `w` is not given, all edges will be considered to have weight one
(results in max cardinality b-matching).

Edges `e` with no associated value in `w` or with `w[e] > cutoff`
will not be considered for the matching.

The algorithm uses Linear Programming to solve the linear relaxation of the
problem first, then eventually refines the solution with an Integer Programming solver.

The efficiency of the algorithm depends on the input graph:
- If the graph is bipartite, then the LP relaxation is integral.
- If the graph is not bipartite, then an IP may be required to refine the solution
 and the computation time may grow exponentially.

The package JuMP.jl and one of its supported solvers are required.

Returns a tuple `(status, W, match)` containing:
- a solve `status` (indicating whether the problem was solved to optimality)
- the tototal weight `W` of the matching
- a vector of vectors `match`, where `match[v]`  contains  the `b` neighbors of `v` in the optimal matching.

**Example**
```juliarepl
julia> g = CompleteGraph(30)
julia> w = EdgeMap(g, e -> rand())
julia> status, W, match = minimum_weight_perfect_bmatching(g, 2, w)
```
"""
function minimum_weight_perfect_bmatching(g::G, b::Integer,
                w::AEdgeMap=ConstEdgeMap(g,1); cutoff=Inf, verb=true) where {G<:AGraph}
        h = G(nv(g))
        for e in edges(g)
            haskey(w, e) && w[e] <= cutoff && add_edge!(h, e)
        end

        return _solve_bmatching(h, b, w, verb)
end

function _solve_bmatching(g::AGraph, b::Integer, w, verb)
    elist = collect(edges(g))

    ## Linear Programming
    # model = Model(with_optimizer(LP_OPTIMIZER))
    # @variable(model, 0 <= x[elist] <= 1)
    # @objective(model, Min, sum(x[e]*w[e] for e in elist))
    # @constraint(model, c1[i=1:nv(g)], sum(x[sort(e)] for e in edges(g, i)) == b)
    # optimize!(model)
    # status = termination_status(model)
    # sol = value(y)
    # cost = objective_value(model)
    # isintegral(sol, verb) && return status, cost, mates(nv(g), b, sol)
    # verb && warn("Using Integer Programming.")

    ## Integer Programming
    model = Model(with_optimizer(MIP_OPTIMIZER))

    @variable(model, y[elist], Bin)
    @objective(model, Min, sum(y[e]*w[e] for e in elist))
    @constraint(model, c[i=1:nv(g)], sum(y[sort(e)] for e in edges(g, i)) == b)

    # Bootstrap from LP
    # for e in elist
    #     setvalue(y[e], round(Int, getvalue(x[e])))
    # end
    optimize!(model)
    status = termination_status(model)
    sol = Dict(e => value(y[e]) for e in elist)
    cost = objective_value(model)
    return status, cost, mates(nv(g), b, sol)
end

function isintegral(sol, verb; atol=1e-8)
    f(x) = abs(x - round(Int, x)) > atol
    n = count(f(sol[e]) for (e,) in keys(sol))
    verb && n > 0 && warn("$n non integer variables out of $(length(sol)) in linear relaxation.")
    return n == 0
end

function mates(n, b::Integer, sol; atol=1e-8)
    mate = [sizehint!(zeros(Int,0), b) for i=1:n]
    for e in keys(sol)
        sol[e] == 0 && continue
        if abs(sol[e] - 1) < atol
            push!(mate[src(e)], dst(e))
            push!(mate[dst(e)], src(e))
        end
    end
    for v in mate
        @assert length(v) == b
        sort!(v)
    end
    return mate
end


#! In this folder I collect functions that are for the version of the model with depreciating information.

"""
    Calculate prior uncertainty after n periods of a stable data regulation regime.
"""
function CalcPriorUncertainty(n, past_posterior; par::ParsGrowth)
    if n == 0
        par.σ²_θ
    else
        par.ρ^2 * past_posterior + par.σ²_η
    end
end

function CalcPosteriorUncertainty(past_prior, par::ParsGrowth)
    #1/(past_prior^-1 + par.σ²_ϵ^-1 + par.σ²_η^-1)
    1/(past_prior^-1 + par.σ²_ϵ^-1)
end

function CalcAllUncertainty(n_max; par::ParsGrowth)
    prior_unc = zeros(n_max + 1)
    post_unc = zeros(n_max + 1)
    for n in 0:n_max
        if n == 0
            prior_unc[n+1] = CalcPriorUncertainty(n, 0; par)
        else
            prior_unc[n+1] = CalcPriorUncertainty(n, post_unc[n]; par)
        end
        post_unc[n+1] = CalcPosteriorUncertainty(prior_unc[n+1], par::ParsGrowth)
    end

    return prior_unc, post_unc
end

function Transition_Probability_dep(par::ParsGrowth, bpar::BParsGrowth; normalize::Symbol = :yes)
    @unpack_ParsGrowth par
    @unpack_BParsGrowth bpar
    Π    = zeros(N, N)

    # Rows: Today's state i
    # Columns: Tomorrow's state j
    for i = 1:N, j = 1:N
        if j == 1
            d = a_grid[j+1] - a_grid[j]

            c_plus = (σ²_ϵ * ((V_n^-1 + σ²_ϵ^-1) * (a_grid[j] + d / 2) / ρ - V_n^-1 * a_grid[i]) - a_grid[i]) / √(V_n + σ²_ϵ)

            Π[i, j] = normcdf(c_plus)
        elseif j == N
            d = a_grid[j] - a_grid[j-1]

            c_minus = (σ²_ϵ * ((V_n^-1 + σ²_ϵ^-1) * (a_grid[j] - d / 2) / ρ - V_n^-1 * a_grid[i]) - a_grid[i]) / √(V_n + σ²_ϵ)

            Π[i, j] = 1 - normcdf(c_minus)
        else
            d_up = a_grid[j+1] - a_grid[j]
            d_down = a_grid[j] - a_grid[j-1]

            c_plus = (σ²_ϵ * ((V_n^-1 + σ²_ϵ^-1) * (a_grid[j] + d_up / 2) / ρ - V_n^-1 * a_grid[i]) - a_grid[i]) / √(V_n + σ²_ϵ)

            c_minus = (σ²_ϵ * ((V_n^-1 + σ²_ϵ^-1) * (a_grid[j] - d_down / 2) / ρ - V_n^-1 * a_grid[i]) - a_grid[i]) / √(V_n + σ²_ϵ)

            Π[i, j] = normcdf(c_plus) - normcdf(c_minus)
        end
    end
    # Renormalize the transition matrix
    if normalize == :yes
        for i = 1:N
            Π[i, :] = Π[i, :] ./ sum(Π[i, :])
        end
    end
    return Π
end


using Revise
using JuMP
using GLPK
using ModelGraphs
using .DualDecompositionSolver

const DDSolver = DualDecompositionSolver

m1 = Model(with_optimizer(GLPK.Optimizer))
@variable(m1, xm[i in 1:2],Bin)
@constraint(m1, xm[1] + xm[2] <= 1)
@objective(m1, Max, 16xm[1] + 10xm[2])

m2 = Model(with_optimizer(GLPK.Optimizer))
@variable(m2, xs[i in 1:2], Bin)
@variable(m2, y[i in 1:2], Bin)
@constraint(m2, y[1] + y[2] <= 1)
@constraint(m2, 8xs[1] + 2xs[2] + y[1] + 4y[2] <= 10)
@objective(m2, Max, 4y[2])

m3 = Model(with_optimizer(GLPK.Optimizer))
@variable(m3, x3[i in 1:2], Bin)
@variable(m3, y[i in 1:2], Bin)
@constraint(m3, y[1] + y[2] <= 1)
@constraint(m3, 8x3[1] + 2x3[2] + y[1] + 4y[2] <= 10)
@objective(m3, Max, y[2])

## Model Graph
graph = ModelGraph()

#Add nodes
n1 = add_node!(graph,m1)
n2 = add_node!(graph,m2)
n3 = add_node!(graph,m3)

#link constraints between models
@linkconstraint(graph, [i in 1:2], n1[:xm][i] == n2[:xs][i])
@linkconstraint(graph, n3[:x3][1] + n1[:xm][1] + n2[:xs][1] <= 2)
@linkconstraint(graph, n1[:xm][2] <= n3[:y][1])

dd_model = DDModel(graph)
DDSolver.optimize!(dd_model)

#print solution
println("Dual decomposition solution: ")
solution = Float64[]
for subproblem in dd_model.subproblems
    append!(solution,value.(all_variables(subproblem)))
end
println(solution)

# #verify with full problem
glpk = with_optimizer(GLPK.Optimizer)
m,ref_map = aggregate(graph)
optimize!(m,glpk)

println()
println("Pure glpk solution:")
println(JuMP.value.(all_variables(m)))


## Process measures

function integrate_measure(qs, integrand, domain, δ::DiracWeight, properties...)
    x = point(δ)
    if x ∈ domain
        I = integrand(x)
        E = zero(I)
    else
        I, E = zero_result(integrand)
    end
    I, E
end

function integrate_measure(qs, integrand, domain, μ::DiscreteWeight, properties...)
    if domain == covering(μ)
        I = sum(w*integrand(x) for (w,x) in zip(weights(μ), points(μ)))
        I, zero_error(I)
    else
        # Only sum over the elements that lie in the domain
        I, E = zero_result(integrand)
        x = points(μ)
        w = weights(μ)
        for i in 1:length(x)
            if x[i] ∈ domain
                I += w[i]*integrand(x[i])
            end
        end
        I, zero_error(I)
    end
end

# Lebesgue measures can pass through unaltered
integrate_measure(qs, integrand, domain, measure::LebesgueMeasure, properties...) =
    integrate_domain(qs, integrand, domain, measure, properties...)

process_measure(qs, integrand, domain, measure::LebesgueMeasure, properties...) =
    (integrand, domain, measure, properties...)

process_measure(qs, integrand, domain, measure::Measure, properties...) =
    process_measure_default(qs, integrand, domain, measure, properties...)

# By default we replace all measures by the Lebesgue on the space
process_measure_default(qs, integrand, domain, measure::Measure{T}, properties...) where {T} =
    ((t->unsafe_weightfun(measure, t)) * integrand, domain, Lebesgue{T}(), properties...)

function process_measure(qs, integrand, domain::MappedDomain, μ::MappedWeight, properties...)
    map1 = forward_map(μ)
    map2 = forward_map(domain)
    if map1 == map2
        # f -> f(m1(x))
        process_measure(qs, integrand ∘ map1, superdomain(domain), supermeasure(μ), properties...)
    else
        process_measure_default(qs, integrand, domain, μ, properties...)
    end
end

function process_measure(qs, integrand, domain, μ::MappedWeight, properties...)
    # f -> f(m(x))
    map1 = forward_map(μ)
    process_measure(qs, integrand ∘ map1, mapped_domain(map1, domain), supermeasure(μ), properties...)
end

# Truncate an infinite domain to a finite one for numerical evaluation of Hermite integrals
function process_measure(qs::AdaptiveStrategy, integrand, domain::FullSpace{T}, measure::HermiteWeight{T}, properties...) where {T}
    U = sqrt(-log(eps(T)))
    (hermite_weightfun * integrand, -U..U, Lebesgue{T}(), properties...)
end

# apply the cosine map for integrals with the ChebyshevT weight, to avoid the singularities
function process_measure(qs::AdaptiveStrategy, integrand, domain::ChebyshevInterval, measure::ChebyshevTWeight{T}, properties...) where {T}
    # Transformation is: f(t) -> pi*f(cos(pi*t))
    Tpi = convert(T, pi)
    map = t -> cos(Tpi*t)
    (Tpi * (integrand ∘ map), UnitInterval{T}(), LebesgueUnit{T}(), properties...)
end

# same as above, but on a subinterval
function process_measure(qs::AdaptiveStrategy, integrand, domain::AbstractInterval, measure::ChebyshevTWeight{T}, properties...) where {T}
    Tpi = convert(T, pi)
    map = t -> cos(Tpi*t)
    a, b = extrema(domain)
    a < -1.001 && throw(BoundsError(measure, a))
    b > 1.001 && throw(BoundsError(measure, b))
    # Set a and b to be within [-1,1] in order to avoid errors with acos below
    a = max(a, -1)
    b = min(b, 1)
    (Tpi * (integrand ∘ map), acos(b)/pi..acos(a)/pi, Lebesgue{T}(), properties...)
end

# apply the cosine map for integrals with the ChebyshevU weight as well
function process_measure(qs::AdaptiveStrategy, integrand, domain::ChebyshevInterval, measure::ChebyshevUWeight{T}, properties...) where {T}
    Tpi = convert(T, pi)
    prefactor = t -> Tpi * sin(Tpi*t)^2
    map = t -> cos(Tpi*t)
    (prefactor * (integrand ∘ map), UnitInterval{T}(), LebesgueUnit{T}(), properties...)
end

function process_measure(qs::AdaptiveStrategy, integrand, domain::AbstractInterval, measure::ChebyshevUWeight{T}, properties...) where {T}
    Tpi = convert(T, pi)
    prefactor = t -> Tpi * sin(Tpi*t)^2
    map = t -> cos(Tpi*t)
    a, b = extrema(domain)
    a < -1.001 && throw(BoundsError(measure, a))
    b > 1.001 && throw(BoundsError(measure, b))
    # Set a and b to be within [-1,1] in order to avoid errors with acos below
    a = max(a, -1)
    b = min(b, 1)
    (prefactor * (integrand ∘ map), acos(b)/pi..acos(a)/pi, Lebesgue{T}(), properties...)
end

function measure_fetch_transformations(qs, domain, weight, properties...)
    T = promote_type(numtype(domain),numtype(weight))
    I = TransformationLogIntegrand{T}()
    I_trans, domain_trans, weight_trans = process_measure(qs, I, domain, weight)
    I_trans.prefactor, I_trans.fun, domain_trans, weight_trans
end

# For product domains, process the measures dimension per dimension
function process_measure(qs::AdaptiveStrategy, integrand, domain::ProductDomain, μ::ProductWeight, properties...)
    if ncomponents(domain) == ncomponents(μ)
        if ncomponents(domain) == 2
            pre1, map1, domain1, μ1 = measure_fetch_transformations(qs, component(domain, 1), component(μ, 1), properties...)
            pre2, map2, domain2, μ2 = measure_fetch_transformations(qs, component(domain, 2), component(μ, 2), properties...)
            prefactor = t -> pre1(t[1])*pre2(t[2])
            map = t -> SA[map1(t[1]), map2(t[2])]
            prefactor * (integrand ∘ map), productdomain(domain1, domain2), productmeasure(μ1, μ2)
        elseif ncomponents(domain) == 3
            pre1, map1, domain1, μ1 = measure_fetch_transformations(qs, component(domain, 1), component(μ, 1), properties...)
            pre2, map2, domain2, μ2 = measure_fetch_transformations(qs, component(domain, 2), component(μ, 2), properties...)
            pre3, map3, domain3, μ3 = measure_fetch_transformations(qs, component(domain, 3), component(μ, 3), properties...)
            prefactor = t -> pre1(t[1])*pre2(t[2])*pre3(t[3])
            map = t -> SA[map1(t[1]), map2(t[2]), map3(t[3])]
            prefactor * (integrand ∘ map), productdomain(domain1, domain2, domain3), productmeasure(μ1, μ2, μ3)
        else
            process_measure_default(qs, integrand, domain, μ, properties...)
        end
    else
        process_measure_default(qs, integrand, domain, μ, properties...)
    end
end

# The "best" rule for certain measures becomes a Gauss rule
integrate_measure(qs::BestRule, integrand, domain::ChebyshevInterval{T}, μ::LegendreWeight, properties...) where {T} =
    integrate_measure(Q_GaussLegendre(T, qs.n), integrand, domain, μ, properties...)

integrate_measure(qs::BestRule, integrand, domain::ChebyshevInterval, μ::JacobiWeight{T}, properties...) where {T} =
    integrate_measure(Q_GaussJacobi(qs.n, μ.α, μ.β), integrand, domain, LegendreWeight{T}(), properties...)

integrate_measure(qs::BestRule, integrand, domain::HalfLine, μ::LaguerreWeight{T}, properties...) where {T} =
    integrate_measure(Q_GaussLaguerre(qs.n, μ.α), integrand, domain, Lebesgue{T}(), properties...)

integrate_measure(qs::BestRule, integrand, domain::FullSpace{T}, μ::HermiteWeight{T}, properties...) where {T} =
    integrate_measure(Q_GaussHermite(qs.n), integrand, domain, Lebesgue{T}(), properties...)


using LogDensities
using Base.Test
using ForwardDiff

struct TestObj{p,T} <: parameters{T,1}
  x::Vector{T}
  prob::ProbabilityVector{p,T}
  pos::PositiveVector{p,T}
  real::RealVector{p,T}
  cov::CovarianceMatrix{p,T}
end

function inv_transform(::Type{TestObj{p,T2} where T2}, x::Vector{T}) where {T <: Real, p}
  Θ = construct(TestObj{p,T}, x)
  ConstrainedParameters.update_Σ!(Θ.cov)
  out = similar(x)
  k = 3p
  for i ∈ 1:p
    out[i] = Θ.prob[i]
    out[i+p] = Θ.pos[i]
    out[i+2p] = Θ.real[i]
    for j ∈ 1:i
      k += 1
      out[k] = Θ.cov.Σ[j,i]
    end
  end
  out
end

p = 2
m1 = Model(TestObj{p})
@testset begin
  @test type_length(typeof(m1.Θ)) == length(m1.Θ) == length(m1.Θ.x) == round(Int, p*(3 + (p+1)/2))

  x = randn(length(m1.Θ))
  copy!(m1.Θ.x, x)
  update!(m1.Θ)

  #The CovarianceMatrix log_jacobian! function drops the constant p*log(2) term.
  @test log_jacobian!(m1.Θ) + p*log(2) ≈ logabsdet(ForwardDiff.jacobian(x -> inv_transform(TestObj{2}, x), x))[1]
end


using ApproxFun, SpectralMeasures

# randn and 1/n^2 decay gives good error bounds
n = 50
a = randn(n)./(1:n).^2
b = max(.05,.5+randn(n-1)./(1:n-1).^2)
spectrum, maxerr = validatedspectrum(a,b)
maxerr

# randn and 1/n decay gives okay error bounds
n = 50
a = randn(n)./(1:n)
b = max(.05,.5+randn(n-1)./(1:n-1))
spectrum, maxerr = validatedspectrum(a,b)
maxerr

# randn with no decay gives terrible error bounds
n = 20
a = randn(n)
b = max(.05,.5+randn(n-1))
spectrum, maxerr = validatedspectrum(a,b)
maxerr


@testset "Esat_from_Tair" begin
  Tair = 15
  Esat_formula=Sonntag1990()
  constants=BigleafConstants()
  eSat = Esat_from_Tair(Tair; Esat_formula, constants)
  # regression test
  @test isapprox(eSat, 1.702, atol=1e-3)
  eSat2 = Esat_from_Tair(Tair; Esat_formula = Alduchov1996(), constants)
  eSat3 = Esat_from_Tair(Tair; Esat_formula = Allen1998(), constants)
  # different results with different methods
  @test !(eSat2 ≈ eSat)
  @test !(eSat3 ≈ eSat)
  @test !(eSat3 ≈ eSat2)
  # but very similar
  @test ≈(eSat2, eSat, rtol = 1e-2)
  @test ≈(eSat3, eSat, rtol = 1e-2)
end

@testset "Esat_from_Tair_slope" begin
    step = 0.1
    Tair = range(20.0, 22.0, step = step)
    Esat_formula=Sonntag1990()
    constants=BigleafConstants()
    Esat = Esat_from_Tair.(Tair)
    delta = Esat_from_Tair_deriv.(Tair)
    delta2 = diff(Esat)/step
    @test all(isapprox.(delta[2:end] - delta2, 0, atol=1e-3))
    Esat3, delta3 = Esat_slope(Tair[1])
    @test Esat3 ≈ Esat[1]
    @test delta3 ≈ delta[1]
end

@testset "LE_to_ET" begin
  ET0 = 200.0
  Tair = 25.0
  res = LE_to_ET(ET0, Tair)
  # regression test
  @test ≈(res, 8.19e-5, atol =1e-7)
  ET = ET_to_LE(res, Tair)
  @test ET ≈ ET0
end

@testset "LE_to_ET" begin
  G_ms,Tair,pressure = 0.005,25.0,100.0
  rmol = ms_to_mol(G_ms,Tair,pressure)
  # regression test
  @test ≈(rmol, 0.2017, atol =1e-4)
  rms = mol_to_ms(rmol, Tair, pressure)
  @test rms ≈ G_ms
end

@testset "e_to_rH" begin
  Tair = 25.0
  eSat = Esat_from_Tair(Tair)
  e = range(0,eSat, length = 5)
  vpd = e_to_VPD.(e, Tair)
  @test VPD_to_e.(vpd, Tair) ≈ e
  rH0 = VPD_to_rH.(vpd,Tair)
  @test rH_to_VPD.(rH0, Tair) ≈ vpd
  rH = e_to_rH.(e, Tair)
  @test  rH ≈ rH0
  #
  rHOversat = @test_logs (:warn,r"higher than saturation") e_to_rH(eSat + 1e-3, Tair)
  @test rHOversat == 1.0
  #
  vpd2 = @test_logs (:warn,r"between 0 and 1") rH_to_VPD(1.2, Tair)
  @test vpd2 < 0 # garbage in garbage out
end

@testset "e_to_q" begin
  pressure = 0.1 #kPa
  Tair = 25.0
  eSat = Esat_from_Tair(Tair)
  e = range(0,eSat, length = 5)
  q = e_to_q.(e, pressure)
  @test q_to_e.(q, pressure) ≈ e
  vpd = q_to_VPD.(q, Tair, pressure)
  @test VPD_to_q.(vpd, Tair, pressure) ≈ q
end

@testset "Rg_to_PPFD" begin
  Rg = 500.0
  PPFD = Rg_to_PPFD(Rg)
  @test PPFD_to_Rg(PPFD) ≈ Rg
end

@testset "kg_to_mol" begin
  mass = 10.0
  molarMass =BigleafConstants().H2Omol
  amountOfSubstance = kg_to_mol(mass, molarMass)
  @test amountOfSubstance == mass/molarMass
end

@testset "umolCO2_to_gC" begin
  CO2_flux = 50.0
  C_flux = umolCO2_to_gC(CO2_flux)
  @test gC_to_umolCO2(C_flux) ≈ CO2_flux
end


"""
    forward(hmm, obs, approx, k=3)

Compute forward probabilities for `hmm` with observations `obs` using likelihood approximation `approx` of order `k`. Returns an [`HMMApproximator`](@ref) object.
"""
function forward(hmm::T, obs::Vector, approx::Type{L}, k::Int = 3) where {T<:HMM,L<:LikelihoodApproximation}
  @unpack logP, logπ₀ = hmm

  n = length(obs)
  nk = nstates(hmm)
  r = k - 1

  # initialize likelihood approximation
  likelihood = approx(hmm)

  # iterator over subsequences of length (k-1)
  subseqs = Iterators.product([1:nk for _ in 1:r]...)

  # normalization constants and likelihoods allocation
  zi = [Vector{Float64}(undef, nk^r) for _ in 1:n-r]
  ll = [Vector{Float64}(undef, nk^k) for _ in 1:n-r]

  q   = Vector{Float64}(undef, nk)
  seq = Vector{Int}(undef, k)

  # top sequence
  @inbounds for s in subseqs
    for s1 = 1:nk
      seq = vcat(s1, s...)
      q[s1] = logπ₀[s1] + likelihood(seq[1:1], obs, 1:1) / k
      for i = 2:k
        q[s1] += logP[seq[i-1], seq[i]] + likelihood(seq[1:i], obs, 1:i) / k
      end
      ll[1][encode(seq, nk)] = q[s1]
    end
    zi[1][encode(s, nk)] = logsumexp(q)
  end

  # intermediate subsequences
  @inbounds for t = 2:(n-k)
    range = t:t+r
    for s in subseqs
      for s1 = 1:nk
        seq = vcat(s1, s...)
        q[s1] = likelihood(seq, obs, range) / k + zi[t-1][encode(seq[1:end-1], nk)]
        ll[t][encode(seq, nk)] = q[s1]
      end

      zi[t][encode(s, nk)] = logsumexp(q) + logP[s[end-1], s[end]]
    end
  end

  # last element
  @inbounds for s in subseqs
    for s1 = 1:nk
      seq = vcat(s1, s...)
      q[s1] = likelihood(seq[end:end], obs, n:n) / k

      for i = 1:r
        range = n-k+i:n
        q[s1] += likelihood(seq[i:end], obs, range) / k
      end

      q[s1] += zi[end-1][encode(seq[1:end-1], nk)]
      ll[end][encode(seq, nk)] = q[s1]
    end

    zi[end][encode(s, nk)] = logsumexp(q) + logP[s[end-1], s[end]]
  end

  # normalization constant
  z = zi[end] .- logsumexp(zi[end])

  HMMApproximator(hmm, obs, zi, ll, z, n, nk, k)
end