# Dunavant, D. A. (1985), High degree efficient symmetrical Gaussian quadrature
# rules for the triangle. Int. J. Numer. Meth. Engng., 21: 1129–1148. doi:
# 10.1002/nme.1620210612
function _get_gauss_tridata(n::Int)
    if (n == 1)
    xw=[0.33333333333333 0.33333333333333 1.00000000000000 / 2.0];
    elseif (n == 2)
    xw=[0.16666666666667 0.16666666666667 0.33333333333333 / 2.0
    0.16666666666667 0.66666666666667 0.33333333333333 / 2.0
    0.66666666666667 0.16666666666667 0.33333333333333 / 2.0];
    elseif (n == 3)
    xw=[0.33333333333333 0.33333333333333 -0.56250000000000 / 2.0
    0.20000000000000 0.20000000000000 0.52083333333333 / 2.0
    0.20000000000000 0.60000000000000 0.52083333333333 / 2.0
    0.60000000000000 0.20000000000000 0.52083333333333 / 2.0];
    elseif (n == 4)
    xw=[0.44594849091597 0.44594849091597 0.22338158967801 / 2.0
    0.44594849091597 0.10810301816807 0.22338158967801 / 2.0
    0.10810301816807 0.44594849091597 0.22338158967801 / 2.0
    0.09157621350977 0.09157621350977 0.10995174365532 / 2.0
    0.09157621350977 0.81684757298046 0.10995174365532 / 2.0
    0.81684757298046 0.09157621350977 0.10995174365532 / 2.0];
    elseif (n == 5)
    xw=[0.33333333333333 0.33333333333333 0.22500000000000 / 2.0
    0.47014206410511 0.47014206410511 0.13239415278851 / 2.0
    0.47014206410511 0.05971587178977 0.13239415278851 / 2.0
    0.05971587178977 0.47014206410511 0.13239415278851 / 2.0
    0.10128650732346 0.10128650732346 0.12593918054483 / 2.0
    0.10128650732346 0.79742698535309 0.12593918054483 / 2.0
    0.79742698535309 0.10128650732346 0.12593918054483 / 2.0];
    elseif (n == 6)
    xw=[0.24928674517091 0.24928674517091 0.11678627572638 / 2.0
    0.24928674517091 0.50142650965818 0.11678627572638 / 2.0
    0.50142650965818 0.24928674517091 0.11678627572638 / 2.0
    0.06308901449150 0.06308901449150 0.05084490637021 / 2.0
    0.06308901449150 0.87382197101700 0.05084490637021 / 2.0
    0.87382197101700 0.06308901449150 0.05084490637021 / 2.0
    0.31035245103378 0.63650249912140 0.08285107561837 / 2.0
    0.63650249912140 0.05314504984482 0.08285107561837 / 2.0
    0.05314504984482 0.31035245103378 0.08285107561837 / 2.0
    0.63650249912140 0.31035245103378 0.08285107561837 / 2.0
    0.31035245103378 0.05314504984482 0.08285107561837 / 2.0
    0.05314504984482 0.63650249912140 0.08285107561837 / 2.0];
    elseif (n == 7)
    xw=[0.33333333333333 0.33333333333333 -0.14957004446768 / 2.0
    0.26034596607904 0.26034596607904 0.17561525743321 / 2.0
    0.26034596607904 0.47930806784192 0.17561525743321 / 2.0
    0.47930806784192 0.26034596607904 0.17561525743321 / 2.0
    0.06513010290222 0.06513010290222 0.05334723560884 / 2.0
    0.06513010290222 0.86973979419557 0.05334723560884 / 2.0
    0.86973979419557 0.06513010290222 0.05334723560884 / 2.0
    0.31286549600487 0.63844418856981 0.07711376089026 / 2.0
    0.63844418856981 0.04869031542532 0.07711376089026 / 2.0
    0.04869031542532 0.31286549600487 0.07711376089026 / 2.0
    0.63844418856981 0.31286549600487 0.07711376089026 / 2.0
    0.31286549600487 0.04869031542532 0.07711376089026 / 2.0
    0.04869031542532 0.63844418856981 0.07711376089026 / 2.0];
    elseif (n == 8)
    xw=[0.33333333333333 0.33333333333333 0.14431560767779 / 2.0
    0.45929258829272 0.45929258829272 0.09509163426728 / 2.0
    0.45929258829272 0.08141482341455 0.09509163426728 / 2.0
    0.08141482341455 0.45929258829272 0.09509163426728 / 2.0
    0.17056930775176 0.17056930775176 0.10321737053472 / 2.0
    0.17056930775176 0.65886138449648 0.10321737053472 / 2.0
    0.65886138449648 0.17056930775176 0.10321737053472 / 2.0
    0.05054722831703 0.05054722831703 0.03245849762320 / 2.0
    0.05054722831703 0.89890554336594 0.03245849762320 / 2.0
    0.89890554336594 0.05054722831703 0.03245849762320 / 2.0
    0.26311282963464 0.72849239295540 0.02723031417443 / 2.0
    0.72849239295540 0.00839477740996 0.02723031417443 / 2.0
    0.00839477740996 0.26311282963464 0.02723031417443 / 2.0
    0.72849239295540 0.26311282963464 0.02723031417443 / 2.0
    0.26311282963464 0.00839477740996 0.02723031417443 / 2.0
    0.00839477740996 0.72849239295540 0.02723031417443 / 2.0];
    else
        throw(ArgumentError("unsupported order for triangle gauss-legendre integration"))
    end
    return xw
end


using CmdStan, Turing, ContinuousBenchmarks

using Mamba: describe

const binomialstanmodel = "
// Inferring a Rate
data {
  int<lower=1> n;
  int<lower=0> k;
}
parameters {
  real<lower=0,upper=1> theta;
  real<lower=0,upper=1> thetaprior;
}
model {
  // Prior Distribution for Rate Theta
  theta ~ beta(1, 1);
  thetaprior ~ beta(1, 1);
  // Observed Counts
  k ~ binomial(n, theta);
}
generated quantities {
  int<lower=0> postpredk;
  int<lower=0> priorpredk;
  postpredk <- binomial_rng(n, theta);
  priorpredk <- binomial_rng(n, thetaprior);
}
"

#global stanmodel, rc, sim
#stanmodel = Stanmodel(Sample(algorithm=CmdStan.Hmc(CmdStan.Static(0.75 * 5), CmdStan.diag_e(), 0.75, 0.0),
#  save_warmup=true, adapt=CmdStan.Adapt(engaged=false)),
#  num_samples=2000, num_warmup=0, thin=1,
#  name="binomial", model=binomialstanmodel, nchains=1);

const binomialdata = [
  Dict("n" => 10, "k" => 5)
]

#rc, sim = stan(stanmodel, binomialdata, CmdStanDir=ContinuousBenchmarks.CMDSTAN_HOME, summary=false)

#describe(sim)

@model binomial_turing(n, k) = begin
  theta ~ Beta(1, 1)
  thetaprior ~ Beta(1, 1)
  k ~ Binomial(n, theta)
end

chn = sample(binomial_turing(binomialdata[1]["n"], binomialdata[1]["k"]), HMC(2000, 0.75, 5))

descibe(chn)


using Plots, Printf
pyplot()
viz = true

@views function heat_1D()
	# physics
	lx  = 10.0
	λ   = 1.0
	ρCp = 1.0
	nt  = 200
	# numerics
	nx  = 127
	∂x  = lx/nx
	xc  = LinRange(∂x/2, lx-∂x/2, nx)
	🔥  = zeros(nx  )
	qx  = zeros(nx+1)
	🔥  = exp.(.-(xc.-lx./2.0).^2)
	∂t  = ∂x^2/ρCp/λ/2.1
	# action
	t0  = Base.time()
	for it = 1:nt
		qx[2:end-1] .= .-λ.*diff(🔥)./∂x
		🔥          .= 🔥 .- ∂t./ρCp.*diff(qx)./∂x
	end
	@printf("T_eff = %1.2e GB/s \n", (2/1e9*nx*sizeof(lx))/((Base.time()-t0)/nt))
	if viz display(plot(xc, 🔥, legend=false, xlabel="lx", ylabel="heat", title="diffusion")) end
	return
end

@time heat_1D()


function gaussian_pdf(n, μ, σ)
    x = collect(1:1<<n)
    pl = @. 1 / sqrt(2pi * σ^2) * exp(-(x - μ)^2 / (2 * σ^2))
    pl / sum(pl)
end


#__precompile__()
module KoalaTrees

export TreeRegressor

# development only:
# import Koala: @dbg, @colon
# export Node, is_stump, is_leaf, has_left, has_right, make_leftchild!, make_rightchild!
# export is_left, is_right
# export unite!, child

# needed for this module:
import Koala
import Koala: Regressor, SupervisedMachine, softwarn, clean!
import Koala: params, keys_ordered_by_values
import DataTableaux
import DataTableaux:  DataTableau, FrameToTableauScheme, IntegerSet
import StatsBase: sample
import DataFrames: AbstractDataFrame
import UnicodePlots
import KoalaTransforms

# to be extended:
import Base: show, showall

# to be extended (but not explicitly rexported):
import Koala: default_transformer_X, default_transformer_y, fit, transform, inverse_transform
import Koala: setup, predict

# constants:
const Small = UInt8

## HELPERS:

"""
# `function mean_and_ss_after_add(mean, ss, n, x)`

Returns the mean, and the sum-of-square deviations from the mean, of
`n+1` numbers, given the corresponding two quantities for the first
`n` numbers (the inputs `mean` and `ss`) and the value of the `n+1`th
number, `x`.
"""
function mean_and_ss_after_add(mean, ss, n, x)
    n >= 0 || throw(DomainError)
    mean_new = (n*mean + x)/(n + 1)
    ss_new = ss + (x - mean_new)*(x - mean)
    return mean_new, ss_new
end

"""
# `function mean_and_ss_after_omit(mean, ss, n, x)`

Given `n` numbers, their mean `mean` and sum-of-square deviations from
the mean `ss`, this function returns the new mean and corresponding
sum-of-square deviations of the same numbers when one of the numbers,
`x` is omitted from the list.

"""
function mean_and_ss_after_omit(mean, ss, n, x)
    n > 1 || throw(DomainError)
    mean_new = (n*mean - x)/(n-1)
    ss_new = ss - (x - mean)*(x - mean_new)
    return mean_new, ss_new
end

"""
    histogram(bag, x, log2_nbins)

Assumes bag is an `AbstractVector` of integers such that `x[i]` is
`Real` for all `i` in `bag`, and sorts `bag` into `n` subvectors
`bin[1], bin[2], ..., bin[n]` based on the values of `x`, using a
uniform subdivision of the interval `(minimum(x[bag]),
maximum(x[bag])` (whose boundaries are placed in a vector called
`boundaries` below). Here `n=2^log2_nbins`. The bins are
"left-closed", except the last which is closed at both ends.

### Return value

The normal return value is

`(bins, boundaries)`

where `bins = [bin[k] for k in 1:n]`. However, in the special case
that `x[bag]` takes on a *single* value `x0`, `bins` is an unassigned
vector of length one, and boundaries is replaced with `[x0]`.

, [x0]`.

"""
function histogram(bag::AbstractVector{Int},
                   x::AbstractVector{T} where T<:Real,
                   log2_nbins)
    nbins = 2^log2_nbins
    nbins > 1 || throw(DomainError)
    boundaries = linspace(minimum(x[bag]), maximum(x[bag]), nbins + 1) |> collect

    if boundaries[1] == boundaries[end]
        nbins = 1
    end

    # create bins
    bin = Vector{Vector{Int}}(nbins)

    # return unassigned bins if x takes constant value:
    if nbins == 1
        return bin, boundaries[1:1]
    end

    # initialize bins as empty:
    for k in 1:nbins
        bin[k] = Int[]
    end

    # For each `i` in `bag` we add `i` to vector `bin[k]` if `x[i]`
    # is between `boundaries[k]` and `boundaries[k+1]` (left included,
    # right excluded unless `k=n_bins`). We sort using binary
    # search. In particular, the bin number `k` is represented as a
    # sum powers of 2 in the inner loop below.

    for i in bag
        k = 1 # initialize bin number
        for power in reverse(0:(log2_nbins - 1))
            δk = 2^power
            if x[i] >= boundaries[k + δk]
                k += δk
            end
        end
        push!(bin[k], i)
    end

    return bin, boundaries

end

## `Node` type - data structure for building binary trees

# Binary trees are identified with their top (stump) nodes, so only a
# `Node` type, with the appropriate possiblities for connection, is
# defined. Connections are established with the methods
# `make_leftchild!` and `make_rightchild!`. Nodes have a `depth`
# field; when a new connection is made, the child's depth is declared
# to be one more than the parent. When a node `N` is created it is
# initially its own parent (equivalently, `is_stump(N) = true`) and is
# its own left and right child; its depth is initially zero.

mutable struct Node{T}
    parent::Node{T}
    left::Node{T}
    right::Node{T}
    data::T
    depth::Int
    function Node{T}(datum) where T
        node = new{T}()
        node.parent = node
        node.left = node
        node.right = node
        node.data = datum
        node.depth = 0
        return node
    end
end

Node(data::T) where T = Node{T}(data)

# Testing connectivity:

is_stump(node) = node.parent == node
is_left(node) =  (node.parent != node) && (node.parent.left == node)
is_right(node) = (node.parent != node) && (node.parent.right == node)
has_left(node) =  (node.left  != node)
has_right(node) = (node.right != node)
is_leaf(node) = node.left == node && node.right == node

# Connecting nodes:

"""
`make_leftchild!(child, parent)` makes `child` the left child of
`parent` and returns the depth of `child`

"""
function make_leftchild!(child, parent)
    parent.left = child
    child.parent = parent
    child.depth = parent.depth + 1
end

"""
`make_rightchild!(child, parent)` makes `child` the right child of
`parent` and returns the depth of `child`

"""
function make_rightchild!(child, parent)
    parent.right = child
    child.parent = parent
    child.depth = parent.depth + 1
end

# Locating children

"""
## `child(parent, gender)`

Returns the left child of `parent` of a `Node` object if `gender` is 1
and right child if `gender is 2. If `gender` is `0` the routine throws
an error if the left and right children are different and otherwise
returns their common value.  For all other values of gender an error
is thrown.

"""
function child(parent, gender)
    if gender == 1
        return parent.left
    elseif gender == 2
        return parent.right
    elseif gender == 0
        if parent.left != parent.right
            throw(Base.error("Left and right children different."))
        else
            return parent.left
        end
    end
    throw(Base.error("Only genders 0, 1 or 2 allowed."))
end

"""
# `unite!(child, parent, gender)`

Makes `child` the `left` or `right` child of a `Node` object `parent`
in case `gender` is `1` or `2` respectively; and makes `parent` the
parent of `child`. For any other values of `gender` the routine makes
`child` simultaneously the left and right child of `parent`, and
`parent` the parent of `child`. Returns `nothing`.

"""
function unite!(child, parent, gender)
    if gender == 1
        make_leftchild!(child, parent)
    elseif gender == 2
        make_rightchild!(child, parent)
    else
        make_leftchild!(child, parent)
        make_rightchild!(child, parent)
    end
end

# Display functionality:

function spaces(n)
    s = ""
    for i in 1:n
        s = string(s, " ")
    end
    return s
end

function get_gender(node)
    if is_stump(node)
        return 'A' # androgenous
    elseif is_left(node)
        return 'L'
    else
        return 'R'
    end
end

tail(n) = "..."*string(n)[end-3:end]

function Base.show(stream::IO, node::Node)
    print(stream, "Node{$(typeof(node).parameters[1])}@$(tail(hash(node)))")
end

function Base.showall(stream::IO, node::Node)
    gap = spaces(node.depth + 1)
    println(stream, string(get_gender(node), gap, node.data))
    if has_left(node)
        showall(stream, node.left)
    end
    if has_right(node)
        showall(stream, node.right)
    end
    return
end

showall(node::Node)=showall(STDOUT, node)

# for testing purposes:
# Node(data) = Node{typeof(data)}(data)

function Node(data, parent::Node)
    child = Node(data)
    make_leftchild!(child, parent)
    return child
end

function Node(parent::Node, data)
    child = Node(data)
    make_rightchild!(child, parent)
    return child
end

## `TreeRegressor` - regression decision tree with nearest neighbor regularization

immutable NodeData
    feature::Int
    kind::Int8   # 0: root, 1: ordinal, 2: categorical, 3: leaf
    r::Float64   # A *threshold*, *float repr. of integer subset*, or
                 # *prediction*, according to kind above
end

const RegressorNode = Node{NodeData}

function should_split_left(pattern, node::RegressorNode)
    j, r, kind = node.data.feature, node.data.r, node.data.kind
    if kind == 1     # ordinal
        return pattern[j] <= r
    elseif kind == 2 # categorical
        small = round(Small, pattern[j])
        return small in round(IntegerSet, r)
    else
        throw(Base.error("Expecting an ordinal or categorical node here."))
    end
end

mutable struct TreeRegressor <: Regressor{RegressorNode}
    max_features::Int
    min_patterns_split::Int
    penalty::Float64
    extreme::Bool
    regularization::Float64
    max_height::Int
    max_bin::Int
    bin_factor::Int
end

function clean!(model::TreeRegressor)
    messg = ""
    if model.min_patterns_split <= 1
        model.min_patterns_split = 2
        messg = messg * "`min_patterns_split` must be at least 2. Resetting to 2. "
    end
    if model.regularization >= 1.0
        model.regularization = 0.0
        messg = messg * "`regularization` must be less than 1. Resetting to 0. "
    end
    if model.max_bin == 1
        model.max_bin = 2
        messg = messg * "`max_bin` must be 0 (exact splits) " *
              "or more than 1. Resetting to 2. "
    end
    return messg
end

function TreeRegressor(;max_features=0, min_patterns_split=2,
                       penalty=0.0, extreme=false,
                       regularization=0.0, max_height=1000, max_bin=0, bin_factor=90)
    model = TreeRegressor(max_features, min_patterns_split,
                          penalty,extreme,
                          regularization, max_height, max_bin, bin_factor)
    softwarn(clean!(model)) # only issues warning if `clean!` changes `model`
    return model
end

function feature_importance_curve(popularity_given_feature, names)

    importance_given_name = Dict{Symbol, Float64}()
    kys = keys(popularity_given_feature)
    N = sum(popularity_given_feature[j] for j in kys)
    for j in kys
        importance_given_name[names[j]] = popularity_given_feature[j]/N
    end
    x = Symbol[]
    y = Float64[]
    for name in reverse(keys_ordered_by_values(importance_given_name))
        push!(x, name)
        push!(y, round(Int, 1000*importance_given_name[name])/1000)
    end
    return x, y
end

# transformer for learning algorithms requiring DataTableau inputs:
struct TreeTransformer_X <: Koala.Transformer end
fit(transformer::TreeTransformer_X, X::AbstractDataFrame, parallel, verbosity) =
    FrameToTableauScheme(X)
function transform(transformer::TreeTransformer_X, scheme, X)
    features = scheme.encoding.names
    issubset(Set(features), Set(names(X))) ||
        error("Attempting to transform DataFrame with incompatible feature labels.")
    return DataTableaux.transform(scheme, X[features])
end

default_transformer_X(model::TreeRegressor) = TreeTransformer_X()
default_transformer_y(model::TreeRegressor) =
    KoalaTransforms.RegressionTargetTransformer(standardize=false)

#####################################################################
# For readability we use `X` and `y` for `Xt` and `yt` from now on. #
#####################################################################

mutable struct Cache

    # independent of model parameters:
    X::DataTableau
    y::Vector{Float64}
    n_patterns::Int
    n_features::Int

    # dependent on model parameters:
    max_features::Int
    log2_max_bin::Int
    max_bin::Int # true value to be used (a power of 2)
    popularity::Vector{Float64} # representing popularity of each feature

    Cache(X, y, n_patterns, n_features) = new(X, y, n_patterns, n_features)
end

function setup(rgs::TreeRegressor, X, y, scheme_X, parallel, verbosity)

    n_features = size(X, 2) # = length(scheme_X.encoding.names)
    n_patterns = length(y)

    return Cache(X, y, n_patterns, n_features)

end

"""
Following function is for splitting on feature with index `j` when
the feature is *categorical*.

Returns `(gain, left_values)` where gain is the lowering of the error
(sum of square deviations) obtained for the best split based on sample
inputs `cache.X[j]` with corresponding target values `cache.y`;
`left_values` is a floating point representation of the set of values
of x for splitting left in the optimal split. If the feature is
constant (takes on a single value) then no split can improve the error
and `gain=0`.

Note that only inputs with indices in the iterator `bag` are
considered.

"""
function split_on_categorical(rgs, j, bag, no_split_error, cache)

    if rgs.extreme

        vals2 = unique(Small[round(Small, cache.X.raw[i,j]) for i in bag])
        n_vals = length(vals2)
        if n_vals == 1
            return 0.0, 0.0 # cache.X[j] is constant in this bag, so no gain
        end
        n_select = rand(1:(n_vals - 1))
        left_selection = sample(vals2, n_select; replace=false)
        left_values_encoded = IntegerSet(left_selection)
        left_count = 0
        right_count = 0
        left_mean = 0.0
        right_mean = 0.0
        for i in bag
            v = round(Small, cache.X.raw[i,j])
            if v in left_selection
                left_count += 1
                left_mean += cache.y[i]
            else
                right_count += 1
                right_mean += cache.y[i]
            end
        end
        left_mean = left_mean/left_count
        right_mean = right_mean/right_count

        # Calcluate the split error, denoted `err` (the sum of squares of
        # deviations of target values from the relevant mean - namely
        # left or right
        err = 0.0
        for i in bag
            v = round(Small, cache.X.raw[i,j])
            if v in left_selection
                err += (left_mean - cache.y[i])^2
            else
                err += (right_mean - cache.y[i])^2
            end
        end
        gain = no_split_error - err
        left_values = Float64(IntegerSet(left_selection))
        return gain, left_values

    end

    # Non-extreme case:

    # 1. Determine a Vector{Small} of values taken by `cache.X[j]`,
    # called `values` and order it according to the mean values of
    # cache.y:

    count = Dict{Small,Int}()   # counts of `cache.X[j]` values, keyed on
    # values of `cache.X[j]`.
    mu = Dict{Small,Float64}() # mean values of target `cache.y` keyed on
    # values of `cache.X[j]`, initially just the
    # unnormalized sum of `cache.y` values.
    for i in bag
        value = round(Small, cache.X.raw[i,j])
        if !haskey(count, value)
            count[value] = 1
            mu[value] = cache.y[i]
        else
            count[value] += 1
            mu[value] += cache.y[i]
        end
    end
    if length(count) == 1          # feature is constant for data in bag
        return 0.0, 0.0            # so no point in splitting;
        # second return value is irrelevant
    end

    # normalize sums to get the means:
    for v in keys(count)
        mu[v] = mu[v]/count[v]
    end
    vals = keys_ordered_by_values(mu) # vals is a Vector{Small}

    # 2. Let the "kth split" correspond to the left-values vals[1],
    # vals[2], ... vals[k] (so that the last split is no
    # split).

    n_vals = length(vals)

    # do first split outside loop handling others:
    left_count = 0
    right_count = 0
    left_mean = 0.0
    right_mean = 0.0
    for i in bag
        v = round(Small, cache.X.raw[i,j])
        if v == vals[1]
            left_count += 1
            left_mean += cache.y[i]
        else
            right_count += 1
            right_mean += cache.y[i]
        end
    end
    left_mean = left_mean/left_count
    right_mean = right_mean/right_count
    left_ss = 0.0
    right_ss = 0.0
    for i in bag
        v = round(Small, cache.X.raw[i,j])
        if v == vals[1]
            left_ss += (left_mean - cache.y[i])^2
        else
            right_ss += (right_mean - cache.y[i])^2
        end
    end

    error = left_ss + right_ss
    position = 1 # updated if error is improved in loop below

    for k in 2:(n_vals - 1)

        # Update the means and sum-of-square deviations:
        for i in bag
            if round(Small, cache.X.raw[i,j]) == vals[k]
                left_mean, left_ss = mean_and_ss_after_add(left_mean, left_ss, left_count, cache.y[i])
                left_count += 1
                right_mean, right_ss = mean_and_ss_after_omit(right_mean, right_ss, right_count, cache.y[i])
                right_count += -1
            end
        end

        # Calcluate the kth split error:
        err = left_ss + right_ss

        if err < error
            error = err
            position = k
        end
    end

    gain = no_split_error - error

    # println("pos error ",position," ", err)
    left_values = Float64(IntegerSet(vals[1:position]))
    return gain, left_values

end # of function `split_on_categorical`

"""
For splitting on feature with index `j` when this is an *ordinal*
feature.

Returns (gain, threshold) where `gain` is the lowering of the error
(sum of square deviations) obtained for the best split based on sample
inputs `cache.X[j]` with corresponding target values 'cache.y`;
`threshold` is the maximum value the feature for a left split in the
best case. If the feature is constant (takes on a single value) then
no split can improve the error and `gain = 0.0`.

Note that only inputs with indices in the iterator `bag` are
considered.

"""
function split_on_ordinal(rgs, j, bag, no_split_error, cache)

    if rgs.extreme
        # determine a Vector{Float64} of values taken by cache.X[j], called `vals` below:
        vals = unique(cache.X.raw[bag,j])
        min_val = minimum(vals)
        max_val = maximum(vals)
        if min_val == max_val      # feature is constant for data in bag
            return 0.0, 0.0        # so no point in splitting; second
            # value irrelevant
        end
        threshold = min_val + rand()*(max_val - min_val)

        # Calculate the left and right mean values of target
        left_count = 0
        right_count = 0
        left_mean = 0.0
        right_mean = 0.0
        for i in bag
            v = cache.X.raw[i,j]
            if v <= threshold
                left_count += 1
                left_mean += cache.y[i]
            else
                right_count += 1
                right_mean += cache.y[i]
            end
        end
        left_mean = left_mean/left_count
        right_mean = right_mean/right_count

        # Calcluate the split error, denoted `err` (the sum of
        # squares of deviations of target values from the relevant
        # mean - namely left or right)
        err = 0.0
        for i in bag
            v = cache.X.raw[i,j]
            if v <= threshold
                err += (left_mean - cache.y[i])^2
            else
                err += (right_mean - cache.y[i])^2
            end
        end
        gain = no_split_error - err
        return  gain, threshold
    end

    # Non-extreme case:

    nbins = cache.max_bin

    if nbins > 0 && length(bag) > rgs.bin_factor*nbins

        # Use histogram splitting.

        # calcluate the histogram:

        bins, boundaries = histogram(bag, view(cache.X.raw, :, j), cache.log2_max_bin)

        # deal with special case that feature takes on constant value:
        if length(boundaries) == 1
            return 0.0, 0.0 # no point in splitting
        end

        # We consider splits at each boundary point except the first
        # and last. We do the first split by hand and use recursive
        # formulas to update means and sum-of-square deviations for
        # the others (for speed enhancement)

        # mean and ss (sum-of-square deviations) for first split (k = 1):
        left_targets = cache.y[bins[1]]
        left_mean = mean(left_targets)
        left_count = length(left_targets)
        left_ss = sum((η - left_mean)^2 for η in left_targets)
        right_targets = cache.y[vcat((bins[k] for k in 2:nbins)...)]
        right_mean = mean(right_targets)
        right_count = length(right_targets)
        right_ss = sum((η - right_mean)^2 for η in right_targets)

        # error for first split:
        error = left_ss + right_ss
        best_k = 1 # to be updated if better split found

        for k in 2:(nbins - 1)

            # update the means and ss values recursively:
            for i in bins[k]
                left_mean, left_ss =
                    mean_and_ss_after_add(left_mean, left_ss, left_count, cache.y[i])
                left_count += 1
                right_mean, right_ss =
                    mean_and_ss_after_omit(right_mean, right_ss, right_count, cache.y[i])
                right_count += -1
            end

            # Calcluate the kth split error:
            err = left_ss + right_ss

            # Note value and split position if there is improvement
            if err < error
                error = err
                best_k = k
            end
        end

        gain = no_split_error - error
        threshold = boundaries[best_k + 1]

        return gain, threshold

    else

        # Use exact splitting.

        # 1. Determine a Vector{Float64} of values taken by cache.X[j],
        # called `vals` below:
        vals = unique(cache.X.raw[bag,j])
        n_vals = length(vals)
        if n_vals == 1             # feature is constant for data in bag
            return 0.0, 0.0        # so no point in splitting; second
                                   # value irrelevant
        end
        sort!(vals)

        # 2. Let the "jth split" correspond to threshold = vals[j], (so
        # that the last split is no split). The error for the jth split
        # will be error[j]. We calculate these errors now:

        # println("len = $(length(vals))")

        # we do the first split outside of the loop that considering the
        # others because we will use recursive formulas to update
        # means and sum-of-square deviations (for speed enhancement)

        # mean and ss (sum-of-square deviations) for first split:
        left_mean = 0.0
        left_count = 0
        right_mean = 0.0
        right_count = 0
        for i in bag
            if cache.X.raw[i,j] == vals[1]
                left_mean += cache.y[i]
                left_count += 1
            else
                right_mean += cache.y[i]
                right_count += 1
            end
        end
        left_mean = left_mean/left_count
        right_mean = right_mean/right_count
        left_ss = 0.0
        right_ss = 0.0
        for i in bag
            if cache.X.raw[i,j] == vals[1]
                left_ss += (cache.y[i] - left_mean)^2
            else
                right_ss += (cache.y[i] - right_mean)^2
            end
        end

        # error for first split:
        error = left_ss + right_ss
        position = 1 # its position, to be updated if better split found

        for k in 2:(n_vals - 1)

            # Update the means and sum-of-square deviations:
            for i in bag # Todo: really need to go through entire bag every time?
                x = cache.X.raw[i,j]
                if x == vals[k] # (ie, x > vals[k-1]) && (x < vals[k])
                    left_mean, left_ss =
                        mean_and_ss_after_add(left_mean, left_ss, left_count, cache.y[i])
                    left_count += 1
                    right_mean, right_ss =
                        mean_and_ss_after_omit(right_mean, right_ss, right_count, cache.y[i])
                    right_count += -1
                end
            end

            # Calcluate the kth split error:
            err = left_ss + right_ss

            # Note value and split position if there is improvement
            if err < error
                error = err
                position = k
            end
        end

        gain = no_split_error - error

        threshold = 0.5*(vals[position] + vals[position + 1])
        return gain, threshold

    end

end # of function `split_on_ordinal`

"""
The following returns `split_failed`, `bag_left`, `bag_right` as
follows:

Computes the error for the best split on each feature within a random
sample of features of size `max_features` (set in `TreeRegressor()`)
and, if splitting improves the error (postive `gain`), create a new
splitting node of appropriate type and connect to `parent`. In
evaluating what is the *best* feature, those features not previously
selected have their gains penalized by muliplying by `rgs.penalty`.

`split_failed` is true if no new node is created.

`bag_left` and `bag_right` are splits of `bag` based on the optimum
splitting found, or are empty if `split_failed`.

Note that splitting is based only on patterns with indices in `bag`.

"""
function attempt_split(rgs, bag, F, parent, gender, no_split_error, cache)

    n_features, max_features = cache.n_features, cache.max_features

    if max_features == n_features
        feature_sample_indices = collect(1:n_features)
    else
        feature_sample_indices = sample(1:n_features, max_features; replace=false)
    end

    max_gain = -Inf # max gain so far
    opt_index = 0 # index of `feature_sample_indices`
    # delivering feature index for max_gain
    # (mythical at present)
    opt_crit = 0.0       # criterion delivering that max_gain

    # println("sample indices = $feature_sample_indices")

    for i in 1:max_features
        j = feature_sample_indices[i]

        if cache.X.encoding.is_ordinal[j]
            gain, crit = split_on_ordinal(rgs, j, bag, no_split_error, cache)
        else
            gain, crit = split_on_categorical(rgs, j, bag, no_split_error, cache)
        end

        if !(j in F) && rgs.penalty != 0.0
            gain = (1 - rgs.penalty)*gain
        end

        if gain > max_gain
            max_gain = gain
            opt_crit = crit
            opt_index = i
        end
    end

    # If no gain, return to calling function with `split_failed=true`:
    if max_gain == 0.0
        yvals = [cache.y[i] for i in bag]
        return true, Int[], Int[]
    end

    # Otherwise, create a new node with a splitting criterion based on
    # the optimal feature and unite with `parent`:
    j = feature_sample_indices[opt_index] # feature with minimum error

    if cache.X.encoding.is_ordinal[j]
        data = NodeData(j, 1, opt_crit)
    else
        data = NodeData(j, 2, opt_crit)
    end
    baby = RegressorNode(data) # new decision node
    unite!(baby, parent, gender)

    # Update the set of unpenalised features and the feature popularity vector:
    push!(F,j)
    cache.popularity[j] += length(bag)

    # Split `bag` accordingly:
    bag_left  = Int[]
    bag_right = Int[]
    for i in bag
        if should_split_left(cache.X[i,:], baby)
            push!(bag_left, i)
        else
            push!(bag_right, i)
        end
    end

    # Return the bag splits with decleration `split_failed=false`:
    return false, bag_left, bag_right

end # of function `attempt_split`

"""
Recursive function to grow the decision tree. Has no return value
but generally alters 2nd and 3rd arguments.

"""
function grow(rgs,
              bag,                   # Patterns to be considerd for splitting
              F,                     # set of penalty feature indices
              parent::RegressorNode, # Node to which any child will be
                                     # connected in successful call to
                                     # `attempt_split` above.
              gender,                # Determines how any child will
                                     # be connected to parent (as
                                     # left, right, or if this is the
                                     # first call to grow,
                                     # androgynous)
              cache)

    n_patterns = length(bag)

    # Compute mean of targets in bag:
    target_mean = 0.0
    for i in bag
        target_mean += cache.y[i]
    end
    target_mean = target_mean/n_patterns

    # Do not split node if insufficient samples, but create and
    # connect leaf node with above target_mean as prediction:
    if n_patterns < rgs.min_patterns_split
        leaf = RegressorNode(NodeData(0, 3, target_mean))
        unite!(leaf, parent, gender)
        return nothing
    end

    # Find sum of square deviations for targets in bag:
    no_split_error = 0.0
    for i in bag
        no_split_error += (cache.y[i] - target_mean)^2
    end

    # If the following is succesful, it creates a child and connects
    # it to `parent`.  In that case it also returns new bags according
    # to the optimal splitting criterion computed there (empty bags
    # otherwise) and updates `F` and `popularity` fields of `cache`:
    split_failed, bag_left, bag_right = attempt_split(rgs, bag, F, parent, gender,
                                                      no_split_error, cache)

    # If split makes no (significant) difference, then create and
    # connect prediction node:
    if split_failed
        leaf = RegressorNode(NodeData(0, 3, target_mean))
        unite!(leaf, parent, gender)
        # println("Leaf born, n_samples = $n_patterns, target prediction = $target_mean")
        return nothing
    end

    # Otherwise continue growing branches left and right:
    baby = child(parent, gender)
    F_left =  copy(F)
    F_right = copy(F)
    grow(rgs, bag_left, F_left, baby, 1, cache)   # grow a left branch
    grow(rgs, bag_right, F_right, baby, 2, cache) # grow a right branch
    # F = union(F_left, F_right)

    return nothing
end

function fit(rgs::TreeRegressor, cache, add, parallel, verbosity)

    # Note: add, parallel are ignored in this method

    cache.max_features = (rgs.max_features == 0 ? cache.n_features : rgs.max_features)
    cache.log2_max_bin = rgs.max_bin != 0 ? round(Int, floor(log2(rgs.max_bin))) : 0
    cache.max_bin = rgs.max_bin !=0 ? 2^cache.log2_max_bin : 0
    cache.popularity = zeros(Float64, cache.n_features)

    # Create a root node to get started. Its unique child is the true
    # stump of the decision tree:
    root = RegressorNode(NodeData(0, 0, 0.0))

    F = Set{Int}() # set of feature indices not penalized (initially empty)

    grow(rgs, 1:cache.n_patterns, F, root, 0, cache) # 0 means `root` is to
                                              # be androgynous,
                                              # meaning `root.left =
                                              # root.right`.

    report = Dict{ Symbol, Tuple{Vector{Symbol},Vector{Float64}} }()
    popularity_given_feature = Dict{Int,Int}()
    for j in 1:cache.n_features
        pop = cache.popularity[j]
        if pop != 0.0
            popularity_given_feature[j] = pop
        end
    end
    report[:feature_importance_curve] =
            feature_importance_curve(popularity_given_feature, cache.X.encoding.names)
    predictor = root.left

    return predictor, report, cache

end

function predict(node::RegressorNode, pattern::Vector)
    while !is_leaf(node)
        if should_split_left(pattern, node)
            node = node.left
        else
            node = node.right
        end
    end
    return node.data.r
end

function predict(rgs::TreeRegressor, predictor, X::DataTableau, parallel, verbosity)

    # Note: `parallel` and `verbosity` currently ignored

    if rgs.regularization == 0.0
        return [predict(predictor, X[i,:]) for i in 1:size(X,1)]
    else
        tree = predictor
        lambda = rgs.regularization

        ret = Array{Float64}(size(X,1))
        k = 1 # counter for index of `ret` (different from bag index)
        for i in 1:size(X,1)

            pattern = X[i,:]

            # Pass pattern down tree from top (stump), recording branching on way
            branchings = Char[] # ['l','r','r', etc]
            node = tree
            while !is_leaf(node)
                if should_split_left(pattern, node)
                    push!(branchings, 'l')
                    node = node.left
                else
                    push!(branchings, 'r')
                    node = node.right
                end
            end
            depth = length(branchings)

            # Passing (partway) back up the tree, collect and sum predictions of
            # nearby leaves each weighted by "distance" away:
            prediction = node.data.r
            height = min(depth, rgs.max_height)
            for h in 1:height
                node = node.parent
                if branchings[depth + 1 - h] == 'l'
                    node = node.right
                else
                    node = node.left
                end
                prediction += (lambda^h)*predict(node, pattern)
                node = node.parent
            end

            # normalize the summed prediction:
            ret[k] = prediction*(1-lambda)/(1-lambda^(height+1))
            k += 1

        end
        return ret

    end

end

## Displaying machines with `TreeRegressor` models

function Base.showall(stream::IO, mach::SupervisedMachine{RegressorNode , TreeRegressor})
    if isdefined(mach,:report) && :feature_importance_curve in keys(mach.report)
        features, importance = mach.report[:feature_importance_curve]
        plt = UnicodePlots.barplot(features, importance,
              title="Feature importance at penalty=$(mach.model.penalty)")
    end
    dict = params(mach)
    dict[:Xt] = string(typeof(mach.Xt), " of shape ", size(mach.Xt))
    dict[:yt] = string(typeof(mach.yt), " of shape ", size(mach.yt))
    delete!(dict, :cache)
    report_items = sort(collect(keys(dict[:report])))
    dict[:report] = "Dict with keys: $report_items"
    showall(stream, mach, dict=dict)
    println(stream, "\nModel detail:")
    showall(stream, mach.model)
    if isdefined(mach,:report) && :feature_importance_curve in keys(mach.report)
        show(stream, plt)
    end
end

end # module


### A Pluto.jl notebook ###
# v0.14.6

using Markdown
using InteractiveUtils

# ╔═╡ cabdb8f7-3d78-422f-bc78-b87fd22eb1d8
begin
    import Pkg
    Pkg.activate(mktempdir())
    Pkg.add([
        Pkg.PackageSpec(name="Plots", version="1"),
        Pkg.PackageSpec(name="PlutoUI", version="0.7"),
    ])
    using Plots, PlutoUI

	Plots.gr(fmt=:png, lw=2)

	PlutoUI.TableOfContents()
end

# ╔═╡ 73adcc40-bdcc-11eb-06eb-970be2047c7c
md"""
# Fig 2.04 Exponential decay
"""

# ╔═╡ ad29e65b-99e2-4fe9-af31-4c29c97aed9b
plot([t-> 3 * exp(-t) t->3 * exp(-2t) t-> 3 * exp(-3t)], 0.0, 5.0, xlim = (0, 5), ylim=(0, 3.2), xlabel="Time", ylabel="Concentration", label = ["exp(-t)" "exp(-2t)" "exp(-3t)"], title= "Figure 2.4")

# ╔═╡ 0f5b8cf2-9b58-40d2-8db1-f61baaf353fe
md"""
## Appendix
Running environment and some auxillary functions.

**You can use [this helper tool](https://fonsp.com/article-test-3/pkghelper.html) to generate these commands!**
"""

# ╔═╡ fcfca624-1ff7-4a60-bd17-d7d32e46de40
begin
	# Convenience functions
    hill(x, k) = x / (x + k)
    hill(x, k, n) = hill(x^n, k^n)
    exprel(x) = ifelse(x≈zero(x), one(x), x / expm1(x))
end

# ╔═╡ Cell order:
# ╠═73adcc40-bdcc-11eb-06eb-970be2047c7c
# ╠═ad29e65b-99e2-4fe9-af31-4c29c97aed9b
# ╠═0f5b8cf2-9b58-40d2-8db1-f61baaf353fe
# ╠═fcfca624-1ff7-4a60-bd17-d7d32e46de40
# ╠═cabdb8f7-3d78-422f-bc78-b87fd22eb1d8


struct MyPoly
    c
    var
end

p = MyPoly([3,-5,2], 'x')

struct MyPoly
    c
    var
    MyPoly(c, var='x') = new(c, var)
end

p.c

p.var

resize!(p.c, 4)
p.c[1:4] = [-1,3,-5,2]
p

p = MyPoly([-1,3,-5,2], p.var)

function times_x(p)
   return MyPoly(vcat(p.c,0), p.var)
end

times_x(p)

function degree(p::MyPoly)
    ix1 = findfirst(p.c .!= 0)
    if ix1 == nothing
        return 0
    else
        return length(p.c) - ix1
    end
end

println(degree(MyPoly([0,0,0,0,0])))
println(degree(MyPoly([0,0,0,0,1])))
println(degree(MyPoly([0,0,0,1,0])))
println(degree(MyPoly([1,0,0,0,0])))

function degree(p::Int)
    println("degree function called with Int argument")
end

function degree(p)
    println("degree function called with any other argument")
end

degree(1.234)
degree([1,2])
degree(-123)
degree(MyPoly([1,2,3]))

function Base.show(io::IO, p::MyPoly)
    d = degree(p)
    print(io, "MyPoly(")
    for k = d:-1:0
        coeff = p.c[end-k]
        if coeff == 0 && d > 0
            continue
        end
        if k < d
            if isa(coeff, Real)
                if coeff > 0
                    print(io, " + ")
                else
                    print(io, " - ")
                end
                coeff = abs(coeff)
            else
                print(io, " + ")
            end
        end
        if isa(coeff, Real)
            print(io, coeff)
        else
            print(io, "($coeff)")
        end
        if k == 0
            continue
        end
        print(io, "⋅", p.var)
        if k > 1
            print(io, "^", k)
        end
    end
    print(io, ")")
end

p

MyPoly([-1.234,0,0,0,4.321], 's')

function (p::MyPoly)(x)
    d = degree(p)
    v = p.c[end-d]
    for cc = p.c[end-d+1:end]
        v = v*x + cc
    end
    return v
end

println(p(1))
println(p(1.234))
println(p.([1,-2,1.3]))  # Note: Broadcasting automatically defined

using PyPlot
function PyPlot.plot(p::MyPoly, xlim=[-2,2])
    xx = collect(range(xlim[1], xlim[2], length=100))
    plot(xx, p.(xx))
    xlabel(string(p.var))
end

p = MyPoly([1,1,-5,-5,4,4])
plot(p)
p

function Base.:+(p1::MyPoly, p2::MyPoly)
    if p1.var != p2.var
        error("Cannot add polynomials of different independent variables.")
    end
    d1 = degree(p1)
    d2 = degree(p2)
    d = max(d1,d2)
    c = [fill(0, d-d1); p1.c] + [fill(0, d-d2); p2.c]
    return MyPoly(c, p1.var)
end

println(p)
println(p + p)
println(p + MyPoly([1.1,2.2]))
println(p + MyPoly([1], 's')) # will trigger our error message

function Base.:-(p1::MyPoly, p2::MyPoly)
    return p1 + MyPoly(-p2.c)
end

function Base.:*(a, p::MyPoly)
    newc = a * p.c
    return MyPoly(newc, p.var)
end

function Base.:*(p::MyPoly, a)
    return a*p
end

p1 = 0.4p
p2 = p1 - .3*MyPoly([-2,1,1])
p3 = -1p2 + p
plot.([p1,p2,p3]);

p = MyPoly([1, -2//3, 6//7])

p.([1, -7//2])

p = MyPoly([1, im, -1, -im, 1])

p.([0, im, -1 - im])

p = MyPoly(collect(-1.5:3.5))

p(BigFloat(-π))


using TestImages, Images, OffsetArrays, CoordinateTransformations, Rotations

# This is an alternative means of accomplishing the same rotation as
# described in the blog post, here by translating the starting image
# directly so that 0,0 is the center of the rotation.
img = OffsetArray(testimage("cameraman"), (-125, -250))
tfm = LinearMap(RotMatrix(-pi/6))
img_rotated = warp(img, tfm)
save("cameraman.png", img)
save("cameraman_rotated.png", clamp01nan.(img_rotated))
cv = colorview(RGB, paddedviews(0, img, img_rotated, img)...)
save("cameraman_overlay.png", clamp01nan.(cv))
pimg = parent(img)
tfmcntr = recenter(RotMatrix(-pi/6), center(pimg))
img_rotated_cntr = warp(pimg, tfmcntr)
cvc = colorview(RGB, paddedviews(0, pimg, img_rotated_cntr, pimg)...)
save("cameraman_overlay_center.png", clamp01nan.(cvc))
inds = map(intersect, indices(img), indices(img_rotated))
cvi = colorview(RGB, img[inds...], img_rotated[inds...], img[inds...])
save("cameraman_overlay_interior.png", clamp01nan.(cvi))

using Plots
pyplot()
ps = []
isfirst = true
for (i, kern) in enumerate((centered([1]), OffsetArray([1], 2:2), OffsetArray([1], -5:-5)))
    a = 1:8
    af = imfilter(a, kern, Fill(0))
    p = plot(a, color="black", xlims=(-3,9), ylims=(0,8), grid=false, linewidth=2, label=isfirst?"input":"", xticks=i == 3 ? (-3:3:9) : nothing)
    plot!(p, -indices(kern, 1), 0:1, color="blue", linewidth=2, label=isfirst?"kernel":"")
    plot!(p, af, color="red", label=isfirst?"output":"")
    push!(ps, p)
    isfirst = false
end
savefig(plot(ps..., layout=(3,1)), "filtering.eps")


#=

Example demonstrating sparsity control by means of the `lazy_tolerance`-factor passed to the lazy AFW variant

A larger lazy_tolerance >= 1 favors sparsity by favoring optimization over the current active set rather than
adding a new FW vertex.

The default for AFW is lazy_tolerance = 2.0

=#

include("activate.jl")

using LinearAlgebra
using Random

n = Int(1e4)
k = 1000

s = rand(1:100)
@info "Seed $s"
Random.seed!(s)

xpi = rand(n);
total = sum(xpi);

# here the optimal solution lies in the interior if you want an optimal solution on a face and not the interior use:
# const xp = xpi;

const xp = xpi ./ total;

f(x) = norm(x - xp)^2
function grad!(storage, x)
    @. storage = 2 * (x - xp)
end

const lmo = FrankWolfe.KSparseLMO(100, 1.0)

## other LMOs to try
# lmo_big = FrankWolfe.KSparseLMO(100, big"1.0")
# lmo = FrankWolfe.LpNormLMO{Float64,5}(1.0)
# lmo = FrankWolfe.ProbabilitySimplexOracle(1.0);
# lmo = FrankWolfe.UnitSimplexOracle(1.0);

const x00 = FrankWolfe.compute_extreme_point(lmo, rand(n))

## example with BirkhoffPolytopeLMO - uses square matrix.
# const lmo = FrankWolfe.BirkhoffPolytopeLMO()
# cost = rand(n, n)
# const x00 = FrankWolfe.compute_extreme_point(lmo, cost)

function build_callback(trajectory_arr)
    return function callback(state)
        return push!(trajectory_arr, (Tuple(state)[1:5]..., state.active_set_length))
    end
end

FrankWolfe.benchmark_oracles(f, grad!, () -> randn(n), lmo; k=100)

x0 = deepcopy(x00)
@time x, v, primal, dual_gap, trajectorySs = FrankWolfe.frank_wolfe(
    f,
    grad!,
    lmo,
    x0,
    max_iteration=k,
    line_search=FrankWolfe.Shortstep(),
    L=2,
    print_iter=k / 10,
    emphasis=FrankWolfe.memory,
    verbose=true,
    trajectory=true,
);

trajectoryAda = []
callback = build_callback(trajectoryAda)

x0 = deepcopy(x00)
@time x, v, primal, dual_gap, _ = FrankWolfe.away_frank_wolfe(
    f,
    grad!,
    lmo,
    x0,
    max_iteration=k,
    line_search=FrankWolfe.Adaptive(),
    print_iter=k / 10,
    emphasis=FrankWolfe.memory,
    verbose=true,
    trajectory=true,
    callback=callback,
);

println("\n==> Lazy AFW.\n")

trajectoryAdaLoc15 = []
callback = build_callback(trajectoryAdaLoc15)

x0 = deepcopy(x00)
@time x, v, primal, dual_gap, _ = FrankWolfe.away_frank_wolfe(
    f,
    grad!,
    lmo,
    x0,
    max_iteration=k,
    line_search=FrankWolfe.Adaptive(),
    print_iter=k / 10,
    emphasis=FrankWolfe.memory,
    verbose=true,
    lazy=true,
    lazy_tolerance=1.5,
    trajectory=true,
    callback=callback,
);

trajectoryAdaLoc2 = []
callback = build_callback(trajectoryAdaLoc2)

x0 = deepcopy(x00)
@time x, v, primal, dual_gap, trajectoryAdaLoc2 = FrankWolfe.away_frank_wolfe(
    f,
    grad!,
    lmo,
    x0,
    max_iteration=k,
    line_search=FrankWolfe.Adaptive(),
    print_iter=k / 10,
    emphasis=FrankWolfe.memory,
    verbose=true,
    lazy=true,
    lazy_tolerance=2.0,
    trajectory=true,
    callback=callback,
);

trajectoryAdaLoc4 = []
callback = build_callback(trajectoryAdaLoc4)

x0 = deepcopy(x00)
@time x, v, primal, dual_gap, _ = FrankWolfe.away_frank_wolfe(
    f,
    grad!,
    lmo,
    x0,
    max_iteration=k,
    line_search=FrankWolfe.Adaptive(),
    print_iter=k / 10,
    emphasis=FrankWolfe.memory,
    verbose=true,
    lazy_tolerance=4.0,
    lazy=true,
    trajectory=true,
    callback=callback,
);

trajectoryAdaLoc10 = []
callback = build_callback(trajectoryAdaLoc10)

x0 = deepcopy(x00)
@time x, v, primal, dual_gap, _ = FrankWolfe.away_frank_wolfe(
    f,
    grad!,
    lmo,
    x0,
    max_iteration=k,
    line_search=FrankWolfe.Adaptive(),
    print_iter=k / 10,
    emphasis=FrankWolfe.memory,
    lazy=true,
    lazy_tolerance=10.0,
    verbose=true,
    trajectory=true,
    callback=callback,
);

# Reduction primal/dual error vs. sparsity of solution

dataSparsity =
    [trajectoryAda, trajectoryAdaLoc15, trajectoryAdaLoc2, trajectoryAdaLoc4, trajectoryAdaLoc10]
labelSparsity = ["AFW", "LAFW-K-1.5", "LAFW-K-2.0", "LAFW-K-4.0", "LAFW-K-10.0"]

FrankWolfe.plot_sparsity(dataSparsity, labelSparsity, legend_position=:topright)


using Vectorize
VML = Vectorize.VML

if VERSION >= v"0.5-"
    using Base.Test
else
    using BaseTestNext
    const Test = BaseTestNext
end

println("===== Testing Intel's VML =====")
# Length of test array
N = 1000
srand(13)

# Test basic utility functionality
@testset "VML: Basic Functionality" begin
    mode = VML.getmode()
    @test mode == VML.setmode(VML.VML_HA)
    @test typeof(VML.setmode(VML.VML_HA)) <: Integer
end

# Real Arithmetic
for T in [Float32, Float64]
    @testset "VML: Basic Arithmetic::$T" begin
        X = convert(Vector{T}, abs(randn(N)))
        Y = convert(Vector{T}, abs(randn(N)))
        @testset "Testing $f::$T" for (f, fb) in [(:add, .+), (:sub, .-), (:div, ./), (:mul, .*),
                                                  (:hypot, :hypot), (:atan2, :atan2), (:pow, .^)]
            @eval fbase = $fb
            @eval fvml = Vectorize.VML.$f
            @test fbase(X, Y) ≈ fvml(X, Y)
        end
    end
end

# Complex Arithmetic
for T in [Complex{Float32}, Complex{Float64}]
    @testset "VML: Complex Arithmetic::$T" begin
        X = convert(Vector{T}, randn(N)) + convert(Vector{T}, randn(N))*im
        Y = convert(Vector{T}, randn(N)) + convert(Vector{T}, randn(N))*im
        @testset "Testing $f::$T" for (f, fb) in [(:add, .+), (:sub, .-), (:div, ./), (:mul, .*),
                                                  (:pow, .^)]
            @eval fbase = $fb
            @eval fvml = Vectorize.VML.$f
            @test fbase(X, Y) ≈ fvml(X, Y)
        end

        @testset "Testing $f::$T" for f in [:mulbyconj]
            @eval fvml = Vectorize.VML.$f
            @test X .* conj(Y) ≈ fvml(X, Y)
        end
    end
end

# Real Vector Math
for T in [Float32, Float64]
    @testset "VML: Real Vector Math::$T" begin
        X = convert(Vector{T}, randn(N)) # unrestricted
        Y = convert(Vector{T}, abs(randn(N))) .+ 1 # greater than 1
        Z = clamp(convert(Vector{T}, randn(N)), -1,  1) # [-1, 1]
        W = clamp(convert(Vector{T}, randn(N)), 0,  2) # [0, 2]
        @testset "Testing $f::$T" for f in [:exp, :abs, :ceil, :floor, :round,
                                            :trunc, :cos, :sin, :tan, :cosh, :sinh, :tanh,
                                            :cbrt, :erf, :erfc, :lgamma, :gamma]
            @eval fb = $f
            @eval fvml = Vectorize.VML.$f
            @test fb(X) ≈ fvml(X)
        end

        @testset "Testing $f::$T" for f in [:sqrt, :log10, :acosh, :asinh, :log]
            @eval fb = $f
            @eval fvml = Vectorize.VML.$f
            @test fb(Y) ≈ fvml(Y)
        end

        # @testset "Testing $f::$T" for f in [:erfcinv]
        #     @eval fb = $f
        #     @eval fvml = Vectorize.VML.$f
        #     @test fb(W) ≈ fvml(W)
        # end

        @testset "Testing $f::$T" for f in [:acos, :asin, :atan, :atanh] #, :erfinv]
            @eval fb = $f
            @eval fvml = Vectorize.VML.$f
            @test fb(Z) ≈ fvml(Z)
        end
    end
end

# Complex Vector Math
for T in [Complex{Float32}, Complex{Float64}]
    @testset "VML: Complex Vector Functions::$T" begin
        X = convert(Vector{T}, complex(randn(N)))
        @testset "Testing $f::$T" for f in [:exp, :abs, :cos, :sin, :tan, :cosh, :sinh, :tanh,
                                            :sqrt, :log10, :acosh, :asinh, :log, :acos, :asin,
                                            :atan, :atanh, :conj]
            @eval fb = $f
            @eval fvml = Vectorize.VML.$f
            @test fb(X) ≈ fvml(X)
        end

        @testset "Testing $f::$T" for f in []
            @eval fb = $f
            @eval fvml = Vectorize.VML.$f
            @test fb(Y) ≈ fvml(Y)
        end

        @testset "Testing $f::$T" for f in []
            @eval fb = $f
            @eval fvml = Vectorize.VML.$f
            @test fb(Z) ≈ fvml(Z)
        end
    end
end

println("===== VML Tests Successful =====\n\n")


using Test, DiscreteExteriorCalculus, DiscretePDEs
using UniqueVectors: UniqueVector

radius, height, offset1, offset2, height_fraction = 3, 2, 1, 1.5, .1
file_name = joinpath(@__DIR__, "current_carrying_wire_cylinder.geo")
geo_write!(file_name, characteristic_length_factor=.3,
    footer="""
    Cylinder(1) = {0, 0, 0, 0, 0, $height, $radius, 2*Pi};
    Point(3) = {0, 0, 0, 1.0};
    Point(4) = {0, 0, $height, 1.0};
    Point(5) = {$offset1, 0, $(height_fraction * height), 1.0};
    Point(6) = {$offset1, 0, $((1-height_fraction) * height), 1.0};
    Point(7) = {$offset2, 0, $(height_fraction * height), 1.0};
    Point(8) = {$offset2, 0, $((1-height_fraction) * height), 1.0};
    Point(9) = {$(-offset1), 0, 0, 1.0};
    Point(10) = {$(-offset1), 0, $height, 1.0};
    Line(4) = {3, 4};
    Line(5) = {5, 6};
    Line(6) = {6, 8};
    Line(7) = {8, 7};
    Line(8) = {7, 5};
    Line(9) = {9, 10};
    Point{3, 9} In Surface{3};
    Point{4, 10} In Surface{2};
    Point{6, 8, 7, 5} In Volume{1};
    Curve{4, 5, 6, 7, 8, 9} In Volume{1};
    Physical Curve("Wire center") = {4};
    Physical Curve("Loop") = {6, 7, 8, 5};
    Physical Curve("Wire offset") = {9};
    """)
initialize!()
gmsh_open!(file_name)
N, K = 3, 4
mesh!(K)

node_tags, points, tcomp = get_triangulated_complex(N, K)
group_dict = get_physical_groups(node_tags, points)
@test typeof(tcomp) <: TriangulatedComplex{N, K}
comp = tcomp.complex
orient!(comp)
sources = [get_current_source(comp, group_dict[k]) for k in
    ["Wire center", "Loop", "Wire offset"]]
m = Metric(N)
mesh = Mesh(tcomp, circumcenter(m))

μ⁻, Λ⁻ = 2, 0
μ⁻_form = get_material(comp, μ⁻, 3)
Λ⁻_form = get_material(comp, Λ⁻, 2)

bbox, null_basis = magnetostatics_blackbox(m, mesh, sources, boundary(comp),
    μ⁻_form, Λ⁻_form)
inductance = impedance_matrix(bbox, null_basis)
@testset "mutual inductance of wires and loops" begin
    @test isapprox(abs(inductance[1,3]),
        abs((log(radius) - log(offset1)) * height/(2π*μ⁻)), rtol=3e-2)
    @test isapprox(abs(inductance[1,2]),
        abs((log(offset1) - log(offset2)) * height * (1-2*height_fraction)/(2π*μ⁻)), rtol=3e-2)
end

if false # set to true to display plots with Gmsh
    A, _ = solve_statics(bbox, null_basis, [0.0, 1.0, 0.0])
    vec_A = sharp(m, comp, A)
    comp_points = UniqueVector([c.points[1] for c in comp.cells[1]])
    ordering = [findfirst(isequal(p), comp_points) for p in points]
    add_field!("Vector potential", node_tags, vec_A[ordering])
    gui!()
end
rm(file_name)


function ethmult(a::Integer, b::Integer)
    r = 0
    while a > 0
        r += b * !even(a)
        a = halve(a)
        b = double(b)
    end
    return r
end

@show ethmult(17, 34)


function derive_pulse(average_power, rep_rate, pulse_time, pulse_type::Symbol=:gaussian, chirp=0)
    pulse_energy = sqrt(average_power / rep_rate / pulse_time)
    #u0(x) = (1+0im) * pulse_energy * exp(x^2/pulse_time^2)
    u0(x) = (1+0im) * pulse_energy * sech(x/pulse_time)
    return u0
end

function derive_pulse(peak_power, pulse_FWHM)
    sqrtpower = sqrt(peak_power)
    pulse_0 = pulse_FWHM/(2*log(1+sqrt(2)))
    u0(x) = (1+0im)*sqrtpower*sech(x/pulse_0)
end


@testset "FBM" begin
    rng = MersenneTwister(42)
    h = 0.3
    k = FBMKernel(h = h)
    v1 = rand(rng, 3)
    v2 = rand(rng, 3)
    @test k(v1,v2) ≈ (sqeuclidean(v1, zero(v1))^h + sqeuclidean(v2, zero(v2))^h - sqeuclidean(v1-v2, zero(v1-v2))^h)/2 atol=1e-5
    @test repr(k) == "Fractional Brownian Motion Kernel (h = $(h))"

    test_interface(k)
    @test repr(k) == "Fractional Brownian Motion Kernel (h = $(h))"
    test_ADs(FBMKernel, ADs = [:ReverseDiff, :Zygote])
    @test_broken "Tests failing for kernelmatrix(k, x) for ForwardDiff"
    test_params(k, ([h],))
end


@polly function kernel_durbin(r, y)
    n = size(r,1)
    z = zeros(eltype(r),n)
    y[1] = -r[1]
    beta = one(eltype(y))
    alpha = -r[1]

    for k = 2:n
        beta = (1 - alpha * alpha) * beta
        sum = zero(eltype(y))
        for i = 1:(k-1)
            sum += r[k-i] * y[i]
        end
        alpha = - (r[k] + sum)/beta

        for i = 1:(k-1)
            z[i] = y[i] + alpha * y[k-i]
        end
        for i = 1:(k-1)
            y[i] = z[i]
        end
        y[k] = alpha
    end
end

let
    n = 2000

    r = zeros(Float32, n)
    y = zeros(Float32, n)

    for i = 1:n
        r[i] = (n+1-i)
    end

    SUITE["durbin"] = @benchmarkable kernel_durbin(r, y) setup = (r = copy($r); y = copy($y))
end


# This file includes and example file for HInterp2D interpolation

using FractalTools
using GeometryBasics
using Makie

# Generate data
f(x, y) = [
    x^2 + y^2 + 1,
    x^2 - y^2
    ]
ngon = Triangle(
    Point(BigFloat(0.), BigFloat(0.)),
    Point(BigFloat(1.), BigFloat(0.)),
    Point(BigFloat(0.5), BigFloat(1.)))
npts = 100
pts = getdata(f, ngon, npts)
interp = interpolate(pts, HInterp2D(0.01 * ones(2,2)))

function err(x, y)
    fval = f(x, y)
    ival = interp(x, y)
    map(item -> abs.(item), fval - ival) ./ map(item -> abs.(item), fval) * 100
end

tpts = getdata(ngon, npts)

fig, ax, plt = trisurf(tpts, (x, y) -> f(x, y)[1], meshcolor3=:red)
trisurf!(ax, tpts, (x, y) -> interp(x, y)[1], meshcolor3=:blue)
fig, ax, plt = trisurf(tpts, (x, y) -> err(x, y)[1], meshcolor3=:red)


#=
  de Bruijn sequence in Julia ConstraintSolver

  Implementation of de Bruijn sequences in Comet, both "classical"
  and "arbitrary".

  Compare with the the web based programs:
    http://www.hakank.org/comb/debruijn.cgi
    http://www.hakank.org/comb/debruijn_arb.cgi

  For Base = 2, N = 3, M = 8 there are 2 solutions:
    x : [](0, 1, 3, 7, 6, 5, 2, 4)
    bincode : [0, 0, 0, 1, 1, 1, 0, 1]

    x : [](0, 1, 2, 5, 3, 7, 6, 4)
    bincode : [0, 0, 0, 1, 0, 1, 1, 1]

  There are two symmetry breaking constraints:
  - if possible, we require that the occurrences of number are the same,
    i.e if M mod Base == 0.
  - the sequence X always start with the minumum value (0).

  Model created by Hakan Kjellerstrand, hakank@gmail.com
  See also my Julia page: http://www.hakank.org/julia/

=#
using ConstraintSolver, JuMP
using Cbc, GLPK, Ipopt
const CS = ConstraintSolver
include("constraints_utils.jl")

function debruijn(base=2,n=3,m=base^n,all_solutions=true,print_solutions=true)
    println("base:$base n:$n m:$m")
    cbc_optimizer = optimizer_with_attributes(Cbc.Optimizer, "logLevel" => 0)
    glpk_optimizer = optimizer_with_attributes(GLPK.Optimizer)
    ipopt_optimizer = optimizer_with_attributes(Ipopt.Optimizer)

    model = Model(optimizer_with_attributes(CS.Optimizer,   "all_solutions"=> all_solutions,
                                                            # "all_optimal_solutions"=>true,
                                                            "logging"=>[],

                                                            # "traverse_strategy"=>:BFS,
                                                            # "traverse_strategy"=>:DFS,
                                                            # "traverse_strategy"=>:DBFS,

                                                            # "branch_split"=>:Smallest,
                                                            # "branch_split"=>:Biggest,
                                                            # "branch_split"=>:InHalf,

                                                            # "simplify"=>false,
                                                            # "simplify"=>true, # default

                                                            "time_limit"=>30,

                                                            # "lp_optimizer" => cbc_optimizer,
                                                            # "lp_optimizer" => glpk_optimizer,
                                                            "lp_optimizer" => ipopt_optimizer,
                                        ))

    # x: list of the integers to use which are converted to
    #    'base-ary' numbers below
    @variable(model, 0 <= x[1:m] <= (base^n)-1, Int)

    # binary: the matrix of the "fully expanded" integers in X
    @variable(model, 0 <= binary[1:m,1:n] <= base-1, Int)

    # number of occurrences for each number
    @variable(model, 0 <= gcc[1:base] <= m, Int)

    @constraint(model, x in CS.AllDifferentSet())

    # Connect x and binary
    for i in 1:m
        # convert the integer to base-ary representation
        to_num(model, [binary[i,j] for j in 1:n], base, x[i])
    end

    # The de Bruijn criterion: Connect one element to the next...
    for i in 2:m, j in 2:n
        @constraint(model,binary[i-1, j] == binary[i, j-1])
    end

    # ... and around the corner.
    for j in 2:n
        @constraint(model, binary[m, j] == binary[1, j-1])
    end

    # bin_code: The de Bruijn sequence, i.e. the first
    #           elements in each row in Binary
    #           Note: this is just a slice.
    bin_code = [binary[i,1] for i in 1:m]

    # GCC: Count the number of different elements in the bin code
    # (poor man's global cardinarlity count...)
    for i in 1:base
        count_ctr(model,bin_code, :(==), i-1, gcc[i])
    end

    # If possible, we require that the occurrences of number are the same.
    # Gcc
    if m % base == 0
        for i in 2:base
            @constraint(model,gcc[i] == gcc[i-1])
            @constraint(model,gcc[i] == round(Int,m / base))
        end
    end

    # symmetry breaking
    # my_min is defined in constraints_utils.jl
    my_min(model, x, x[1])

    println("solve")
    # Solve the problem
    optimize!(model)

    status = JuMP.termination_status(model)
    # println("status:$status")
    if status == MOI.OPTIMAL
        num_sols = MOI.get(model, MOI.ResultCount())
        println("num_sols:$num_sols\n")
        if print_solutions
            for sol in 1:num_sols
                println("solution #$sol")
                xx = convert.(Integer,JuMP.value.(x; result=sol))
                binaryx = convert.(Integer,JuMP.value.(binary; result=sol))
                gccx = convert.(Integer,JuMP.value.(gcc; result=sol))
                if base < 10
                    bin_codex = join([binaryx[i,1] for i in 1:m],"")
                else
                    bin_codex = [binaryx[i,1] for i in 1:m]
                end

                println("x: $xx")
                println("binary:")
                for row in eachrow(binaryx)
                    println(row)
                end
                println("bin_code: $bin_codex")
                println("gcc: $gccx")
                println()

            end
            println("num_sols: $num_sols\n")
        end
    else
        println("status: $status")
    end

    return status
end

base=2
n=3
m=base^n
all_solutions= true
print_solutions=true
@time debruijn(base,n,m,all_solutions,print_solutions)

# Test a non standard version: m = 27 (instead of 2^5=32)
# It should be 2000 solutions. And it is, but it takes ≈ 29s!
base = 2
n = 5
m = 27
@time debruijn(base,n,m,all_solutions,false)


##############################################################################
##
## Type with stored qr
##
##############################################################################
struct DenseQRAllocatedSolver{Tqr <: StridedMatrix, Tu <: AbstractVector} <: AbstractAllocatedSolver
    qrm::Tqr
    u::Tu
    function DenseQRAllocatedSolver{Tqr, Tu}(qrm, u) where {Tqr <: StridedMatrix, Tu <: AbstractVector}
        length(u) == size(qrm, 1) || throw(DimensionMismatch("u must have length size(J, 1)"))
        new(qrm, u)
    end
end

function DenseQRAllocatedSolver(qrm::Tqr, u::Tu) where {Tqr <: StridedMatrix, Tu <: AbstractVector}
    DenseQRAllocatedSolver{Tqr, Tu}(qrm, u)
end

##############################################################################
##
## solve J'J \ J'y by QR
##
##############################################################################

function AbstractAllocatedSolver(nls::LeastSquaresProblem{Tx, Ty, Tf, TJ, Tg}, optimizer::Dogleg{QR}) where {Tx, Ty, Tf, TJ <: StridedVecOrMat, Tg}
    return DenseQRAllocatedSolver(similar(nls.J), _zeros(nls.y))
end

function LinearAlgebra.ldiv!(x::AbstractVector, J::StridedMatrix,  y::AbstractVector, A::DenseQRAllocatedSolver)
    u, qrm = A.u, A.qrm
    copyto!(qrm, J)
    copyto!(u, y)
    if VERSION >= v"1.7.0"
        ldiv!(qr!(qrm, ColumnNorm()), u)
    else
        ldiv!(qr!(qrm, Val(true)), u)
    end
    for i in 1:length(x)
        x[i] = u[i]
    end
    return x, 1
end

##############################################################################
##
## solve (J'J + diagm(damp)) \ J'y by QR
##
##############################################################################

function AbstractAllocatedSolver(nls::LeastSquaresProblem{Tx, Ty, Tf, TJ, Tg}, optimizer::LevenbergMarquardt{QR}) where {Tx, Ty, Tf, TJ <: StridedVecOrMat, Tg}
    qrm = zeros(eltype(nls.J), length(nls.y) + length(nls.x), length(nls.x))
    u = zeros(eltype(nls.y), length(nls.y) + length(nls.x))
    return DenseQRAllocatedSolver(qrm, u)
end

function LinearAlgebra.ldiv!(x::AbstractVector, J::StridedMatrix, y::AbstractVector,
                damp::AbstractVector, A::DenseQRAllocatedSolver, verbose::Bool = false)
    u, qrm = A.u, A.qrm

    # transform dammp
    length(u) ==  length(y) + length(x) || throw(DimensionMismatch("length(u) should equal length(x) + length(y)"))

    # update qr as |J; diagm(damp)|
    fill!(qrm, zero(eltype(qrm)))
    for j in 1:size(J, 2)
        for i in 1:size(J, 1)
            qrm[i, j] = J[i, j]
        end
    end

    leny = length(y)
    for i in 1:length(damp)
        qrm[leny + i, i] = sqrt(damp[i])
    end

    # update u as |J; 0|
    fill!(u, zero(eltype(u)))
    for i in 1:length(y)
        u[i] = y[i]
    end
    if verbose
        @show mean(u)
        sleep(1)
    end

    # solve
    if VERSION >= v"1.7.0"
        ldiv!(qr!(qrm, ColumnNorm()), u)
    else
        ldiv!(qr!(qrm, Val(true)), u)
    end
    if verbose
        @show mean(u)
        sleep(1)
    end
    for i in 1:length(x)
        x[i] = u[i]
    end
    return x, 1
end


# This file is a part of AstroLib.jl. License is MIT "Expat".
# Copyright (C) 2016 Mosè Giordano.

function airtovac(wave_air::AbstractFloat)
    if wave_air >= 2000
        wave_vac = wave_air
        for iter= 1:2
            sigma2 = (1e4/wave_vac)^2 # Convert to wavenumber squared
            # Computer conversion factor.
            fact = 1 + 5.792105e-2/(238.0185 - sigma2) +
                1.67917e-3/(57.362 - sigma2)
            wave_vac = wave_air*fact # Convert Wavelength
        end
        return wave_vac
    else
        return wave_air
    end
end

"""
    airtovac(wave_air) -> wave_vacuum

### Purpose ###

Converts air wavelengths to vacuum wavelengths.

### Explanation ###

Wavelengths are corrected for the index of refraction of air under standard conditions.
Wavelength values below \$2000 Å\$ will *not* be altered, take care within \$[1 Å, 2000
Å]\$.  Uses relation of Ciddor (1996).

### Arguments ###

* `wave_air`: the wavelength in air.

### Output ###

Vacuum wavelength in angstroms.

### Method ###

Uses relation of Ciddor (1996), Applied Optics 62, 958.

### Example ###

If the air wavelength is `w = 6056.125` (a Krypton line), then `airtovac(w)`
yields a vacuum wavelength of `6057.8019`.

```jldoctest
julia> using AstroLib

julia> airtovac(6056.125)
6057.801930991426
```

### Notes ###

`vactoair` converts vacuum wavelengths to air wavelengths.

Code of this function is based on IDL Astronomy User's Library.
"""
airtovac(wave_air::Real) = airtovac(float(wave_air))


# Ari Huttunen # HIDE
using Pkg # HIDE
Pkg.activate(@__DIR__) # HIDE # ?confused about global dir or relative dir, this works for now running from the dir.
Pkg.add(["Makie","CairoMakie","PalmerPenguins", "Colors", "Chain", "DataFrames", "GLM"]) # HIDE
using CairoMakie, PalmerPenguins, Colors, Chain, DataFrames, GLM
CairoMakie.activate!() # HIDE
let
    ENV["DATADEPS_ALWAYS_ACCEPT"] = "true" # Don't ask if the datasets should be downloaded
	penguins = dropmissing(DataFrame(PalmerPenguins.load()))
    set_theme!(Theme(
        Axis = (
            leftspinevisible = false,
            rightspinevisible = false,
            bottomspinevisible = false,
            topspinevisible = false,
            xticksvisible = false,
            yticksvisible = false,
        ),
    ))

    # These colors were taken from https://allisonhorst.github.io/palmerpenguins/
	adeliecolor = colorant"#FC7808"
	chinstrapcolor = colorant"#8C00EC"
	gentoocolor = colorant"#107A78"
	(adeliecolor, chinstrapcolor, gentoocolor)

    style = Dict(
        "Adelie" => (
            color=(adeliecolor,0.7),
            strokecolor=adeliecolor,
            label="Adelie",
            marker=:circle,
            markersize=20,
			linewidth = 5
        ),
        "Chinstrap" => (
            color=(chinstrapcolor,0.7),
            strokecolor=chinstrapcolor,
            label="Chinstrap",
            marker=:utriangle,
            markersize=20,
			linewidth = 5
        ),
        "Gentoo" => (
            color=(gentoocolor,0.7),
            strokecolor=gentoocolor,
            label="Gentoo",
            marker=:rect,
            markersize=20,
			linewidth = 5
        )
    )

	fig = Figure(resolution = (1000, 700))
	ax1 = fig[1, 1] = Axis(fig, title = "Flipper and bill length",
		xlabel="Flipper length (mm)", ylabel="Bill length (mm)")
	ax1.xticks = 160:10:240
	ax1.yticks = 30:5:60

	specie = @chain penguins begin
		filter(:species => ==("Adelie"), _)
	end
	X = specie[!,:flipper_length_mm]
	Y = specie[!,:bill_length_mm]
	scatter!(ax1, X, Y; style["Adelie"]...)
	ols = lm(@formula(Y ~ X), DataFrame(X=X, Y=Y))
	y = predict(ols)
	lines!(ax1, X, y; style["Adelie"]...)

	chinstrap = @chain penguins begin
		filter(:species => ==("Chinstrap"), _)
	end
	X = chinstrap[!,:flipper_length_mm]
	Y = chinstrap[!,:bill_length_mm]
	scatter!(ax1, X,Y; style["Chinstrap"]...)
	ols = lm(@formula(Y ~ X), DataFrame(X=X, Y=Y))
	y = predict(ols)
	lines!(ax1, X, y; style["Chinstrap"]...)

	gentoo = @chain penguins begin
		filter(:species => ==("Gentoo"), _)
	end
	X = gentoo[!,:flipper_length_mm]
	Y = gentoo[!,:bill_length_mm]
	scatter!(ax1,X, Y; style["Gentoo"]...)
	ols = lm(@formula(Y ~ X), DataFrame(X=X, Y=Y))
	y = predict(ols)
	lines!(ax1, X, y; style["Gentoo"]...)
	axislegend("Penguin species", position = :rb)

    save(joinpath(@__DIR__, "output", "penguinsDataSetWithChain.png"), fig)
end

using Pkg # HIDE
Pkg.status() # HIDE
Pkg.activate() # HIDE


export opt_linear_fit!

"""
    opt_linear_fit!(
        graph,
        objfun,
        discr,
        linear_cref;
        input = :A,
        errtype = :abserr,
        linlsqr = :backslash,
        droptol = 0,
    )

Linear fitting of a `graph` of the form

    c_1 g_1(x) + c_2 g_2(x) + … + c_n g_n(x)

to the values of `objfun`, in the points `discr`. Reference to the coefficients
`c_1,…,c_n` should be given `linear_cref`.

The variable `graph` is modified during the iterations and the function has no
return value.

See [`opt_gauss_newton!`](@ref) for a description the kwarg `errtype` and
`input`, and [`solve_linlsqr!`](@ref) for the kwargs `linlsqr and `droptol`.
"""
function opt_linear_fit!(
    graph,
    objfun,
    discr,
    linear_cref;
    input = :A,
    errtype = :abserr,
    linlsqr = :backslash,
    droptol = 0,
)
    objfun_vals = objfun.(discr)

    vals = init_vals_eval_graph!(graph, discr, nothing, input)
    eval_graph(graph, discr, vals = vals, input = input)

    n = length(linear_cref)
    T = eltype(valtype(vals))
    A = ones(T, size(discr, 1), n)
    for k = 1:n
        parent = graph.parents[linear_cref[k][1]][linear_cref[k][2]]
        A[:, k] = vals[parent]
    end

    adjust_for_errtype!(A, objfun_vals, objfun_vals, errtype)

    c = solve_linlsqr!(A, objfun_vals, linlsqr, droptol)
    set_coeffs!(graph, c, linear_cref)

    return nothing
end


# copyright (c) 2017 Ralph Smith
# SPDX-License-Identifier: MIT-Expat
# License-Filename: LICENSE.md

eye(A::Matrix{T}) where T = Matrix{eltype(A)}(I,size(A))

function gnsdcheck(A,B,V,ν,μ,tol=1e-8; quiet=false)
    n = size(B,1)
    @assert size(A) == (n,n)
    @assert size(B) == (n,n)
    @assert size(V) == (n,n)
    @assert ν == length(μ)

    ok = true
    t = norm(A - V * B * V') / norm(A)
    if t > tol
        ok = false
        warn("decomposition error: $t")
    elseif !quiet
        println("decomposition error: ",t)
    end
    t = norm(eye(V) - V' * V)
    if t > 10n*eps()
        ok = false
        warn("unitarity error: $t")
    elseif !quiet
        println("unitarity error: ",t)
    end

    # compute start indices of blocks
    bs = cumsum(vcat([1],μ))
    (bs[end] == n+1) || push!(bs,n+1)
    nb = length(bs)-1

    for j=1:ν
        if norm(B[bs[j]:n, bs[j]:bs[j+1]-1]) != 0
            warn("subdiag block $j is not null")
            ok=false
        end
    end
    for ib=1:nb
        Bii = B[bs[ib]:bs[ib+1]-1,bs[ib]:bs[ib+1]-1]
        if ib>ν
            F = svd!(Bii)
            if any(F.S.< tol)
                warn("diag block $ib (> ν) is singular")
                ok=false
            end
        else
            if norm(Bii) != 0
                warn("nonzero diag block $ib")
                ok=false
            end
        end
        for jb=ib+1:nb
            Bij = B[bs[ib]:bs[ib+1]-1,bs[jb]:bs[jb+1]-1]
            F = svd!(Bij)
            if any(F.S.< tol)
                warn("block $ib,$jb does not have full column rank")
                ok=false
            end
        end
    end
    ok && (quiet || println("all tests pass."))
    ok
end

"""
    makegnb(ν,μx) -> B

construct a matrix with specified generalized nullspace structure

This makes the block-structured part `B`; apply a unitary transform to taste.
"""
function makegnb(ν,μx; cplx=false)
    nb = length(μx)
    (nb ∈ [ν,ν+1]) || throw(ArgumentError("must specify ν or ν+1 blocks"))
    for j=2:ν
        (μx[j-1] >= μx[j]) ||
            throw(ArgumentError("singular block sizes must be nonincreasing"))
    end
    bs = vcat([1],1 .+ cumsum(μx)) # starting indices of blocks
    n = bs[end] -1
    if cplx
        B = randn(n,n) + 1im*randn(n,n)
    else
        B = randn(n,n)
    end
    for i=1:ν
        B[bs[i]:n, bs[i]:bs[i+1]-1] .= 0
    end
    rmul!(B,1/norm(B))
    B
end


include("./utilities.jl")
include("../../src/CGPO.jl")

# ==============================================================================
# Floats on the exact matrix.

function FloatsOnExactMatrix()
    N = 64
    function PerformForRho(rho)
        Eigens = EigDistribution(N, rho=rho, eigen_min=1e-5)
        A, _ = TinyIntervalTestMatrices(
            Eigens,
            scale=N*1e-5(maximum(Eigens)/minimum(Eigens))
        )
        b = ones(size(A, 1))
        RelErrors = PerformCGFor(A, b, exact=false, epsilon=1e-10)
    return RelErrors end

    RelErrors = PerformForRho(0.8)
    fig = plot(
        RelErrors,
        yaxis=:log10,
        xlabel="iterations: k",
        ylabel="\$\\frac{\\Vert e_k \\Vert_A}{\\Vert e_0\\Vert_A}\$",
        label="\$\\rho=0.8\$",
        left_margin = 10Plots.mm,
        size=(750, 500), dpi=300,
    )
    RelErrors = PerformForRho(0.9)
    plot!(
        fig,
        RelErrors,
        label="\$\\rho=0.9\$",
        linestyle=:dot
    )
    RelErrors = PerformForRho(0.7)
    plot!(
        fig,
        RelErrors,
        label="\$\\rho=0.7\$",
        linestyle=:dashdot
    )
    RelErrors = PerformForRho(1)
    plot!(
        fig,
        RelErrors,
        label="\$\\rho=1\$",
        linestyle=:dash
    )
    title!(fig, "Float CG on Exact Matrix")
    fig |> display
    savefig(fig, "$(@__DIR__)/plots/floats_cg_on_exact_matrix")
return nothing end

FloatsOnExactMatrix()

# ==============================================================================
# Exact on the Smeared out matrix.

function ExactOnSmearedMatrix()
    N = 64
    function PerformForRho(rho)
        Eigens = EigDistribution(N, rho=rho, eigen_min=1e-5)
        ConditionNumb = maximum(Eigens)/minimum(Eigens)
        _, A = TinyIntervalTestMatrices(
            Eigens,
            scale=2e-5ConditionNumb,
            m=100
        )
        b = ones(size(A, 1))
        RelErrors = PerformCGFor(A, b, exact=true, epsilon=1e-10)
    return RelErrors end

    RelErrors = PerformForRho(0.8)
    fig = plot(
        RelErrors,
        yaxis=:log10,
        xlabel="iterations: k",
        ylabel="\$\\frac{\\Vert e_k \\Vert_A}{\\Vert e_0\\Vert_A}\$",
        label="\$\\rho=0.8\$",
        left_margin = 10Plots.mm,
        size=(750, 500), dpi=300,
    )
    RelErrors = PerformForRho(0.9)
    plot!(
        fig,
        RelErrors,
        label="\$\\rho=0.9\$",
        linestyle=:dot
    )
    RelErrors = PerformForRho(0.7)
    plot!(
        fig,
        RelErrors,
        label="\$\\rho=0.7\$",
        linestyle=:dashdot
    )
    RelErrors = PerformForRho(1)
    plot!(
        fig,
        RelErrors,
        label="\$\\rho=1\$",
        linestyle=:dash
    )
    title!(fig, "Exact CG on Smeared Matrix")
    savefig(fig, "$(@__DIR__)/plots/exact_cg_on_smeared_matrix")
    fig |> display
return nothing end

ExactOnSmearedMatrix()


immutable Chisq <: ContinuousUnivariateDistribution
    df::Float64 # non-integer degrees of freedom are meaningful
    function Chisq(d::Real)
        d > zero(d) || error("df must be positive")
        new(float64(d))
    end
end

@_jl_dist_1p Chisq chisq

function entropy(d::Chisq)
    x = d.df / 2.0 + log(2.0) + lgamma(d.df / 2.0)
    x + (1.0 - d.df / 2.0) * digamma(d.df / 2.0)
end

insupport(::Chisq, x::Real) = zero(x) <= x < Inf
insupport(::Type{Chisq}, x::Real) = zero(x) <= x < Inf

kurtosis(d::Chisq) = 12.0 / d.df

mean(d::Chisq) = d.df

# TODO: Switch to using quantile?
function median(d::Chisq)
    k = d.df
    k * (1.0 - 2.0 / (9.0 * k))^3
end

function mgf(d::Chisq, t::Real)
    k = d.df
    (1.0 - 2.0 * t)^(-k / 2.0)
end

cf(d::Chisq, t::Real) = (1.0 - 2.0 * im * t)^(-d.df / 2.0)

mode(d::Chisq) = d.df > 2.0 ? d.df - 2.0 : 0.0
modes(d::Chisq) = [mode(d)]

# rand - the distribution chi^2(df) is 2 * gamma(df / 2)
# for integer n, a chi^2(n) is the sum of n squared standard normals
function rand(d::Chisq)
    d.df == 1 ? randn()^2 : 2.0 * rand(Gamma(d.df / 2.0))
end

function rand!(d::Chisq, A::Array{Float64})
    if d.df == 1
        for i in 1:length(A)
            A[i] = randn()^2
        end
        return A
    end
    if d.df >= 2
        dpar = d.df / 2.0 - 1.0 / 3.0
    else
        error("require degrees of freedom df >= 2")
    end
    cpar = 1.0 / sqrt(9.0 * dpar)
    for i in 1:length(A)
        A[i] = 2.0 * randg2(dpar, cpar)
    end
    A
end

skewness(d::Chisq) = sqrt(8.0 / d.df)

var(d::Chisq) = 2.0 * d.df


function normout(A::SparseMatrixCSC{T,Int64}) where T
  # NORMOUT Normalize the outdegrees of the matrix A.
  #
  # P = normout(A)
  #
  #   P has the same non-zero structure as A, but is normalized such that the
  #   sum of each row is 1, assuming that A has non-negative entries.
  #

  # compute the row-sums/degrees
  #   d = sum(A,2)
  #   d = squeeze(d',1)
  #   id = 1./d
  #   P = spdiagm(id)*A
  #   return P

    S = vec(sum(A,dims=2))
    ei,ej,ev = findnz(A)
    m,n = size(A)
    P = sparse(ei,ej,ev./S[ei],m,n)
    return P

end


using RandomCorrelationMatrices
using Base.Test

srand(12345)

println("Random correlation matrix")
n = 4
for η in [2,8,32,128]
    r = randcormatrix(n,η)
    @show η
    for i in 1:n
        for j in 1:n
            @printf("%5.2f ", r[i,j])
        end
        println()
    end

    ranges = Float64[]
    for s in 1:1000
        r = randcormatrix(n,η)
        r0 = r - diagm(diag(r))
        push!(ranges, maximum(vec(r0)) - minimum(vec(r0)))
    end
    @show mean(ranges)
    @show median(ranges)
end

println("\nRandom covariance matrix")
σ = [100, 200, 300, 400, 500]
@show σ
Σ = randcovmatrix(5, 10, σ)
L = chol(Σ,Val{:L})
samples = 100_000
results = zeros(5,samples)
for i in 1:samples
    results[:,i] = L*randn(5)
end
@show round(mean(results,2),1)
@show round(std(results,2),1)


# Covariance Matrix Adaptation Evolution Strategy
# ==================
#
# Implementation: (μ/μ_I,λ)-CMA-ES
#
# μ is the number of parents
# λ is the number of offspring.
#
@with_kw struct CMAES <: Optimizer
    N::Int
    initPopulation::Individual = ones(N)
    initStrategy::Strategy = strategy(τ = sqrt(N), τ_c = N^2, τ_σ = sqrt(N))
    μ::Integer = 1
    λ::Integer = 1
end

function optimize(objfun::Function, opt::CMAES;
                    iterations::Integer = 1_000,
                    tol::Float64 = 1e-10,
                    verbose = false)
    @unpack N,initPopulation,initStrategy,μ,λ = opt
    @assert μ < λ "Offspring population must be larger then parent population"

    # Initialize parent population
    individual = getIndividual(initPopulation, N)
    population = fill(individual, μ)
    offspring = Array{typeof(individual)}(undef, λ)
    fitpop = fill(Inf, μ)
    fitoff = fill(Inf, λ)

    parent = copy(individual)
    C = Diagonal{Float64}(I, N)
    s = zeros(N)
    s_σ = zeros(N)
    σ = 1.0
    E = zeros(N, λ)
    W = zeros(N, λ)

    # Generation cycle
    count = 0
    while true
        SqrtC = (C + C') / 2.0
        try
            SqrtC = cholesky(SqrtC).U
        catch
            println("Break on Cholesky")
            break
        end

        for i in 1:λ
            # offspring are generated by transforming standard normally distributed random vectors using a transformation matrix
            E[:,i] = randn(N)
            W[:,i] = σ * (SqrtC * E[:,i])
            offspring[i] = parent + W[:,i]   # (L1)
            fitoff[i] = objfun(offspring[i]) # Evaluate fitness
        end

        # Select new parent population
        idx = sortperm(fitoff)[1:μ]
        for i in 1:μ
            population[i] = offspring[idx[i]]
            fitpop[i] = fitoff[idx[i]]
        end

        w = vec(mean(W[:,idx], dims=2))
        ɛ = vec(mean(E[:,idx], dims=2))
        parent += w            #  forming recombinant perent for next generation (L2)
        s = (1.0 - 1.0/initStrategy[:τ])*s +
            (sqrt(μ/initStrategy[:τ] * (2.0 - 1.0/initStrategy[:τ]))/σ)*w      # (L3)
        C = (1.0 - 1.0/initStrategy[:τ_c]).*C + (s./initStrategy[:τ_c])*s'     # (L4)
        s_σ = (1.0 - 1.0/initStrategy[:τ_σ])*s_σ +
            sqrt(μ/initStrategy[:τ_σ]*(2.0 - 1.0/initStrategy[:τ_σ]))*ɛ        # (L5)
        σ = σ*exp(((s_σ'*s_σ)[1] - N)/(2*N*sqrt(N)))                           # (L6)

        # termination condition
        count += 1
        if count == iterations || σ <= tol
            break
        end
        verbose && println("BEST: $(fitpop[1]): $(σ)")
    end

    return population[1], fitpop[1], count
end


using FractionalDiffEq, Plots

# Analytical solution
analytical(x) = x.^1.8 .*mittleff(1.8, 2.8, -x.^1.8)
# Numerical solution
fun(x, y) = 1-y
prob = SingleTermFODEProblem(fun, 1.8, [0, 0], (0, 20))
sol = solve(prob, 0.01, PECE())
tspan = collect(0:0.01:20)
target = analytical.(tspan)

plot(sol, linewidth=5, label="Numerical", legend=:bottomright)
plot!(tspan, target, lw=3, ls=:dash, label="Analytical")


using Parameters
include("common.jl")
# ATP-dependent K channel (KATP) parameters
@with_kw struct KATPParams
    k_o = 5.4
    # Temperature adjustment
    FT = 1.0266
    # Default Michaelis constant for sodium
    KM_NA = 25.9
    # Default Michaelis constant for sodium
    KM_MG = 0.65 * sqrt(k_o + 5.0)
    # electrical distance of magnesium
    δMG = 0.32
    # electrical distance of sodium
    δNA = 0.35
    # Channel density (/cm²)
    σ = 6.0E8
    # Max channel open probability
    P₀ = 0.91
    # Unitary conductance of KATP channel
    γ = 35.375E-9 * (k_o / 5.4)^2.4
    # Mg factor of KATP channel
    IMAX = σ * P₀ * FT * γ
end

#=
ATP-inhibited K current from Gauthier et. al, 2012, based on Ferrero et al. 1996
Scalar version
=#
function ikatp(vm, eK, na_i, atp_i, adp_i, mg_i, KM_MG, KM_NA, δMG, δNA, IMAX)
    f_mg = _mm(KM_MG, mg_i * _ra(2 * δMG * vm))
    f_na = _hills(KM_NA, na_i * _ra(δNA * vm), 2)
    km_atp = 35.8e-3 + 17.9 * 1000^(-0.44) * adp_i^0.56
    h = 1.9 + 0.74 * exp(-adp_i)
    f_atp = _hills(km_atp, atp_i, h)
    iKatp = IMAX * f_mg * f_na * f_atp * (vm - eK)
end

function ikatp(vm, eK, na_i, atp_i, adp_i, mg_i, pKATP::KATPParams)
    @unpack KM_MG, KM_NA, δMG, δNA, IMAX = pKATP
    ikatp(vm, eK, na_i, atp_i, adp_i, mg_i, KM_MG, KM_NA, δMG, δNA, IMAX)
end


"""
    Lx,Ly,Lz = makeangularmomentum(c,M,a,basis="Hermite")

Construct a compact representation of angular momentum operators in the given basis using step operators.

`basis`: name of the orthonomal mode space. Currently only `"Hermite"` is implemented.

`c`: cfield array on which `L_j` are to act.

`M`: vector of cutoff quantum numbers, i.e. `Mx,My = M` in 2D.

`a`: vector of oscillator lengths, i.e. `ax,ay = a` in 2D.

"""
function makeangularmomentum(c,M,a,basis="Hermite")
if basis=="Hermite"
  dim=length(size(c))
  if dim==1
    error("L not defined in 1D")
  elseif dim==2
    ax,ay = a
    Mx,My = M
    X,Px = ladderops(Mx,ax,basis)
    Y,Py = ladderops(My,ay,basis)
    Lz = -im*(X*c*Py' - Px*c*Y') #decide if return a function or matrices
    return Lz
  elseif dim==3
    ax,ay,az = a
    Mx,My,Mz = M
    X,Px = ladderops(Mx,ax,basis)
    Y,Py = ladderops(My,ay,basis)
    Z,Pz = ladderops(My,ay,basis)
    Lx = -im*(Y.*Pz - Z.*P̂y)
    Ly = -im*(Ẑ.*P̂x - X̂.*P̂z)
    Lz = -im*(X̂.*P̂y - Ŷ.*P̂x)
    return Lx,Ly,Lz
  end
else error("basis not implemented")
end
end


alpha = 0.85
list = "test/fixtures/dangling.adj"
order = 6

M, order, max_ordinal = fastadj(Float64, Int64, alpha, list, 8)

expected = sparse([
 0.0  0.85     0.0      0.0      0.0      0.0
 0.0  0.0      0.425    0.425    0.0      0.0
 0.0  0.28333  0.0      0.28333  0.28333  0.0
 0.0  0.28333  0.28333  0.0      0.28333  0.0
 0.0  0.0      0.28333  0.28333  0.0      0.28333
 0.0  0.0      0.0      0.0      0.0      0.0
])

difference = (max(dense(expected) - dense(M)))

@assert isequal(6, order)
@assert isequal(6, max_ordinal)
@assert isequal(0, difference)


@with_kw mutable struct AdagradParameters{T <: AbstractFloat} <: AbstractPolicyParameters
    ϵ::T = 1e-6
end

struct Adagrad <: AbstractSmoothing
    params::AdagradParameters{Float64}
    grms::Vector{Float64}

    function (::Type{Adagrad})(; kwargs...)
        return new(AdagradParameters(; kwargs...),Vector{Float64}())
    end
end

function initialize!(adagrad::Adagrad,x₀::Vector{Float64})
    resize!(adagrad.grms,length(x₀))
    adagrad.grms .= zeros(length(x₀))
end

function smooth!(adagrad::Adagrad,klocal::Integer,kglobal::Integer,x::AbstractVector,gprev::AbstractVector,gcurr::AbstractVector)
    @unpack ϵ = adagrad.params
    adagrad.grms .+= gprev .* gprev
    gcurr .= gprev ./ (.√adagrad.grms .+ ϵ)
end


using LinearAlgebra
using Nystrom
using StaticArrays
using Random
using Plots
plotlyjs()
Random.seed!(1)

## general definitions
xout = SVector(3,3)
ops = (
    Laplace(;dim=2),
    Helmholtz(;dim=2,k=1.2),
    Elastostatic(;dim=2,μ=2,λ=3)
)
methods = (
    Nystrom.assemble_dim,
    Nystrom.assemble_gk
)

pde      = ops[1]     # choose the pde
assemble = methods[1] # choose the singular integration method

T    = Nystrom.default_density_eltype(pde)
c    = rand(T)

## place to store dofs and error
ndofs = []
ee = []

npatches = [2,4,8,16,32,64,128,256,512]
qorder   =  5

for n in npatches
    ## generate mesh
    Geometry.clear_entities!()
    Ω   = ParametricSurfaces.Circle() |> Geometry.Domain
    Γ   = boundary(Ω)
    M   = meshgen(Γ,(n,))
    mesh = NystromMesh(view(M,Γ),order=qorder)
    push!(ndofs,length(Nystrom.dofs(mesh)))
    ## create exact solution
    u    = (qnode) -> SingleLayerKernel(pde)(xout,qnode)*c
    dudn = (qnode) -> transpose(DoubleLayerKernel(pde)(xout,qnode))*c
    γ₀u   = Density(u,mesh)
    γ₁u   = Density(dudn,mesh)
    γ₀u_norm = norm(norm.(γ₀u,Inf),Inf)
    # define integral operators
    Sop = SingleLayerOperator(pde,mesh)
    Dop = DoubleLayerOperator(pde,mesh)
    ## compute error
    S = assemble(Sop)
    D = assemble(Dop)
    e1    = Nystrom.error_interior_green_identity(S,D,γ₀u,γ₁u)/γ₀u_norm
    push!(ee,norm(e1,Inf))
end

fig = plot(ndofs,ee,xscale=:log10,yscale=:log10,m=:o,label="error",lc=:black);
plot!(fig,xlabel="n",ylabel="error");
for p in 1:5
    c = ee[end]*ndofs[end]^p
    plot!(fig,ndofs,c ./ ndofs.^(p),xscale=:log10,yscale=:log10,label="h^$p",ls=:dash);
end
display(fig)