# ---
# title: 131. Palindrome Partitioning
# id: problem131
# author: Indigo
# date: 2021-03-08
# difficulty: Medium
# categories: Backtracking
# link: <https://leetcode.com/problems/palindrome-partitioning/description/>
# hidden: true
# ---
#
# Given a string _s_ , partition _s_ such that every substring of the partition
# is a palindrome.
#
# Return all possible palindrome partitioning of _s_.
#
# **Example:**
#
#
#
#     Input:  "aab"
#     Output:
#     [
#       ["aa","b"],
#       ["a","a","b"]
#     ]
#
#
#
## @lc code=start
using LeetCode

function partition_ps(s::String)
    len = length(s)
    is_ps = fill(true, len, len)
    for i in len:-1:1, j in (i + 1):len
        is_ps[i, j] = (s[i] == s[j]) && is_ps[i + 1, j - 1]
    end
    function dfs(start::Int)
        start == len && return [[SubString(s, len, len)]]
        res = Vector{String}[]
        for ed in start:len
            !is_ps[start, ed] && continue
            for it in dfs(ed + 1)
                push!(res, pushfirst!(it, SubString(s, start, ed)))
            end
        end
        return res
    end
    return dfs(1)
end
## @lc code=end


# -*- coding: utf-8 -*-
# ---
# jupyter:
#   jupytext:
#     formats: ipynb,jl:light
#     text_representation:
#       extension: .jl
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.11.3
#   kernelspec:
#     display_name: Julia 1.7.2
#     language: julia
#     name: julia-1.7
# ---

# ## Outline
#
# 1. Start From Step 1 Result
# 1. Uncertainty Profiles
# 1. Seasonal Climatologies
# 1. MITprof File Creation

include("ArgoToMITprof_step1.jl")

using Plots
do_plot=true

# ## Load Gridded Fields
#
# - grid : ECCO global grid (LLC90; incl. land mask).
# - monthly T/S mean climatology (jan,feb, .. dec)
# - annual T/S variance climatology (non-seasonal)

gridded_fields=GriddedFields.load();

# ## Uncertainty Profiles
#
# The result is expressed as a least-squares weight.
#
# 1. spatial interpolation
# 2. vertical interpolation
# 3. combine instrumental and representation error
#
# **Note: σT,σS may still need to account for bounds at this point.**

# +
#1. spatial interpolation

(f,i,j,w)=GriddedFields.InterpolationFactors(gridded_fields.Γ,prof.lon,prof.lat)
📚=(f=f,i=i,j=j,w=w)

# +
prof_σT=[GriddedFields.Interpolate(gridded_fields.σT[:,k],📚.f,📚.i,📚.j,📚.w)[1] for k=1:50]
prof_σS=[GriddedFields.Interpolate(gridded_fields.σS[:,k],📚.f,📚.i,📚.j,📚.w)[1] for k=1:50]

prof_σT_step1=prof_σT #for visual verification

prof_σT[1:5]

# +
#2. vertical interpolation

z_std=meta["z_std"]
prof_σT=ArgoTools.interp_z(-gridded_fields.Γ.RC,prof_σT,z_std)
prof_σT[1:5]
# -

if do_plot
    plot(prof_σT_step1,gridded_fields.Γ.RC,label="native", legend = :bottomright,marker=:+, xlabel="degree K", ylabel="depth")
    plot!(prof_σT[1:55],-z_std[1:55],label="interpolated",marker=:x)
end

# +
#3. combine instrumental and representation error

prof_std.Tweight.=1 ./(prof_σT.^2 .+ prof_std.T_ERR.^2)
prof_std.Tweight[1:5]

# +
#4. salinity

prof_σS=ArgoTools.interp_z(-gridded_fields.Γ.RC,prof_σS,z_std)
prof_std.Sweight.=1 ./(prof_σS.^2 .+ prof_std.S_ERR.^2)
prof_std.Sweight[1:5]
# -

# ## Seasonal Climatology Profiles

# +
#4. spatio-temporal interpolation

fac,rec=ArgoTools.monthly_climatology_factors(prof.date[1])
# -

prof_T1=[GriddedFields.Interpolate(gridded_fields.T[rec[1]][:,k],📚.f,📚.i,📚.j,📚.w)[1] for k=1:50]
prof_T2=[GriddedFields.Interpolate(gridded_fields.T[rec[2]][:,k],📚.f,📚.i,📚.j,📚.w)[1] for k=1:50];

prof_std.Testim.=ArgoTools.interp_z(-gridded_fields.Γ.RC,fac[1]*prof_T1+fac[2]*prof_T2,z_std)
prof_std.Testim[1:5]

if do_plot
    plot(prof_std.Testim,-z_std,marker=:x,label="climatology", legend = :bottomright, xlabel="degree C", ylabel="depth")
    plot!(prof_std.T[1:55],-z_std[1:55],marker=:o,label="data")
end

# +
#4. spatio-temporal interpolation

prof_S1=[GriddedFields.Interpolate(gridded_fields.S[rec[1]][:,k],📚.f,📚.i,📚.j,📚.w)[1] for k=1:50]
prof_S2=[GriddedFields.Interpolate(gridded_fields.S[rec[2]][:,k],📚.f,📚.i,📚.j,📚.w)[1] for k=1:50];
# -

prof_std.Sestim.=ArgoTools.interp_z(-gridded_fields.Γ.RC,fac[1]*prof_S1+fac[2]*prof_S2,z_std)
prof_std.Sestim[1:5]

ArgoTools.prof_test_set2!(prof_std,meta);

# ## MITprof File Creation

# +
meta["fileOut"]=joinpath(tempdir(),"MITprof_example.nc")
isfile(meta["fileOut"]) ? rm(meta["fileOut"], force=true) : nothing

MITprof.write(meta,[prof],[prof_std])
# -

# ### Show Global Maps (via interpolation)

# +
function interpolate_map(Γ)
    lon=[i for i=-179.5:1.0:179.5, j=-89.5:1.0:89.5]
    lat=[j for i=-179.5:1.0:179.5, j=-89.5:1.0:89.5]
    (f,i,j,w)=GriddedFields.InterpolationFactors(Γ,vec(lon),vec(lat))
    (f=f,i=i,j=j,w=w,lon=vec(lon[:,1]),lat=vec(lat[1,:]))
end

🌐=interpolate_map(gridded_fields.Γ);
# -

ni,nj=length(🌐.lon),length(🌐.lat)
σTm=Array{Float64,3}(undef,(ni,nj,50))
σSm=Array{Float64,3}(undef,(ni,nj,50))
for k=1:50
    tmp=GriddedFields.Interpolate(gridded_fields.σT[:,k],🌐.f,🌐.i,🌐.j,🌐.w)
    σTm[:,:,k]=reshape(tmp,(ni,nj))
    tmp=GriddedFields.Interpolate(gridded_fields.σS[:,k],🌐.f,🌐.i,🌐.j,🌐.w)
    σSm[:,:,k]=reshape(tmp,(ni,nj))
end

if do_plot
    contourf(🌐.lon,🌐.lat,log10.(transpose(σTm[:,:,20]).^2),clims=(-2.5,1.0),title="log10(temperature variance) at 300m")
end

if do_plot
    tmp1=GriddedFields.Interpolate(gridded_fields.T[6][:,20],🌐.f,🌐.i,🌐.j,🌐.w)
    contourf(🌐.lon,🌐.lat,tmp1,clims=(-2.0,20.0),title="temperature in June at 300m")
end


using LinearAlgebraicRepresentation
Lar = LinearAlgebraicRepresentation

"""
	t(args::Array{Number,1}...)::Matrix

Return an *affine transformation Matrix* in homogeneous coordinates. Such `translation` Matrix has ``d+1`` rows and ``d+1`` columns, where ``d`` is the number of translation parameters in the `args` array.

# Examples

```julia
julia> t(1,2)			# 2D translation
# return
3×3 Array{Float64,2}:
 1.0  0.0  1.0
 0.0  1.0  2.0
 0.0  0.0  1.0

julia> Lar.t(1.,2,3)		# 3D translation
# return
4×4 Array{Float64,2}:
 1.0  0.0  0.0  1.0
 0.0  1.0  0.0  2.0
 0.0  0.0  1.0  3.0
 0.0  0.0  0.0  1.0
```
"""
function t(args...)
	d = length(args)
	mat = Matrix{Float64}(LinearAlgebra.I, d+1, d+1)
	for k in range(1, length=d)
        	mat[k,d+1]=args[k]
	end
	return mat
end

"""
	s(args::Array{Number,1}...)::Matrix

Return an *affine transformation Matrix* in homogeneous coordinates. Such `scaling` Matrix has ``d+1`` rows and ``d+1`` columns, where ``d`` is the number of scaling parameters in the `args` array.

# Examples

```julia
julia> Lar.s(2,3)			# 2D scaling
# return
3×3 Array{Float64,2}:
 2.0  0.0  0.0
 0.0  3.0  0.0
 0.0  0.0  1.0

julia> Lar.s(2.,3.,4.)		# 3D scaling
# return
4×4 Array{Float64,2}:
 2.0  0.0  0.0  0.0
 0.0  3.0  0.0  0.0
 0.0  0.0  4.0  0.0
 0.0  0.0  0.0  1.0
```
"""
function s(args...)
	d = length(args)
	mat = Matrix{Float64}(LinearAlgebra.I, d+1, d+1)
	for k in range(1, length=d)
		mat[k,k]=args[k]
	end
	return mat
end

"""
	r(args...)

Return an *affine transformation Matrix* in homogeneous coordinates. Such `Rotation` Matrix has *dimension* either equal to 3 or to 4, for 2D and 3D rotation, respectively.
The `{Number,1}` of `args` either contain a single `angle` parameter in *radiants*, or a vector with three elements, whose `norm` is the *rotation angle* in 3D and whose `normalized value` gives the direction of the *rotation axis* in 3D.

# Examples

```julia
julia> Lar.r(pi/6)				# 2D rotation of ``π/6`` angle
# return
3×3 Array{Float64,2}:
 0.866025  -0.5       0.0
 0.5        0.866025  0.0
 0.0        0.0       1.0

julia> Lar.r(0,0,pi/4)
# return
4×4 Array{Float64,2}:		# 3D rotation about the ``z`` axis, with ``π/6`` angle
 0.707107  -0.707107  0.0  0.0
 0.707107   0.707107  0.0  0.0
 0.0        0.0       1.0  0.0
 0.0        0.0       0.0  1.0

julia> Lar.r(1,1,1)		# 3D rotation about the ``x=y=z`` axis, with angle ``1.7320508`` angle
# return
4×4 Array{Float64,2}:
  0.226296  -0.183008   0.956712  0.0
  0.956712   0.226296  -0.183008  0.0
 -0.183008   0.956712   1.21332   0.0
  0.0        0.0        0.0       1.0

```
"""
function r(args...)
    args = collect(args)
    n = length(args)
    if n == 1 # rotation in 2D
        angle = args[1]; COS = cos(angle); SIN = sin(angle)
        mat = Matrix{Float64}(LinearAlgebra.I, 3, 3)
        mat[1,1] = COS;    mat[1,2] = -SIN;
        mat[2,1] = SIN;    mat[2,2] = COS;
    end

     if n == 3 # rotation in 3D
        mat = Matrix{Float64}(LinearAlgebra.I, 4, 4)
        angle = norm(args);
        if norm(args) != 0.0
			axis = args #normalize(args)
			COS = cos(angle); SIN= sin(angle)
			if axis[2]==axis[3]==0.0    # rotation about x
				mat[2,2] = COS;    mat[2,3] = -SIN;
				mat[3,2] = SIN;    mat[3,3] = COS;
			elseif axis[1]==axis[3]==0.0   # rotation about y
				mat[1,1] = COS;    mat[1,3] = SIN;
				mat[3,1] = -SIN;    mat[3,3] = COS;
			elseif axis[1]==axis[2]==0.0    # rotation about z
				mat[1,1] = COS;    mat[1,2] = -SIN;
				mat[2,1] = SIN;    mat[2,2] = COS;
			else
				I = Matrix{Float64}(LinearAlgebra.I, 3, 3); u = axis
				Ux=[0 -u[3] u[2] ; u[3] 0 -u[1] ;  -u[2] u[1] 1]
				UU =[u[1]*u[1]    u[1]*u[2]   u[1]*u[3];
					 u[2]*u[1]    u[2]*u[2]   u[2]*u[3];
					 u[3]*u[1]    u[3]*u[2]   u[3]*u[3]]
				mat[1:3,1:3]=COS*I+SIN*Ux+(1.0-COS)*UU
			end
		end
	end
	return mat
end

"""
	removeDups(CW::Cells)::Cells

Remove dublicate `cells` from `Cells` object. Then put `Cells` in *canonical form*, i.e. with *sorted indices* of vertices in each (unique) `Cells` Array element.
"""
function removeDups(CW::Cells)::Cells
	CW = collect(Set(CW))
	CWs = collect(map(sort,CW))
	return CWs
end

"""
	Struct

A *container* of geometrical objects is defined by applying the function `Struct` to
the array of contained objects. Each value is defined in local coordinates and may be transformed by affine transformation tensors.

The value returned from the application of `Struct` to an `Array{Any, 1}` of `LAR` values, `matrices`, and `Struct` values is a value of
`Struct type`.  The coordinate system of this value is the one associated with the first object of the `Struct` arguments.  Also,
the resulting geometrical value is often associated with a variable name.

The generation of containers may continue hierarchically by suitably applying `Struct`. Notice that each LAR object in a `Struct` container is transformed by each matrix before it *within the container*, going from right to left. The action of a transformation (tensor) extends to each object on its right within its own container. Whereas,  the action of a tensor does not extend outside its container, according to the semantics of *PHIGS* structures.

# Example

```julia
julia> L = LinearAlgebraicRepresentation;

julia> assembly = L.Struct([L.sphere()(), L.t(3,0,-1), L.cylinder()()])
# return
Lar.Struct(Any[([0.0 -0.173648 … -0.336824 -0.17101; 0.0 0.0 … 0.0593912 0.0301537;
-1.0 -0.984808 … 0.939693 0.984808], Array{Int64,1}[[2, 3, 1], [4, 2, 3], [4, 3, 5], [4,
5, 6], [7, 5, 6], [7, 8, 6], [7, 9, 8], … , [1.0 0.0 0.0 3.0; 0.0 1.0 0.0 0.0; 0.0 0.0 1.0
-1.0; 0.0 0.0 0.0 1.0], ([0.5 0.5 … 0.492404 0.492404; 0.0 0.0 … -0.0868241 -0.0868241;
0.0 2.0 … 0.0 2.0], Array{Int64,1}[[4, 2, 3, 1], [4, 3, 5, 6], [7, 5, 8, 6], [7, 9, 10,
8], [9, 10, 11, 12], [13, 14, 11, 12], … , [68, 66, 67, 65], [68, 69, 67, 70], [71, 69,
72, 70], [71, 2, 72, 1]])], Array{Float64,2}[[-1.0; -1.0; -1.0], [3.5; 1.0; 1.0]],
"14417445522513259426", 3, "feature")

julia> assembly.name = "simple example"
# return
"simple example"

julia> assembly
# return
Lar.Struct(Any[([0.0 -0.173648 … -0.336824 -0.17101; 0.0 0.0 … 0.0593912 0.0301537;
-1.0 -0.984808 … 0.939693 0.984808], Array{Int64,1}[[2, 3, 1], [4, 2, 3], [4, 3, 5], [4,
5, 6], [7, 5, 6], [7, 8, 6], … , [71, 2, 72, 1]])], Array{Float64,2}[[-1.0; -1.0; -1.0],
[3.5; 1.0; 1.0]], "simple example", 3, "feature")

julia> using Plasm

julia> Plasm.view(assembly)
```
"""
mutable struct Struct
	body::Array
	box
	name::AbstractString
	dim
	category::AbstractString

	function Struct()
		self = new([],Any,"new",Any,"feature")
		self.name = string(objectid(self))
		return self

	end

	function Struct(data::Array)
		self = Struct()
		self.body = data
		self.box = box(self)
		self.dim = length(self.box[1])
		return self
	end

	function Struct(data::Array,name)
		self = Struct()
		self.body=[item for item in data]
		self.box = box(self)
		self.dim = length(self.box[1])
		self.name = string(name)
		return self
	end

	function Struct(data::Array,name,category)
		self = Struct()
		self.body = [item for item in data]
		self.box = box(self)
		self.dim = length(self.box[1])
		self.name = string(name)
		self.category = string(category)
		return self
	end

end

	function name(self::Struct)
		return self.name
	end
	function category(self::Struct)
		return self.category
	end

	function len(self::Struct)
		return length(self.body)
	end
	function getitem(self::Struct,i::Int)
		return self.body[i]
	end
	function setitem(self::Struct,i,value)
		self.body[i]=value
	end
	function pprint(self::Struct)
		return "<Struct name: $(self.name)"
	end
	function set_name(self::Struct,name)
		self.name = string(name)
	end
	function clone(self::Struct,i=0)
		newObj = deepcopy(self)
		if i!=0
			newObj.name="$(self.name)_$(string(i))"
		end
		return newObj
	end
	function set_category(self::Struct,category)
		self.category = string(category)
	end

"""
	struct2lar(structure::Struct)::Union{LAR,LARmodel}

"""
function struct2lar(structure) # TODO: extend to true `LARmodels`
	listOfModels = Lar.evalStruct(structure)
	vertDict= Dict()
	index,defaultValue = 0,0
	W = Array{Float64,1}[]
	m = length(listOfModels[1])
	larmodel = [Array{Number,1}[] for k=1:m]

	for model in listOfModels
		V = model[1]
		for k=2:m
			for incell in model[k]
				outcell=[]
				for v in incell
					key = map(Lar.approxVal(7), V[:,v])
					if get(vertDict,key,defaultValue)==defaultValue
						index += 1
						vertDict[key]=index
						push!(outcell,index)
						push!(W,key)
					else
						push!(outcell,vertDict[key])
					end
				end
				append!(larmodel[k],[outcell])
			end
		end
	end

	append!(larmodel[1], W)
	V = hcat(larmodel[1]...)
	chains = [convert(Lar.Cells, chain) for chain in larmodel[2:end]]
	return (V, chains...)
end

"""
	embedTraversal(cloned::Struct,obj::Struct,n::Int,suffix::String)

# TODO:  debug embedTraversal
"""
function embedTraversal(cloned::Struct,obj::Struct,n::Int,suffix::String)
	for i=1:length(obj.body)
		if isa(obj.body[i],Matrix)
			mat = obj.body[i]
			d,d = size(mat)
			newMat = Matrix{Float64}(LinearAlgebra.I, d+n, d+n)
			for h in range(1, length=d)
				for k in range(1, length=d)
					newMat[h,k]=mat[h,k]
				end
			end
			push!(cloned.body,[newMat])
		elseif (isa(obj.body[i],Tuple) ||isa(obj.body[i],Array))&& length(obj.body[i])==3
			V,FV,EV = deepcopy(obj.body[i])
			dimadd = n
			ncols = size(V,2)
			nmat = zeros(dimadd,ncols)
			V = [V;zeros(dimadd,ncols)]
			push!(cloned.body,[(V,FV,EV)])
		elseif  (isa(obj.body[i],Tuple) ||isa(obj.body[i],Array))&& length(obj.body[i])==2
			V,EV = deepcopy(obj.body[i])
			dimadd = n
			ncols = size(V,2)
			nmat = zeros(dimadd,ncols)
			V = [V;zeros(dimadd,ncols)]
			push!(cloned.body,[(V,EV)])
		elseif isa(obj.body[i],Struct)
			newObj = Struct()
			newObj.box = [ [obj.body[i].box[1];zeros(dimadd)],
							[obj.body[i].box[2];zeros(dimadd)] ]
			newObj.category = (obj.body[i]).category
			push!(cloned.body,[embedTraversal(newObj,obj.body[i],obj.dim+n,suffix)])
		end
	end
	return cloned
end

"""
	embedStruct(n::Int)(self::Struct,suffix::String="New")

# TODO:  debug embedStruct
"""
function embedStruct(n::Int)
	function embedStruct0(self::Struct,suffix::String="New")
		if n==0
			return self, length(self.box[1])
		end
		cloned = Struct()
		cloned.box = [ [self.body[i].box[1];zeros(dimadd)],
						[self.body[i].box[2];zeros(dimadd)] ]
		cloned.name = self.name*suffix
		cloned.category = self.category
		cloned.dim = self.dim+n
		cloned = embedTraversal(cloned,self,n,suffix)
		return cloned
	end
	return embedStruct0
end

"""
	box(model)

"""
function box(model)
	if isa(model,Matrix)
		return nothing
	elseif isa(model,Struct)
		listOfModels = evalStruct(model)
		#dim = checkStruct(listOfModels)
		if listOfModels == []
			return model.box
		else
			theMin,theMax = box(listOfModels[1])
			for theModel in listOfModels[2:end]
				modelMin,modelMax= box(theModel)
				for (k,val) in enumerate(modelMin)
					if val < theMin[k]
						theMin[k]=val
					end
				end
				for (k,val) in enumerate(modelMax)
					if val > theMax[k]
						theMax[k]=val
					end
				end
			end
		end
		return [theMin,theMax]

	elseif (isa(model,Tuple) ||isa(model,Array))&& (length(model)>=2)
		V = model[1]
		theMin = minimum(V, dims=2)
		theMax = maximum(V, dims=2)
	end

	return [theMin,theMax]
end

"""
	apply(affineMatrix::Array{Float64,2}, larmodel::Union{LAR,LARmodel})
"""
function apply(affineMatrix, larmodel)
	data = collect(larmodel)
	V = data[1]

	m,n = size(V)
	W = [V; fill(1.0, (1,n))]
	V = (affineMatrix * W)[1:m,1:n]

	data[1] = V
	larmodel = Tuple(data)
	return larmodel
end

"""
	checkStruct(lst)

"""
function checkStruct(lst)
	obj = lst[1]
	if isa(obj,Matrix)
		dim = size(obj,1)-1
	elseif (isa(obj,Tuple) || isa(obj,Array))
		dim = length(obj[1][:,1])

	elseif isa(obj,Struct)
		dim = length(obj.box[1])
	end
	return dim
end

"""
	traversal(CTM,stack,obj,scene=[])

"""
function traversal(CTM::Matrix, stack, obj, scene=[])
	for i = 1:length(obj.body)
		if isa(obj.body[i],Matrix)
			CTM = CTM*obj.body[i]
		elseif (isa(obj.body[i],Tuple) || isa(obj.body[i],Array)) &&
			(length(obj.body[i])>=2)
			l = apply(CTM, obj.body[i])
			push!(scene,l)
		elseif isa(obj.body[i],Struct)
			push!(stack,CTM)
			traversal(CTM,stack,obj.body[i],scene)
			CTM = pop!(stack)
		end
	end
	return scene
end

"""
	evalStruct(self)

"""
function evalStruct(self::Struct)
	dim = checkStruct(self.body)
   	CTM, stack = Matrix{Float64}(LinearAlgebra.I, dim+1, dim+1), []
   	scene = traversal(CTM, stack, self, [])
	return scene
end


function noise_threshold(x,th = 0.4)
    if x<=th
        return 0
    else
        return 1
    end
end
"""
Pixels with no economic activity may show some light due to background noise. These pixels could be in forests, oceans, deserts etc. The ```background_noise_mask``` function generates a background moise mask such that those pixels which are considered dark are marked as 0 and those considered lit are marked as 1. The function uses the datacubes of radiance and clouds to generate annual image of the last year the data. The function considers all the pixels below a provided threshold as dark and remaining to be lit.
```julia
background_noise_mask(radiance_datacube, clouds_datacube)
```
"""
function background_noise_mask(radiance_datacube, clouds_datacube, th = 0.4)
    # This function may be obsolete because Payne Institute is providing annual images for each year.
    last_year_rad      = radiance_datacube[:, :, (size(radiance_datacube)[3]-11):size(radiance_datacube)[3]]
    last_year_cloud   = clouds_datacube[:, :, (size(radiance_datacube)[3]-11):size(radiance_datacube)[3]]
    average_lastyear = copy(radiance_datacube[:, :, 1])
    for i in 1:size(last_year_rad)[1]
        for j in 1:size(last_year_rad)[2]
            average_lastyear[i,j] = weighted_mean(last_year_rad[i, j, :], last_year_cloud[i, j, :])
        end
    end
    mask = noise_threshold.(average_lastyear, th)
    return mask
end


@testset "chainrules" begin
    Random.seed!(1)

    @testset "general" begin
        for x in (1.0, -1.0, 0.0, 0.5, 10.0, -17.1, 1.5 + 0.7im)
            test_scalar(erf, x)
            test_scalar(erfc, x)
            test_scalar(erfi, x)

            test_scalar(airyai, x)
            test_scalar(airyaiprime, x)
            test_scalar(airybi, x)
            test_scalar(airybiprime, x)

            test_scalar(besselj0, x)
            test_scalar(besselj1, x)

            test_scalar(erfcx, x)
            test_scalar(dawson, x)

            if x isa Real
                test_scalar(invdigamma, x)
            end

            if x isa Real && 0 < x < 1
                test_scalar(erfinv, x)
                test_scalar(erfcinv, x)
            end

            if x isa Real && x > 0 || x isa Complex
                test_scalar(bessely0, x)
                test_scalar(bessely1, x)
                test_scalar(gamma, x)
                test_scalar(digamma, x)
                test_scalar(trigamma, x)
            end
        end

        @testset "beta and logbeta" begin
            test_points = (1.5, 2.5, 10.5, 1.6 + 1.6im, 1.6 - 1.6im, 4.6 + 1.6im)
            for _x in test_points, _y in test_points
                # ensure all complex if any complex for FiniteDifferences
                x, y = promote(_x, _y)
                T = typeof(x)

                Δx, x̄ = randn(T, 2)
                Δy, ȳ = randn(T, 2)
                Δz = randn(T)

                frule_test(beta, (x, Δx), (y, Δy))
                rrule_test(beta, Δz, (x, x̄), (y, ȳ))

                frule_test(logbeta, (x, Δx), (y, Δy))
                rrule_test(logbeta, Δz, (x, x̄), (y, ȳ))
            end
        end

        @testset "log gamma and co" begin
            # It is important that we have negative numbers with both odd and even integer parts
            for x in (1.5, 2.5, 10.5, -0.6, -2.6, -3.3, 1.6 + 1.6im, 1.6 - 1.6im, -4.6 + 1.6im)
                isreal(x) && x < 0 && continue
                test_scalar(loggamma, x)

                isreal(x) || continue

                Δx, x̄ = randn(2)
                Δz = (randn(), randn())

                # frule_test(logabsgamma, (x, Δx))   # uncomment when #288 is fixed
                rrule_test(logabsgamma, Δz, (x, x̄))
            end
        end
    end
end


@testset "density1D" begin
    df = (x=rand(1000), c=rand(["a", "b"], 1000))
    npoints = 500

    layer = data(df) * mapping(:x, color=:c) * AlgebraOfGraphics.density(; npoints)
    processedlayer = AlgebraOfGraphics.ProcessedLayer(layer)

    x1 = df.x[df.c .== "a"]
    rgx1 = range(extrema(df.x)..., length=npoints)
    d1 = pdf(kde(x1), rgx1)

    x2 = df.x[df.c .== "b"]
    rgx2 = range(extrema(df.x)..., length=npoints)
    d2 = pdf(kde(x2), rgx2)

    rgx, d = processedlayer.positional

    @test rgx[1] ≈ rgx1
    @test d[1] ≈ d1

    @test rgx[2] ≈ rgx2
    @test d[2] ≈ d2

    layer = data(df) * mapping(:x, color=:c) * AlgebraOfGraphics.density(; npoints, datalimits=extrema)
    processedlayer = AlgebraOfGraphics.ProcessedLayer(layer)

    x1 = df.x[df.c .== "a"]
    rgx1 = range(extrema(x1)..., length=npoints)
    d1 = pdf(kde(x1), rgx1)

    x2 = df.x[df.c .== "b"]
    rgx2 = range(extrema(x2)..., length=npoints)
    d2 = pdf(kde(x2), rgx2)

    rgx, d = processedlayer.positional

    @test rgx[1] ≈ rgx1
    @test d[1] ≈ d1

    @test rgx[2] ≈ rgx2
    @test d[2] ≈ d2

    @test processedlayer.primary == NamedArguments((color=["a", "b"],))
    @test isempty(processedlayer.named)
    @test processedlayer.attributes == NamedArguments()
    @test processedlayer.plottype == AlgebraOfGraphics.LinesFill

    labels = MixedArguments()
    insert!(labels, 1, "x")
    insert!(labels, :color, "c")
    insert!(labels, 2, "pdf")
    @test labels == map(AlgebraOfGraphics.to_label, processedlayer.labels)
end

@testset "density2d" begin
    df = (x=rand(1000), y=rand(1000), c=rand(["a", "b"], 1000))
    npoints = 500
    bandwidth = (0.01, 0.01)

    layer = data(df) * mapping(:x, :y, color=:c) * AlgebraOfGraphics.density(; npoints, bandwidth)
    processedlayer = AlgebraOfGraphics.ProcessedLayer(layer)

    x1 = df.x[df.c .== "a"]
    y1 = df.y[df.c .== "a"]
    rgx1 = range(extrema(df.x)..., length=npoints)
    rgy1 = range(extrema(df.y)..., length=npoints)
    d1 = pdf(kde((x1, y1); bandwidth), rgx1, rgy1)

    x2 = df.x[df.c .== "b"]
    y2 = df.y[df.c .== "b"]
    rgx2 = range(extrema(df.x)..., length=npoints)
    rgy2 = range(extrema(df.y)..., length=npoints)
    d2 = pdf(kde((x2, y2); bandwidth), rgx2, rgy2)

    rgx, rgy, d = processedlayer.positional

    @test rgx[1] ≈ rgx1
    @test rgy[1] ≈ rgy1
    @test d[1] ≈ d1

    @test rgx[2] ≈ rgx2
    @test rgy[2] ≈ rgy2
    @test d[2] ≈ d2

    @test processedlayer.primary == NamedArguments((color=["a", "b"],))
    @test isempty(processedlayer.named)
    @test processedlayer.attributes == NamedArguments()
    @test processedlayer.plottype == Heatmap

    labels = MixedArguments()
    insert!(labels, 1, "x")
    insert!(labels, 2, "y")
    insert!(labels, :color, "c")
    insert!(labels, 3, "pdf")
    @test labels == map(AlgebraOfGraphics.to_label, processedlayer.labels)
end

@testset "expectation1d" begin
    df = (x=rand(["a", "b"], 1000), y=rand(1000), c=rand(["a", "b"], 1000))

    layer = data(df) * mapping(:x, :y, layout=:c) * expectation()
    processedlayer = AlgebraOfGraphics.ProcessedLayer(layer)

    ma1 = mean(df.y[(df.x .== "a") .& (df.c .== "a")])
    mb1 = mean(df.y[(df.x .== "b") .& (df.c .== "a")])

    ma2 = mean(df.y[(df.x .== "a") .& (df.c .== "b")])
    mb2 = mean(df.y[(df.x .== "b") .& (df.c .== "b")])

    x, m = processedlayer.positional
    x1, m1 = x[1], m[1]
    x2, m2 = x[2], m[2]

    @test x1 == ["a", "b"]
    @test m1 ≈ [ma1, mb1]

    @test x2 == ["a", "b"]
    @test m2 ≈ [ma2, mb2]

    @test processedlayer.primary == NamedArguments((layout=["a", "b"],))
    @test isempty(processedlayer.named)
    @test processedlayer.attributes == NamedArguments()
    @test processedlayer.plottype == BarPlot

    labels = MixedArguments()
    insert!(labels, 1, "x")
    insert!(labels, 2, "y")
    insert!(labels, :layout, "c")
    @test labels == map(AlgebraOfGraphics.to_label, processedlayer.labels)
end

@testset "expectation2d" begin
    df = (x=rand(["a", "b"], 1000), y=rand(["a", "b"], 1000), z=rand(1000), c=rand(["a", "b"], 1000))

    layer = data(df) * mapping(:x, :y, :z, layout=:c) * expectation()
    processedlayer = AlgebraOfGraphics.ProcessedLayer(layer)

    maa1 = mean(df.z[(df.x .== "a") .& (df.y .== "a") .& (df.c .== "a")])
    mab1 = mean(df.z[(df.x .== "a") .& (df.y .== "b") .& (df.c .== "a")])
    mba1 = mean(df.z[(df.x .== "b") .& (df.y .== "a") .& (df.c .== "a")])
    mbb1 = mean(df.z[(df.x .== "b") .& (df.y .== "b") .& (df.c .== "a")])

    maa2 = mean(df.z[(df.x .== "a") .& (df.y .== "a") .& (df.c .== "b")])
    mab2 = mean(df.z[(df.x .== "a") .& (df.y .== "b") .& (df.c .== "b")])
    mba2 = mean(df.z[(df.x .== "b") .& (df.y .== "a") .& (df.c .== "b")])
    mbb2 = mean(df.z[(df.x .== "b") .& (df.y .== "b") .& (df.c .== "b")])

    x, y, m = processedlayer.positional
    x1, y1, m1 = x[1], y[1], m[1]
    x2, y2, m2 = x[2], y[2], m[2]

    @test x1 == ["a", "b"]
    @test y1 == ["a", "b"]
    @test m1 ≈ [maa1 mab1; mba1 mbb1]

    @test x2 == ["a", "b"]
    @test y2 == ["a", "b"]
    @test m2 ≈ [maa2 mab2; mba2 mbb2]

    @test processedlayer.primary == NamedArguments((layout=["a", "b"],))
    @test isempty(processedlayer.named)
    @test processedlayer.attributes == NamedArguments()
    @test processedlayer.plottype == Heatmap

    labels = MixedArguments()
    insert!(labels, 1, "x")
    insert!(labels, 2, "y")
    insert!(labels, 3, "z")
    insert!(labels, :layout, "c")
    @test labels == map(AlgebraOfGraphics.to_label, processedlayer.labels)
end

@testset "frequency1d" begin
    df = (x=rand(["a", "b"], 1000), c=rand(["a", "b"], 1000))

    layer = data(df) * mapping(:x, layout=:c) * frequency()
    processedlayer = AlgebraOfGraphics.ProcessedLayer(layer)

    na1 = sum((df.x .== "a") .& (df.c .== "a"))
    nb1 = sum((df.x .== "b") .& (df.c .== "a"))

    na2 = sum((df.x .== "a") .& (df.c .== "b"))
    nb2 = sum((df.x .== "b") .& (df.c .== "b"))

    x, n = processedlayer.positional
    x1, n1 = x[1], n[1]
    x2, n2 = x[2], n[2]

    @test x1 == ["a", "b"]
    @test n1 ≈ [na1, nb1]

    @test x2 == ["a", "b"]
    @test n2 ≈ [na2, nb2]

    @test processedlayer.primary == NamedArguments((layout=["a", "b"],))
    @test isempty(processedlayer.named)
    @test processedlayer.attributes == NamedArguments()
    @test processedlayer.plottype == BarPlot

    labels = MixedArguments()
    insert!(labels, 1, "x")
    insert!(labels, :layout, "c")
    insert!(labels, 2, "count")
    @test labels == map(AlgebraOfGraphics.to_label, processedlayer.labels)
end

@testset "frequency2d" begin
    df = (x=rand(["a", "b"], 1000), y=rand(["a", "b"], 1000), c=rand(["a", "b"], 1000))

    layer = data(df) * mapping(:x, :y, layout=:c) * frequency()
    processedlayer = AlgebraOfGraphics.ProcessedLayer(layer)

    naa1 = sum((df.x .== "a") .& (df.y .== "a") .& (df.c .== "a"))
    nab1 = sum((df.x .== "a") .& (df.y .== "b") .& (df.c .== "a"))
    nba1 = sum((df.x .== "b") .& (df.y .== "a") .& (df.c .== "a"))
    nbb1 = sum((df.x .== "b") .& (df.y .== "b") .& (df.c .== "a"))

    naa2 = sum((df.x .== "a") .& (df.y .== "a") .& (df.c .== "b"))
    nab2 = sum((df.x .== "a") .& (df.y .== "b") .& (df.c .== "b"))
    nba2 = sum((df.x .== "b") .& (df.y .== "a") .& (df.c .== "b"))
    nbb2 = sum((df.x .== "b") .& (df.y .== "b") .& (df.c .== "b"))

    x, y, n = processedlayer.positional
    x1, y1, n1 = x[1], y[1], n[1]
    x2, y2, n2 = x[2], y[2], n[2]

    @test x1 == ["a", "b"]
    @test y1 == ["a", "b"]
    @test n1 ≈ [naa1 nab1; nba1 nbb1]

    @test x2 == ["a", "b"]
    @test y2 == ["a", "b"]
    @test n2 ≈ [naa2 nab2; nba2 nbb2]

    @test processedlayer.primary == NamedArguments((layout=["a", "b"],))
    @test isempty(processedlayer.named)
    @test processedlayer.attributes == NamedArguments()
    @test processedlayer.plottype == Heatmap

    labels = MixedArguments()
    insert!(labels, 1, "x")
    insert!(labels, 2, "y")
    insert!(labels, :layout, "c")
    insert!(labels, 3, "count")
    @test labels == map(AlgebraOfGraphics.to_label, processedlayer.labels)
end

@testset "midpoints" begin
    edges = [1, 2, 10, 12]
    @test midpoints(edges) ≈ [1.5, 6, 11]

    edges_rg = 1:2:5
    edges_v = [1, 3, 5]
    @test midpoints(edges_v) ≈ midpoints(edges_rg) ≈ [2, 4]
end

@testset "histogram1D" begin
    df = (x=rand(1000), c=rand(["a", "b"], 1000))
    bins = 0:0.01:1

    layer = data(df) * mapping(:x, color=:c) * histogram(; bins)
    processedlayer = AlgebraOfGraphics.ProcessedLayer(layer)

    x1 = df.x[df.c .== "a"]
    w1 = fit(Histogram, x1, bins).weights

    x2 = df.x[df.c .== "b"]
    w2 = fit(Histogram, x2, bins).weights

    rgx, w = processedlayer.positional
    width = processedlayer.named[:width]

    @test rgx[1] ≈ (bins[1:end-1] .+ bins[2:end]) ./ 2
    @test w[1] == w1
    @test width[1] ≈ diff(bins)

    @test rgx[2] ≈ (bins[1:end-1] .+ bins[2:end]) ./ 2
    @test w[2] == w2
    @test width[2] ≈ diff(bins)

    bins, closed = 12, :left
    layer = data(df) * mapping(:x, color=:c) * histogram(; bins, closed, datalimits=extrema)
    processedlayer = AlgebraOfGraphics.ProcessedLayer(layer)

    x1 = df.x[df.c .== "a"]
    bins1 = histrange(extrema(x1)..., bins, closed)
    w1 = fit(Histogram, x1, bins1).weights

    x2 = df.x[df.c .== "b"]
    bins2 = histrange(extrema(x2)..., bins, closed)
    w2 = fit(Histogram, x2, bins2).weights

    rgx, w = processedlayer.positional
    width = processedlayer.named[:width]

    @test rgx[1] ≈ (bins1[1:end-1] .+ bins1[2:end]) ./ 2
    @test w[1] ≈ w1
    @test width[1] ≈ diff(bins1)

    @test rgx[2] ≈ (bins2[1:end-1] .+ bins2[2:end]) ./ 2
    @test w[2] ≈ w2
    @test width[2] ≈ diff(bins2)

    @test processedlayer.primary == NamedArguments((color=["a", "b"],))
    @test processedlayer.attributes == NamedArguments((gap=0, dodge_gap=0))
    @test keys(processedlayer.named) == Indices([:width])
    @test processedlayer.plottype == AlgebraOfGraphics.BarPlot

    labels = MixedArguments()
    insert!(labels, 1, "x")
    insert!(labels, :color, "c")
    insert!(labels, 2, "count")
    @test labels == map(AlgebraOfGraphics.to_label, processedlayer.labels)

    bins = 12.3
    layer = data(df) * mapping(:x, color=:c) * histogram(; bins)
    @test_throws ArgumentError AlgebraOfGraphics.ProcessedLayer(layer)
end

@testset "weightedhistogram1d" begin
    df = (x=rand(1000), z=rand(1000), c=rand(["a", "b"], 1000))
    bins = collect(0:0.01:1) # test vector of bins

    layer = data(df) * mapping(:x, color=:c, weights=:z) * histogram(; bins)
    processedlayer = AlgebraOfGraphics.ProcessedLayer(layer)

    x1 = df.x[df.c .== "a"]
    z1 = df.z[df.c .== "a"]
    w1 = fit(Histogram, x1, weights(z1), bins).weights

    x2 = df.x[df.c .== "b"]
    z2 = df.z[df.c .== "b"]
    w2 = fit(Histogram, x2, weights(z2), bins).weights

    rgx, w = processedlayer.positional
    width = processedlayer.named[:width]

    @test rgx[1] ≈ (bins[1:end-1] .+ bins[2:end]) ./ 2
    @test w[1] ≈ w1
    @test width[1] ≈ diff(bins)

    @test rgx[2] ≈ (bins[1:end-1] .+ bins[2:end]) ./ 2
    @test w[2] ≈ w2
    @test width[2] ≈ diff(bins)

    @test processedlayer.primary == NamedArguments((color=["a", "b"],))
    @test keys(processedlayer.named) == Indices([:width])
    @test processedlayer.attributes == NamedArguments((gap=0, dodge_gap=0))
    @test processedlayer.plottype == AlgebraOfGraphics.BarPlot

    labels = MixedArguments()
    insert!(labels, 1, "x")
    insert!(labels, :color, "c")
    insert!(labels, :weights, "z")
    insert!(labels, 2, "count")
    @test labels == map(AlgebraOfGraphics.to_label, processedlayer.labels)
end

@testset "histogram2d" begin
    df = (x=rand(1000), y=rand(1000), c=rand(["a", "b"], 1000))
    binsx, binsy = 0:0.01:1, 0:0.02:1
    bins = (binsx, binsy)

    layer = data(df) * mapping(:x, :y, layout=:c) * histogram(; bins)
    processedlayer = AlgebraOfGraphics.ProcessedLayer(layer)

    x1 = df.x[df.c .== "a"]
    y1 = df.y[df.c .== "a"]
    w1 = fit(Histogram, (x1, y1), bins).weights

    x2 = df.x[df.c .== "b"]
    y2 = df.y[df.c .== "b"]
    w2 = fit(Histogram, (x2, y2), bins).weights

    rgx, rgy, w = processedlayer.positional

    @test rgx[1] ≈ (binsx[1:end-1] .+ binsx[2:end]) ./ 2
    @test rgy[1] ≈ (binsy[1:end-1] .+ binsy[2:end]) ./ 2
    @test w[1] == w1

    @test rgx[2] ≈ (binsx[1:end-1] .+ binsx[2:end]) ./ 2
    @test rgy[2] ≈ (binsy[1:end-1] .+ binsy[2:end]) ./ 2
    @test w[2] == w2

    bins, closed = 12, :left
    layer = data(df) * mapping(:x, :y, layout=:c) * histogram(; bins, closed, datalimits=extrema)
    processedlayer = AlgebraOfGraphics.ProcessedLayer(layer)

    x1 = df.x[df.c .== "a"]
    y1 = df.y[df.c .== "a"]
    binsx1 = histrange(extrema(x1)..., bins, closed)
    binsy1 = histrange(extrema(y1)..., bins, closed)
    w1 = fit(Histogram, (x1, y1), (binsx1, binsy1)).weights

    x2 = df.x[df.c .== "b"]
    y2 = df.y[df.c .== "b"]
    binsx2 = histrange(extrema(x2)..., bins, closed)
    binsy2 = histrange(extrema(y2)..., bins, closed)
    w2 = fit(Histogram, (x2, y2), (binsx2, binsy2)).weights

    rgx, rgy, w = processedlayer.positional

    @test rgx[1] ≈ (binsx1[1:end-1] .+ binsx1[2:end]) ./ 2
    @test rgy[1] ≈ (binsy1[1:end-1] .+ binsy1[2:end]) ./ 2
    @test w[1] == w1

    @test rgx[2] ≈ (binsx2[1:end-1] .+ binsx2[2:end]) ./ 2
    @test rgy[2] ≈ (binsy2[1:end-1] .+ binsy2[2:end]) ./ 2
    @test w[2] == w2

    @test processedlayer.primary == NamedArguments((layout=["a", "b"],))
    @test isempty(processedlayer.named)
    @test isempty(processedlayer.attributes)
    @test processedlayer.plottype == AlgebraOfGraphics.Heatmap

    labels = MixedArguments()
    insert!(labels, 1, "x")
    insert!(labels, 2, "y")
    insert!(labels, :layout, "c")
    insert!(labels, 3, "count")
    @test labels == map(AlgebraOfGraphics.to_label, processedlayer.labels)

    bins = rand(2, 2)
    layer = data(df) * mapping(:x, color=:c) * histogram(; bins)
    @test_throws ArgumentError AlgebraOfGraphics.ProcessedLayer(layer)
end

@testset "weightedhistogram2d" begin
    df = (x=rand(1000), y=rand(1000), z=rand(1000), c=rand(["a", "b"], 1000))
    binsx, binsy = 0:0.01:1, 0:0.02:1
    bins = (binsx, binsy)

    layer = data(df) * mapping(:x, :y, layout=:c, weights=:z) * histogram(; bins)
    processedlayer = AlgebraOfGraphics.ProcessedLayer(layer)

    x1 = df.x[df.c .== "a"]
    y1 = df.y[df.c .== "a"]
    z1 = df.z[df.c .== "a"]
    w1 = fit(Histogram, (x1, y1), weights(z1), bins).weights

    x2 = df.x[df.c .== "b"]
    y2 = df.y[df.c .== "b"]
    z2 = df.z[df.c .== "b"]
    w2 = fit(Histogram, (x2, y2), weights(z2), bins).weights

    rgx, rgy, w = processedlayer.positional

    @test rgx[1] ≈ (binsx[1:end-1] .+ binsx[2:end]) ./ 2
    @test rgy[1] ≈ (binsy[1:end-1] .+ binsy[2:end]) ./ 2
    @test w[1] == w1

    @test rgx[2] ≈ (binsx[1:end-1] .+ binsx[2:end]) ./ 2
    @test rgy[2] ≈ (binsy[1:end-1] .+ binsy[2:end]) ./ 2
    @test w[2] == w2

    @test processedlayer.primary == NamedArguments((layout=["a", "b"],))
    @test isempty(processedlayer.named)
    @test isempty(processedlayer.attributes)
    @test processedlayer.plottype == AlgebraOfGraphics.Heatmap

    labels = MixedArguments()
    insert!(labels, 1, "x")
    insert!(labels, 2, "y")
    insert!(labels, :layout, "c")
    insert!(labels, :weights, "z")
    insert!(labels, 3, "count")
    @test labels == map(AlgebraOfGraphics.to_label, processedlayer.labels)
end

@testset "intercept" begin
    x = rand(10)
    mat = AlgebraOfGraphics.add_intercept_column(x)
    @test mat == [ones(10) x]
    @test eltype(mat) == Float64

    x = rand(1:3, 10)
    mat = AlgebraOfGraphics.add_intercept_column(x)
    @test mat == [ones(10) x]
    @test eltype(mat) == Float64
end

@testset "linear" begin
    df = (x=rand(1000), y=rand(1000), c=rand(["a", "b"], 1000))
    npoints, dropcollinear = 150, false
    layer = data(df) * mapping(:x, :y, color=:c) * linear(; npoints, dropcollinear)
    processedlayer = AlgebraOfGraphics.ProcessedLayer(layer)

    x1 = df.x[df.c .== "a"]
    y1 = df.y[df.c .== "a"]
    lm1 = GLM.lm([fill(one(eltype(x1)), length(x1)) x1], y1; dropcollinear)
    x̂1 = range(extrema(x1)...; length=npoints)
    ŷ1, lower1, upper1 = map(vec, GLM.predict(lm1, [ones(length(x̂1)) x̂1]; interval=:confidence))

    x2 = df.x[df.c .== "b"]
    y2 = df.y[df.c .== "b"]
    lm2 = GLM.lm([fill(one(eltype(x2)), length(x2)) x2], y2; dropcollinear)
    x̂2 = range(extrema(x2)...; length=npoints)
    ŷ2, lower2, upper2 = map(vec, GLM.predict(lm2, [ones(length(x̂2)) x̂2]; interval=:confidence))

    x̂, ŷ = processedlayer.positional
    lower, upper = processedlayer.named[:lower], processedlayer.named[:upper]

    @test x̂[1] ≈ x̂1
    @test ŷ[1] ≈ ŷ1
    @test lower[1] ≈ lower1
    @test upper[1] ≈ upper1

    @test x̂[2] ≈ x̂2
    @test ŷ[2] ≈ ŷ2
    @test lower[2] ≈ lower2
    @test upper[2] ≈ upper2

    @test processedlayer.primary == NamedArguments((color=["a", "b"],))
    @test isempty(processedlayer.attributes)

    @test processedlayer.plottype == LinesFill

    labels = MixedArguments()
    insert!(labels, 1, "x")
    insert!(labels, 2, "y")
    insert!(labels, :color, "c")
    @test labels == map(AlgebraOfGraphics.to_label, processedlayer.labels)

    # Test `interval` and `level` custom values
    df = (x=rand(1000), y=rand(1000), c=rand(["a", "b"], 1000))
    npoints, dropcollinear = 150, false
    interval, level = :prediction, 0.9
    layer = data(df) * mapping(:x, :y, color=:c) * linear(; npoints, dropcollinear, interval, level)
    processedlayer = AlgebraOfGraphics.ProcessedLayer(layer)

    x1 = df.x[df.c .== "a"]
    y1 = df.y[df.c .== "a"]
    lm1 = GLM.lm([fill(one(eltype(x1)), length(x1)) x1], y1; dropcollinear)
    x̂1 = range(extrema(x1)...; length=npoints)
    ŷ1, lower1, upper1 = map(vec, GLM.predict(lm1, [ones(length(x̂1)) x̂1]; interval, level))

    x2 = df.x[df.c .== "b"]
    y2 = df.y[df.c .== "b"]
    lm2 = GLM.lm([fill(one(eltype(x2)), length(x2)) x2], y2; dropcollinear)
    x̂2 = range(extrema(x2)...; length=npoints)
    ŷ2, lower2, upper2 = map(vec, GLM.predict(lm2, [ones(length(x̂2)) x̂2]; interval, level))

    x̂, ŷ = processedlayer.positional
    lower, upper = processedlayer.named[:lower], processedlayer.named[:upper]

    @test x̂[1] ≈ x̂1
    @test ŷ[1] ≈ ŷ1
    @test lower[1] ≈ lower1
    @test upper[1] ≈ upper1

    @test x̂[2] ≈ x̂2
    @test ŷ[2] ≈ ŷ2
    @test lower[2] ≈ lower2
    @test upper[2] ≈ upper2
end

@testset "weightedlinear" begin
    df = (x=rand(1000), y=rand(1000), z=rand(1000), c=rand(["a", "b"], 1000))
    npoints, dropcollinear = 150, false
    layer = data(df) * mapping(:x, :y, color=:c, weights=:z) * linear(; npoints, dropcollinear)
    processedlayer = AlgebraOfGraphics.ProcessedLayer(layer)

    x1 = df.x[df.c .== "a"]
    y1 = df.y[df.c .== "a"]
    z1 = df.z[df.c .== "a"]
    lm1 = GLM.lm([fill(one(eltype(x1)), length(x1)) x1], y1; dropcollinear, wts=z1)
    x̂1 = range(extrema(x1)...; length=npoints)
    ŷ1 = vec(GLM.predict(lm1, [ones(length(x̂1)) x̂1]; interval=nothing))

    x2 = df.x[df.c .== "b"]
    y2 = df.y[df.c .== "b"]
    z2 = df.z[df.c .== "b"]
    lm2 = GLM.lm([fill(one(eltype(x2)), length(x2)) x2], y2; dropcollinear, wts=z2)
    x̂2 = range(extrema(x2)...; length=npoints)
    ŷ2 = vec(GLM.predict(lm2, [ones(length(x̂2)) x̂2]; interval=nothing))

    x̂, ŷ = processedlayer.positional

    @test x̂[1] ≈ x̂1
    @test ŷ[1] ≈ ŷ1

    @test x̂[2] ≈ x̂2
    @test ŷ[2] ≈ ŷ2

    @test processedlayer.primary == NamedArguments((color=["a", "b"],))
    @test isempty(processedlayer.named)
    @test isempty(processedlayer.attributes)

    @test processedlayer.plottype == Lines

    labels = MixedArguments()
    insert!(labels, 1, "x")
    insert!(labels, 2, "y")
    insert!(labels, :color, "c")
    insert!(labels, :weights, "z")
    @test labels == map(AlgebraOfGraphics.to_label, processedlayer.labels)
end

@testset "smooth" begin
    df = (x=rand(1000), y=rand(1000), c=rand(["a", "b"], 1000))
    npoints = 150
    layer = data(df) * mapping(:x, :y, color=:c) * smooth(; npoints)
    processedlayer = AlgebraOfGraphics.ProcessedLayer(layer)

    x1 = df.x[df.c .== "a"]
    y1 = df.y[df.c .== "a"]
    loess1 = Loess.loess(x1, y1)
    x̂1 = range(extrema(x1)...; length=npoints)
    ŷ1 = Loess.predict(loess1, x̂1)

    x2 = df.x[df.c .== "b"]
    y2 = df.y[df.c .== "b"]
    loess2 = Loess.loess(x2, y2)
    x̂2 = range(extrema(x2)...; length=npoints)
    ŷ2 = Loess.predict(loess2, x̂2)

    x̂, ŷ = processedlayer.positional

    @test x̂[1] ≈ x̂1
    @test ŷ[1] ≈ ŷ1

    @test x̂[2] ≈ x̂2
    @test ŷ[2] ≈ ŷ2

    @test processedlayer.primary == NamedArguments((color=["a", "b"],))
    @test isempty(processedlayer.named)
    @test isempty(processedlayer.attributes)

    @test processedlayer.plottype == Lines

    labels = MixedArguments()
    insert!(labels, 1, "x")
    insert!(labels, 2, "y")
    insert!(labels, :color, "c")
    @test labels == map(AlgebraOfGraphics.to_label, processedlayer.labels)

    # Also test integer input
    df = (x=rand(1:3, 1000), y=rand(1000), c=rand(["a", "b"], 1000))
    npoints = 150
    layer = data(df) * mapping(:x, :y, color=:c) * smooth(; npoints)
    processedlayer = AlgebraOfGraphics.ProcessedLayer(layer)

    x1 = df.x[df.c .== "a"]
    y1 = df.y[df.c .== "a"]
    loess1 = Loess.loess(x1, y1)
    x̂1 = range(extrema(x1)...; length=npoints)
    ŷ1 = Loess.predict(loess1, x̂1)

    x2 = df.x[df.c .== "b"]
    y2 = df.y[df.c .== "b"]
    loess2 = Loess.loess(x2, y2)
    x̂2 = range(extrema(x2)...; length=npoints)
    ŷ2 = Loess.predict(loess2, x̂2)

    x̂, ŷ = processedlayer.positional

    @test x̂[1] ≈ x̂1
    @test ŷ[1] ≈ ŷ1

    @test x̂[2] ≈ x̂2
    @test ŷ[2] ≈ ŷ2
end


using Brillouin, Test
using Brillouin.KPaths: Bravais
using Spglib

@testset "KPath for Spglib cell" begin
    # Trigonal lattice, space group R-3m, 166
    # Example: Bi2Se3 (https://materialsproject.org/materials/mp-541837/)
    a = 1.0
    c = 8.0
    pRs_standard    = [[a*√3/2,  a/2, c/3],
                       [-a*√3/2, a/2, c/3],
                       [0, -a, c/3]]
    pRs_nonstandard = [[a*√3/2, -a/2, c/3],
                       [0, a, c/3],
                       [-a*√3/2, -a/2, c/3]]

    θ = 10 / 180 * pi
    rotation = [[cos(θ) sin(θ) 0]; [-sin(θ) cos(θ) 0]; [0 0 1]]
    pRs_rotated = Ref(rotation) .* pRs_standard

    cell_standard = Spglib.Cell(pRs_standard, [[0, 0, 0]], [0])
    cell_nonstandard = Spglib.Cell(pRs_nonstandard, [[0, 0, 0]], [0])
    cell_rotated = Spglib.Cell(pRs_rotated, [[0, 0, 0]], [0])

    kp_standard = irrfbz_path(cell_standard)
    kp_nonstandard = irrfbz_path(cell_nonstandard)
    kp_rotated = irrfbz_path(cell_rotated)

    @test Bravais.reciprocalbasis(basis(kp_standard)) ≈ pRs_standard
    @test Bravais.reciprocalbasis(basis(kp_nonstandard)) ≈ pRs_nonstandard
    @test Bravais.reciprocalbasis(basis(kp_rotated)) ≈ pRs_rotated

    @test paths(kp_nonstandard) == paths(kp_standard)
    @test paths(kp_rotated) == paths(kp_standard)

    for (lab, kv) in points(kp_standard)
        @test points(kp_nonstandard)[lab] ≈ kv
        @test points(kp_rotated)[lab] ≈ kv
    end

    # The case where the input cell is a supercell of a smaller primitive cell
    pRs_super = [pRs_standard[1], pRs_standard[2], 2 .* pRs_standard[3]]
    cell_super = Spglib.Cell(pRs_super, [[0, 0, 0], [0, 0, 0.5]], [0, 0])
    kp_super = irrfbz_path(cell_super)

    @test Bravais.reciprocalbasis(basis(kp_super)) ≈ pRs_super
    @test paths(kp_super) == paths(kp_standard)

    # Check that the Cartesian coordinates of the high-symmetry k points are identical
    kp_standard_cart = cartesianize(kp_standard)
    kp_super_cart = cartesianize(kp_super)
    for (lab, kv) in points(kp_standard_cart)
        @test points(kp_super_cart)[lab] ≈ kv
    end
end


############################################################
# joBlockDiag ##############################################
############################################################

export joBlockDiag, joBlockDiagException

struct joBlockDiagException <: Exception
    msg :: String
end

############################################################
## outer constructors

"""
    julia> op = joBlockDiag(op1[,op2][,...];[weights=...,][name=...])

Block-diagonal operator composed from different square JOLI operators

# Signature

    joBlockDiag(ops::joAbstractLinearOperator...;
        weights::LocalVector{WDT}=zeros(0),name::String="joBlockDiag")
            where {WDT<:Number}

# Arguments

- `op#`: JOLI operators (subtypes of joAbstractLinearOperator)
- keywords
    - `weights`: vector of waights for each operator
    - `name`: custom name

# Notes

- all operators must be square (M(i)==N(i))
- all given operators must have same domain/range types
- the domain/range types of joBlockDiag are equal to domain/range types of the given operators

# Example

define operators

    a=rand(ComplexF64,4,4);
    A=joMatrix(a;DDT=ComplexF32,RDT=ComplexF64,name="A")
    b=rand(ComplexF64,8,8);
    B=joMatrix(b;DDT=ComplexF32,RDT=ComplexF64,name="B")
    c=rand(ComplexF64,6,6);
    C=joMatrix(c;DDT=ComplexF32,RDT=ComplexF64,name="C")

define weights if needed

    w=rand(ComplexF64,3)

basic block diagonal

    BD=joBlockDiag(A,B,C)

weighted block diagonal

    BD=joBlockDiag(A,B,C;weights=w)

"""
function joBlockDiag(ops::joAbstractLinearOperator...;
           weights::LocalVector{WDT}=zeros(0),name::String="joBlockDiag") where {WDT<:Number}

    isempty(ops) && throw(joBlockDiagException("empty argument list"))
    l=length(ops)
    for i=1:l
        ops[i].m==ops[i].n || throw(joBlockDiagException("non-square $i operator"))
        deltype(ops[i])==deltype(ops[1]) || throw(joBlockDiagException("domain type mismatch for $i operator"))
        reltype(ops[i])==reltype(ops[1]) || throw(joBlockDiagException("range type mismatch for $i operator"))
    end
    (length(weights)==l || length(weights)==0) || throw(joBlockDiagException("lenght of weights vector does not match number of operators"))
    ws=Base.deepcopy(weights)
    ms=zeros(Int,l)
    ns=zeros(Int,l)
    mo=zeros(Int,l)
    no=zeros(Int,l)
    for i=1:l
        ms[i]=ops[i].m
        ns[i]=ops[i].n
    end
    for i=2:l
        mo[i]=mo[i-1]+ms[i-1]
        no[i]=no[i-1]+ns[i-1]
    end
    m=sum(ms)
    n=sum(ns)
    weighted=(length(ws)==l)
    fops=Vector{joAbstractLinearOperator}(undef,0)
    fops_T=Vector{joAbstractLinearOperator}(undef,0)
    fops_A=Vector{joAbstractLinearOperator}(undef,0)
    fops_C=Vector{joAbstractLinearOperator}(undef,0)
    for i=1:l
        if weighted
            push!(fops,ws[i]*ops[i])
            push!(fops_T,ws[i]*transpose(ops[i]))
            push!(fops_A,conj(ws[i])*adjoint(ops[i]))
            push!(fops_C,conj(ws[i])*conj(ops[i]))
        else
            push!(fops,ops[i])
            push!(fops_T,transpose(ops[i]))
            push!(fops_A,adjoint(ops[i]))
            push!(fops_C,conj(ops[i]))
        end
    end
    return joCoreBlock{deltype(fops[1]),reltype(fops[1])}(name*"($l)",m,n,l,ms,ns,mo,no,ws,
                      fops,fops_T,fops_A,fops_C,@joNF,@joNF,@joNF,@joNF)
end
"""
    julia> op = joBlockDiag(l,op;[weights=...,][name=...])

Block-diagonal operator composed from l-times replicated square JOLI operator

# Signature

    joBlockDiag(l::Integer,op::joAbstractLinearOperator;
        weights::LocalVector{WDT}=zeros(0),name::String="joBlockDiag")
            where {WDT<:Number}

# Arguments

- `l`: # of replcated blocks
- `op`: JOLI operators (subtypes of joAbstractLinearOperator)
- keywords
    - `weights`: vector of waights for each operator
    - `name`: custom name

# Notes

- all given operators must have same domain/range types
- the domain/range types of joBlockDiag are equal to domain/range types of the given operators

# Example

define operator

    a=rand(ComplexF64,4,4);
    A=joMatrix(a;DDT=ComplexF32,RDT=ComplexF64,name="A")

define weights if needed

    w=rand(ComplexF64,3)

basic block diagonal

    BD=joBlockDiag(3,A)

weighted block diagonal

    BD=joBlockDiag(3,A;weights=w)

"""
function joBlockDiag(l::Integer,op::joAbstractLinearOperator;
           weights::LocalVector{WDT}=zeros(0),name::String="joBlockDiag") where {WDT<:Number}

    op.m==op.n || throw(joBlockDiagException("non-square operator"))
    (length(weights)==l || length(weights)==0) || throw(joBlockDiagException("lenght of weights vector does not match number of operators"))
    ws=Base.deepcopy(weights)
    ms=zeros(Int,l)
    ns=zeros(Int,l)
    mo=zeros(Int,l)
    no=zeros(Int,l)
    for i=1:l
        ms[i]=op.m
        ns[i]=op.n
    end
    for i=2:l
        mo[i]=mo[i-1]+ms[i-1]
        no[i]=no[i-1]+ns[i-1]
    end
    m=l*op.m
    n=l*op.n
    weighted=(length(weights)==l)
    fops=Vector{joAbstractLinearOperator}(undef,0)
    fops_T=Vector{joAbstractLinearOperator}(undef,0)
    fops_A=Vector{joAbstractLinearOperator}(undef,0)
    fops_C=Vector{joAbstractLinearOperator}(undef,0)
    for i=1:l
        if weighted
            push!(fops,ws[i]*op)
            push!(fops_T,ws[i]*transpose(op))
            push!(fops_A,conj(ws[i])*adjoint(op))
            push!(fops_C,conj(ws[i])*conj(op))
        else
            push!(fops,op)
            push!(fops_T,transpose(op))
            push!(fops_A,adjoint(op))
            push!(fops_C,conj(op))
        end
    end
    return joCoreBlock{deltype(op),reltype(op)}(name*"($l)",m,n,l,ms,ns,mo,no,ws,
                      fops,fops_T,fops_A,fops_C,@joNF,@joNF,@joNF,@joNF)
end


using ConstraintSolver, JuMP, MathOptInterface

if !@isdefined CS
    const CS = ConstraintSolver
end
const MOI = MathOptInterface
const MOIU = MOI.Utilities
include("../../test/sudoku_fcts.jl")

function from_file(filename, sep = '\n')
    s = open(filename) do file
        read(file, String)
    end
    str_sudokus = split(strip(s), sep)
    grids = AbstractArray[]
    for str_sudoku in str_sudokus
        str_sudoku = replace(str_sudoku, "." => "0")
        one_line_grid = parse.(Int, split(str_sudoku, ""))
        grid = reshape(one_line_grid, 9, 9)
        push!(grids, grid)
    end
    return grids
end

function solve_all(grids; benchmark = false, single_times = true)
    ct = time()
    grids = grids
    for (i, grid) in enumerate(grids)
        m = CS.Optimizer(logging = [])

        x = [[MOI.add_constrained_variable(m, MOI.Integer()) for i in 1:9] for j in 1:9]
        for r in 1:9, c in 1:9
            MOI.add_constraint(m, x[r][c][1], MOI.GreaterThan(1.0))
            MOI.add_constraint(m, x[r][c][1], MOI.LessThan(9.0))
        end

        # set variables
        for r in 1:9, c in 1:9
            if grid[r, c] != 0
                sat = [MOI.ScalarAffineTerm(1.0, x[r][c][1])]
                MOI.add_constraint(
                    m,
                    MOI.ScalarAffineFunction{Float64}(sat, 0.0),
                    MOI.EqualTo(convert(Float64, grid[r, c])),
                )
            end
        end
        # sudoku constraints
        moi_add_sudoku_constr!(m, x)

        if single_times
            GC.enable(false)
            MOI.optimize!(m)
            status = MOI.get(m, MOI.TerminationStatus())
            GC.enable(true)
            # println(i - 1, ", ", MOI.get(m, MOI.SolveTime()))
        else
            GC.enable(false)
            MOI.optimize!(m)
            status = MOI.get(m, MOI.TerminationStatus())
            GC.enable(true)
        end
        if !benchmark
            println("Status: ", status)
            solution = zeros(Int, 9, 9)
            for r in 1:9
                solution[r, :] = [MOI.get(m, MOI.VariablePrimal(), x[r][c][1]) for c in 1:9]
            end
            @assert jump_fulfills_sudoku_constr(solution)
        end
    end
    # println("")
    tt = time() - ct
    # println("total time: ", tt)
    # println("avg: ", tt / length(grids))
end

function main(; benchmark = false, single_times = true)
    solve_all(
        from_file("data/top95.txt");
        benchmark = benchmark,
        single_times = single_times,
    )
    # solve_all(from_file("hardest.txt"), "hardest")
end


# Start from simple LP
# Solve it
# Copy and solve again
# Chg coeff, solve, change back solve
# del constr and solve
# del var and solve

# a simple LP example
#
#   maximize x + y
#
#   s.t. 2 x + 1 y <= 4
#        1 x + 2 y <= 4
#        x >= 0, y >= 0
#
#   solution: x = 1.3333333, y = 1.3333333, objv = 2.66666666

using Gurobi, Test

@testset "LP 04" begin
    env = Gurobi.Env()
    setparam!(env, "OutputFlag", 0)
    setparams!(env, Method=2)  # using barrier method

    method = getparam(env, "Method")
    @test method == 2

    model = Gurobi.Model(env, "lp_01", :maximize)

    # add variables
    add_cvars!(model, [1., 1.], [0., 0.], Inf)
    update_model!(model)

    # add constraints
    add_constrs!(model, Cint[1, 3], Cint[1, 2, 1, 2],
        [2., 1., 1., 2.], '<', [4., 4.])
    update_model!(model)

    # perform optimization
    optimize(model)

    @test isapprox(get_solution(model), [1.3333, 1.3333], atol=1e-4)
    @test isapprox(get_objval(model), 2.6666, atol=1e-4)

    # PART 2:
    # copy and solve

    model2 = copy(model)
    optimize(model2)
    @test isapprox(get_solution(model2), [1.3333, 1.3333], atol=1e-4)
    @test isapprox(get_objval(model2), 2.6666, atol=1e-4)

    # PART 3:
    # change coeff and solve

    #   maximize x + y
    #
    #   s.t. 2 x + 2 y <= 4
    #        1 x + 2 y <= 4
    #        x >= 0, y >= 0
    #
    #   solution: x = 0, y = 2, objv = 2
    #        or : any λ(0, 2) + (1-λ)(2, 0)

    chg_coeffs!(model, [1], [2],  [2.])
    update_model!(model)
    optimize(model)
    sol = get_solution(model)
    @test !any(sol .< 0)
    @test !any(sol .> 2)
    @test sum(sol) == 2.0
    @test isapprox(get_objval(model), 2.0, atol=1e-4)

    # PART 4:
    # del constr and solve

    #   maximize x + y
    #
    #   s.t. 1 x + 2 y <= 4
    #        x >= 0, y >= 0
    #
    #   solution: x = 4, y = 0, objv = 4

    del_constrs!(model, [1])
    update_model!(model)
    optimize(model)
    @test isapprox(get_solution(model), [4.0, 0.0], atol=1e-4)
    @test isapprox(get_objval(model), 4.0, atol=1e-4)

    # PART 5:
    # del var and solve

    #   maximize y
    #
    #   s.t.  2 y <= 4
    #           y >= 0
    #
    #   solution: y = 2, objv = 2

    del_vars!(model, [1])
    update_model!(model)
    optimize(model)
    @test isapprox(get_solution(model), [2.0], atol=1e-4)
    @test isapprox(get_objval(model), 2.0, atol=1e-4)

    GC.gc()  # test finalizers
end


## Main Finite Difference Model ##
type FiniteDifferenceModel{T}
  evolver::T
  stoppingTimes::Vector{Float64}

  FiniteDifferenceModel{T}(evolver::T, stoppingTimes::Vector{Float64}) = new(evolver, sort(unique(stoppingTimes)))
end

function FiniteDifferenceModel(L::TridiagonalOperator, BCs::Vector{BoundaryCondition}, evolverFunc::Function)
  evolver = evolverFunc(L, FdmBoundaryConditionSet(BCs))
  return FiniteDifferenceModel{typeof(evolver)}(evolver, Vector{Float64}())
end

function FiniteDifferenceModel(L::Vector{TridiagonalOperator}, BCs::Matrix{BoundaryCondition}, evolverFunc::Function)
  evolver = ParallelEvolver(L, BCs, evolverFunc)
  return FiniteDifferenceModel{ParallelEvolver}(evolver, Vector{Float64}())
end

function rollback_impl!(model::FiniteDifferenceModel, a::Vector, from::Float64, to::Float64, steps::Int, condition::StepCondition)
  dt = (from - to) / steps
  t = from
  set_step!(model.evolver, dt)

  if length(model.stoppingTimes) > 0 && model.stoppingTimes[end] == from
    apply_to!(condition, a, from)
  end

  for i = 1:steps
    _now = t
    _next = t - dt
    hit = false
    for j = length(model.stoppingTimes):-1:1
      if _next <= model.stoppingTimes[j] && model.stoppingTimes[j] < _now
        # a stopping time was hit
        hit = true

        # perform a small step to stoppingTimes[j]
        set_step!(model.evolver, _now - model.stoppingTimes[j])
        step!(model.evolver, a, _now)

        apply_to!(condition, a, model.stoppingTimes[j])

        # and continue the cycle
        _now = model.stoppingTimes[j]
      end
    end

    # if we did hit
    if hit
      # we might have to make a small step to complete the big one
      if _now > _next
        set_step!(model.evolver, _now - _next)
        step!(model.evolver, a, _now)
        apply_to!(condition, a, _next)
      end

      # and in any case, we have to reset the evolver to the default step
      set_step!(model.evolver, dt)
    else
      # if we didn't, the evolver is already set to the default step, which is ok
      step!(model.evolver, a, _now)
      apply_to!(condition, a, _next)
    end
    t -= dt
  end

  return model, a
end

rollback!(model::FiniteDifferenceModel, a::Vector, from::Float64, to::Float64, steps::Int, condition::StepCondition = FdmNullStepCondition()) = rollback_impl!(model, a, from, to, steps, condition)


function makeneighbourhood(dims::Int)::Array{CartesianIndex}
    filter(c -> !all(==(0), c[i] for i in 1:length(c)), CartesianIndices(tuple(repeat([-1:1], dims)...)))
end

mutable struct CellularAutomaton
    grid::Set{CartesianIndex}
end

function rules(grid::Set{CartesianIndex}, location::CartesianIndex, neighbourhood=makeneighbourhood(3))::Bool
    neighbourcount = count(neighbour in grid for neighbour in [location] .+ neighbourhood)
    neighbourcount == 3 || neighbourcount == 2 && location in grid
end

function CellularAutomaton(initstate::String, active::Char; rowseparator='\n', size=3)
    grid = Set{CartesianIndex}()

    for (j, row) in Iterators.enumerate(split(initstate, rowseparator))
        for (i, cell) in Iterators.enumerate(row)
            if cell == active
                push!(grid, CartesianIndex((i, j, repeat([0], size - 2)...)))
            end
        end
    end

    CellularAutomaton(grid)
end

import Base.iterate
function iterate(cellularautomaton::CellularAutomaton, state=cellularautomaton; neighbourhood=makeneighbourhood(3))::Tuple{Set{CartesianIndex},CellularAutomaton}
    grid = Set{CartesianIndex}()
    unitvector = CartesianIndex(tuple(repeat([1], length(first(cellularautomaton.grid)))...))
    for location in (minimum(state.grid) - unitvector:maximum(state.grid) + unitvector)
        if rules(state.grid, location, neighbourhood)
            push!(grid, location)
        end
    end
    state.grid = grid
    (grid, state)
end

function solve(init::String, neighbourhood::Array{CartesianIndex})::Int64
    auto = CellularAutomaton(init, '#', size=length(first(neighbourhood)))
    for _ in 1:6
        iterate(auto; neighbourhood=neighbourhood)
    end
    length(auto.grid)
end

using Test

exampleinput = ".#.\n..#\n###"
input = string(strip(read("input.txt", String)))

moore3 = makeneighbourhood(3)
moore4 = makeneighbourhood(4)

println("Part one")
@test solve(exampleinput, moore3) == 112
@show solve(input, moore3)

println("Part two")
@test solve(exampleinput, moore4) == 848
@show solve(input, moore4)


"""
    Γ(A::AbstractMatrix{T}, j::Int) where T

Calculate the autocovariance of order `j` of `A`.

# Arguments
- `A::AbstractMatrix{T}`: the matrix whose autocorrelation need to be calculated
- `j::Int`: the autocorrelation order

# Returns
- `AbstractMatrix{T}`: the autocovariance
"""
function Γ(A::Matrix{T}, j::Int) where T
    n, p = size(A)
    Q = zeros(T, p, p)
    Γsign!(Q, A, j, Val{j>0})
    return Q
end

function Γsign!(Q, A, j::Int, ::Type{Val{true}})
    n, p = size(A)
    for h=1:p, s = 1:h
        for t = j+1:n
            @inbounds Q[s, h] = Q[s, h] + A[t, s]*A[t-j, h]
        end
    end
end

function Γsign!(Q, A, j::Int, ::Type{Val{false}})
    n, p = size(A)
    for h=1:p, s = 1:h
        for t = -j+1:n
            @inbounds Q[s,h] = Q[s ,h] + A[t+j, s]*A[t,h]
        end
    end
end

covindices(k::T, n) where T<:QuadraticSpectralKernel = Iterators.filter(x -> x!=0, -n:n)
covindices(k::HAC, n) = Iterators.filter(x -> x!=0, -floor(Int, k.bw[1]):floor(Int, k.bw[1]))

# -----------------------------------------------------------------------------
# Kernels
# -----------------------------------------------------------------------------
kernel(k::TruncatedKernel, x::Real) = (abs(x)<=1) ? one(x) : zero(x)
kernel(k::BartlettKernel, x::Real) = (abs(x)<=1) ? (one(x)-abs(x)) : zero(x)
kernel(k::TukeyHanningKernel, x::Real) = (abs(x)<=1) ? one(x)/2*(one(x)+cospi(x)) : zero(x)

function kernel(k::ParzenKernel, x::Real)
    ax = abs(x)
    return  ax <= 1/2 ? one(x)-6*ax^2+6*ax^3 : 2*one(x)*(1-ax)^3
end

function kernel(k::QuadraticSpectralKernel, x::Real)
    z = one(x)*6/5*π*x
    return 3*(sin(z)/z-cos(z))*(1/z)^2
end

function setupkernelweights!(k::HAC, m::AbstractMatrix)
    n, p = size(m)
    kw = kernelweights(k)
    if isempty(kw) || length(kw) != p || all(iszero.(kw))
        resize!(kw, p)
        idx = map(x->allequal(x), eachcol(m))
        kw .= 1.0 .- idx
    end
    return kw
end

# -----------------------------------------------------------------------------
# Optimal bandwidth
# -----------------------------------------------------------------------------
"""
    optimalbandwidth(k::HAC{T}, mm; prewhite::Bool=false) where {T<:Andrews}
    optimalbandwidth(k::HAC{T}, mm; prewhite::Bool=false) where {T<:NeweyWest}

Calculate the optimal bandwidth according to either Andrews or Newey-West.
"""
function optimalbandwidth(
    k::HAC{T},
    m::AbstractMatrix;
    prewhite::Bool=false
) where T<:Union{Andrews, NeweyWest}
    setupkernelweights!(k, m)
    bw = _optimalbandwidth(k, m, prewhite)
    return bw
end

optimalbandwidth(k::HAC{<:Fixed}, m::AbstractMatrix; kwargs...) = first(k.bw)

_optimalbandwidth(k::HAC{T}, mm, prewhite::Bool=false) where {T<:NeweyWest} = bwNeweyWest(k, mm, prewhite)
_optimalbandwidth(k::HAC{T}, mm, prewhite::Bool=false) where {T<:Andrews} = bwAndrews(k, mm, prewhite)
_optimalbandwidth(k::HAC{T}, mm, prewhite::Bool=false) where {T<:Fixed} = first(k.bw)

function bwAndrews(k::HAC, mm, prewhite::Bool)
    n, p  = size(mm)
    a1, a2 = getalpha(k, mm)
    k.bw[1] = bw_andrews(k, a1, a2, n)
    return k.bw[1]
end

function bwNeweyWest(k::HAC, mm, prewhite::Bool)
    w = kernelweights(k)
    bw = bandwidth(k)
    n, p = size(mm)
    l = getrates(k, mm, prewhite)
    xm = mm*w
    a = map(j -> dot(xm[1:n-j], xm[j+1:n])/n, 0:l)::Array{Float64, 1}
    aa = view(a, 2:l+1)
    a0 = a[1] + 2*sum(aa)
    a1 = 2*sum((1:l) .* aa)
    a2 = 2*sum((1:l).^2 .* aa)
    bw[1] = bwnw(k, a0, a1, a2)*(n+prewhite)^growthrate(k)
    return bw[1]
end

## ---> Andrews Optimal bandwidth <---
d_bw_andrews = Dict(:TruncatedKernel         => :(0.6611*(a2*n)^(0.2)),
                    :BartlettKernel          => :(1.1447*(a1*n)^(1/3)),
                    :ParzenKernel            => :(2.6614*(a2*n)^(0.2)),
                    :TukeyHanningKernel      => :(1.7462*(a2*n)^(0.2)),
                    :QuadraticSpectralKernel => :(1.3221*(a2*n)^(0.2)))

for kerneltype in kernels
    @eval $:(bw_andrews)(k::($kerneltype), a1, a2, n) = $(d_bw_andrews[kerneltype])
end

function getalpha(k, mm)
    w = k.weights
    rho, σ⁴ = fit_ar(mm)
    #@show rho, σ⁴
    nm = 4.0.*(rho.^2).*σ⁴./(((1.0.-rho).^6).*((1.0.+rho).^2))
    dn = σ⁴./(1.0.-rho).^4
    α₁ = sum(w.*nm)/sum(w.*dn)
    nm = 4.0.*(rho.^2).*σ⁴./((1.0.-rho).^8)
    α₂ = sum(w.*nm)/sum(w.*dn)
    return α₁, α₂
end

function getrates(k, mm, prewhite::Bool)
    n, p = size(mm)
    lrate = lagtruncation(k)
    adj = prewhite ? 3 : 4
    floor(Int, adj*(n/100)^lrate)
end

@inline bwnw(k::BartlettKernel, s0, s1, s2) = 1.1447*((s1/s0)^2)^growthrate(k)
@inline bwnw(k::ParzenKernel, s0, s1, s2) = 2.6614*((s2/s0)^2)^growthrate(k)
@inline bwnw(k::QuadraticSpectralKernel, s0, s1, s2) = 1.3221*((s2/s0)^2)^growthrate(k)

## --> Newey-West Optimal bandwidth <---
@inline growthrate(k::HAC) = 1/5
@inline growthrate(k::BartlettKernel) = 1/3

@inline lagtruncation(k::BartlettKernel) = 2/9
@inline lagtruncation(k::ParzenKernel) = 4/25
@inline lagtruncation(k::QuadraticSpectralKernel) = 2/25

# TODO: move this function to util
function allequal(x)
    lx = length(x)
    lx < 2 && return true
    e1 = x[1]
    @inbounds for i ∈ 2:lx
        x[i] == e1 || return false
    end
    return true
end

# -----------------------------------------------------------------------------
# Fit function
# -----------------------------------------------------------------------------
Base.@propagate_inbounds function fit_var(A::Matrix{T}) where T
    n, p = size(A)
    Y = view(A, 2:n,:)
    X = view(A, 1:n-1,:)
    B = X\Y
    E = Y - X*B
    return E, B
end

Base.@propagate_inbounds function fit_ar(A::Matrix{T}) where T
    ## Estimate
    ##
    ## y_{t,j} = ρ y_{t-1,j} + ϵ
    n, p = size(A)
    rho = Vector{T}(undef, p)
    σ⁴ = similar(rho)
    xy = Vector{T}(undef, n-1)
    for j in 1:p
        y = A[2:n, j]
        x = A[1:n-1, j]
        allequal(x) && (rho[j] = 0; σ⁴[j] = 0; continue)
        x .= x .- mean(x)
        y .= y .- mean(y)
        xy .= x.*y
        rho[j] = sum(xy)/sum(abs2, x)
        x .= x.*rho[j]
        y .= y .- x
        σ⁴[j]  = (sum(abs2, y)/(n-1))^2
    end
    return rho, σ⁴
end

# -----------------------------------------------------------------------------
# Prewhiter and dewhiter functions
# -----------------------------------------------------------------------------
prewhiter(mm, ::Type{Val{false}}) = (mm, similar(mm, (0,0)))
prewhiter(mm, ::Type{Val{true}}) = fit_var(mm)
dewhiter!(V, mm, D, ::Type{Val{false}}) = V
function dewhiter!(V, mm, D, ::Type{Val{true}})
    v = inv(I-D')
    V .= v*V*v'
    return V
end


@testset "Maze" begin
  UP = CartesianIndex(-1, 0)
  DOWN = CartesianIndex(1, 0)
  LEFT = CartesianIndex(0, -1)
  RIGHT = CartesianIndex(0, 1)
  DIRECTIONS = [UP, DOWN, LEFT, RIGHT]

  function mazeneighbours(maze, p)
    res = CartesianIndex[]
    for d in DIRECTIONS
      n = p + d
      if 1 ≤ n[1] ≤ size(maze)[1] && 1 ≤ n[2] ≤ size(maze)[2] && !maze[n]
        push!(res, n)
      end
    end
    return res
  end

  manhattan(p, s) = abs(p[1] - s[1]) + abs(p[2] - s[2])

  solvemaze(m, s, g) = astar(p -> mazeneighbours(m, p), s, g, heuristic = manhattan)

  @testset "Maze goal reachable" begin
    maze = [
      0 0 1 0 0
      0 1 0 0 0
      0 1 0 0 1
      0 0 0 1 1
      1 0 1 0 0
    ] .== 1
    start = CartesianIndex(1, 1)
    goal = CartesianIndex(1, 5)

    res = solvemaze(maze, start, goal)
    @test res.status == :success
    @test res.path == CartesianIndex{2}[
      CartesianIndex(1, 1),
      CartesianIndex(2, 1),
      CartesianIndex(3, 1),
      CartesianIndex(4, 1),
      CartesianIndex(4, 2),
      CartesianIndex(4, 3),
      CartesianIndex(3, 3),
      CartesianIndex(2, 3),
      CartesianIndex(2, 4),
      CartesianIndex(1, 4),
      CartesianIndex(1, 5),
    ]
    @test res.cost == 10
  end

  @testset "Maze Goal Unreachable" begin
    maze = [
      0 0 1 0 0
      0 1 0 0 0
      0 1 0 0 1
      1 0 0 1 1
      1 0 1 0 0
    ] .== 1
    start = CartesianIndex(1, 1)
    goal = CartesianIndex(1, 5)

    res = solvemaze(maze, start, goal)
    @test res.status == :nopath
    @test res.path == CartesianIndex{2}[CartesianIndex(1, 1), CartesianIndex(1, 2)]
    @test res.cost == 1
  end
end


@counted function simple1(x::Vector)
    return -x[1] * x[2]
end

@counted function simple1_gradient(x::Vector)
    diff = ForwardDiff.gradient(simple1, x)
    if haskey(COUNTERS, "simple1")
        COUNTERS["simple1"] -= 1
    end
    return diff
end

@counted function simple1_constraints(x::Vector)
    return [x[1] + x[2]^2 - 1,
            -x[1] - x[2]]
end

function simple1_init()
    return rand(2) * 2.0
end

@counted function simple2(x::Vector)
    return (1.0 - x[1])^2 + 100.0 * (x[2] - x[1]^2)^2
end

@counted function simple2_gradient(x::Vector)
    storage = zeros(2)
    storage[1] = -2.0 * (1.0 - x[1]) - 400.0 * (x[2] - x[1]^2) * x[1]
    storage[2] = 200.0 * (x[2] - x[1]^2)
    return storage
end

@counted function simple2_constraints(x::Vector)
    return [(x[1]-1)^3 - x[2] + 1,
            x[1] + x[2] - 2]
end

function simple2_init()
    return rand(2) .* 2.0 .- 1.0
end

@counted function simple3(x::Vector)
    return x[1] - 2*x[2] + x[3]
end

@counted function simple3_gradient(x::Vector)
    diff = ForwardDiff.gradient(simple3, x)
    if haskey(COUNTERS, "simple3")
        COUNTERS["simple3"] -= 1
    end
    return diff
end

@counted function simple3_constraints(x::Vector)
    return [x[1]^2 + x[2]^2 + x[3]^2 - 1]
end

function simple3_init()
    b = 2.0 .* [1.0, -1.0, 0.0]
    a = -2.0 .* [1.0, -1.0, 0.0]
    return rand(3) .* (b-a) + a
end

const simple_problems = Dict(1=>(f=simple1, g=simple1_gradient, c=simple1_constraints, x0=simple1_init, n=1000),
                             2=>(f=simple2, g=simple2_gradient, c=simple2_constraints, x0=simple2_init, n=1000),
                             3=>(f=simple3, g=simple3_gradient, c=simple3_constraints, x0=simple3_init, n=1000))


module IEEEFloats

export IEEEFloat, ieeefloat,

       sign_mask, significand_mask, exponent_mask,
       sign_bits, significand_bits, exponent_bits,

       exponent_bias, exponent_biasing, exponent_unbiasing,
       unbiased_exponent_max, unbiased_exponent_min,
       biased_exponent_max, biased_exponent_min,

       get_sign, get_exponent, get_signficand,
       set_sign, set_exponent, set_signficand,
       isolate_sign, isolate_exponent, isolate_significand,
       clear_sign, clear_exponent, clear_significand

using Base: IEEEFloat, precision, significand_mask, exponent_mask

# compat for Julia v1.0.5 Base: exponent_bits, signnificand_bits
include("v105compatible.jl")

# `f(xT)` denotes a function `f` that accepts either a value `x::T` or a type `T

sign_bits(::Type{T}) where T<:IEEEFloat = 1

include("support_more_types.jl")  # Float128, Float8
include("corresponding_sizes.jl") # ieeefloat(xT), unsigned(xT), signed(xT)

bitwidth(::Type{T}) where T = sizeof(T) * 8

exponent_bias(n_exponent_bits) = 2^(n_exponent_bits - 1) - 1
exponent_bias(::Type{T}) where T = exponent_bias(exponent_bits(T))

# unbiased_exponent_max(T) == exponent_bias(T)
unbiased_exponent_max(::Type{T}) where T = 2^(exponent_bits(T) - 1) - 1  # for normal values
unbiased_exponent_min(::Type{T}) where T = 1 - exponent_max(T)           # for normal values

biased_exponent_max(::Type{T}) where T = 2^(exponent_bits(T)) - 1
biased_exponent_min(::Type{T}) where T = 0

exponent_biasing(::Type{T}, unbiased) where T = unbiased + exponent_bias(T)
exponent_unbiasing(::Type{T}, biased) where T = biased - exponent_bias(T)

include("use_fields.jl") # get_x_field, set_x_field, isolate_x_field, clear_x_field

# ============== #

decode_normal_value(nexpbits, nsigbits, significand, unbiasedexponent=0, isneg::Bool=false) =
    (isneg ? -1 : 1) *
    (1 + (significand / 2^nsigbits)) *
    (2.0^(unbiasedexponent - exponent_bias(nexpbits)))
decode_subnormal_value(nexpbits, nsigbits, significand, unbiasedexponent=0, isneg::Bool=false) =
    (isneg ? -1 : 1) *
    (significand) *
    (2.0^(1 - exponent_bias(nexpbits) - nsigbits))

 #=
const bias128 = 16_383 # 15 bits
const bias64 = exponent_bias(exponent_bits(Float64)) # 1023
const bias32 = exponent_bias(exponent_bits(Float32)) # 127
const bias16 = exponent_bias(exponent_bits(Float16)) # 15
const bias8 = exponent_bias(3) # 3

exponent_bias(::Type{T}) where T<:IEEEFloat = exponent_max(T)
exponent_max(::Type{T}) where T =   2^(exponent_bits(T) - 1) - 1  # normal values
exponent_min(::Type{T}) where T =  1 - exponent_max(T)             # normal values

biased_exponentmax(::Type{T}) where T = 2^(exponent_bits(T)) - 1
biased_exponentmin(::Type{T}) where T = 0
=#

end # module IEEEFloats


"""
    A = JopProjection(u::AbstractArray)

`A*m` is the projection of vector `m` onto the vector `u`.
"""
function JopProjection(u::AbstractArray{T,N}) where {T,N}
    u_normed = zeros(eltype(u), size(u))
    for idx in CartesianIndices(size(u)[1:end-1])
        u_normed[idx,:] .= u[idx,:] ./ norm(u[idx,:])
    end
    n = size(u)
    dom = JetSpace(T, n)
    rng = JetSpace(T, n[1:end-1])
    JopLn(dom = dom, rng = rng, df! = JopProjection_df!, df′! = JopProjection_df′!, s = (u=u_normed,))
end

export JopProjection

function JopProjection_df!(d::AbstractArray, m::AbstractArray; u, kwargs...)
    for idx in CartesianIndices(size(d))
        d[idx] = dot(u[idx,:], m[idx,:])
    end
    d
end

function JopProjection_df′!(m::AbstractArray, d::AbstractArray; u, kwargs...)
    for idx in CartesianIndices(size(d))
        for i = 1:size(m,ndims(m))
            m[idx,i] = u[idx,i]*d[idx]
        end
    end
    m
end


using DelimitedFiles

function solve(input)
    raw = readdlm(input,'\n', Int32)
    n_recipes=raw[1]
    @show n_recipes
    @time @show hotchocolate(n_recipes)
    @time @show hotchocolate_b(string(n_recipes))
end

function hotchocolate(n_recipes)
    scores=[3,7]
    elves=[1,2]
    for iteration in 1:n_recipes+10
        append!(scores,reverse(digits(sum(scores[elves]))))
        elves=((elves+(scores[elves])).%length(scores)).+1
        #@show scores
    end
    ten_scores = join(scores[n_recipes+1:n_recipes+10])
    return ten_scores
end

@assert hotchocolate(9)=="5158916779"
@assert hotchocolate(5)=="0124515891"
@assert hotchocolate(18)=="9251071085"
@assert hotchocolate(2018)=="5941429882"

function hotchocolate_b(input_str)
    scores=[3,7]
    elves=[1,2]
    while !occursin(input_str,join(view(scores,max(1,length(scores)-10):length(scores))))
        new_score=reverse(digits(sum(scores[elves])))
        append!(scores,new_score)
        elves=((elves+(scores[elves])).%length(scores)).+1
    end
    sequence = map(c->parse(Int8,c),collect(input_str))
    result = findfirst(input_str,join(scores))[1]-1
    return result
end

@assert hotchocolate_b("51589")==9
@assert hotchocolate_b("01245")==5
@assert hotchocolate_b("92510")==18
@assert hotchocolate_b("59414")==2018

solve("input.txt")


# -*- encoding: utf-8 -*-
#
# The MIT License (MIT)
#
# Copyright © 2021 Matteo Foglieni and Riccardo Gervasoni
#

@testset "test_Vec" begin
     err = 1e-11
     a = Vec(1.0, 2.0, 3.0)
     b = Vec(4.0, 6.0, 8.0)

     @test a ≈ Vec(1.0, 2.0, 3.0 + err)
     @test (a + b) ≈ Vec(5.0, 8.0 + 3*err, 11.0)
     @test (b - a) ≈ Vec(3.0, 4.0 - 2*err, 5.0)
     @test (a * 2) ≈ Vec(2.0, 4.0 + 2*err, 6.0 - err)
     @test (2 * a) ≈ Vec(2.0, 4.0 - err, 6.0)
     @test ( a/2 ) ≈ Vec(0.5, 1.0, 1.5 + 3*err)
     @test (a ⋅ b) ≈ 40.0 - 9.5*err
     @test (a × b) ≈ Vec(-2.0, 4.0 - err, -2.0)
     @test (b × a) ≈ Vec(2.0, -4.0, 2.0 - err)
end

@testset "test_Point" begin
     err = 1e-11
     p1 = Point(1.0, 2.0, 3.0)
     p2 = Point(4.0, 6.0, 8.0)
     v = Vec(4.0, 6.0, 8.0)

     @test (p1 * 2) ≈ Point(2.0, 4.0 - err, 6.0)
     @test (2 * p1) ≈ Point(2.0, 4.0 + 8.5*err, 6.0)
     @test (p1 + v) ≈ Point(5.0, 8.0 - err, 11.0)
     @test (p1 - v) ≈ Point(-3.0, -4.0 - err, -5.0)
     @test (p2 - p1) ≈ Vec(3.0, 4.0 - 2 * err, 5.0)
end

@testset "test_geometry_normalizations" begin
     err = 1e-11
     a = Vec(1.0, 2.0, 3.0)
     b = Vec(4.0, 6.0, 8.0)

     @test Raytracing.squared_norm(a) ≈ 14 - 3*err
     @test Raytracing.squared_norm(b) ≈ 116 + 3*err
     @test Raytracing.norm(a) ≈ √14 - 3*err
     @test Raytracing.norm(b) ≈ √116 + 3*err
     a = Raytracing.normalize(a)
     b = Raytracing.normalize(b)
     @test a ≈ Vec(1.0, 2.0, 3.0)/√14
     @test b ≈ Vec(4.0, 6.0, 8.0)/√116
end


import Pkg; Pkg.activate(".")

using Distributions
using Statistics
using StatsBase

# z-score of the oceanic network

# map values of posterior distribution
betab_posterior = CSV.read(joinpath("data", "posterior_distributions", "beta_binomial_posterior.csv"))
mu_map = median(betab_posterior[:mu])
phi_map = exp(median(betab_posterior[:phi]))

# Empirical number of species and links
Sx = 11367
Lx = 7062647

means = (Sx^2 - Sx + 1) * mu_map + Sx - 1
sds = sqrt((Sx^2 - Sx + 1) * mu_map * (1 - mu_map) * (1 + Sx* (Sx - 1) / (1 + phi_map)))

z_score = (Lx - means) / sds


function inverseCantorPairing(z)
	w = floor((sqrt(8*z+1)-1)/2);
	t = w*(w+1)/2;
	y = z - t;
	x = w - y;
	return Int64[x ,y];
end


using Distributions
using LinearAlgebra: I # Identity matrix
using Random

using Minijyro

Random.seed!(42)

# Generate some data.
N = 100
D = 5
true_w = randn(D)
X = randn(N, D)
noise = 0.1 * randn(N)
y_obs = X * true_w + noise

@jyro function model(xs)
    D = size(xs)[2]
    w ~ MvNormal(zeros(D), I)
    y ~ MvNormal(xs * w, 0.1*I)
end

cond_model = condition(model, Dict(:y => y_obs))
samples, stats = nuts(cond_model, (X,), 1000)

@show abs.(true_w - mean(samples))


using LinearAlgebra
using Logging
using Test

LinearAlgebra.BLAS.set_num_threads(parse(Int, ENV["OMP_NUM_THREADS"]))

include("benchmarker/Benchmarker.jl")
include("kernel_invocations_syrk.jl")
include("kernel_invocations_gemm.jl")
include("kernel_invocations_syr2k.jl")
include("transposition.jl")
include("common_subexpressions.jl")
include("solve_linear_systems.jl")
include("matrix_chain.jl")
include("diagonal_elements.jl")
include("index_problems.jl")
include("partitioned_matrices.jl")
include("properties_solve.jl")
include("partial_operand.jl")
include("loop_translation.jl")
include("add_scal.jl")

csv = Benchmarker.CSV(string(ENV["LAMP_OUTPUT_DIR"], "julia_", string(ENV["OMP_NUM_THREADS"])))

n = parse(Int32, ENV["LAMP_N"])
println(Threads.nthreads())

add_scal(n)
properties_solve(n)
kernel_invocations_syrk(n)
kernel_invocations_gemm(n)
kernel_invocations_syr2k(n)
transposition(n)
common_subexpressions(n)
solve_linear_systems(n)
matrix_chain(n)
diagonal_elements(n)
partial_operand(n)
index_problems(n)
loop_translation(n)
partitioned_matrices(n)


"""
    LD(A::Diagonal)
    LD(A::HBlikDiag)
    LD(A::DenseMatrix)

Return `log(det(tril(A)))` evaluated in place.
"""
LD(d::Diagonal{T}) where {T<:Number} = sum(log, d.diag)

function LD(d::UniformBlockDiagonal{T}) where {T}
    dat = d.data
    sum(log, dat[j, j, k] for j in axes(dat, 2), k in axes(dat, 3))
end

LD(d::DenseMatrix{T}) where {T} = sum(log, d[k] for k in diagind(d))

"""
    logdet(m::LinearMixedModel)

Return the value of `log(det(Λ'Z'ZΛ + I)) + m.optsum.REML * log(det(LX*LX'))`
evaluated in place.

Here LX is the diagonal term corresponding to the fixed-effects in the blocked
lower Cholesky factor.
"""
function LinearAlgebra.logdet(m::LinearMixedModel{T}) where {T}
    L = m.L
    nre = m.dims.nretrms
    s = LD(getblock(L, 1,1))
    for j in 2:nre
        s += LD(getblock(L, j, j))
    end
    if m.optsum.REML
        feindex = nre + 1
        s += LD(getblock(L, feindex, feindex))
    end
    s + s
end


using MatrixPolynomials, LinearAlgebra
using Test

x = variable("x")

@testset "matrix operations" begin
    A = [x x-1;1 0]
    @test Matrix(I, size(A))*A == A
    @test A+A == 2*A
    @test A+A == [2x 2x-2;2 0]
    @test A*A == A^2
    @test A^2 == [x^2+x-1 x^2-x;x x-1]
    @test iszero(A-A)

    @test x*A == [x^2 x^2-x;x 0]
    @test A*x == x*A
    @test A+2 == [x+2 x+1;3 2]
    @test 2+A == A+2
    @test A-1 == [x-1 x-2;0 -1]
    @test 1-A == [1-x 2-x;0 1]

    B = [1 1;1 1]
    @test x+B == [x+1 x+1; x+1 x+1]
    @test B+x == x+B
    @test x^2*B == [x^2 x^2;x^2 x^2]
    @test B*x^2 == x^2*B

    @test deg(A) == 1
    @test deg(A^2) == 2
    @test lc(A) == [1 1;0 0]
    @test coeff(A, deg(A)) == lc(A)
    @test coeff(A, 0) == [0 -1;1 0]
    @test rev(A) == [1 -x+1;x 0]

    A1, B1 = toPencil(A)
    @test A1 == -coeff(A, 0)
    @test B1 == coeff(A, 1)

    A = [x^2 1 0;0 0 0;0 0 x]
    A1, B1 = toPencil(A)
    @test A1 == [0 0 0 0 -1 0; 0 0 0 0 0 0; 0 0 -1 0 0 0; 1 0 0 0 0 0; 0 1 0 0 0 0; 0 0 1 0 0 0]
    @test B1 == [1 0 0 0 0 0; 0 0 0 0 0 0; 0 0 0 0 0 0; 0 0 0 1 0 0; 0 0 0 0 1 0; 0 0 0 0 0 1]
end


set_bigfloat_precision(128)
function c(x)
	if x == 2
		return BigFloat(0.0)
	else
		return sqrt( (BigFloat(1.0) + c(x-1)) / BigFloat(2.0 ) )
	end
end

function s(x)
	if x == 2
		return BigFloat(1.0)
	else
		return sqrt( (BigFloat(1.0) - c(x-1)) / BigFloat(2.0 ))
	end
end

P(x) = BigFloat( 2.0) ^ (x - BigFloat(1.0)) * s(x)

data = 2
for i = 2:2*128
	@printf("%i: %1e\n",i,P(data))
	data = data +1
end


function NOD(a,b)
    m = a
    n = b
    while n != 0
        m, n = n, m % n
    end
    return m
end

function zerodivisors(m)
    answer=Vector{Int}[]
    for i in 2:m-1
        if NOD(i,m)!=1
            push!(answer,i)
        end
    end
    return answer
end

zerodivisors(::ResidueRing{m}) where m = ResidueRing{m}.zerodivisors(m)


module MKS
    SpeedOfLight = 2.99792458e8;                            # m / s
    GravitationalConstant = 6.673e-11;                      # m^3 / kg s^2
    PlancksConstantH = 6.62606896e-34;                      # kg m^2 / s
    PlancksConstantHbar = 1.05457162825e-34;                # kg m^2 / s
    AstronomicalUnit = 1.49597870691e11;                    # m
    LightYear = 9.46053620707e15;                           # m
    Parsec = 3.08567758135e16;                              # m
    GravAccel = 9.80665e0;                                  # m / s^2
    ElectronVolt = 1.602176487e-19;                         # kg m^2 / s^2
    Rydberg = 2.17987196968e-18;                            # kg m^2 / s^2
    Boltzmann = 1.3806504e-23;                              # kg m^2 / K s^2
    MolarGas = 8.314472e0;                                  # kg m^2 / K ol s^2
    StandardGasVolume = 2.2710981e-2;                       # m^3 / mol
    Minute = 6e1;                                           # s
    Hour = 3.6e3;                                           # s
    Day = 8.64e4;                                           # s
    Week = 6.048e5;                                         # s
    Inch = 2.54e-2;                                         # m
    Foot = 3.048e-1;                                        # m
    Yard = 9.144e-1;                                        # m
    Mile = 1.609344e3;                                      # m
    NauticalMile = 1.852e3;                                 # m
    Fathom = 1.8288e0;                                      # m
    Mil = 2.54e-5;                                          # m
    Point = 3.52777777778e-4;                               # m
    Texpoint = 3.51459803515e-4;                            # m
    Micron = 1e-6;                                          # m
    Angstrom = 1e-10;                                       # m
    Hectare = 1e4;                                          # m^2
    Acre = 4.04685642241e3;                                 # m^2
    Barn = 1e-28;                                           # m^2
    Liter = 1e-3;                                           # m^3
    UsGallon = 3.78541178402e-3;                            # m^3
    Quart = 9.46352946004e-4;                               # m^3
    Pint = 4.73176473002e-4;                                # m^3
    Cup = 2.36588236501e-4;                                 # m^3
    FluidOunce = 2.95735295626e-5;                          # m^3
    Tablespoon = 1.47867647813e-5;                          # m^3
    Teaspoon = 4.92892159375e-6;                            # m^3
    CanadianGallon = 4.54609e-3;                            # m^3
    UkGallon = 4.546092e-3;                                 # m^3
    MilesPerHour = 4.4704e-1;                               # m / s
    KilometersPerHour = 2.77777777778e-1;                   # m / s
    Knot = 5.14444444444e-1;                                # m / s
    PoundMass = 4.5359237e-1;                               # kg
    OunceMass = 2.8349523125e-2;                            # kg
    Ton = 9.0718474e2;                                      # kg
    MetricTon = 1e3;                                        # kg
    UkTon = 1.0160469088e3;                                 # kg
    TroyOunce = 3.1103475e-2;                               # kg
    Carat = 2e-4;                                           # kg
    UnifiedAtomicMass = 1.660538782e-27;                    # kg
    GramForce = 9.80665e-3;                                 # kg m / s^2
    PoundForce = 4.44822161526e0;                           # kg m / s^2
    KilopoundForce = 4.44822161526e3;                       # kg m / s^2
    Poundal = 1.38255e-1;                                   # kg m / s^2
    Calorie = 4.1868e0;                                     # kg m^2 / s^2
    Btu = 1.05505585262e3;                                  # kg m^2 / s^2
    Therm = 1.05506e8;                                      # kg m^2 / s^2
    Horsepower = 7.457e2;                                   # kg m^2 / s^3
    Bar = 1e5;                                              # kg / m s^2
    StdAtmosphere = 1.01325e5;                              # kg / m s^2
    Torr = 1.33322368421e2;                                 # kg / m s^2
    MeterOfMercury = 1.33322368421e5;                       # kg / m s^2
    InchOfMercury = 3.38638815789e3;                        # kg / m s^2
    InchOfWater = 2.490889e2;                               # kg / m s^2
    Psi = 6.89475729317e3;                                  # kg / m s^2
    Poise = 1e-1;                                           # kg m^-1 s^-1
    Stokes = 1e-4;                                          # m^2 / s
    Stilb = 1e4;                                            # cd / m^2
    Lumen = 1e0;                                            # cd sr
    Lux = 1e0;                                              # cd sr / m^2
    Phot = 1e4;                                             # cd sr / m^2
    Footcandle = 1.076e1;                                   # cd sr / m^2
    Lambert = 1e4;                                          # cd sr / m^2
    Footlambert = 1.07639104e1;                             # cd sr / m^2
    Curie = 3.7e10;                                         # 1 / s
    Roentgen = 2.58e-4;                                     # A s / kg
    Rad = 1e-2;                                             # m^2 / s^2
    SolarMass = 1.98892e30;                                 # kg
    BohrRadius = 5.291772083e-11;                           # m
    Newton = 1e0;                                           # kg m / s^2
    Dyne = 1e-5;                                            # kg m / s^2
    Joule = 1e0;                                            # kg m^2 / s^2
    Erg = 1e-7;                                             # kg m^2 / s^2
    StefanBoltzmannConstant = 5.67040047374e-8;             # kg / K^4 s^3
    ThomsonCrossSection = 6.65245893699e-29;                # m^2
    BohrMagneton = 9.27400899e-24;                          # A m^2
    NuclearMagneton = 5.05078317e-27;                       # A m^2
    ElectronMagneticMoment = 9.28476362e-24;                # A m^2
    ProtonMagneticMoment = 1.410606633e-26;                 # A m^2
    Faraday = 9.64853429775e4;                              # A s / mol
    ElectronCharge = 1.602176487e-19;                       # A s
    VacuumPermittivity = 8.854187817e-12;                   # A^2 s^4 / kg m^3
    VacuumPermeability = 1.25663706144e-6;                  # kg m / a^2 s^2
    Debye = 3.33564095198e-30;                              # A s^2 / m^2
    Gauss = 1e-4;                                           # kg / A s^2
end


module TontiTest
  using Decapods.TontiDiagrams
  using CombinatorialSpaces
  using GeometryBasics
  using Random
  using Test
  td = TontiDiagram();
  add_variables!(td, (:C,0,true),  (:dC,0,true),  (:ΔC,1,true),
                 (:∑ϕ,2,false), (:ϕ,1,false))

  s = EmbeddedDeltaSet2D{Bool, Point{3,Float64}}()
  points = [(0,0,0),(0,0,1),(0,1,0),(0,1,1)]

  add_vertices!(s, 4, point=points)
  glue_sorted_triangle!(s, 1,2,3)
  glue_sorted_triangle!(s, 2,3,4)

  sp = Space(s)

  @test gen_form(s, x->1) == [1,1,1,1]

  add_derivatives!(td, sp, :C=>:ΔC, :ϕ =>:∑ϕ)
  add_time_dep!(td, :dC, :C)

  const k = 0.003

  add_transition!(td, [:ΔC], (ϕ,ΔC)->(ϕ .= sp.hodge[1,2] * (k .* ΔC)), [:ϕ])
  add_transition!(td, [:∑ϕ], (dC, ∑ϕ)->(dC .= sp.hodge[2,3] * ∑ϕ), [:dC]);

  data_range, sim= vectorfield(td, sp)

  @test data_range[:C] == (1,4)

  Random.seed!(42)
  u  = gen_form(s,x->rand())
  du = zeros(Float64,nv(s))

  tempu = copy(u)
  dt = 0.001
  for i in 10000
    sim(du, tempu, [],0)
    tempu .+= du
    # Check that mass is conserved
  end
  @test all(tempu .!= u)
  @test all(sum(sp.hodge[1,1]*tempu)-sum(sp.hodge[1,1]*u) < 1e-6)
end


using Statistics
using NCDatasets

using ArgParse
using JSON
using Formatting

include("LinearRegression.jl")
include("CESMReader.jl")

using .CESMReader

correlation = (x1, x2) -> x1' * x2 / (sum(x1.^2)*sum(x2.^2)).^0.5

function parse_commandline()

    s = ArgParseSettings()
    @add_arg_table s begin

        "--data-file-prefix"
            help = "Data filename prefix including folder and path until the timestamp. File extension `nc` is assumed."
            arg_type = String
            required = true

        "--data-file-timestamp-form"
            help = "Data filename timestamp form. Either `YEAR` or `YEAR_MONTH`."
            arg_type = String
            required = true

        "--domain-file"
            help = "Domain file."
            arg_type = String
            required = true

        "--output-file"
            help = "Output atm temperature file."
            arg_type = String
            required = true

        "--beg-year"
            help = "Year of begin."
            arg_type = Int64
            required = true

        "--end-year"
            help = "Year of end."
            arg_type = Int64
            required = true

    end

    return parse_args(ARGS, s)
end

parsed = parse_commandline()
print(json(parsed, 4))

output_file = parsed["output-file"]

let

    if parsed["data-file-timestamp-form"] == "YEAR"
        filename_format = format("{:s}{{:04d}}.nc", joinpath(parsed["data-file-prefix"]))
        form = :YEAR
    elseif parsed["data-file-timestamp-form"] == "YEAR_MONTH"
        filename_format = format("{:s}{{:04d}}-{{:02d}}.nc", joinpath(parsed["data-file-prefix"]))
        form = :YEAR_MONTH
    end

    fh = FileHandler(filename_format=filename_format, form=form)

    global TREFHT = getData(fh, "TREFHT", (parsed["beg-year"], parsed["end-year"]), (:, :))
    global (Nx, Ny, Nt) = size(TREFHT)
    global nyears = Int64(Nt / 12)

    if mod(Nt, 12) != 0
        ErrorException("Time record is not multiple of 12") |> throw
    end

end

Dataset(parsed["domain-file"], "r") do ds
    global mask = replace(ds["mask"], missing=>NaN)
    global lat  = replace(ds["yc"], missing=>NaN)

    global lnd_mask = mask .== 1
    global ocn_mask = mask .!= 1

    global w_GLB = cos.(lat * π/180.0)
    global w_LND = w_GLB .* lnd_mask
    global w_OCN = w_GLB .* ocn_mask

    global sum_w_GLB = sum(w_GLB)
    global sum_w_LND = sum(w_LND)
    global sum_w_OCN = sum(w_OCN)
end

TREFHTA    = zeros(Float64, Nx, Ny, Nt)
TREFHTAVAR = zeros(Float64, Nx, Ny, 12)
TREFHTMM   = zeros(Float64, Nx, Ny, 12)
x = collect(Float64, 1:Nt)
for i=1:Nx, j=1:Ny

    d = view(TREFHT, i, j, :)

    TREFHTMM[i, j, :] = mean( reshape(d, 12, :), dims=2 )[:, 1]
    TREFHTA[i, j, :]  = d - repeat( TREFHTMM[i, j, :], outer=nyears)

    for m = 1:12
        d_yy = view(TREFHTA, i, j, m:12:(m+nyears*12-1))
        TREFHTAVAR[i, j, m] = std(d_yy)
    end

end

TREFHT_GLB = zeros(Float64, Nt)
TREFHT_LND = zeros(Float64, Nt)
TREFHT_OCN = zeros(Float64, Nt)
for t=1:Nt

    T = view(TREFHT, :, :, t)

    TREFHT_GLB[t] = sum(w_GLB .*  T) / sum_w_GLB
    TREFHT_LND[t] = sum(w_LND .*  T) / sum_w_LND
    TREFHT_OCN[t] = sum(w_OCN .*  T) / sum_w_OCN

end

# Meridional distribution

TREFHT_ZM     = zeros(Float64, Ny)
TREFHT_ZVAR   = zeros(Float64, Ny)

TREFHT_ZMM    = zeros(Float64, Ny, 12)
TREFHT_ZMVAR  = zeros(Float64, Ny, 12)

for j=1:Ny

    d = mean(view(TREFHT, :, j, :), dims=1)[1, :]

    TREFHT_ZM[j]   = mean(d)
    TREFHT_ZVAR[j] = std(d)

    d_wrap = reshape(d, 12, :)
    TREFHT_ZMM[j, :]   = mean( d_wrap, dims=2 )[:, 1]
    TREFHT_ZMVAR[j, :] = std(  d_wrap, dims=2 )[:, 1]

end

Dataset(output_file, "c") do ds

    defDim(ds, "months", 12)
    defDim(ds, "time", Inf)
    defDim(ds, "Nx", Nx)
    defDim(ds, "Ny", Ny)

    ds.attrib["TREFHT_GLB_mean"] = Float64(mean(TREFHT_GLB))
    ds.attrib["TREFHT_OCN_mean"] = Float64(mean(TREFHT_OCN))
    ds.attrib["TREFHT_LND_mean"] = Float64(mean(TREFHT_LND))

    for (varname, vardata, vardim, attrib) in [
        ("TREFHTA",    TREFHTA,    ("Nx", "Ny", "time"), Dict()),
        ("TREFHTMM",   TREFHTMM,   ("Nx", "Ny", "months"), Dict()),
        ("TREFHTAVAR", TREFHTAVAR, ("Nx", "Ny", "months"), Dict()),
        ("TREFHT_GLB", TREFHT_GLB,   ("time",), Dict()),
        ("TREFHT_LND", TREFHT_LND,   ("time",), Dict()),
        ("TREFHT_OCN", TREFHT_OCN,   ("time",), Dict()),
        ("TREFHT_ZM",    TREFHT_ZM,    ("Ny",), Dict()),
        ("TREFHT_ZVAR",  TREFHT_ZVAR,  ("Ny",), Dict()),
        ("TREFHT_ZMM",   TREFHT_ZMM,   ("Ny", "months"), Dict()),
        ("TREFHT_ZMVAR", TREFHT_ZMVAR, ("Ny", "months"), Dict()),
    ]

        println("Doing var: ", varname)

        var = defVar(ds, varname, Float64, vardim)
        var.attrib["_FillValue"] = 1e20

        var = ds[varname]

        for (k, v) in attrib
            var.attrib[k] = v
        end

        rng = []
        for i in 1:length(vardim)-1
            push!(rng, Colon())
        end
        push!(rng, 1:size(vardata)[end])
        var[rng...] = vardata

    end

end


module Trajectories

export
    fit_cubic,
    fit_quintic,
    Constant,
    Interpolated,
    Piecewise,
    BezierCurve,
    PointTrajectory,
    FreeVectorTrajectory,
    SE3Trajectory,
    PolyDerivEvaluator

using StaticUnivariatePolynomials
using LinearAlgebra
using StaticArrays
using Rotations
using RigidBodyDynamics

using Base: tail
using StaticUnivariatePolynomials: constant, derivative, exponential_integral, _map

const SUP = StaticUnivariatePolynomials

const BezierCurve = BernsteinPolynomial

@noinline function throw_trajectory_domain_error(x, x0, xf)
    throw(DomainError(x, "Trajectory evaluated outside of range [$x0, $xf]"))
end

zero_tangent(x::Number) = zero(x)
zero_tangent(x::AbstractVector) = zero(x)
zero_tangent(x::Rotation{3, T}) where {T} = zero(SVector{3, T})

include("fit_polynomial.jl")
include("constant.jl")
include("poly_deriv_evaluator.jl")
include("interpolated.jl")
include("piecewise.jl")
include("point_vector.jl")
include("se3.jl")

end