#=
# 6. Variable free stream
In this notebook we will simulate the flow with a time-varying free stream past a
stationary body. To demonstrate this, we will solve for oscillatory flow past a
rectangular object, in which the $x$ component of the free stream is

$$U_\infty(t) = A \sin(\Omega t)$$
=#
using ViscousFlow
#-
using Plots

#=
### Problem specification
We will set the Reynolds number to 200
=#
my_params = Dict()
my_params["Re"] = 200

#=
In order to set a time-varying free stream, we have to define a function
that provides the instantaneous free stream components and pass that
function into the system definition. In this function, we will
use the `Sinusoid` function (available via the `RigidBodyTools` module)
to create the modulated free stream. To demonstrate its possibilities,
we will pass in the parameters for the sinusoid via the `my_params` dictionary.
The "freestream average" specifies a mean free stream, if desired.
=#
my_params["freestream average"] = 0.0
my_params["freestream frequency"]  = 2.0
my_params["freestream amplitude"] = 1.0
my_params["freestream phase"] = 0.0

#=
Now we define the function. We can call it anything we want,
but it has to have the argument signature as shown. The
`Sinusoid` function is used, with the shift operator `>>`
to apply any phase lag.
=#
function my_freestream(t,phys_params)
    U = phys_params["freestream average"]
    Ω = phys_params["freestream frequency"]
    Ax = phys_params["freestream amplitude"]
    ϕx = phys_params["freestream phase"]
    Vinfmag = Ax*(RigidBodyTools.Sinusoid(Ω) >> (ϕx/Ω))
    Vinf_angle = get(phys_params,"freestream angle",0.0)

    Uinf = (U + Vinfmag(t))*cos(Vinf_angle)
    Vinf = (U + Vinfmag(t))*sin(Vinf_angle)
    return Uinf, Vinf
end

#=
The freestream function is generically considered a forcing function,
so we pass it in via the "freestream" key in the forcing dictionary.
=#
forcing_dict = Dict("freestream" => my_freestream)

# Now let us carry on with the other usual steps:

# ### Discretize
xlim = (-2.0,2.0)
ylim = (-1.5,1.5)
my_params["grid Re"] = 4.0
g = setup_grid(xlim,ylim,my_params)

#=
### Set up bodies
Here, we will set up an ellipse in the center of the domain
=#
Δs = surface_point_spacing(g,my_params)
body = Ellipse(0.25,0.5,Δs)

T = RigidTransform((0.0,0.0),0.0)
T(body)
#-
plot(body,xlim=xlim,ylim=ylim)

#=
### Construct the system structure
This step is like the previous notebook, but now we also provide the body and the freestream:
=#
sys = viscousflow_system(g,body,phys_params=my_params,forcing=forcing_dict);
#=
### Initialize
Now, we initialize with zero vorticity
=#
u0 = init_sol(sys)
# and create the integrator
tspan = (0.0,10.0)
integrator = init(u0,tspan,sys)

#=
### Solve
Now we are ready to solve the problem. Let's advance the solution to $t = 2.5$.
=#
@time step!(integrator,2.5)

#=
### Examine
Let's look at the flow field at the end of this interval
=#
sol = integrator.sol
plt = plot(layout = (2,2), size = (800, 600), legend=:false)
tsnap = 1.0:0.5:2.5
for (i, t) in enumerate(tsnap)
    plot!(plt[i],vorticity(sol,sys,t),sys,layers=false,title="t = $(round(t,digits=2))",clim=(-10,10),levels=range(-10,10,length=30),color = :RdBu)
    plot!(plt[i],surfaces(sol,sys,t))
end
plt

#=
#### Compute the force history
Just as we did for the stationary body in a constant free stream
=#
fx, fy = force(sol,sys,1);
# Plot them
plot(
plot(sol.t,2*fx,xlim=(0,Inf),ylim=(-6,6),xlabel="Convective time",ylabel="\$C_D\$",legend=:false),
plot(sol.t,2*fy,xlim=(0,Inf),ylim=(-6,6),xlabel="Convective time",ylabel="\$C_L\$",legend=:false),
    size=(800,350)
)

# The mean drag and lift coefficients are
meanCD = GridUtilities.mean(2*fx[3:end])
#-
meanCL = GridUtilities.mean(2*fy[3:end])


using Test, YaoBase, LuxurySparse

@testset "batch normalize" begin
    s = rand(3, 4)
    batch_normalize!(s, 1)
    for i = 1:4
        @test sum(s[:, i]) ≈ 1
    end

    s = rand(3, 4)
    ss = batch_normalize(s, 1)
    for i = 1:4
        @test sum(s[:, i]) != 1
        @test sum(ss[:, i]) ≈ 1
    end
end

@testset "hilbertkron" begin
    A,B,C,D = [randn(2,2) for i = 1:4]
    II = IMatrix(2)
    ⊗ = kron
    @test hilbertkron(4, [A, B], [3, 1]) ≈ II ⊗ A ⊗ II ⊗ B
    @test hilbertkron(4, [A ⊗ B, C], [3, 1]) ≈ A ⊗ B ⊗ II ⊗ C
    @test hilbertkron(4, [A ⊗ B], [1]) ≈ II ⊗ II ⊗ A ⊗ B
    @test hilbertkron(4, [A ⊗ B, C ⊗ D], [1, 3]) ≈ C ⊗ D ⊗ A ⊗ B

    U = randn(2,2)
    U2 = randn(4,4)
    m = U2 ⊗ II ⊗ U ⊗ II
    @test m == hilbertkron(5, [U, U2], [2, 4])
end

@testset "batched kron" begin
    A, B, C  = rand(ComplexF64, 4, 4, 3), rand(ComplexF64, 4, 4, 3), rand(ComplexF64, 4, 4, 3)
    D = batched_kron(A, B, C)

    tD = zeros(ComplexF64, 64, 64, 3)
    for k in 1:3
        tD[:, :, k] = kron(A[:, :, k], B[:, :, k], C[:, :, k])
    end

    @test tD ≈ D

    B2 = reshape(transpose(reshape(permutedims(B, (3,1,2)), 3,16)), 4,4,3)
    @test B2 isa Base.ReshapedArray
    @test Array(B2) ≈ B
    D2 = batched_kron(A, B2, C)
    @test tD ≈ D2
end

@testset "random matrices" begin
    mat = rand_unitary(8)
    @test isunitary(mat)
    mat = rand_hermitian(8)
    @test ishermitian(mat)

    @test ishermitian(sprand_hermitian(8, 0.5))
    @test isunitary(sprand_unitary(8, 0.5))
end

@testset "rotmat" begin
    theta = 0.5
    @test rot_mat(ComplexF64, Const.X, theta) ≈ ComplexF64[cos(theta/2) -im * sin(theta/2); -im * sin(theta/2) cos(theta/2)]
    @test rot_mat(ComplexF64, Const.X, theta) |> eltype == ComplexF64
    @test rot_mat(ComplexF32, Const.X, theta) |> eltype == ComplexF32
end


function hyperbolic_cross_polynomial(node::AbstractArray{R,1},multi_index::Array{S,2}) where {T<:AbstractFloat,R<:Number,S<:Integer}

  n = 2*maximum(multi_index,dims=1).+1
  d = length(n) # d is the number of dimensions

  base_polynomials        = Array{Array{R,2},1}(undef,d)
  unique_base_polynomials = Array{Array{R,1},2}(undef,size(multi_index))

  # Construct the base polynomials

  @inbounds for i = 1:d
    base_polynomials[i] = chebyshev_polynomial(n[i]-1,node[i])
  end

  # Compute the unique polynomial terms from the base polynomials

  @inbounds for i in axes(multi_index,1)
    for j = 1:d
      if multi_index[i,j] == 0
        unique_base_polynomials[i,j] = [base_polynomials[j][1]]
      else
        unique_base_polynomials[i,j] = [base_polynomials[j][2*multi_index[i,j]], base_polynomials[j][2*multi_index[i,j]+1]]
      end
    end
  end

  # Construct the first row of the interplation matrix

  n = determine_grid_size(multi_index)
  polynomial = Array{R,1}(undef,n[1])
  l = 1
  @inbounds for j in axes(multi_index,1)
    new_polynomials = unique_base_polynomials[j,1]
    for i = 2:d
      new_polynomials = kron(new_polynomials,unique_base_polynomials[j,i])
    end
    m = length(new_polynomials)
    polynomial[l:l+m-1] = new_polynomials
    l += m
  end

  return polynomial

end

function hyperbolic_cross_polynomial(node::AbstractArray{R,1},multi_index::Array{S,2},domain::Union{Array{T,1},Array{T,2}}) where {T<:AbstractFloat,R<:Number,S<:Integer}

  if size(domain,2) != length(node)
    error("domain is inconsistent with the number of dimensions")
  end

  node = copy(node)

  d = length(node)
  for i = 1:d
    node[i] = normalize_node(node[i],domain[:,i])
  end

  polynomial = hyperbolic_cross_polynomial(node,multi_index)

  return polynomial

end

function hyperbolic_cross_evaluate(weights::Array{T,1},node::AbstractArray{R,1},multi_index::Array{S,2}) where {T<:AbstractFloat,R<:Number,S<:Integer}

  poly = hyperbolic_cross_polynomial(node,multi_index)

  estimate = weights'poly

  return estimate

end

function hyperbolic_cross_evaluate(weights::Array{T,1},node::AbstractArray{R,1},multi_index::Array{S,2},domain::Union{Array{T,1},Array{T,2}}) where {T<:AbstractFloat,R<:Number,S<:Integer}

if size(domain,2) != length(node)
  error("domain is inconsistent with the number of dimensions")
end

node = copy(node)

d = length(node)
for i = 1:d
  node[i] = normalize_node(node[i],domain[:,i])
end

estimate = hyperbolic_cross_evaluate(weights,node,multi_index)

return estimate

end

function hyperbolic_cross_evaluate(weights::Array{T,1},polynomial::Array{R,1}) where {T<:AbstractFloat,R<:Number}

  estimate = weights'polynomial

  return estimate

end

function hyperbolic_cross_evaluate(weights::Array{T,1},multi_index::Array{S,2}) where {T<:AbstractFloat,S<:Integer}

function goo(x::Array{R,1}) where {R<:Number}

  return hyperbolic_cross_evaluate(weights,x,multi_index)

end

return goo

end

function hyperbolic_cross_evaluate(weights::Array{T,1},multi_index::Array{S,2},domain::Union{Array{T,1},Array{T,2}}) where {T<:AbstractFloat,S<:Integer}

function goo(x::Array{R,1}) where {R<:Number}

  return hyperbolic_cross_evaluate(weights,x,multi_index,domain)

end

return goo

end


struct LogLoss{M<:AbstractMatrix,V<:AbstractVector} <: AbstractLoss
  A::M
  b::V

  function (::Type{LogLoss})(A::AbstractMatrix, b::AbstractVector)
    @assert size(A, 1) == length(b) "`A` and `b` must have compatible dimensions"
    T = float(promote_type(eltype(A), eltype(b)))
    A_ = convert(AbstractMatrix{T}, A)
    b_ = convert(AbstractVector{T}, b)

    new{typeof(A_),typeof(b_)}(A_, b_)
  end
end

nsamples(loss::LogLoss)   = size(loss.A, 1)
nfeatures(loss::LogLoss)  = size(loss.A, 2)

function loss!(loss::LogLoss, x::AbstractVector{T}, g::AbstractVector{T}) where T
  val = -loss.b .* (loss.A * x)
  fval = zero(T)

  for idx in 1:length(val)
    temp = exp(val[idx])
    @inbounds fval += isinf(temp) ? val[idx] : log1p(temp)
    @inbounds val[idx] = isinf(temp) ? -loss.b[idx] : -loss.b[idx]*temp/(one(T) + temp)
  end
  mul!(g, transpose(loss.A), val)
  return fval
end


#= -------------------------------------------------------------------------
# @file problema1_sin_disipador.jl
#
# @date 11/23/17 20:37:44
# @author Martin Noblia
# @email mnoblia@disroot.org
#
# @brief
# Simulacion de las ecuaciones difenciales del problema 1 sin disipador
# @detail
#
#  Licence:
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
---------------------------------------------------------------------------=#
# imports
using DifferentialEquations
using PyPlot
#=------------------------------------------------------------------------------
                           constants
------------------------------------------------------------------------------=#
const  C_j   =  0.5
const  C_c   =  6.8
const  R_jc  =  1.0
const  R_ca  =  30.0
const  T_a   =  30.0
const  T_j_max = 150.0
const  P_d  = (T_j_max - T_a) / (R_jc + R_ca)
a_11 = (-1 / C_c) * ((1 / R_jc) + (1 / R_ca))
a_12 = (1 / (R_jc * C_c))
a_21 = (1 / (C_j * R_jc))
a_22 = (-1/ (C_j * R_jc))
#=------------------------------------------------------------------------------
                        system matrix
------------------------------------------------------------------------------=#
A = [a_11 a_12
     a_21 a_22]

T₀ = [0.0, 0.0] # initial condition
tspan = (0.0, 1500.0) # simulation time
#=------------------------------------------------------------------------------
                           inputs
------------------------------------------------------------------------------=#
step_Pd(t) = (t >=0.0) ? P_d: 0.0
pulse_Pd(t) = (P_d * sin(2.0*π*100.0*t) + P_d) / 2.0
#=------------------------------------------------------------------------------
                  function to modelate the system
------------------------------------------------------------------------------=#
f(t, T) = A * T + [0, step_Pd(t) / C_j] + [T_a / (R_ca * C_c), 0]
prob = ODEProblem(f, T₀, tspan)
sol = solve(prob)
final_value_T_c = sol[1, end]
final_value_T_j = sol[2, end]
println("el valor final de T_J es:", sol[2, end])
#=------------------------------------------------------------------------------
                           Ploting
------------------------------------------------------------------------------=#
plot(sol.t, sol[1, :], label=L"T_c", linewidth=3)
plot(sol.t, sol[2, :], label=L"T_j", linewidth=2)
axhline(final_value_T_j, color="k", linestyle="--")
text(0, final_value_T_j + 3, latexstring("T_J="*"$final_value_T_j"), fontsize=10)
xlabel(L"t\,[seg]")
ylabel(L"T\,[^{\circ}C]")
grid("on")
matplotlib["rcParams"][:update](["font.size" => 14, "font.family" => "serif"])
legend()


@testset "paretoset" begin
    p = hcat(([x,y] for x in 0:0.2:1 for y in 0:0.2:1 if (x-1)^2+(y-1)^2<=1)...)
    @test p[:,ParetoSet(p)] == [0.0 0.2 0.4 1.0; 1.0 0.4 0.2 0.0]
    @test p[:,ParetoSet(p,[:Min, :Min])] == [0.0 0.2 0.4 1.0; 1.0 0.4 0.2 0.0]
    @test p[:,ParetoSet(p,[EfficientGlobalOptimization.Min, EfficientGlobalOptimization.Min])] == [0.0 0.2 0.4 1.0; 1.0 0.4 0.2 0.0]
    p = hcat(([x,y] for x in 0:0.2:1 for y in 0:0.2:1 if x^2+y^2<=1)...)
    @test p[:,ParetoSet(p, [:Max, :Max])] == [0.0 0.6 0.8 1.0; 1.0 0.8 0.6 0.0]
    @test p[:,ParetoSet(p, [EfficientGlobalOptimization.Max, EfficientGlobalOptimization.Max])] == [0.0 0.6 0.8 1.0; 1.0 0.8 0.6 0.0]
end


# Orthogonalisation

```
Name - gmprd
The matriz R is formed as AB = R
A - The first factor in the product (n x m), unchanged on output
B - Second factor in product (m x l), unchanged on output
R - Product matrix (n x l)
n - number of rows in A (and in R)
m - number of columns in A and rows in B
l - number of columns in B (and in R)

see also: gtprd

```
function gmprd(A, B, R, n, m, l)
  ir =  0
  ik = -m
  for k in 1:l
    ik = ik + m
    for j in 1:n
      ir = ir + 1
      ji = j - n
      ib = ik
      R[ir] = 0.
      for i in 1:m
        ji = ji + n
        ib = ib + 1
        R[ir] = R[ir] + A[ji]*B[ib]
      end
    end
  end
end
export gmprd

```
Manual page is similar to the function gmprd

```
function gtprd(A, B, R, n, m, l)
  ir =  0
  ik = -n
  for k in 1:l
    ij = 0
    ik = ik + m
    for j in 1:m
      ir = ir + 1
      ib = ik
      R[ir] = 0.
      for i in 1:n
        ij = ij + 1
        ib = ib + 1
        R[ir] = R[ir] + A[ij]*B[ib]
      end
    end
  end
end

export gtprd


# following Joe Fowler's drift_correct from mass

    # """Compute a drift correction that minimizes the spectral entropy of
    # <uncorrected> (a filtered pulse height vector) after correction.
    # The <indicator> vector should be of equal length and should have some
    # linear correlation with the pulse gain. Generally it will be the
    # pretrigger mean of the pulses, but you can experiment with other choices.

    # The entropy will be computed on corrected values only in the range
    # [0, <limit>], so <limit> should be set to a characteristic large
    # value of <uncorrected>. If <limit> is None (the default), then
    # in will be compute as 25% larger than the 99%ile point of
    # <uncorrected>

    # The model is that the filtered pulse height PH should be scaled by
    # (1 + a*PTM) where a is an arbitrary parameter computed here, and
    # PTM is the difference between each record's pretrigger mean and the
    # median value of all pretrigger means. (Or replace "pretrigger mean"
    # with whatever quantity you passed in as <indicator>.)
    # """
using KernelDensity, Distributions, Optim

abstract PerPulseTransformation

immutable LinearDriftCorrect <: PerPulseTransformation
    indicator_median::Float64
    param::Float64
end
uvhist(edges, y) = UnivariateKDE(edges, convert(Array{Float64,1},hist(y,edges)[2]))
density(h::UnivariateKDE) = h.density
edges(h::UnivariateKDE) = h.x
Base.midpoints(h::UnivariateKDE) = midpoints(edges(h))
Base.conv(h::Histogram, y) = conv(UnivariateKDE(edges(h), convert(Vector{Float64},counts(h))), y)
function drift_correct(indicator, uncorrected, σ_smooth, limit=NaN)
    median_indicator = median(indicator)
    centered_indicator = indicator-median_indicator
    if isnan(limit)
        limit = 1.25*StatsBase.percentile(uncorrected, 99)
    end
    edges = pow2edges(σ_smooth*0.4, (0, limit))
    smoother = Normal(0,σ_smooth)
    function tooptimize_entropy(param)
        corrected = uncorrected+centered_indicator*param[1]
        uvh = uvhist(edges, corrected)
        entropy(uvh,σ_smooth)
    end
    optimum = optimize(tooptimize_entropy,-1.0,1.0) # optimize(f(x), lower_bound_x, upper_bound_x) minimizes f with respect to x using brent method
    #check for sucess?
    LinearDriftCorrect(median_indicator, optimum.minimum)
end

function pow2edges(stepsize_guess, limits)
    nbins_guess = int(round( 0.5+(last(limits)-first(limits))/stepsize_guess ))
    pow2=1024
    while pow2<nbins_guess
        pow2*=2
    end
    nbins = pow2
    edges = first(limits):(last(limits)-first(limits))/pow2:last(limits)
end
function entropy(h::UnivariateKDE, σ)
    hsmooth=conv(h, Normal(0,σ))
    -sum(StatsBase.xlogx(density(hsmooth)))
end

using Base.Test

@test length(pow2edges(0.1,1:100))==1025
@test isapprox(first(pow2edges(0.1,1:100)),1)
@test isapprox(last(pow2edges(0.1,1:100)),100)

n=10000
indicator = randn(n)*10
uncorrected = 1000+randn(n)+indicator*0.1
dc = drift_correct(indicator, uncorrected, 0.5)
@test abs(dc.indicator_median)<0.5
@test abs(dc.param+0.1)<0.005


module KernelStat
#-----------------------------------------------------------
#PURPOSE
#	Colin T. Bowers module for kernel functions and statistics
#NOTES
#	Currently this module only really contains a function for estimating bandwidths (needed for estimating block lengths in dependent bootstraps)
#LICENSE
#	MIT License (see github repository for more detail: https://github.com/colintbowers/KernelStat.jl.git)
#-----------------------------------------------------------

#Load any entire modules that are needed (use import ModuleName1, ModuleName2, etc)
using 	StatsBase

#Load any specific variables/functions that are needed (use import ModuleName1.FunctionName1, ModuleName2.FunctionName2, etc)
import 	Base.string,
		Base.show,
		Base.copy

#Specify the variables/functions to export (use export FunctionName1, FunctionName2, etc)
export	evaluate, #Evaluate kernel function at given input
		activedomain, #Function that returns a UnitRange indicating the range over which kernel output is non-zero
		paramdomain, #Function that returns a UnitRange indicating the feasible domain kernel parameters
		setbandwidth!, #Function to adjust bandwidth of input kernel function
		KernelFunction, #Abstract type that nests all kernel function types
		KernelDummy, #Dummy type (useful for update! routines)
		KernelUniform, #Kernel function type
		KernelBartlett, #Kernel function type
		KernelTriangular, #Identical to KernelBartlett
		KernelEpanechnikov, #Kernel function type
		KernelGaussian, #Kernel function type
		KernelParzen2009RK, #Kernel function type
		KernelPR1993FlatTop, #Kernel function type
		KernelP2003FlatTop, #Kernel function type
		KernelPP2002Trap, #Kernel function type
		KernelPP2002Smooth, #Kernel function type
		KernelPR1994SB, #Kernel function for variance estimator from Politis, Romano (1994) "The Stationary Bootstrap"
		hacvariance, #Function for estimating heteroskedasticity and auto-correlation consistent variance
		HACVarianceMethod, #Abstract type for nesting all HAC variance methods
		HACVarianceBasic, #Basic HAC variance estimator
		bandwidth, #Function for estimating bandwidth
		BandwidthMethod, #Abstract type that nests all bandwidth method types
		BandwidthDummy, #Dummy type (useful for update! functions)
		BandwidthMax, #Maximum possible bandwidth
		BandwidthWhiteNoise, #Bandwidth type that uses Gaussian white noise assumption to get correlation confidence bounds
		BandwidthBartlett, #Bandwidth type that uses Bartlett formula to get correlation confidence bounds
		BandwidthP2003 #Bandwidth type implements the bandwidth estimation method of Politis (2003) "Adaptive Bandwidth Choice"

#******************************************************************************

#----------------------------------------------------------
#SET CONSTANTS FOR MODULE
#----------------------------------------------------------
const gaussianPremult = (1 / sqrt(2*pi))::Float64

#----------------------------------------------------------
#TYPE
#	abstract super-type and one type for each kernel function
#PURPOSE
#	These types are used to define different kernel functions, parameterised by the fields of the type, and all of which can be called via the evaluate function.
#----------------------------------------------------------
#Abstract super type
abstract KernelFunction
#Dummy type for evaluate function that ditches the indicator function check
type KernelNoIndCheck; end
#General dummy type
type KernelDummy <: KernelFunction ; end
#------- TYPE DEFINITIONS ---------------
#Common kernel functions
type KernelUniform <: KernelFunction
	LB::Float64
	UB::Float64
	function KernelUniform(LB::Float64, UB::Float64)
		LB >= UB && error("lower bound greater than upper bound")
		new(LB, UB)
	end
end
KernelUniform() = KernelUniform(-1.0, 1.0)
KernelUniform(LB::Number, UB::Number) = KernelUniform(convert(Float64, LB), convert(Float64, UB))
type KernelBartlett <: KernelFunction
	bandwidth::Float64
	KernelBartlett(bandwidth::Float64) = bandwidth <= 0 ? error("Bandwidth must be strictly positive") : new(bandwidth)
end
KernelBartlett() = KernelBartlett(1.0)
KernelBartlett(bw::Number) = KernelBartlett(convert(Float64, bw))
type KernelTriangular <: KernelFunction #Identical to KernelBartlett
	bandwidth::Float64
	KernelTriangular(bandwidth::Float64) = bandwidth <= 0 ? error("Bandwidth must be strictly positive") : new(bandwidth)
end
KernelTriangular() = KernelTriangular(1.0)
KernelTriangular(bw::Number) = KernelTriangular(convert(Float64, bw))
type KernelEpanechnikov <: KernelFunction
	bandwidth::Float64
	KernelEpanechnikov(bandwidth::Float64) = bandwidth <= 0 ? error("Bandwidth must be strictly positive") : new(bandwidth)
end
KernelEpanechnikov() = KernelEpanechnikov(1.0)
KernelEpanechnikov(bw::Number) = KernelEpanechnikov(convert(Float64, bw))
type KernelGaussian <: KernelFunction
	bandwidth::Float64
	KernelGaussian(bandwidth::Float64) = (bandwidth <= 0) ? error("Bandwidth must be strictly positive") : new(bandwidth)
end
KernelGaussian() = KernelGaussian(1.0) #Constructor for the most common use-case
KernelGaussian(bw::Number) = KernelGaussian(convert(Float64, bw))
#Kernel functions from specific papers
#Barndorff-Nielsen, Hansen, Lunde, Shephard (2009) "Realized Kernels in Practice"
type KernelParzen2009RK <: KernelFunction
end
#Politis, Romano (1993) "On a family of smoothing kernels of infinite order"
type KernelPR1993FlatTop <: KernelFunction
	m::Float64
	M::Float64
	function KernelPR1993FlatTop(m::Float64, M::Float64)
		(m <= 0 || M <= 0) && error("Parameters of this kernel function must be strictly positive")
		(m >= M) && error("First parameter of this kernel function must be stricty smaller than second parameter")
		new(m, M)
	end
end
KernelPR1993FlatTop() = KernelPR1993FlatTop(0.5, 1.0) #A popular specification of this kernel function
#Politis, Romano (2003) "Adaptive Bandwidth Choice" [NOTE, equivalent to KernelPR1993FlatTop with m = 1/2, and M = 1]
type KernelP2003FlatTop <: KernelFunction; end
#Paparoditis, Politis (2002) "The tapered block bootstrap for general statistics from stationary sequences"
type KernelPP2002Trap <: KernelFunction
	p1::Float64
	KernelPP2002Trap(p1::Float64) = !(0 < p1 <= 0.5) ? error("Invalid kernel function parameter") : new(p1)
end
KernelPP2002Trap() = KernelPP2002Trap(0.43) #Optimal value from Paparoditis, Politis (2002)
#Paparoditis, Politis (2002) "The tapered block bootstrap for general statistics from stationary sequences"
type KernelPP2002Smooth <: KernelFunction
	p1::Float64
	KernelPP2002Smooth(p1::Float64) = !(1 <= p1 < Inf) ? error("Invalid kernel function parameter") : new(p1)
end
KernelPP2002Smooth(p1::Int) = KernelPP2002Smooth(convert(Float64, p1))
KernelPP2002Smooth() = KernelPP2002Smooth(1.3) #Optimal value from Paparoditis, Politis (2002)
#Politis, Romano (1994) "The Stationary Bootstrap" (kernel function for variance estimator, see eqn 7)
type KernelPR1994SB <: KernelFunction
	bandwidth::Float64 #The upper bound (usually the number of observations)
	p::Float64 #parameter of geometric distribution (1 / expected block length)
	function KernelPR1994SB(bandwidth::Float64, p::Float64)
		bandwidth <= 0 && error("Bandwidth must be strictly positive")
		!(0 < p < 1) && error("Geometric distribution parameter must lie on the interval (0, 1)")
		new(bandwidth, p)
	end
end
KernelPR1994SB() = KernelPR1994SB(1.0, 0.1)
KernelPR1994SB(bandwidth::Number) = KernelPR1994SB(convert(Float64, bandwidth), 0.1)
KernelPR1994SB(bandwidth::Number, p::Number) = KernelPR1994SB(convert(Float64, bandwidth), convert(Float64, p))
#------- METHODS ------
#string method gets string representation of type
string(kT::KernelDummy) = "dummy"
string(kT::KernelUniform) = "uniform"
string(kT::KernelBartlett) = "bartlett"
string(kT::KernelTriangular) = "triangular"
string(kT::KernelEpanechnikov) = "epanechnikov"
string(kT::KernelGaussian) = "gaussian"
string(kT::KernelParzen2009RK) = "parzen2009RK"
string(kT::KernelPR1993FlatTop) = "PR1993FlatTop"
string(kT::KernelP2003FlatTop) = "P2003FlatTop"
string(kT::KernelPP2002Trap) = "PP2002Trap"
string(kT::KernelPP2002Smooth) = "PP2002Smooth"
string(kT::KernelPR1994SB) = "PR1994SB"
#show method
function show(io::IO, kF::KernelFunction)
	println(io, "Kernel function = " * string(kF))
	fieldNames = names(kF)
	for n = 1:length(fieldNames)
		println("    field " * string(fieldNames[n]) * " = " * string(getfield(kF, n)))
	end
end
show{T<:KernelFunction}(k::T) = show(STDOUT, k)
#deepcopy method
# function deepcopy(kF::KernelFunction)
# 	tempArgs = [ deepcopy(getfield(kF, i)) for i = 1:length(names(kF)) ]
# 	return(eval(parse(string(typeof(kF)) * "(tempArgs...)")))
# end
#------ EVALUATE method for evaluating kernel function at a given value ------------------------------
#Common Kernel functions
evaluate(x::Number, kT::KernelUniform) = indicator(x, UnitRange(kT.LB, kT.UB)) * (1 / (kT.UB - kT.LB))
evaluate(x::Number, kT::KernelUniform, ::KernelNoIndCheck) = 1 / (kT.UB - kT.LB)
evaluate{T<:Union(KernelBartlett, KernelTriangular)}(x::Number, kT::T) = indicator(x / kT.bandwidth, UnitRange(-1, 1)) * (1 - abs(x / kT.bandwidth))
evaluate{T<:Union(KernelBartlett, KernelTriangular)}(x::Number, kT::T, ::KernelNoIndCheck) = 1 - abs(x / kT.bandwidth)
evaluate(x::Number, kT::KernelEpanechnikov) = indicator(x / kT.bandwidth, UnitRange(-1, 1)) * (0.75 / kT.bandwidth) * (1 - (x^2 / kT.bandwidth^2))
evaluate(x::Number, kT::KernelEpanechnikov, ::KernelNoIndCheck) = (0.75 / kT.bandwidth) * (1 - (x^2 / kT.bandwidth^2))
evaluate(x::Number, kT::KernelGaussian) = (1 / kT.bandwidth) * gaussianPremult * exp(-1 * (x^2 / (2 * kT.bandwidth^2)))
evaluate(x::Number, kT::KernelGaussian, ::KernelNoIndCheck) = evaluate(x, kT)
function evaluate{T<:Number}(x::T, kT::KernelParzen2009RK)
	if x < 0; return(zero(T))
	elseif x <= 0.5; return(1 - 6*x^2 + 6*x^3)
	elseif x <= 1; return(2*(1 - x)^3)
	else return(zero(T))
	end
end
function evaluate{T<:Number}(x::T, kT::KernelParzen2009RK, ::KernelNoIndCheck)
	if x <= 0.5; return(1 - 6*x^2 + 6*x^3)
	else; return(2*(1 - x)^3)
	end
end
#Kernel functions from specific papers (note, may not satisfy standard kernel function properties)
#KernelPR1993FlatTop
function evaluate{T<:Number}(x::T, kT::KernelPR1993FlatTop)
	if abs(x) <= kT.m; return(one(T))
	elseif abs(x) <= kT.M; return(1 - (abs(x) - kT.m) / (kT.M - kT.m))
	else; return(zero(T))
	end
end
evaluate(x::Number, kT::KernelPR1993FlatTop, ::KernelNoIndCheck) = evaluate(x, kT)
#KernelP2003FlatTop (equivalent to KernelPR1993FlatTop with m=0.5 and M=1)
function evaluate{T<:Number}(x::T, kT::KernelP2003FlatTop)
	if abs(x) <= 0.5; return(one(T))
	elseif abs(x) <= 1; return(2 * (1 - abs(x)))
	else; return(zero(T))
	end
end
evaluate(x::Number, kT::KernelP2003FlatTop, ::KernelNoIndCheck) = evaluate(x, kT)
#KernelPP2002Trap
function evaluate{T<:Number}(x::T, kT::KernelPP2002Trap)
	if x < 0; return(zero(T))
	elseif x < kT.p1; return(x / kT.p1)
	elseif x < 1 - kT.p1; return(one(T))
	elseif x < 1; return((1 - x) / kT.p1)
	else; return(zero(T))
	end
end
evaluate(x::Number, kT::KernelPP2002Trap, ::KernelNoIndCheck) = evaluate(x, kT)
#KernelPP2002Smooth
function evaluate{T<:Number}(x::T, kT::KernelPP2002Smooth)
	if 0 <= x <= 1; return(1 - abs(2*x - 1)^kT.p1)
	else; return(zero(T))
	end
end
evaluate(x::Number, kT::KernelPP2002Smooth, ::KernelNoIndCheck) = 1 - abs(2*x - 1)^kT.p1
#KernelPR1994SB
evaluate(x::Number, kT::KernelPR1994SB) = indicator(x, UnitRange(0.0, kT.bandwidth)) * ((1 - x/kT.bandwidth)*(1 - kT.p)^x + (x/kT.bandwidth)*(1 - kT.p)^(kT.bandwidth - x))
evaluate(x::Number, kT::KernelPR1994SB, ::KernelNoIndCheck) = (1 - x/kT.bandwidth)*(1 - kT.p)^x + (x/kT.bandwidth)*(1 - kT.p)^(kT.bandwidth - x)
#Array input wrappers
evaluate{T<:Number}(x::AbstractVector{T}, kT::KernelFunction) = [ evaluate(x[n], kT) for n = 1:length(x) ]
evaluate{T<:Number}(x::AbstractMatrix{T}, kT::KernelFunction) = [ evaluate(x[n, m], kT) for n = 1:size(x, 1), m = 1:size(x, 2) ]
#-----kernelweight ------------------------
#This function transforms a kernelFunction into a set of weights
function kernelweight(kF::KernelFunction, numWeight::Int)
	r = activedomain(kF)
	r.start > 0 && error("Kernel function active domain start point too high to construct weights")
	r.stop < numWeight - 1 && error("Kernel function active domain end point not high enough to construct weights")
	w = [ evaluate(n, kF) for n = 0:numWeight-1 ]
	w *= (1 / sum(w)) #scale weights so they sum to unity
	return(w)
end
#-----activedomain and paramdomain ------------------------
#The purpose of activedomain is to return the domain (of kernel function input) over which the output of the kernel function is active
#The purpose of paramdomain is to return the domain over which the indicated parameter can be defined.
activedomain(kT::KernelUniform) = UnitRange(kT.LB, kT.UB)
activedomain(kT::KernelEpanechnikov) = UnitRange(-kT.bandwidth, kT.bandwidth)
activedomain{T<:Union(KernelBartlett, KernelTriangular)}(kT::T) = UnitRange(-kT.bandwidth, kT.bandwidth)
activedomain(kT::KernelGaussian) = UnitRange(-Inf, Inf)
activedomain(kT::KernelParzen2009RK) = UnitRange(0, 1)
activedomain(kT::KernelPR1993FlatTop) = UnitRange(-kT.M, kT.M)
activedomain(kT::KernelP2003FlatTop) = UnitRange(-1, 1)
activedomain(kT::KernelPP2002Trap) = UnitRange(0, 1)
activedomain(kT::KernelPP2002Smooth) = UnitRange(0, 1)
activedomain(kT::KernelPR1994SB) = UnitRange(0, kT.bandwidth)
function paramdomain(kT::KernelUniform, pNum::Int)
	pNum == 1 && return(UnitRange(-Inf, kT.UB))
	pNum == 2 && return(UnitRange(kT.LB, Inf))
	error("Invalid parameter number")
end
paramdomain(kT::KernelEpanechnikov, pNum::Int) = pNum == 1 ? UnitRange(nextfloat(0.0), prevfloat(Inf)) : error("Invalid parameter number")
paramdomain{T<:Union(KernelBartlett, KernelTriangular)}(kT::T, pNum::Int) = pNum == 1 ? UnitRange(nextfloat(0.0), prevfloat(Inf)) : error("Invalid parameter number")
paramdomain(kT::KernelGaussian, paramNum::Int) = pNum == 1 ? UnitRange(nextfloat(0.0), prevfloat(Inf)) : error("Invalid parameter number")
paramdomain(kT::KernelParzen2009RK, pNum::Int) = error("Kernel function has no parameters")
function paramdomain(kT::KernelPR1993FlatTop, paramNum::Int)
	paramNum == 1 && return(UnitRange(nextfloat(0.0), Inf))
	paramNum == 2 && return(UnitRange(nextfloat(0.0), Inf))
	error("Invalid parameter number")
end
paramdomain(kT::KernelP2003FlatTop, pNum::Int) = error("Kernel function has no parameters")
paramdomain(kT::KernelPP2002Trap, pNum::Int) = (pNum == 1) ? UnitRange(nextfloat(0.0), 0.5) : error("Invalid parameter number")
paramdomain(kT::KernelPP2002Smooth) = (pNum == 1) ? UnitRange(1, Inf) : error("Invalid parameter number")
function paramdomain(kT::KernelPR1994SB, pNum::Int)
	pNum == 1 && return(UnitRange(0, Inf))
	pNum == 2 && return(UnitRange(nextfloat(0.0), prevfloat(1.0)))
	error("Invalid parameter number")
end
function setbandwidth!(kT::KernelUniform, LB::Number, UB::Number)
	kT.LB = convert(Float64, LB)
	kT.UB = convert(Float64, UB)
end
function setbandwidth!(kT::KernelUniform, bandwidth::Number)
	kT.LB = 0.0
	kT.UB = convert(Float64, bandwidth)
end
function setbandwidth!{T<:Union(KernelEpanechnikov, KernelGaussian, KernelPR1994SB, KernelTriangular, KernelBartlett)}(kT::T, bandwidth::Number)
	kT.bandwidth = convert(Float64, bandwidth)
end
setbandwidth!(kT::KernelFunction, bandwidth::Number) = error("The input kernel function does not have a natural bandwidth measure")

#---------- TYPES FOR DIFFERENT METHODS OF BANDWIDTH ESTIMATION
#Abstract super-type
abstract BandwidthMethod
#General dummy type (useful for update! functions)
type BandwidthDummy <: BandwidthMethod; end
#Dummy type for using maximum possible bandwidth
type BandwidthMax <: BandwidthMethod; end
#type definitions
type BandwidthStockWatsonDefault
	adjustmentTerm::Float64
	BandwidthStockWatsonDefault(adjustmentTerm::Float64) = (adjustmentTerm <= 0) ? error("Adjustment scalar must be strictly greater than zero") : new(adjustmentTerm)
end
BandwidthStockWatsonDefault() = BandwidthStockWatsonDefault(1.0)
type BandwidthWhiteNoise <: BandwidthMethod
	adjustmentTerm::Float64
	BandwidthWhiteNoise(adjustmentTerm::Float64) = (adjustmentTerm <= 0) ? error("Adjustment scalar must be strictly greater than zero") : new(adjustmentTerm)
end
BandwidthWhiteNoise() = BandwidthWhiteNoise(1.0)
type BandwidthBartlett <:BandwidthMethod
	adjustmentTerm::Float64
	BandwidthBartlett(adjustmentTerm::Float64) = (adjustmentTerm <= 0) ? error("Adjustment scalar must be strictly greater than zero") : new(adjustmentTerm)
end
BandwidthBartlett() = BandwidthBartlett(1.0)
type BandwidthP2003 <:BandwidthMethod
	adjustmentTerm::Float64
	c::Float64
	K::Int
	function BandwidthP2003(adjustmentTerm::Float64, c::Float64, K::Int)
		adjustmentTerm <= 0 && error("Adjustment scalar must be strictly greater than zero")
		c <= 0 && error("c must be strictly positive")
		K < 1 && error("K must be strictly positive")
		new(adjustmentTerm, c, K)
	end
end
BandwidthP2003() = BandwidthP2003(1.0, 2.0, 5)
BandwidthP2003(numObs::Int) = BandwidthP2003(1.0, 2.0, max(5, convert(Int, ceil(sqrt(log(10, numObs))))))
#-------------- METHODS---------
#string
string(x::BandwidthMax) = "maximumBandwidth"
string(x::BandwidthStockWatsonDefault) = "stockWatsonDefault"
string(x::BandwidthWhiteNoise) = "whiteNoise"
string(x::BandwidthBartlett) = "bartlett"
string(x::BandwidthP2003) = "P2003"
#show method for BandwidthMethod
function show(io::IO, b::BandwidthMethod)
	println(io, "bandwidth method = " * string(b))
	fieldNames = names(b)
	for n = 1:length(fieldNames)
		println("    field " * string(fieldNames[n]) * " = " * string(getfield(b, n)))
	end
end
show{T<:BandwidthMethod}(b::T) = show(STDOUT, b)
#deepcopy
function deepcopy(x::BandwidthMethod)
	tempArgs = [ deepcopy(getfield(x, i)) for i = 1:length(names(x)) ]
	return(eval(parse(string(typeof(x)) * "(tempArgs...)")))
end
#---------- TYPES FOR DIFFERENT METHODS OF HAC-VARIANCE ESTIMATION
#Abstract super-type
abstract HACVarianceMethod
#type definitions
type HACVarianceBasic <: HACVarianceMethod
	kernelFunction::KernelFunction
	bandwidthMethod::BandwidthMethod
	function HACVarianceBasic(kernelFunction::KernelFunction, bandwidthMethod::BandwidthMethod)
		!(typeof(kernelFunction) <: Union(KernelUniform, KernelGaussian, KernelEpanechnikov, KernelPR1994SB, KernelBartlett, KernelTriangular)) && error("This kernel function does not have appropriate properties for HAC variance estimation")
		new(kernelFunction, bandwidthMethod)
	end
end
HACVarianceBasic() = HACVarianceBasic(KernelEpanechnikov(), BandwidthP2003())
HACVarianceBasic(numObs::Int) = HACVarianceBasic(KernelEpanechnikov(), BandwidthP2003(numObs))
HACVarianceBasic{T<:Number}(x::Vector{T}) = HACVarianceBasic(KernelEpanechnikov(), BandwidthP2003(length(x)))
#---------- METHODS ----------------
string(::HACVarianceBasic) = "hacVarianceBasic"
function show(io::IO, x::HACVarianceBasic)
	println(io, "HAC variance estimator type. Parameters are:")
	show(io, x.kernelFunction)
	show(io, x.bandwidthMethod)
end
#deepcopy
function deepcopy(x::HACVarianceMethod)
	tempArgs = [ deepcopy(getfield(x, i)) for i = 1:length(names(x)) ]
	return(eval(parse(string(typeof(x)) * "(tempArgs...)")))
end

#----------------------------------------------------------
#FUNCTION
#	bandwidth
#INPUT
#	(x::Vector{T<:Number}, method::BandwidthMethod): Estimate bandwidth using specified method on input data x
#OUTPUT
#	Output is the tuple (M::Int, xVar::Float64, covVec::Vector{Float64})
#		M is the bandwidth estimator. Minimum possible estimate is 1, maximum possible estimate is length(x) - 1.
#		xVar is the variance of the input data x but scaled using (1/length(x)) instead of (1/(length(x)-1))
#		covVec is the vector of sample autocovariances (NOT autocorrelations) from lag 1 to lag K of input x. K is determined within the function.
#	We pass all three of these variables as output because xVar and covVec are frequently used in the calling function, and so this saves duplicating computations.
#NOTES
#	M can be close to length(x). Typically, this is not a good situation, so the user will probably want to control for this in the calling function and bind maximum M to a more reasonable value.
#	M has an enforced minimum value of 2. Allowing results of M < 2 typically result in kernel function evaluations that always equal 0.
#	Autocorrelations are calculated using autocor (e.g. fft methods) in blocks of 20 (i.e. if bandwidth estimate is not found using first 20 autocorrelations, the next 20 will be calculated, and so on)
#----------------------------------------------------------
function bandwidth{T<:Number}(x::AbstractVector{T}, method::BandwidthMax)
	length(x) < 2 && error("Input data must have at least two observations")
	return(length(x)-1, ((length(x)-1) / length(x)) * var(x), Array(Float64, 0))
end
function bandwidth{T<:Number}(x::AbstractVector{T}, method::BandwidthStockWatsonDefault)
	length(x) < 2 && error("Input data must have at least two observations")
	return(0.75 * length(x)^(1/3), ((length(x)-1) / length(x)) * var(x), Array(Float64, 0))
end
function bandwidth{T<:Number}(x::AbstractVector{T}, method::BandwidthWhiteNoise)
	length(x) < 2 && error("Input data must have at least two observations")
	mHat = 1
	corVec = Array(Float64, 0)
	append!(corVec, autocor(x, 1:min(20, length(x)-1)))
	whiteNoiseBound = sqrt(1 / length(x))
	nullRejected = false
	if abs(corVec[1]) > 1.96 * whiteNoiseBound #"1.96*" implies we are hard-coding hypothesis test at ~95% confidence level. Note, if this condition is not triggered, then we detect no autocorrelation and so leave mHat at 1
		mSt = 1
		while mSt < length(x)
			for m = mSt:length(corVec)-1
				if abs(corVec[m+1]) < 1.96 * whiteNoiseBound
					nullRejected = true
					mHat = m + 1
					break
				end
			end
			if nullRejected == true
				break
			else
				mSt = length(corVec)
				append!(corVec, autocor(x, mSt+1:min(mSt+20, length(x)))) #Grow auto-correlation vector by 20 and try again
			end
		end
		if nullRejected == false
			mHat = length(x)
		end
	end
	M = convert(Int, ceil(method.adjustmentTerm * 2 * mHat)) #"2*" is a standard rule, see e.g. Politis (2003) "Adaptive Bandwidth Choice". method.adjustmentTerm allows user to tinker with output.
	if M > length(x) - 1
		M = length(x) - 1 #Set maximum value for M
	elseif M < 2
		M = 2 #Set minimum value for M
	end
	xVar = ((length(x)-1) /  length(x)) * var(x) #Used to scale autocorrelations to autocovariances for return argument
	return(M, xVar, xVar * corVec)
end
function bandwidth{T<:Number}(x::AbstractVector{T}, method::BandwidthBartlett)
	length(x) < 2 && error("Input data must have at least two observations")
	mHat = 1
	corVec = Array(Float64, 0)
	append!(corVec, autocor(x, 1:min(20, length(x)-1)))
	bartlettBound = sqrt(1 / length(x))
	nullRejected = false
	if abs(corVec[1]) > 1.96 * bartlettBound #"1.96*" implies we are hard-coding hypothesis test at ~95% confidence level. Note, if this condition is not triggered, then we detect no autocorrelation and so leave mHat at 1
		corSum = 0.0 #Sum of squared autocorrelations. We add lags to the overall sum iteratively within the loop
		mSt = 1
		while mSt < length(x)
			for m = mSt:length(corVec)-1
				corSum += 2*corVec[m]^2
				bartlettBound = sqrt((1 + corSum) / length(x))
				if abs(corVec[m+1]) < 1.96 * bartlettBound
					nullRejected = true
					mHat = m + 1
					break
				end
			end
			if nullRejected == true
				break
			else
				mSt = length(corVec)
				append!(corVec, autocor(x, mSt+1:min(mSt+20, length(x)))) #Grow auto-correlation vector by 20 and try again
			end
		end
		if nullRejected == false
			mHat = length(x)
		end
	end
	M = convert(Int, ceil(method.adjustmentTerm * 2 * mHat)) #"2*" is a standard rule, see e.g. Politis (2003) "Adaptive Bandwidth Choice". method.adjustmentTerm allows user to tinker with output.
	if M > length(x) - 1
		M = length(x) - 1 #Set maximum value for M
	elseif M < 2
		M = 2 #Set minimum value for M
	end
	xVar = ((length(x)-1) /  length(x)) * var(x) #Used to scale autocorrelations to autocovariances for return argument
	return(M, xVar, xVar * corVec)
end
function bandwidth{T<:Number}(x::AbstractVector{T}, method::BandwidthP2003)
	length(x) < 2 && error("Input data must have at least two observations")
	mHat = 1
	corVec = Array(Float64, 0)
	append!(corVec, autocor(x, 1:min(20, length(x)-1))) #Add first block of autocorrelations
	corBound = method.c * sqrt(log(10, length(x)) / length(x)) #Note, use of log base 10 is deliberate and recommended in Politis (2003)
	KCounter = 0
	mHatFound = false
	for k = 1:length(x)-2
		if abs(corVec[k]) < corBound
			KCounter += 1 #Bound is satisfied so add one to counter
		else
			KCounter = 0 #Bound is not satisfied, so reset the counter
		end
		if KCounter >= method.K #We found K autocorrelations in a row that satisfy the bound, so break.
			mHat = k - method.K + 1
			mHatFound = true
			break
		end
		if k == length(corVec) #We've run out of autocorrelations to check, so we need to add another block of them to corVec
			append!(corVec, autocor(x, length(corVec)+1:min(length(corVec)+20, length(x)-1)))
		end
	end
	if mHatFound == false
		mHat = length(x) - 1
	end
	M = convert(Int, ceil(method.adjustmentTerm * 2 * mHat)) #"2*" is a standard rule, see e.g. Politis (2003) "Adaptive Bandwidth Choice". method.adjustmentTerm allows user to tinker with output.
	if M > length(x) - 1
		M = length(x) - 1 #Set maximum value for M
	elseif M < 2
		M = 2 #Set minimum value for M
	end
	xVar = ((length(x)-1) /  length(x)) * var(x) #Used to scale autocorrelations to autocovariances for return argument
	return(M, xVar, xVar * corVec)
end

#----------------------------------------------------------
#FUNCTION
#	hacvariance
#INPUT
#	(x::Vector{T<:Number}, method::HACVarianceMethod): Estimate variance using specified method
#OUTPUT
#	Output is a Float64 variance estimator
#NOTES
#----------------------------------------------------------
function hacvariance{T<:Number}(x::AbstractVector{T}, method::HACVarianceBasic)
	(M, xVar, xCov) = bandwidth(x, method.bandwidthMethod)
	length(xCov) < M && append!(xCov, autocov(x, length(xCov)+1:M)) #Get any additional autocovariances that we might need
	setbandwidth!(method.kernelFunction, M)
	kernelAdjTerm = 1 / evaluate(0.0, method.kernelFunction) #Used to scale kernel functions that don't satisfy k(0) = 1
	v = xVar
	noIndCheck = KernelNoIndCheck() #Since we've set the bandwidth to M, we can safely ignore indicator checks
	for m = 1:M
		v += 2 * kernelAdjTerm * evaluate(m, method.kernelFunction, noIndCheck) * xCov[m]
	end
	v = convert(Float64, max(v, 0.0)) #variance estimators should be non-negative
	return(v, M)
end

#Non-exported indicator function for if a value lies in a specified range
indicator{T<:Number}(x::T, LB::T, UB::T) = (LB <= x <= UB) ? 1 : 0
indicator{T<:Number}(x::T, r::UnitRange) = (r.start <= x <= r.stop) ? 1 : 0

end # module


module mmmmPoiss_06122017
using FinEtools
using FinEtoolsHeatDiff
using Test
import LinearAlgebra: cholesky, Symmetric
function test()

  # println("""
  #
  # Heat conduction example described by Amuthan A. Ramabathiran
  # http://www.codeproject.com/Articles/579983/Finite-Element-programming-in-Julia:
  # Unit square, with known temperature distribution along the boundary,
  # and uniform heat generation rate inside.  Mesh of regular linear TRIANGLES,
  # in a grid of 1000 x 1000 edges (2M triangles, 1M degrees of freedom).
  # Version: 05/29/2017
  # """
  # )
  t0 = time()

  A = 1.0 # dimension of the domain (length of the side of the square)
  thermal_conductivity = [i==j ? one(FFlt) : zero(FFlt) for i=1:2, j=1:2]; # conductivity matrix
  Q = -6.0; # internal heat generation rate
  function getsource!(forceout::FFltVec, XYZ::FFltMat, tangents::FFltMat, fe_label::FInt)
    forceout[1] = Q; #heat source
  end
  tempf(x) = (1.0 .+ x[:,1].^2 .+ 2*x[:,2].^2);#the exact distribution of temperature
  N = 10;# number of subdivisions along the sides of the square domain

  # println("Mesh generation")
  fens,fes =T3block(A, A, N, N)

  geom = NodalField(fens.xyz)
  Temp = NodalField(zeros(size(fens.xyz,1),1))

  # println("Searching nodes  for BC")
  l1 = selectnode(fens; box=[0. 0. 0. A], inflate = 1.0/N/100.0)
  l2 = selectnode(fens; box=[A A 0. A], inflate = 1.0/N/100.0)
  l3 = selectnode(fens; box=[0. A 0. 0.], inflate = 1.0/N/100.0)
  l4 = selectnode(fens; box=[0. A A A], inflate = 1.0/N/100.0)
  List = vcat(l1, l2, l3, l4)
  setebc!(Temp, List, true, 1, tempf(geom.values[List,:])[:])
  applyebc!(Temp)
  numberdofs!(Temp)

  t1 = time()

  material = MatHeatDiff(thermal_conductivity)

  femm = FEMMHeatDiff(IntegDomain(fes, TriRule(1), 100.), material)

  # println("Conductivity")
  K = conductivity(femm, geom, Temp)
  # println("Nonzero EBC")
  @show F2 = nzebcloadsconductivity(femm, geom, Temp);
  # println("Internal heat generation")
  # fi = ForceIntensity(FFlt, getsource!);# alternative  specification
  fi = ForceIntensity(FFlt[Q]);
    F1 = distribloads(femm, geom, Temp, fi, 3);

  # println("Factorization")
  K = cholesky(Symmetric(K))
  # println("Solution of the factorized system")
  U = K\(F1+F2)
  scattersysvec!(Temp,U[:])

  # println("Total time elapsed = $(time() - t0) [s]")
  # println("Solution time elapsed = $(time() - t1) [s]")

  Error= 0.0
  for k=1:size(fens.xyz,1)
    Error = Error.+abs.(Temp.values[k,1].-tempf(reshape(fens.xyz[k,:], (1,2))))
  end
  # println("Error =$Error")

  # File =  "a.vtk"
  # MeshExportModule.vtkexportmesh (File, fes.conn, [geom.values Temp.values], MeshExportModule.T3; scalars=Temp.values, scalars_name ="Temperature")

  @test Error[1]<1.e-5

  true
end
end
using .mmmmPoiss_06122017
mmmmPoiss_06122017.test()


using Knet
include("readdata.jl")

# Similar to model02, trying more fully connected layers

@knet function model03(x; T=51, D=4, W=3, o...)
    c1 = cbf73(x; o..., out=64)
    c2 = cbf73(c1; o..., out=128)
    c3 = cbf73(c2; o..., out=256)
    c = pool(c3; o..., window=T)
    f1 = wbf73(c; o..., out=128)
    f2 = wbf73(f1; o..., out=128)
    return wbf73(f2; o..., out=2, f=:soft)
end

function train03(; N=2^13, lr=0.001, adam=true, nbatch=128, o...)
    global f = compile(:model03; o...)
    setp(f, lr=lr, adam=adam)
    global d = GulayseData(nbatch)
    sloss = zloss = -1; n = nextn = 1
    for (x,ygold) in d
        ypred = forw(f, x)
        sl = softloss(ypred,ygold); sloss = (n==1 ? sl : 0.99 * sloss + 0.01 * sl)
        zl = zeroone(ypred,ygold);  zloss = (n==1 ? zl : 0.99 * zloss + 0.01 * zl)
        n==nextn && (println((n,sloss,1-zloss)); nextn*=2)
        back(f, ygold, softloss)
        update!(f)
        (n += 1) > N && break
    end
end

train03()


### A Pluto.jl notebook ###
# v0.12.6

using Markdown
using InteractiveUtils

# ╔═╡ ce2dfd4a-1deb-11eb-31f5-4d237b2d0610
include("dg_operators.jl");

# ╔═╡ 8e8eb794-1df0-11eb-3f3b-9f2c61ce3c3a
import Printf: @sprintf

# ╔═╡ 8e8f981c-1df0-11eb-2a06-c5686fc02af0
import LinearAlgebra: eigvals

# ╔═╡ d1a69f42-1df0-11eb-09b3-215314f63339
import Plots

# ╔═╡ 8e9477c4-1df0-11eb-188b-2ff136c41a1a
# First we define the computational parameters
begin
  # Floating Point Type
  FT = Float64

  # Polynomial Order
  N = 1

  # number of elements
  num_elems = 11

  # Final time for PDE simulation
  t_final = FT(1 // 10)

  # create the dt data for the simulation
  dg = dg_data(N, num_elems, FT(0), 2FT(π))
end;

# ╔═╡ 8ea3c5d0-1df0-11eb-0d3b-8bfedcbaa3ad
begin
  # First we test the function
  ρ = sin.(dg.x)

  # which has the property that ∂^4 ρ / ∂_x^4 = ρ
  D4ρ = reshape(dg.D4 * ρ[:], size(ρ))
end;

# ╔═╡ e083d63c-1df1-11eb-21c0-f5312717ad4d
begin
  # Plot the solution and the approximation... not so good!
  Plots.plot(dg.x, ρ, color = :red, legend = :none)
  Plots.plot!(dg.x, D4ρ, color = :blue, legend = :none)
end

# ╔═╡ c3740ae6-1df4-11eb-333f-a7ae14f8982e
begin
  # Looking at the error we see the obvious badnees :(
  Plots.plot(dg.x, ρ - D4ρ, color = :red, legend = :none)
end

# ╔═╡ 2ae63aa8-1df4-11eb-00d7-c328b731c322
begin
  # Let's instead evolve the PDE ρ_t = ρ_xxxx some small time
  # with ρ₀(x) = sin(x) the exact solution is ρ(x, t) = exp(-t) sin(x)
  ρ2 = reshape(exp(-t_final * dg.D4) * ρ[:], size(ρ))
  D4ρ2 = reshape(dg.D4 * ρ2[:], size(ρ2))
  D4ρ2_exact = exp(-t_final) * ρ
end;

# ╔═╡ f5a5924e-1df1-11eb-30e4-3908a1ac07cc
begin
  # Plot the solution and the approximation... looking better!
  Plots.plot(dg.x, ρ2, color = :red, legend = :none)
  Plots.plot!(dg.x, D4ρ2, color = :blue, legend = :none)
end

# ╔═╡ d278db02-1df4-11eb-0a90-d90e75af7ea3
begin
  # Looking at the error things are converging
  Plots.plot(dg.x, ρ2 - D4ρ2, color = :red, legend = :none)
end

# ╔═╡ fec8c72a-1e29-11eb-3ce4-af2701a644a7
# Now let's do a refinement study
let
  # These are the levels we will compute
  Ks = 2 .^ (1:8)

  # Loop over refinement levels and compute error
  err = zeros(FT, length(Ks))
  for (iter, K) in enumerate(Ks)
    dg = dg_data(N, K, FT(0), 2FT(π))
    ρ = sin.(dg.x)
    ρ2 = reshape(exp(-t_final * dg.D4) * ρ[:], size(ρ))
    err[iter] = sqrt(sum(dg.M .* (ρ2 - ρ * exp(-t_final)).^2))
  end

  # Compute the rates
  rate = (log2.(err[1:end-1]) - log2.(err[2:end])) ./
  (log2.(Ks[2:end])    - log2.(Ks[1:end-1]))

  # Make a nice ouput string
  str = @sprintf "```polynomial order = %d```\n" N
  str *= @sprintf "
  ```num elements = %03d :: rate (error) = XXXX (%.3e)```\n
  " Ks[1] err[1]
  for (K, e, r) in zip(Ks[2:end], err[2:end], rate)
    str *= @sprintf "```num elements = %03d :: rate (error) = %7.2f (%.3e)```\n\n" K r e
  end
  Markdown.parse(str)
end

# ╔═╡ Cell order:
# ╠═ce2dfd4a-1deb-11eb-31f5-4d237b2d0610
# ╠═8e8eb794-1df0-11eb-3f3b-9f2c61ce3c3a
# ╠═8e8f981c-1df0-11eb-2a06-c5686fc02af0
# ╠═d1a69f42-1df0-11eb-09b3-215314f63339
# ╠═8e9477c4-1df0-11eb-188b-2ff136c41a1a
# ╠═8ea3c5d0-1df0-11eb-0d3b-8bfedcbaa3ad
# ╠═e083d63c-1df1-11eb-21c0-f5312717ad4d
# ╠═c3740ae6-1df4-11eb-333f-a7ae14f8982e
# ╠═2ae63aa8-1df4-11eb-00d7-c328b731c322
# ╠═f5a5924e-1df1-11eb-30e4-3908a1ac07cc
# ╠═d278db02-1df4-11eb-0a90-d90e75af7ea3
# ╠═fec8c72a-1e29-11eb-3ce4-af2701a644a7


using Plexifications
using Base.Test: @test, @test_throws

@test_throws ErrorException begin
    inv(HyperComplex(0, 0.0))
end

@test begin
    iszerodivisor(HyperComplex(0,0,1,2))
end

@test begin
    !iszerodivisor(HyperComplex(1,2,3,5))
end

@test begin
    a = Plexifications.randomBigFloat()
    isreal(HyperComplex(a))
end

@test begin
    a = 1
    !isreal(HyperComplex(a, a))
end

@test begin
    z = random(HyperComplex{Int})
    z == +(z)
end

@test begin
    a = Plexifications.randomBigFloat()
    real(HyperComplex(a)) == a
end

@test begin
    io = IOBuffer()
    show(io, HyperComplex(1, 2, 3, 4, 5, 6, 7, 8))
    l = String(take!(io))
    r = "[1: 1, H: 2, A: 3, HA: 4, B: 5, HB: 6, AB: 7, HAB: 8]"
    l == r
end

@test begin
    x = random(HyperComplex{BigFloat})
    y = random(HyperComplex{BigFloat})
    x != y
end

@test begin
    a = Plexifications.randomBigInt()
    z = random(HyperComplex{BigInt})
    l = a + z
    r = z + a
    l == r
end

@test begin
    a = Plexifications.randomBigInt()
    z = random(HyperComplex{BigInt})
    l = a - z
    r = -(z - a)
    l == r
end

@test begin
    x = random(HyperComplex{BigInt})
    y = random(HyperComplex{BigInt})
    l = x + y
    r = y + x
    l == r
end

@test begin
    x = random(DualComplex{BigInt})
    y = random(HyperComplex{BigInt})
    l = x + y
    r = y + x
    l == r
end

@test begin
    x = random(HyperComplex{BigInt})
    y = random(HyperComplex{BigInt})
    l = x * y
    r = y * x
    l == r
end

@test begin
    x = random(DualComplex{BigInt})
    y = random(HyperComplex{BigInt})
    l = x * y
    r = y * x
    l == r
end

@test begin
    x = random(HyperComplex{BigInt})
    y = random(HyperComplex{BigInt})
    l = x - y
    r = -(y - x)
    l == r
end

@test begin
    x = random(DualComplex{BigInt})
    y = random(HyperComplex{BigInt})
    l = x - y
    r = -(y - x)
    l == r
end

@test begin
    x = random(HyperComplex{BigInt})
    y = random(HyperComplex{BigInt})
    z = random(HyperComplex{BigInt})
    l = (x + y) + z
    r = x + (y + z)
    l == r
end

@test begin
    x = random(HyperComplex{BigInt})
    y = random(HyperComplex{BigInt})
    z = random(HyperComplex{BigInt})
    l = (x * y) * z
    r = x * (y * z)
    l == r
end

@test begin
    x = random(HyperComplex{BigInt})
    barA(barA(x)) == x
end

@test begin
    x = random(HyperComplex{BigInt})
    barB(barB(x)) == x
end

@test begin
    x = random(HyperComplex{BigInt})
    barH(barH(x)) == x
end

@test begin
    x = random(HyperComplex{BigInt})
    barB(barA(x)) == barA(barB(x))
end

@test begin
    x = random(HyperComplex{BigInt})
    barH(barA(x)) == barA(barH(x))
end

@test begin
    x = random(HyperComplex{BigInt})
    barB(barH(x)) == barH(barB(x))
end

@test begin
    x = random(HyperComplex{Rational{BigInt}})
    inv(inv(x)) == x
end

@test begin
    x = random(HyperComplex{BigInt})
    y = random(HyperComplex{BigInt})
    l = barA(x * y)
    r = barA(x) * barA(y)
    l == r
end

@test begin
    x = random(HyperComplex{BigInt})
    y = random(HyperComplex{BigInt})
    l = barA(x + y)
    r = barA(x) + barA(y)
    l == r
end

@test begin
    x = random(HyperComplex{BigInt})
    y = random(HyperComplex{BigInt})
    l = barA(x - y)
    r = barA(x) - barA(y)
    l == r
end

@test begin
    x = random(HyperComplex{BigInt})
    y = random(HyperComplex{BigInt})
    l = barB(x * y)
    r = barB(x) * barB(y)
    l == r
end

@test begin
    x = random(HyperComplex{BigInt})
    y = random(HyperComplex{BigInt})
    l = barB(x + y)
    r = barB(x) + barB(y)
    l == r
end

@test begin
    x = random(HyperComplex{BigInt})
    y = random(HyperComplex{BigInt})
    l = barB(x - y)
    r = barB(x) - barB(y)
    l == r
end

@test begin
    x = random(HyperComplex{BigInt})
    y = random(HyperComplex{BigInt})
    l = barH(x * y)
    r = barH(x) * barH(y)
    l == r
end

@test begin
    x = random(HyperComplex{BigInt})
    y = random(HyperComplex{BigInt})
    l = barH(x + y)
    r = barH(x) + barH(y)
    l == r
end

@test begin
    x = random(HyperComplex{BigInt})
    y = random(HyperComplex{BigInt})
    l = barH(x - y)
    r = barH(x) - barH(y)
    l == r
end

@test begin
    x = random(HyperComplex{Rational{BigInt}})
    y = random(HyperComplex{Rational{BigInt}})
    l = inv(x * y)
    r = inv(y) * inv(x)
    l == r
end

@test begin
    x = random(HyperComplex{Rational{BigInt}})
    y = random(HyperComplex{Rational{BigInt}})
    l = inv(x / y)
    r = y / x
    l == r
end

@test begin
    x = random(HyperComplex{Rational{BigInt}})
    y = random(HyperComplex{Rational{BigInt}})
    l = inv(x \ y)
    r = y \ x
    l == r
end

@test begin
    x = random(HyperComplex{BigInt})
    y = random(HyperComplex{BigInt})
    z = random(HyperComplex{BigInt})
    l = x * (y + z)
    r = (x * y) + (x * z)
    l == r
end

@test begin
    a = Plexifications.randomBigInt()
    x = random(HyperComplex{BigInt})
    y = random(HyperComplex{BigInt})
    l = a * (x + y)
    r = (a * x) + (a * y)
    l == r
end

@test begin
    a = Plexifications.randomBigInt()
    x = random(HyperComplex{BigInt})
    l = a * x
    r = x * a
    l == r
end

@test begin
    x = random(HyperComplex{BigInt})
    y = random(HyperComplex{BigInt})
    z = random(HyperComplex{BigInt})
    l = (x + y) * z
    r = (x * z) + (y * z)
    l == r
end

@test begin
    x = random(HyperComplex{BigInt})
    y = random(HyperComplex{BigInt})
    z = random(HyperComplex{BigInt})
    l = x * (y - z)
    r = (x * y) - (x * z)
    l == r
end

@test begin
    x = random(HyperComplex{BigInt})
    y = random(HyperComplex{BigInt})
    z = random(HyperComplex{BigInt})
    l = (x - y) * z
    r = (x * z) - (y * z)
    l == r
end

@test begin
    x = random(HyperComplex{BigInt})
    n = norm(x)
    n > big(0) || n == big(0)
end

@test begin
    x = random(HyperComplex{BigInt})
    y = random(HyperComplex{BigInt})
    l = norm(x * y)
    r = norm(x) * norm(y)
    l == r
end

@test_throws ErrorException begin
    inv(HyperComplex(0, 0.0))
end

@test_throws ErrorException begin
    random(HyperComplex{Int}) / 0
end

@test_throws ErrorException begin
    0 \ random(HyperComplex{Int})
end

@test begin
    a = Plexifications.randomBigRational()
    z = random(HyperComplex{Rational{BigInt}})
    l = z / a
    r = a \ z
    l == r
end

@test begin
    a = Plexifications.randomBigRational()
    z = random(HyperComplex{Rational{BigInt}})
    l = a / z
    r = z \ a
    l == r
end

@test begin
    z = random(HyperComplex{BigInt})
    z == z + zero(HyperComplex{BigInt})
end

@test begin
    z = random(HyperComplex{BigInt})
    z == z * one(HyperComplex{BigInt})
end

@test begin
    z = random(HyperComplex{BigInt})
    z == z + zero(z)
end

@test begin
    z = random(HyperComplex{BigInt})
    z == z * one(z)
end

@test begin
    x = random(HyperComplex{BigInt})
    y = random(HyperComplex{BigInt})
    l = commutator(x, y)
    r = zero(HyperComplex{BigInt})
    l == r
end

@test begin
    x = random(HyperComplex{BigInt})
    y = random(HyperComplex{BigInt})
    l = alternatorL(x, y)
    r = zero(HyperComplex{BigInt})
    l == r
end

@test begin
    x = random(HyperComplex{BigInt})
    y = random(HyperComplex{BigInt})
    l = alternatorR(x, y)
    r = zero(HyperComplex{BigInt})
    l == r
end

@test begin
    x = random(HyperComplex{BigInt})
    y = random(HyperComplex{BigInt})
    l = flexator(x, y)
    r = zero(HyperComplex{BigInt})
    l == r
end

@test begin
    x = random(HyperComplex{BigInt})
    y = random(HyperComplex{BigInt})
    z = random(HyperComplex{BigInt})
    l = associator(x, y, z)
    r = zero(HyperComplex{BigInt})
    l == r
end

@test begin
    x = random(HyperComplex{BigInt})
    y = random(HyperComplex{BigInt})
    z = random(HyperComplex{BigInt})
    l = jacobiator(x, y, z)
    r = 3 * x * y * z
    l == r
end

@test begin
    l = HyperComplex(Cmplex(1,2))
    r = HyperComplex(1,2)
    l == r
end

@test begin
    l = HyperComplex(DualComplex(1,2))
    r = HyperComplex(1,2,0)
    l == r
end

@test begin
    l = HyperComplex(1, 2, 3.0)
    r = HyperComplex(1.0, 2.0, 3.0)
    l == r
end

@test begin
    l = HyperComplex(1, 2, 3, 4.0)
    r = HyperComplex(1.0, 2.0, 3.0, 4.0)
    l == r
end

@test begin
    l = HyperComplex(1, 2, 3, 4, 5.0)
    r = HyperComplex(1.0, 2.0, 3.0, 4.0, 5.0)
    l == r
end

@test begin
    l = HyperComplex(1, 2, 3, 4, 5, 6.0)
    r = HyperComplex(1.0, 2.0, 3.0, 4.0, 5.0, 6.0)
    l == r
end

@test begin
    l = HyperComplex(1, 2, 3, 4, 5, 6, 7.0)
    r = HyperComplex(1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0)
    l == r
end

@test begin
    l = HyperComplex(1, 2, 3, 4, 5, 6, 7, 8.0)
    r = HyperComplex(1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0)
    l == r
end


using LinearAlgebra
using CVChannel
using Test
using DelimitedFiles

"""
In this script we look at the PPT relaxation of the communication value of the
Werner-Holevo channel for multiple copies and its relation to the parameters
λ and the number of copies.
"""

println("Here we look at how the PPT relaxation of cv scales in number of copies.")
println("We do this for n ≤ 8 and λ in [0,0.1,...,1].") #It takes a lot more time if n = 9 & λ != 0
println("This also allows us to generate our non-multiplicativity results.")
@testset "Generate cv as function of n and lambda" begin
    d = 3
    n_list = [1:8;]
    λ_list = [0:0.01:1;]
    data_table = zeros(length(n_list),length(λ_list))
    for n_val in n_list
        println("Now obtaining values for n = ", n_val)
        λ_ctr = 1
        for λ_val in λ_list
            if λ_ctr == Int(ceil(length(λ_list)/2))
                println("Now halfway through this value of n.")
            end
            A,B,g,a = generalWHLPConstraints(n_val,d,λ_val*ones(n_val))
            cv_ppt, opt_var = wernerHolevoCVPPT(n_val,d,A,B,g,a)
            data_table[n_val,λ_ctr] = cv_ppt
            λ_ctr += 1
        end
    end
    #This generates the table about multiplicativity
    mult_table =[
        (data_table[2,:] - data_table[1,:].^2)';
        (data_table[4,:] - data_table[2,:].^2)';
        (data_table[8,:] - data_table[4,:].^2)';
    ]
    λ_header = zeros(1,length(λ_list))
    spacing = ""
    for i in 1:length(λ_list)
        λ_header[i] = λ_list[i]
        spacing = [spacing ""]
    end
    header_p_1 = hcat("n\\lambda:", λ_header)
    header_p_2 = hcat("lambda:", λ_header)
    col_inf_p_2 = ["cv(N^2) - cv(N)^2";"cv(N^4) - cv(N^2)^2";"cv(N^8) - cv(N^4)^2"]
    data_to_save_p_1 = vcat(hcat(n_list, data_table),spacing)
    data_to_save_p_2 = vcat(header_p_2,hcat(col_inf_p_2,mult_table))
    data_to_save = vcat(header_p_1,vcat(data_to_save_p_1,data_to_save_p_2))
    println("Here are the results:")
    show(stdout, "text/plain", data_to_save)

    info_vec = Vector{Union{Nothing,String}}(nothing, size(data_to_save)[1])
    info_vec[1] = "INFO:"
    info_vec[2] = "Generated by multi-copy-werner-holevo-cv.jl"
    info_vec[3] = "d = " * string(d)
    info_vec[4] = "n_list = " * string(n_list)
    info_vec[5] = "lambda_list = " * string(λ_list)
    data_to_save = hcat(data_to_save,info_vec)
    print("\nPlease name the file you'd like to write the results to: \n")
    file_name = readline()
    file_to_open = string(file_name,".csv")
    writedlm(file_to_open, data_to_save, ',')

    @test isapprox(data_table[2,1], 3, atol = 1e-6)
end


__precompile__()
"""
Module to apply coordinate transformations.
"""
module Transformations

using LazySets, ..Utils, MathematicalSystems

export transform

"""
    transform(S; [method])

Interface function that calls the respective transformation function.

### Input

- `S`      -- discrete or continuous system
- `method` -- (optional, default: `'schur'`) transformation method name; valid
              otions are:

    * `'schur'`

### Output

A tuple containing:

- transformed discrete or continuous system
- inverse transformation matrix for reverting the transformation

### Notes

The functions that are called in the background should return a the transformed
system components `A`, `X0`, and `U`, and also an inverse transformation matrix `M`.
"""
function transform(S::InitialValueProblem; method::String="schur")

    if method == "schur"
        return schur_transform(S)
    else
        error("the transformation method $method is undefined")
    end
end

"""
    schur_transform(S)

Applies a Schur transformation to a discrete or continuous system.

### Input

- `S` -- discrete or continuous system

### Output

A tuple containing:

- transformed discrete or continuous system
- inverse transformation matrix for reverting the transformation

### Algorithm

We use Julia's default `schurfact` function to compute a
[Schur decomposition](https://en.wikipedia.org/wiki/Schur_decomposition)
of the coefficients matrix ``A``.
"""
function schur_transform(S::InitialValueProblem)

    A_new, T_new = schur(full(S.A)) # full (dense) matrix is required

    # recall that for Schur matrices, inv(T) == T'
    Z_inverse = T_new.'

    # apply transformation to the initial states
    X0_new = Z_inverse * S.x0

    # apply transformation to the inputs
    U_new = Z_inverse * S.s.U

    # obtain the transformation matrix for reverting the transformation again
    T_inverse = F[:vectors]

    if S.s isa DiscreteSystem
        return (DiscreteSystem(A_new, X0_new, U_new), T_inverse)
    elseif S.s isa ContinuousSystem
        return (ContinuousSystem(A_new, X0_new, U_new), T_inverse)
    end
end

end # module


"""
Base abstract class for implementing regions in `N`-dimensional space with
dimensions of type `T`.
"""
abstract type Region{T,N} end

dimtype(::Type{<:Region{T}}) where T = T
dimtype(r::Region) = dimtype(typeof(r))
Base.ndims(::Type{<:Region{<:Any, N}}) where N = N
Base.ndims(r::Region) = ndims(typeof(r))

"""
`N`-dimensional point.
"""
struct Point{T, N} <: Region{T, N}
    coord::NTuple{N, T}
end

"""
    empty(::Type{T}) where T<:Region

Generate empty (uninitialized) region of type `T`.
"""
empty(::Type{Point{T,N}}) where {T,N} =
    Point{T,N}(ntuple(_ -> convert(T, NaN), N))
empty(::Type{Point{T,N}}) where {T<:Integer,N} =
    Point{T,N}(ntuple(_ -> zero(T), N))

"""
    area(a::Region)

`N`-dimensional "area" (volume etc) of `a`.
"""
area(a::Point{T}) where T = zero(typeof(zero(T)*zero(T)))

"""
    perimeter(a::Region)

The sum of the `a` sides.
"""
perimeter(a::Point{T}) where T = zero(T)

"""
    isvalid(a::Region)

Check that the parameters of `a` are valid and it defines a proper region.
"""
isvalid(a::Point) = all(!isnan, a.coord)

"""
Rectangular region constrained by `low[i]`...`high[i]` in each of `N` dimensions.
"""
struct Rect{T, N} <: Region{T, N}
    low::NTuple{N, T}
    high::NTuple{N, T}
end

Rect(low::NTuple{N, T1}, high::NTuple{N, T2}) where {N, T1<:Number, T2<:Number} =
    Rect{Base.promote_type(T1, T2), N}(low, high)

Rect{T,N}(pt::Point{T,N}) where {T,N} = Rect{T,N}(pt.coord, pt.coord)
Rect(pt::Point) = Rect(pt.coord, pt.coord)

# "empty" (uninitialized) rectangles
empty(::Type{Rect{T,N}}) where {T,N} =
    Rect{T,N}(ntuple(_ -> convert(T, NaN), N), ntuple(_ -> convert(T, NaN), N))
empty(::Type{Rect{T,N}}) where {T<:Integer,N} =
    Rect{T,N}(ntuple(_ -> typemax(T), N), ntuple(_ -> typemax(T), N))

"""
    combine(a::Region, b::Region)

MBR that contains both `a` and `b` regions.
"""
combine(a::Rect{T,N}, b::Rect{T,N}) where {T,N} =
    @inbounds Rect{T,N}(ntuple(i -> min(a.low[i], b.low[i]), N),
                        ntuple(i -> max(a.high[i], b.high[i]), N))

# check that coords are not NaN and side lengths not negative.
@generated isvalid(a::Rect{T,N}) where {T,N} =
    quote
        Base.Cartesian.@nall $N i -> !isnan(a.low[i]) && !isnan(a.high[i]) && a.low[i] <= a.high[i]
    end

"""
    intersect(a::Region, b::Region)

MBR that is an intersection of `a` and `b` regions.
"""
function intersect(a::Rect{T,N}, b::Rect{T,N}) where {T,N}
    res = @inbounds Rect{T,N}(ntuple(i -> max(a.low[i], b.low[i]), N),
                              ntuple(i -> min(a.high[i], b.high[i]), N))
    return isvalid(res) ? res : empty(typeof(a))
end

@generated area(a::Rect{T,N}) where {T,N} =
    quote
        @inbounds res = a.high[1] - a.low[1]
        @inbounds Base.Cartesian.@nexprs $(N-1) i -> res *= (a.high[i+1] - a.low[i+1])
        return res
    end

@generated perimeter(a::Rect{T,N}) where {T,N} =
    quote
        @inbounds res = a.high[1] - a.low[1]
        @inbounds Base.Cartesian.@nexprs $(N-1) i -> res += (a.high[i+1] - a.low[i+1])
        return res*ndims(a)
    end

"""
    overlap_area(a::Region, b::Region)

The area of MBR for `a` and `b` intersection.
"""
@generated overlap_area(a::Rect{T,N}, b::Rect{T,N}) where {T,N} =
    quote
        @inbounds res = min(a.high[1], b.high[1]) - max(a.low[1], b.low[1])
        res <= 0.0 && return zero(typeof(res))
        @inbounds Base.Cartesian.@nexprs $(N-1) i -> begin
            width_i = min(a.high[i+1], b.high[i+1]) - max(a.low[i+1], b.low[i+1])
            width_i <= zero(typeof(width_i)) && return zero(typeof(res))
            res *= width_i
        end
        return res
    end

"""
    combined_area(a::Region, b::Region)

The area of MBR for the union of `a` and `b`.
"""
@generated combined_area(a::Rect{T,N}, b::Rect{T,N}) where {T,N} =
    quote
        @inbounds res = max(a.high[1], b.high[1]) - min(a.low[1], b.low[1])
        @inbounds Base.Cartesian.@nexprs $(N-1) i -> res *= max(a.high[i+1], b.high[i+1]) - min(a.low[i+1], b.low[i+1])
        return res
    end

"""
    enlargement(a::Region, b::Region)

How much `a` grows when combined with `b`.

The difference between the MBR area of the `a` and `b` union and the MBR area of `a`.
"""
enlargement(a::Rect{T,N}, b::Rect{T,N}) where {T,N} = combined_area(a, b) - area(a)

"""
    center(a::Region)

The center point of `a`.
"""
center(a::Rect{T,N}) where {T,N} =
    Point(ntuple(i -> 0.5*(a.low[i] + a.high[i]), N))

@generated sqrdistance(a::Point{T,N}, b::Point{T,N}) where {T,N} =
    quote
        @inbounds res = abs2(a.coord[1] - b.coord[1])
        @inbounds Base.Cartesian.@nexprs $(N-1) i -> res += abs2(a.coord[i+1] - b.coord[i+1])
        return res
    end

"""
    intersects(a::Region, b::Region)

Check whether `a` intersects with `b`.
"""
@generated intersects(a::Rect{T,N}, b::Rect{T,N}) where {T,N} =
    quote
        @inbounds Base.Cartesian.@nall $N i -> (a.low[i] <= b.low[i] <= a.high[i]) || (b.low[i] <= a.low[i] <= b.high[i])
    end

"""
    contains(a::Region, b::Region)

Check whether `a` contains `b`.
"""
@generated contains(a::Rect{T,N}, b::Rect{T,N}) where {T,N} =
    quote
        @inbounds Base.Cartesian.@nall $N i -> (a.low[i] <= b.low[i]) && (a.high[i] >= b.high[i])
    end

@generated contains(a::Rect{T,N}, b::Point{T,N}) where {T,N} =
    quote
        @inbounds Base.Cartesian.@nall $N i -> (a.low[i] <= b.coord[i] <= a.high[i])
    end

"""
    in(a::Region, b::Region)

Check whether `a` is contained inside `b`.
"""
Base.in(a::Rect, b::Rect) = contains(b, a)
Base.in(a::Point, b::Rect) = contains(b, a)

# point is equal to the MBR if its low and high corners coincides
Base.:(==)(a::Point, b::Rect) = a.coord == b.low == b.high
Base.:(==)(a::Rect, b::Point) = b == a

"""
    touches(a::Rect, b::Rect)

Check whether `a` and `b` rectangles touches from the inside
(i.e. any `low` side touches `low` or `high` touches `high`).
"""
@generated touches(a::Rect{T,N}, b::Rect{T,N}) where {T,N} =
    quote
        @inbounds Base.Cartesian.@nany $N i -> (a.low[i] == b.low[i]) || (a.high[i] == b.high[i])
    end


#Script to make a series of demonstration plots
#27/07/2017

using Plots, JLD, LaTeXStrings
pyplot()
include("functions.jl")
PyPlot.close("all")

#Input information
atom = "87Rb"
nn = 50
ll = 0
jj = 0.5
mj = 0.5

alpha_c = getalpha(atom)

#Calculate wave function
normY_sol, rr = numerovfunc(atom,nn,ll,jj)

#Rescale for plotting
plotscale = sqrt(rr)
probamp = (normY_sol.*plotscale).^2

PyPlot.figure()
if nn>20
  plot(plotscale[plotscale .> sqrt(alpha_c^(1/3))],normY_sol[plotscale .> sqrt(alpha_c^(1/3))],linewidth=2)
else
  plot(plotscale,normY_sol,linewidth=2)
end
plot!(xlabel=L"(r/a_0)^{1/2}")
plot!(ylabel=L"r^{1/2}R(r) \,(a_0^{-1})")
plot!(title=string(atom," radial wavefunction |n,l,j⟩ = |",string(nn),",",string(ll),",",string(jj),"⟩"))
plot!(leg=false)
gui()

PyPlot.figure()
if nn>20
  plot(rr[rr .> alpha_c^(1/3)],probamp[rr .> alpha_c^(1/3)],linewidth=2)
else
  plot(rr,probamp,linewidth=2)
end
plot!(xlabel=L"r/a_0")
plot!(ylabel=L"|rR(r)|^2 \,(a_0^{-1})")
plot!(title=string(atom," radial probability density |n,l,j⟩ = |",string(nn),",",string(ll),",",string(jj),"⟩"))
plot!(leg=false)
gui()

#Compare groundstate to self-consistent field method
scfdict = load("Data/SCFcalc.jld")
plotscale2 = scfdict["plotscale2"]
sqrtrR = scfdict["sqrtrR"]
normY_solg, rrg = numerovfunc(atom,5,0,0.5)

PyPlot.figure()
plot(sqrt(rrg),-normY_solg,linewidth=2,label="Numerov")
scatter!(plotscale2,sqrtrR,label="Callaway")
plot!(xlabel=L"r/a_0")
plot!(ylabel=L"|r^{1/2}R(r)|^2 \,(a_0^{-1})")
plot!(title="Rubidium Groundstate Radial Wavefunction")
gui()

#Lifetimes
ltdict = load("Data/Lifetimes.jld")
sexpt = ltdict["sexpt"]
sexpterr = ltdict["sexpterr"]
pexpt = ltdict["pexpt"]
pexpterr = ltdict["pexpterr"]
dexpt = ltdict["dexpt"]
dexpterr = ltdict["dexpterr"]

#S1_2
nns = collect(28:45)
calcs = zeros(length(nns))
for kk = 1:length(nns)
  calcs[kk] = Radiative_lifetimes(atom,nns[kk],0,0.5)
end
calcs = calcs*1e6

#P3_2
nnp = collect(34:44)
calcp = zeros(length(nnp))
for kk = 1:length(nnp)
  calcp[kk] = Radiative_lifetimes(atom,nnp[kk],1,1.5)
end
calcp = calcp*1e6

#D5_2
nnd = collect(29:44)
calcd = zeros(length(nnd))
for kk = 1:length(nnd)
  calcd[kk] = Radiative_lifetimes(atom,nnd[kk],2,2.5)
end
calcd = calcd*1e6

PyPlot.figure()
scatter(nns,calcs,label="Calculated")
scatter!(nns,sexpt[:],yerr=repmat(sexpterr[:],1,2),label="Experimental")
plot!(xlabel=L"n")
plot!(ylabel=L"τ\, (μ\mathrm{s})")
plot!(title="Plot of calculated and experimental lifetimes for \n |n,l,j⟩=|n,0,0.5⟩ states")
gui()

PyPlot.figure()
scatter(nnp,calcp,label="Calculated")
scatter!(nnp,pexpt[:],yerr=repmat(pexpterr[:],1,2),label="Experimental")
plot!(xlabel=L"n")
plot!(ylabel=L"τ\, (μ\mathrm{s})")
plot!(title="Plot of calculated and experimental lifetimes for \n |n,l,j⟩=|n,1,1.5⟩ states")
gui()

PyPlot.figure()
scatter(nnd,calcd,label="Calculated")
scatter!(nnd,dexpt[:],yerr=repmat(dexpterr[:],1,2),label="Experimental")
plot!(xlabel=L"n")
plot!(ylabel=L"τ\, (μ\mathrm{s})")
plot!(title="Plot of calculated and experimental lifetimes for \n |n,l,j⟩=|n,2,2.5⟩ states")
gui()

#Blockade shift
RRSI, θ, blockadeshiftmeshGHz, C_6val = BlockadeShift(atom,nn,ll,jj,mj)

PyPlot.figure()
heatmap(RRSI*1e6,θ,blockadeshiftmeshGHz*1e3)
plot!(xlabel=L"R\, (μ\mathrm{m})")
plot!(ylabel=L"θ")
plot!(zlabel=L"ΔW\, (\mathrm{MHz})")
plot!(title=string("Calculated Rydberg blockade shift for \n |n,l,j,mⱼ⟩ = |",string(nn),",",string(ll),",",string(jj),",",string(mj),"⟩ state of ",atom," [MHz]"))
gui()

#C₆
c6dict = load("Data/C6data.jld")
nnc6 = c6dict["nnc6"]
#Convert from GHz⋅m⁻⁶ to GHz⋅μm⁻⁶
c6S1_2 = c6dict["c6S1_2"]*1e36
c6D3_2 = c6dict["c6D3_2"]*1e36
c6D5_2 = c6dict["c6D5_2"]*1e36

PyPlot.figure()
plot(nnc6,abs(c6S1_2),linewidth=2,marker=(:circle))
plot!(yaxis=(:log))
plot!(xlabel=L"n")
plot!(ylabel=L"|C₆|\, (\mathrm{GHz}⋅μ\mathrm{m}⁻⁶)")
plot!(title="Calculated C₆ for |n,l,j⟩ = |n,0,0.5⟩")
plot!(leg=false)
gui()

PyPlot.figure()
plot(nnc6,abs(c6D3_2),linewidth=2,marker=(:circle))
plot!(yaxis=(:log))
plot!(xlabel=L"n")
plot!(ylabel=L"|C₆|\, (\mathrm{GHz}⋅μ\mathrm{m}⁻⁶)")
plot!(title="Calculated C₆ for |n,l,j⟩ = |n,2,1.5⟩")
plot!(leg=false)
gui()

PyPlot.figure()
plot(nnc6,abs(c6D5_2),linewidth=2,marker=(:circle))
plot!(yaxis=(:log))
plot!(xlabel=L"n")
plot!(ylabel=L"|C₆|\, (\mathrm{GHz}⋅μ\mathrm{m}⁻⁶)")
plot!(title="Calculated C₆ for |n,l,j⟩ = |n,2,2.5⟩")
plot!(leg=false)
gui()


using Plots

MAXIMUMGRIDSIZE=198

function calculateState(Grid, i, j, gridsize)

    if i > 1 && j > 1 && i < gridsize && j < gridsize
        numneighbors = Grid[i-1, j] + Grid[i+1, j] + Grid[i, j-1] + Grid[i, j+1]
    elseif i == 1 && j == 1 #Top Left Corner
        numneighbors = Grid[i+1, j] + Grid[i, j+1]
    elseif i == 1 && j > 1 && j < gridsize #Top Row
        numneighbors = Grid[i, j-1] + Grid[i, j+1] + Grid[i+1, j]
    elseif j == 1 && i > 1 && i < gridsize #Left Column
        numneighbors = Grid[i-1, j] + Grid[i+1, j] + Grid[i, j+1]
    elseif i == gridsize && j > 1 && j < gridsize #Bottom Row
        numneighbors = Grid[i, j-1] + Grid[i, j+1] + Grid[i-1, j]
    elseif j == gridsize && i > 1 && i < gridsize #Right Column
        numneighbors = Grid[i-1, j] + Grid[i+1, j] + Grid[i, j-1]
    elseif i == gridsize && j == 1 #Bottom left Corner
        numneighbors = Grid[i-1, j] + Grid[i, j+1]
    elseif i == gridsize && j == gridsize #Bottom right Corner
        numneighbors = Grid[i-1, j] + Grid[i, j-1]
    elseif i == 1 && j == gridsize #Top right Corner
        numneighbors = Grid[i+1, j] + Grid[i, j-1]
    end

    if numneighbors == 3
        return 1
    elseif numneighbors < 2 || numneighbors > 3
        return 0
    else
        return Grid[i, j]
    end
end

function optimizedCalculateState(Grid, i, j, gridsize)

    if i == 1
        if j == 1 #Top Left Corner
            numneighbors = Grid[i+1, j] + Grid[i, j+1]
        elseif j < gridsize #Top Row
            numneighbors = Grid[i, j-1] + Grid[i, j+1] + Grid[i+1, j]
        else #Top right Corner
            numneighbors = Grid[i+1, j] + Grid[i, j-1]
        end
    elseif i < gridsize
        if j == 1 #Left Column
        numneighbors = Grid[i-1, j] + Grid[i+1, j] + Grid[i, j+1]
        elseif j < gridsize
            numneighbors = Grid[i-1, j] + Grid[i+1, j] + Grid[i, j-1] + Grid[i, j+1]
        else #Right Column
            numneighbors = Grid[i-1, j] + Grid[i+1, j] + Grid[i, j-1]
        end
    else
        if j == 1 #Bottom left Corner
            numneighbors = Grid[i-1, j] + Grid[i, j+1]
        elseif j < gridsize #Bottom Row
            numneighbors = Grid[i, j-1] + Grid[i, j+1] + Grid[i-1, j]
        else #Bottom right Corner
            numneighbors = Grid[i-1, j] + Grid[i, j-1]
        end
    end

    if numneighbors == 3
        return 1
    elseif numneighbors < 2 || numneighbors > 3
        return 0
    else
        return Grid[i, j]
    end
end

function calculateStateSwap(Grid, i::Int, j::Int, CurrentIndex::Int, gridsize::Int)

    if i > 1 && j > 1 && i < gridsize && j < gridsize
        numneighbors = Grid[i-1, j, CurrentIndex] + Grid[i+1, j, CurrentIndex] + Grid[i, j-1, CurrentIndex] + Grid[i, j+1, CurrentIndex]
    elseif i == 1 && j == 1 #Top Left Corner
        numneighbors = Grid[i+1, j, CurrentIndex] + Grid[i, j+1, CurrentIndex]
    elseif i == 1 && j > 1 && j < gridsize #Top Row
        numneighbors = Grid[i, j-1, CurrentIndex] + Grid[i, j+1, CurrentIndex] + Grid[i+1, j, CurrentIndex]
    elseif j == 1 && i > 1 && i < gridsize #Left Column
        numneighbors = Grid[i-1, j, CurrentIndex] + Grid[i+1, j, CurrentIndex] + Grid[i, j+1, CurrentIndex]
    elseif i == gridsize && j > 1 && j < gridsize #Bottom Row
        numneighbors = Grid[i, j-1, CurrentIndex] + Grid[i, j+1, CurrentIndex] + Grid[i-1, j, CurrentIndex]
    elseif j == gridsize && i > 1 && i < gridsize #Right Column
        numneighbors = Grid[i-1, j, CurrentIndex] + Grid[i+1, j, CurrentIndex] + Grid[i, j-1, CurrentIndex]
    elseif i == gridsize && j == 1 #Bottom left Corner
        numneighbors = Grid[i-1, j, CurrentIndex] + Grid[i, j+1, CurrentIndex]
    elseif i == gridsize && j == gridsize #Bottom right Corner
        numneighbors = Grid[i-1, j, CurrentIndex] + Grid[i, j-1, CurrentIndex]
    elseif i == 1 && j == gridsize #Top right Corner
        numneighbors = Grid[i+1, j, CurrentIndex] + Grid[i, j-1, CurrentIndex]
    end

    if numneighbors == 3
        return 1
    elseif numneighbors < 2 || numneighbors > 3
        return 0
    else
        return Grid[i, j, CurrentIndex]
    end
end

function simulateGoL(Grid, gridsize::Int)
    for turn in 1:100
        Grid_copy = deepcopy(Grid)
        for i in 1:gridsize
            for j in 1:gridsize
                Grid_copy[i, j] = calculateState(Grid, i, j, gridsize)
            end
        end
        Grid = Grid_copy
    end
end

function simulateGoLOpt(Grid, gridsize::Int)
    for turn in 1:100
        Grid_copy = deepcopy(Grid)
        for i in 1:gridsize
            for j in 1:gridsize
                Grid_copy[i, j] = optimizedCalculateState(Grid, i, j, gridsize)
            end
        end
        Grid = Grid_copy
    end
end

function simulateGoLGridSwap(Grid1, Grid2, gridsize::Int)
    for turn in 1:100
        if mod(turn, 2) == 0
            for i in 1:gridsize
                for j in 1:gridsize
                    Grid2[i, j] = calculateState(Grid1, i, j, gridsize)
                end
            end
        else
            for i in 1:gridsize
                for j in 1:gridsize
                    Grid1[i, j] = calculateState(Grid2, i, j, gridsize)
                end
            end
        end
    end
end

function simulateGoLChange(Grid, gridsize::Int, changearray)
    for turn in 1:100
        changecounter = 1
        for i in 1:gridsize
            for j in 1:gridsize
                newvalue = calculateState(Grid, i, j, gridsize)
                if Grid[i, j] != newvalue
                    changearray[changecounter, 1] = i
                    changearray[changecounter, 2] = j
                    changearray[changecounter, 3] = newvalue
                    changecounter += 1
                end
            end
        end
        for i in 1:changecounter-1
            Grid[changearray[i, 1], changearray[i, 2]] = changearray[i, 3]
        end
    end
end

time_per_simulation = Array{Float32}(undef, MAXIMUMGRIDSIZE)

for gridsize_iterator in 1:MAXIMUMGRIDSIZE
    Grid = rand(0:1, gridsize_iterator+2, gridsize_iterator+2)
    stats = @timed simulateGoL(Grid, gridsize_iterator+2)
    time_per_simulation[gridsize_iterator] = stats.time
end
plot([3:1:MAXIMUMGRIDSIZE+2], time_per_simulation, xlim=[3, MAXIMUMGRIDSIZE+2], dpi=600, xlabel="Grid Size", ylabel="Time Per Simulation", label="Nested For Loop")

time_per_simulation = Array{Float32}(undef, MAXIMUMGRIDSIZE)
changearray = zeros(Int16, MAXIMUMGRIDSIZE^2, 3)
for gridsize_iterator in 1:MAXIMUMGRIDSIZE
    Grid = rand(0:1, gridsize_iterator+2, gridsize_iterator+2)
    stats = @timed simulateGoLChange(Grid, gridsize_iterator+2, changearray)
    time_per_simulation[gridsize_iterator] = stats.time
end
plot!([3:1:MAXIMUMGRIDSIZE+2], time_per_simulation, xlim=[3, MAXIMUMGRIDSIZE+2], dpi=600, label="Change Track")

time_per_simulation = Array{Float32}(undef, MAXIMUMGRIDSIZE)
for gridsize_iterator in 1:MAXIMUMGRIDSIZE
    GridTemp = rand(0:1, gridsize_iterator+2, gridsize_iterator+2)
    Grid1 = deepcopy(GridTemp)
    Grid2 = deepcopy(GridTemp)
    stats = @timed simulateGoLGridSwap(Grid1, Grid2, gridsize_iterator+2)
    time_per_simulation[gridsize_iterator] = stats.time
end
plot!([3:1:MAXIMUMGRIDSIZE+2], time_per_simulation, xlim=[3, MAXIMUMGRIDSIZE+2], dpi=600, label="Grid Swap")

time_per_simulation = Array{Float32}(undef, MAXIMUMGRIDSIZE)
for gridsize_iterator in 1:MAXIMUMGRIDSIZE
    Grid = rand(0:1, gridsize_iterator+2, gridsize_iterator+2)
    stats = @timed simulateGoLOpt(Grid, gridsize_iterator+2)
    time_per_simulation[gridsize_iterator] = stats.time
end
plot!([3:1:MAXIMUMGRIDSIZE+2], time_per_simulation, xlim=[3, MAXIMUMGRIDSIZE+2], dpi=600, label="Nested For Loop Opt")

png("GoLdoubleLoop.png")


################################################################################
#
#          EllCrv/EllCrvDiv.jl : Divisors on Elliptic Curves and Logarithms of
#                                   Divisors
#
################################################################################

################################################################################
#
#  Submodules
#
################################################################################

include("EllCrv.jl")

################################################################################
#
#  Exports
#
################################################################################

export AbstractDivisor, EllCrvDivisor, EllCrvModel, Logarithm
export EllipticCurveDivisor, ord, assoc

################################################################################
#
#  Abstract Types/Structs
#
################################################################################

abstract type AbstractDivisor{EllCrvPt} end

mutable struct EllCrvDivisor{EllCrvPt} <: AbstractDivisor{EllCrvPt}
    degree::Int
    coeff::Array{Int, 1}
    points::Tuple{Vararg{EllCrvPt, N} where N}
    func_field_without_zero::AbstractAlgebra.Field
    rat_func::AbstractAlgebra.FieldElem
    principal::Bool

    # blowup(i::AbstractVarietyMap) --> AbstractVariety
    # if exceptionalDiv(blowup) --> EllCrvDivisor
    exceptional::Bool

    # if prod(coeff::Array{T, 1}) >= 0
    effective::Bool

    function EllCrvDivisor{EllCrvPt}(coeffs::Array{Int, 1},
        points::Tuple{Vararg{EllCrvPt, N} where N}, check::Bool = true) where
        EllCrvPt
        if check
            if assoc(rat_func, coeffs, points)
                ECD = new{EllCrvPt}()
                ECD = +(coeffs[1]*EllCrvPt...)
                ECD.principal = true
            elseif prod(coeffs) >= 0
                ECD = new{EllCrvPt}()
                ECD = +(coeffs[1]*EllCrvPt...)
                ECD.effective = true
            end
        else
            ECD = new{EllCrvPt}()
            ECD = +(coeffs[1]*EllCrvPt...)
        end
        return ECD
    end
end

mutable struct EllCrvModel
    curve::EllCrv
    point::EllCrvPt
    divisor::EllCrvDivisor
end

# Logarithms of Divisors
mutable struct Logarithm
    base_ec_model::EllCrvModel
    divisor::EllCrvDivisor
    #...
end

################################################################################
#
#  Constructors for users
#
################################################################################

function EllipticCurveDivisor(coeff::Array{Int, 1},
    points::Tuple{Vararg{EllCrvPt, N} where N}, check::Bool = true)
    if check
        ECD = EllCrvDivisor{EllCrvPt}(coeff, points, check)
        return ECD
    end
end

################################################################################
#
#  Divisorial Type Methods
#
################################################################################

function ord(rat_func::AbstractAlgebra.FieldElem,
    points::Tuple{Vararg{EllCrvPt, N} where N})

    if AbstractAlgebra.denominator(rat_func) == 0

    end

    return
end

function assoc(rat_func::AbstractAlgebra.FieldElem,
    coeff::Array{Int, 1}, points::Tuple{Vararg{EllCrvPt, N} where N})

    coeff = ord(rat_func, points)
    return EllipticCurveDivisor(coeff, points, true)
end


"""
    solve_pattern_generation(d::StudentHousingData)

Solve generalized assignment problem with column generation.
If `suboptimal_rc` is true, we generate the first column we find with negative
reduced cost.
"""
function solve_pattern_generation(d::StudentHousingData, suboptimal_rc::Bool=false)

    nhouses = length(d.houses)
    npatterns = length(d.patterns_allow)

    p_init = collect(1:npatterns)
    # Create an initial set of columns by randomly picking the first entity
    # allowed for any house
    U = zeros(nhouses, npatterns)
    for p = 1:npatterns
        # If there is a pattern that doesn't match any houses, skip it
        if isempty(d.patterns_allow[p]) || (d.demands[p, 1, 1] == 0)
            continue
        end
        # A naive initial assignment
        # Find the first house allowed
        for i = 1:nhouses
            if house_fits_pattern(i, p, d)
                U[i, p] = 1.0
                break
            end
            # If we don't want to check all patterns, only add 1 col
            if suboptimal_rc
                break
            end
        end
    end

    # Our cost vector
    c = ones(nhouses)

    # =========================================================================
    # Master problem
    # =========================================================================
    m = Model(solver = GurobiSolver(OutputFlag=0))
    @variable(m, 0 <= λ[i in p_init] <= 1)
    @objective(m, Max, sum(λ[p] * dot(c, U[:, p]) for p in p_init))
    @constraints(m, begin
        beds[i = 1:nhouses], sum(λ[p] * U[i, p] * beds_needed(p, d) for p in p_init) <= beds_avail(i, d.houses)
        convexity[p in p_init], sum(λ[p]) <= 1
    end)

    # =========================================================================
    # The knapsack subproblem for each pattern
    # =========================================================================
    function get_kp(p::Int, π::Vector{Float64})
        sp = Model(solver = GurobiSolver(OutputFlag=0))
        @variable(sp, u[i = 1:nhouses] >= 0, Int)
        @constraints(sp, begin
            sum(u[i] for i = 1:nhouses) <= d.demands[p, 1, 1]
            [i = 1:nhouses], u[i] <= house_fits_pattern(i, p, d)
        end)
        @objective(sp, Max, dot((c - π * beds_needed(p, d)), u))
        sp
    end

    # Cache information about columns we generate
    U_generated = Vector{Float64}[]
    pattern_choice = Int[]
    λ_generated = Variable[]

    best_sp  = Model()
    best_p = 0
    best_rc = Inf
    iter = 1

    while best_rc > 0
        # Solve the restricted master problem
        @assert solve(m) == :Optimal
        π = getdual(m[:beds])
        w = getdual(m[:convexity])

        # Solve a subproblem for each pattern, get the column with the best reduced
        # cost over all patterns
        best_rc = -Inf
        for p = 1:npatterns
            # Get the subproblem
            sp = get_kp(p, π)
            @assert solve(sp) == :Optimal
            col_rc = getobjectivevalue(sp) - w[p]
            if col_rc > best_rc
                best_rc = col_rc
                best_sp = sp
                best_p = p
            end
            # If we don't care about the optimal column
            if suboptimal_rc
                # ... and we found one with positive reduced cost
                if col_rc > 1e-4
                    # just add the one we found
                    break
                end
            end
        end

        best_u = getvalue(best_sp[:u])
        push!(U_generated, best_u)
        push!(pattern_choice, best_p)

        # Add the best column
        push!(λ_generated, @variable(m,
                lowerbound = 0.0,
                upperbound = 1.0,
                basename="_λnew_$iter",
                objective = dot(c, best_u),
                inconstraints = [beds; convexity[best_p]],
                coefficients = [best_u * beds_needed(best_p, d); 1.0]
              )
        )
        iter += 1
    end
    # Solve one last time
    @assert solve(m) == :Optimal

    # Return the master problem and information about generated columns
    m, U_generated, λ_generated, pattern_choice
end


"""
    Find the best question to ask by iterating over every feature / value
    and calculating the information gain.
"""
function find_best_split(rows)::Tuple{Float64, Union{Question, Nothing}}
    best_gain = 0.0  # keep track of the best information gain
    best_question::Union{Nothing, Question} = nothing  # keep track of the feature / value that produced it
    current_uncertainty = gini(rows)
    n_features = length(rows[1]) - 1  # number of columns

    for col in 1:n_features  # for each feature
        values::Set{Feature} = Set([row[col] for row in rows])  # unique values in the column
        for val in values  # for each value
            question = Question(col, val)

            # try splitting the dataset
            true_rows, false_rows = partition(rows, question)

            # Skip this split if it doesn't divide the dataset.
            if length(true_rows) == 0 || length(false_rows) == 0
                continue
            end
            # Calculate the information gain from this split
            gain = info_gain(true_rows, false_rows, current_uncertainty)

            # You actually can use ">" instead of ">=" here
            # but I wanted the tree to look a certain way for our
            # toy dataset.
            if gain >= best_gain
                best_gain, best_question = gain, question
            end # if
        end # for val in values
    end # for col in 1:n_features
    best_gain, best_question
end


using Random, Turing, PGFPlotsX, NNlib, SpecialFunctions, LaTeXStrings
using StatsFuns: logistic, norminvcdf

using Memoization
using ReverseDiff

using Zygote

N = 50
X = vcat(-rand(N) .- 0.5, rand(N) .+ 0.5)
l = vcat(ones(N), zeros(N))
y = l + 0.1*randn(2*N)

xt = range(-5., 5., length = 100);

const triangleconst = π/(2*sqrt(2))
const preluconst = π/4
_trianglewave(x) = (x - π * floor(x/π + 0.5)) * (-1)^floor(x/π + 0.5)
_trianglewave(x::ReverseDiff.TrackedReal) = ReverseDiff.ForwardOptimize(_trianglewave)(x)

heaviside(x) = x < zero(x) ? zero(x) : x >= zero(x) ? one(x) : oftype(x,0.5)
lsmatern(x, q, ν, λ) = q*heaviside(x)*x^(ν-1/2)*exp(-λ*x)
heaviside(x::ReverseDiff.TrackedReal) = ReverseDiff.ForwardOptimize(heaviside)(x)

abstract type PeriodicFunction end

struct SinActivation <: PeriodicFunction end
const sineconst = sqrt(2)
(σ::SinActivation)(x::T) where T = T(sineconst) * sin(x)

struct SinCosActivation <: PeriodicFunction end
(σ::SinCosActivation)(x) = sin(x) + cos(x)

struct TriangleWave <: PeriodicFunction end
(σ::TriangleWave)(x) = triangleconst * _trianglewave(x)

struct PeriodicReLU <: PeriodicFunction end
(σ::PeriodicReLU)(x) = preluconst * (_trianglewave(x) + _trianglewave(x + π/2))

abstract type ActivationFunction end

struct ReLU <: ActivationFunction end
(σ::ReLU)(x) = relu(x)
prior_weights(::ReLU) = Normal()
prior_bias(::ReLU) = Normal()
getname(::ReLU) = "relu"

struct Step <: ActivationFunction end
(σ::Step)(x) = heaviside(x)
prior_weights(::Step) = Normal()
prior_bias(::Step) = Normal()
getname(::Step) = "step"

struct Sigmoid <: ActivationFunction end
(σ::Sigmoid)(x) = sigmoid(x)
prior_weights(::Sigmoid) = Normal()
prior_bias(::Sigmoid) = Normal()
getname(::Sigmoid) = "sigmoid"

struct RBFNN <: ActivationFunction end
(σ::RBFNN)(x) = exp(-x^2)
prior_weights(::RBFNN) = Normal()
prior_bias(::RBFNN) = Normal()
getname(::RBFNN) = "RBF_NN"

struct MaternNN <: ActivationFunction
    q # precomputed constant
    λ # precomputed constant
    ν # smoothness
    function MaternNN(ν)
        λf(ν) = sqrt(2*ν)
        Af(ν) = sqrt(2 * π^(1/2) * λf(ν)^(2*ν) * gamma(ν + 0.5) / gamma(ν)) / gamma(ν + 0.5)
        return new(Af(ν), λf(ν), ν)
    end
end
(σ::MaternNN)(x) = σ.q*heaviside(x)*x^(σ.ν-1/2)*exp(-σ.λ*x)
prior_weights(::MaternNN) = Normal()
prior_bias(::MaternNN) = Normal()
getname(σ::MaternNN) = "Matern_NN_$(Int(σ.ν*10))"

struct RBF <: ActivationFunction
    σ::PeriodicFunction
end
(σ::RBF)(x) = σ.σ(x)
prior_weights(::RBF) = Normal()
prior_bias(::RBF) = Uniform(-π, π)
getname(::RBF) = "RBF"

struct Matern <: ActivationFunction
    σ::PeriodicFunction
    ν # smoothness
end
(σ::Matern)(x) = σ.σ(x)
prior_weights(σ::Matern) = σ.ν == 0.5 ? Cauchy() : TDist(2*σ.ν)
prior_bias(::Matern) = Uniform(-π, π)
getname(σ::Matern) = "Matern-$(Int(σ.ν*10))"

@pgf p = Axis(
    {
        ylabel= raw"$\sigma(x)$",
        no_markers,
        xticklabels={},
        yticklabels={},
        width = "15cm",
    },
    PlotInc(Table(xt, RBF(SinActivation()).(xt))),
    LegendEntry("sinusiod"),
    PlotInc(Table(xt, RBF(SinCosActivation()).(xt))),
    LegendEntry("sine-cosine"),
    PlotInc(Table(xt, RBF(TriangleWave()).(xt))),
    LegendEntry("triangle wave"),
    PlotInc(Table(xt, RBF(PeriodicReLU()).(xt))),
    LegendEntry("periodic ReLU"),
)

display("image/svg+xml", p)

@model function bnn(x, y, afun, K)

    w ~ filldist(prior_weights(afun), 1, K)
    b ~ filldist(prior_bias(afun), 1, K)

    l ~ FlatPos(0)
    z = afun.(l * x * w .+ b)

    wo ~ filldist(Normal(), K)
    bo ~ Normal()

    f = (z * (wo / K) .+ bo) / 2

    s ~ FlatPos(0)
    Turing.@addlogprob! -sum(abs2((yj-fj)/s)/2 + log(s) for (fj,yj) in zip(f,y))
end

Turing.setadbackend(:reversediff)
Turing.setrdcache(true);
Turing.setprogress!(true);

function predict(x, afun, w::Chains, b::Chains, l, wo::Chains, bo, s; burnin = 5000)
    nsamples = length(w)

    latents = map(s -> begin
            w_ = Array(w[s])
            b_ = Array(b[s])

            l_ = l[s]

            wo_ = vec(Array(wo[s]))
            bo_ = bo[s]

            z = afun.(x * l_ * w_ .+ b_)
            f = z * (wo_ / length(wo_)) .+ bo_
            f ./= 2
        end, burnin:nsamples)

    err_o = 1.96*sqrt.(var(latents)) .+ s
    err = 1.96*std(latents)

    return mean(latents), err, err_o
end

function plot(m, sf, so)

    @pgf p = Axis(
        {
            "scatter/classes" = {
                0 = {"black", mark_size="0.5"},
                1 = {"black", mark_size="0.5"},
            },
            axis_lines = "none",
            ymin = minimum(m .- so) - 0.25,
            ymax = maximum(m .+ so) + 0.25
        },
        PlotInc( { no_markers, white, solid }, Table(xt, m) ),
        PlotInc( { no_markers, white, solid }, Table(xt, m .+ so) ),
        PlotInc( { no_markers, white, solid }, Table(xt, m .- so) ),

        PlotInc( { scatter, "only marks", "scatter src" = "explicit symbolic", },
            Table({ meta = "label" }, x = X, y = y, label = Int.(l)) )
        )

    return p
end

function plot_lines(m, sf, so, filename)

        @pgf p_lines = Axis(
        {
            "scatter/classes" = {
                0 = {"black", mark_size="0.1"},
                1 = {"black", mark_size="0.1"},
            },
            ymin = minimum(m .- so) - 0.2,
            ymax = maximum(m .+ so) + 0.2
        },

        PlotInc( Graphics({xmin=minimum(xt),xmax=maximum(xt), ymin=minimum(m .- so), ymax=maximum(m .+ so)}, "$(pwd())/$(filename)") ),
        PlotInc( { no_markers, gray, dotted }, Table(xt, ones(length(xt))*0.5)),
        PlotInc( { no_markers, red, solid }, Table(xt, m) ),
        PlotInc( { no_markers, black, dashed }, Table(xt, m .+ sf) ),
        PlotInc( { no_markers, black, dashed }, Table(xt, m .- sf) ),
        PlotInc( { no_markers, black, solid }, Table(xt, m .+ so) ),
        PlotInc( { no_markers, black, solid }, Table(xt, m .- so) ),
        )

    return p_lines
end

afuns = [
    ReLU(),
    Step(),
    Sigmoid(),
    Matern(SinActivation(), 1/2),
    Matern(SinActivation(), 3/2),
    RBF(SinActivation()),
    MaternNN(1/2, 1),
    MaternNN(3/2, 1),
    RBFNN()
]

K = 10
for afun in afuns#[k:k]

    @info afun

    model = bnn(X, y, afun, K);
    chain = sample(model, NUTS(1000, 0.75), 5000);

    @info "Finished sampling"

    # extract names

    w_names = filter(n -> contains(string(n), "w["), names(chain))
    wo_names = filter(n -> contains(string(n), "wo["), names(chain))
    b_names = filter(n -> contains(string(n), "b["), names(chain))

    m, sf, so = predict(xt, afun,
        chain[w_names],
        chain[b_names],
        chain["l"],
        chain[wo_names],
        chain["bo"],
        mean(chain["s"]),
        burnin = 1000
    )

    p = plot(m, sf, so)

    @info "Saving plots"

    pgfsave("$(getname(afun))_plot.png", p)

    p = plot_lines(m, sf, so, "$(getname(afun))_plot.png")

    pgfsave("$(getname(afun))_plot.pdf", p)
    pgfsave("$(getname(afun))_plot.tex", p, include_preamble = false)
end