"""
Type: NLPStopping

Methods: start!, stop!, update\\_and\\_start!, update\\_and\\_stop!, fill\\_in!, reinit!, status
KKT, unconstrained\\_check, unconstrained2nd\\_check, optim\\_check\\_bounded

Specialization of GenericStopping. Stopping structure for non-linear programming problems using NLPModels.

Attributes:
- pb         : an AbstractNLPModel
- state      : The information relative to the problem, see GenericState
- (opt) meta : Metadata relative to stopping criterion, see `StoppingMeta`.
- (opt) main_stp : Stopping of the main loop in case we consider a Stopping
                          of a subproblem.
                          If not a subproblem, then nothing.
- (opt) listofstates : ListofStates designed to store the history of States.
- (opt) stopping_user_struct : Contains any structure designed by the user.

`NLPStopping(:: AbstractNLPModel, :: AbstractState; meta :: AbstractStoppingMeta = StoppingMeta(), max_cntrs :: Dict = init_max_counters(), main_stp :: Union{AbstractStopping, Nothing} = nothing, list :: Union{ListofStates, Nothing} = nothing, stopping_user_struct :: Any = nothing, kwargs...)`

 Note:
- designed for `NLPAtX` State. Constructor checks that the State has the
 required entries.

 There is an additional default constructor creating a Stopping where the State is by default and the
 optimality function is the function `KKT()``.

 `NLPStopping(pb :: AbstractNLPModel; kwargs...)`

 Note: Kwargs are forwarded to the classical constructor.
 """
mutable struct NLPStopping{
  Pb, M, SRC, T, MStp, LoS
}  <: AbstractStopping{Pb, M, SRC, T, MStp, LoS}

  # problem
  pb                   :: Pb

  # Common parameters
  meta                 :: M
  stop_remote          :: SRC

  # current state of the problem
  current_state        :: T

  # Stopping of the main problem, or nothing
  main_stp             :: MStp

  # History of states
  listofstates         :: LoS

  # User-specific structure
  stopping_user_struct :: AbstractDict

end

function NLPStopping(pb             :: Pb,
                     meta           :: M,
                     stop_remote    :: SRC,
                     current_state  :: T;
                     main_stp       :: AbstractStopping = VoidStopping(),
                     list           :: AbstractListofStates = VoidListofStates(),
                     user_struct    :: AbstractDict = Dict(),
                     kwargs...
                     ) where {Pb  <: AbstractNLPModel,
                              M   <: AbstractStoppingMeta,
                              SRC <: AbstractStopRemoteControl,
                              T   <: AbstractState}

    return NLPStopping(pb, meta, stop_remote, current_state,
                       main_stp, list, user_struct)
end

function NLPStopping(pb             :: Pb,
                     meta           :: M,
                     current_state  :: T;
                     main_stp       :: AbstractStopping = VoidStopping(),
                     list           :: AbstractListofStates = VoidListofStates(),
                     user_struct    :: AbstractDict = Dict(),
                     kwargs...
                     ) where {Pb <: AbstractNLPModel,
                              M  <: AbstractStoppingMeta,
                              T  <: AbstractState}

  stop_remote = StopRemoteControl() #main_stp == VoidStopping() ? StopRemoteControl() : cheap_stop_remote_control()

  return NLPStopping(pb, meta, stop_remote, current_state,
                     main_stp, list, user_struct)
end

function NLPStopping(pb             :: Pb,
                     current_state  :: T;
                     main_stp       :: AbstractStopping = VoidStopping(),
                     list           :: AbstractListofStates = VoidListofStates(),
                     user_struct    :: AbstractDict = Dict(),
                     kwargs...
                     ) where {Pb <: AbstractNLPModel, T <: AbstractState}

  if :max_cntrs in keys(kwargs)
    mcntrs = kwargs[:max_cntrs]
  else
    mcntrs = init_max_counters()
  end

  if :optimality_check in keys(kwargs)
    oc = kwargs[:optimality_check]
  else
    oc = KKT
  end

    meta = StoppingMeta(;max_cntrs = mcntrs, optimality_check = oc, kwargs...)
    stop_remote = StopRemoteControl()

  return NLPStopping(pb, meta, stop_remote, current_state,
                     main_stp, list, user_struct)
end

function NLPStopping(pb :: AbstractNLPModel;
                     n_listofstates :: Integer = 0,
                     kwargs...)
  #Create a default NLPAtX
  nlp_at_x = NLPAtX(pb.meta.x0)

  if n_listofstates > 0 && :list ∉ keys(kwargs)
    list = ListofStates(n_listofstates, Val{typeof(nlp_at_x)}())
    return NLPStopping(pb, nlp_at_x, list = list, optimality_check = KKT; kwargs...)
  end

  return NLPStopping(pb, nlp_at_x, optimality_check = KKT; kwargs...)
end

"""
init\\_max\\_counters:
initialize the maximum number of evaluations on each of
the functions present in the NLPModels.Counters, e.g.

`init_max_counters(; allevals :: T = typemax(T), obj = allevals, grad = allevals, cons = allevals, jcon = allevals, jgrad = allevals, jac = allevals, jprod = allevals, jtprod = allevals, hess = allevals, hprod = allevals, jhprod = allevals, sum = 11 * allevals, kwargs...)`

`:neval_sum` is by default limited to `|Counters| * allevals`.
"""
function init_max_counters(; allevals :: T = typemax(Int), kwargs...) where {T <: Integer}

  entries = [Meta.parse(split("$(f)", '_')[2]) for f in fieldnames(Counters)]
  lim_fields = keys(kwargs)
  cntrs = Dict{Symbol,T}([
    (Meta.parse("neval_$(t)"), t in lim_fields ? kwargs[t] : allevals) for t in entries
  ])
  push!(cntrs,
    (:neval_sum => :sum in lim_fields ? kwargs[:sum] : typemax(T) )
  )

  return cntrs
end

function max_evals!(stp :: NLPStopping, allevals :: Integer)
  stp.meta.max_cntrs = init_max_counters(allevals = allevals)
  return stp
end

function max_evals!(stp :: NLPStopping; allevals :: T = typemax(Int), kwargs...) where {T <: Integer}
  stp.meta.max_cntrs = init_max_counters(allevals = allevals; kwargs...)
  return stp
end

"""
init\\_max\\_counters\\_NLS:
initialize the maximum number of evaluations on each of
the functions present in the `NLPModels.NLSCounters`, e.g.

`init_max_counters_NLS(; allevals = typemax(T), residual = allevals, jac_residual = allevals, jprod_residual = allevals, jtprod_residual = allevals, hess_residual = allevals, jhess_residual = allevals, hprod_residual = allevals, kwargs...)`
"""
function init_max_counters_NLS(; allevals :: T = typemax(Int), kwargs...) where {T <: Integer}

  cntrs_nlp = init_max_counters(; allevals = allevals, kwargs...)

  entries = [Meta.parse(split("$(f)", '_')[2]) for f in setdiff(fieldnames(NLSCounters),[:counters])]
  lim_fields = keys(kwargs)
  cntrs = Dict{Symbol,T}([
    (Meta.parse("neval_$(t)"), t in lim_fields ? kwargs[t] : allevals) for t in entries
  ])

  return merge(cntrs_nlp, cntrs)
end

"""
fill_in!: (NLPStopping version) a function that fill in the required values in the `NLPAtX`.

`fill_in!( :: NLPStopping, :: Union{AbstractVector, Nothing}; fx :: Union{AbstractVector, Nothing} = nothing, gx :: Union{AbstractVector, Nothing} = nothing, Hx :: Union{MatrixType, Nothing} = nothing, cx :: Union{AbstractVector, Nothing} = nothing, Jx :: Union{MatrixType, Nothing} = nothing, lambda :: Union{AbstractVector, Nothing} = nothing, mu :: Union{AbstractVector, Nothing} = nothing, matrix_info :: Bool = true, kwargs...)`
"""
function fill_in!(stp         :: NLPStopping{Pb, M, SRC, NLPAtX{S, T, HT, JT}, MStp, LoS},
                  x           :: T;
                  fx          :: Union{eltype(T), Nothing} = nothing,
                  gx          :: Union{T, Nothing}         = nothing,
                  Hx          :: Union{HT, Nothing}        = nothing,
                  cx          :: Union{T, Nothing}         = nothing,
                  Jx          :: Union{JT, Nothing}        = nothing,
                  lambda      :: Union{T, Nothing}         = nothing,
                  mu          :: Union{T, Nothing}         = nothing,
                  matrix_info :: Bool                      = true,
                  kwargs...) where {Pb, M, SRC, MStp, LoS, S, T, HT, JT}

  gfx = isnothing(fx)  ? obj(stp.pb, x)   : fx
  ggx = isnothing(gx)  ? grad(stp.pb, x)  : gx

  if isnothing(Hx) && matrix_info
    gHx = hess(stp.pb, x).data
  else
    gHx = Hx
  end

  if stp.pb.meta.ncon > 0
    gJx = isnothing(Jx) ? jac(stp.pb, x)  : Jx
    gcx = isnothing(cx) ? cons(stp.pb, x) : cx
  else
    gJx = stp.current_state.Jx
    gcx = stp.current_state.cx
  end

  #update the Lagrange multiplier if one of the 2 is asked
  if (stp.pb.meta.ncon > 0 || has_bounds(stp.pb)) && (isnothing(lambda) || isnothing(mu))
    lb, lc = _compute_mutliplier(stp.pb, x, ggx, gcx, gJx; kwargs...)
  elseif  stp.pb.meta.ncon == 0 && !has_bounds(stp.pb) && isnothing(lambda)
    lb, lc = mu, stp.current_state.lambda
  else
    lb, lc = mu, lambda
  end

  return update!(stp, x = x, fx = gfx, gx = ggx, Hx = gHx,
                             cx = gcx, Jx = gJx, mu = lb,
                             lambda = lc)
end

function fill_in!(stp         :: NLPStopping{Pb, M, SRC, OneDAtX{S,T}, MStp, LoS},
                  x           :: T;
                  fx          :: Union{T, Nothing} = nothing,
                  gx          :: Union{T, Nothing} = nothing,
                  f₀          :: Union{T, Nothing} = nothing,
                  g₀          :: Union{T, Nothing} = nothing,
                  kwargs...) where {Pb, M, SRC, S, T, MStp, LoS}

 gfx = isnothing(fx) ? obj(stp.pb, x)    : fx
 ggx = isnothing(gx) ? grad(stp.pb, x)   : gx
 gf₀ = isnothing(f₀) ? obj(stp.pb, 0.0)  : f₀
 gg₀ = isnothing(g₀) ? grad(stp.pb, 0.0) : g₀

 return update!(stp.current_state, x=x, fx = gfx, gx = ggx, f₀ = gf₀, g₀ = gg₀)
end

"""
For NLPStopping, `rcounters` set as true also reinitialize the counters.
"""
function reinit!(stp       :: NLPStopping;
                 rstate    :: Bool = false,
                 rlist     :: Bool = true,
                 rcounters :: Bool = false,
                 kwargs...)

  stp.meta.start_time  = NaN
  stp.meta.optimality0 = 1.0

  #reinitialize the boolean status
  reinit!(stp.meta)

  #reinitialize the counter of stop
  stp.meta.nb_of_stop = 0

  #reinitialize the list of states
  if rlist && (typeof(stp.listofstates) != VoidListofStates)
    #TODO: Warning we cannot change the type of ListofStates
    stp.listofstates = rstate ? VoidListofStates() : ListofStates(stp.current_state)
  end

  #reinitialize the state
  if rstate
    reinit!(stp.current_state; kwargs...)
  end

  #reinitialize the NLPModel Counters
  if rcounters && typeof(stp.pb) <: AbstractNLPModel
    NLPModels.reset!(stp.pb)
  end

  return stp
end

"""
\\_resources\\_check!: check if the optimization algorithm has exhausted the resources.
                   This is the NLP specialized version that takes into account
                   the evaluation of the functions following the sum_counters
                   structure from NLPModels.

`_resources_check!(:: NLPStopping, :: T)`

Note:
- function uses counters in `stp.pb`, and update the counters in the state.
- function is compatible with `Counters`, `NLSCounters`, and any type whose entries match the entries of `max_cntrs`.
- all the NLPModels have an attribute `counters` and a function `sum_counters(nlp)`.
"""
function _resources_check!(stp    :: NLPStopping,
                           x      :: T
                           ) where T <: Union{AbstractVector, Number}
  max_cntrs = stp.meta.max_cntrs

  if max_cntrs == Dict{Symbol,Int64}()
    return stp.meta.resources
  end

  # check all the entries in the counter
  max_f = false
  if typeof(stp.pb) <: AbstractNLSModel || (:counters in fieldnames(typeof(stp.pb)) && typeof(stp.pb.counters) == NLSCounters)
    for f in intersect(fieldnames(NLSCounters), keys(max_cntrs))
      max_f = f != :counters ? (max_f || (eval(f)(stp.pb) > max_cntrs[f])) : max_f
    end
    for f in intersect(fieldnames(Counters), keys(max_cntrs))
      max_f = max_f || (eval(f)(stp.pb) > max_cntrs[f])
    end
  elseif typeof(stp.pb) <: AbstractNLPModel || (:counters in fieldnames(typeof(stp.pb)) && typeof(stp.pb.counters) == Counters)
    for f in intersect(fieldnames(Counters), keys(max_cntrs))
      max_f = max_f || (eval(f)(stp.pb) > max_cntrs[f])
    end
  end

  # Maximum number of function and derivative(s) computation
  if :neval_sum in keys(stp.meta.max_cntrs)
    max_evals = sum_counters(stp.pb) > max_cntrs[:neval_sum]
  end

  # global user limit diagnostic
  if (max_evals || max_f) stp.meta.resources = true end

  return stp.meta.resources
end

"""
`_unbounded_problem_check!`: This is the NLP specialized version that takes into account
                   that the problem might be unbounded if the objective or the
                   constraint function are unbounded.

`_unbounded_problem_check!(:: NLPStopping, :: AbstractVector)`

Note:
- evaluate the objective function if `state.fx` for NLPAtX or `state.fx` for OneDAtX is `_init_field` and store in `state`.
- do NOT evaluate the constraint function if `state.cx` is `_init_field` and store in `state`.
- if minimize problem (i.e. nlp.meta.minimize is true) check if
`state.fx <= - meta.unbounded_threshold`,
otherwise check `state.fx ≥ meta.unbounded_threshold`.
- `state.cx` is unbounded if larger than `|meta.unbounded_threshold|`.
"""
function _unbounded_problem_check!(stp  :: NLPStopping{Pb, M, SRC, NLPAtX{S, T, HT, JT}, MStp, LoS},
                                   x    :: AbstractVector
                                  ) where {Pb, M, SRC, MStp, LoS, S, T, HT, JT}

  if isnan(stp.current_state.fx)
	  stp.current_state.fx = obj(stp.pb, x)
  end

  if stp.pb.meta.minimize
    f_too_large = stp.current_state.fx <= - stp.meta.unbounded_threshold
  else
    f_too_large = stp.current_state.fx >=   stp.meta.unbounded_threshold
  end

  c_too_large = false
  #we do not evaluate the constraint if not in the state.
  if stp.pb.meta.ncon != 0 && (stp.current_state.cx != _init_field(typeof(stp.current_state.cx)))
    c_too_large = norm(stp.current_state.cx) >= abs(stp.meta.unbounded_threshold)
  end

  if (f_too_large || c_too_large) stp.meta.unbounded_pb = true end

  return stp.meta.unbounded_pb
end

function _unbounded_problem_check!(stp  :: NLPStopping{Pb, M, SRC, OneDAtX{S,T}, MStp, LoS},
                                   x    :: Union{AbstractVector, Number}
                                  ) where {Pb, M, SRC, MStp, LoS, S, T}
  if isnan(stp.current_state.fx)
	  stp.current_state.fx = obj(stp.pb, x)
  end

  if stp.pb.meta.minimize
    f_too_large = stp.current_state.fx <= - stp.meta.unbounded_threshold
  else
    f_too_large = stp.current_state.fx >=   stp.meta.unbounded_threshold
  end

  return stp.meta.unbounded_pb
end
"""
\\_infeasibility\\_check!: This is the NLP specialized version.

Note:
  - check wether the `current_score` contains Inf.
  - check the feasibility of an optimization problem in the spirit of a convex
  indicator function.
"""
function _infeasibility_check!(stp  :: NLPStopping,
                               x    :: T) where T
#=
#- evaluate the constraint function if `state.cx` is `nothing` and store in `state`.
#- check the Inf-norm of the violation ≤ stp.meta.atol
 if stp.pb.meta.ncon != 0 #if the problems has constraints, check |c(x)|
  cx = stp.current_state.cx
  if cx == _init_field(typeof(stp.current_state.cx))
   cx = cons(stp.pb, x)
  end
  vio = max.(max.(cx - stp.pb.meta.ucon, 0.), max.(stp.pb.meta.lcon - cx, 0.))
  tol = Inf #stp.meta.atol
  stp.meta.infeasible = _inequality_check(vio, stp.meta.atol, 0.) ? true : stp.meta.infeasible
 end
 =#

  if stp.pb.meta.minimize
    vio = any(z-> z ==  Inf, stp.current_state.current_score)
    if vio stp.meta.infeasible = true end
  else
    vio = any(z-> z == -Inf, stp.current_state.current_score)
    if vio stp.meta.infeasible = true end
  end

  return stp.meta.infeasible
end

################################################################################
# Nonlinear problems admissibility functions
# Available: unconstrained_check(...), optim_check_bounded(...), KKT
################################################################################
include("nlp_admissible_functions.jl")

################################################################################
# line search admissibility functions
#
# TODO: change the ls_admissible_functions and use tol_check et tol_check_neg to
# handle the inequality instead of a max.
################################################################################
include("ls_admissible_functions.jl")

#=
"""
"""
function feasibility_optim_check(pb, state; kwargs...)
     vio = _feasibility(pb, state)
     tol = Inf #stp.meta.atol
     return _inequality_check(vio, tol, 0.)
end
=#

################################################################################
# Functions computing Lagrange multipliers of a nonlinear problem
# Available: _compute_mutliplier(...)
################################################################################
include("nlp_compute_multiplier.jl")


module DFUtil

using DataFrames
using Dates
using Pipe
using CSV

export sum_columns, group_data_into_periods, match_row, to_json, to_json_var, to_js_value
export de_miss_rows, include_or_exclude, firstrow, lastrow, nthrow
export pQuarter, pWeek, pYear, pMonth
export to_csv_text, from_csv_text
export concat!, leftjoiner, tryRename, tryRename!, dropByName!, colsToGrid

function force_vector(v)
	if isa(v, String) || isa(v, Symbol)
		return [v]
	end
	v = collect(Iterators.flatten([v]))
	return isa(v, Vector) ? v : [v]
end

"""
	include_or_exclude(df, includes, excludes)

Given a DataFrame either restrict it to the includes list, or remove the exludes list

Sounds a bit daft but it is to provide KW options in functions

If both are given, the `includes` take priority

# Arguments
- `df` the DataFrame
- `includes` the String, Symbol or list of those to include
- `excludes` the String, Symbol or list of those to exclude

# Examples

	df = DataFrame([[1,2,3], [10,20,30]], ["a", "b"])

	julia> include_or_exclude(df, includes=["a"])
	3×1 DataFrame
	Row │ a
		│ Int64
	─────┼───────
	1 │     1
	2 │     2
	3 │     3

	julia> include_or_exclude(df, excludes="a")
	3×1 DataFrame
	Row │ b
		│ Int64
	─────┼───────
	1 │    10
	2 │    20
	3 │    30

	julia> include_or_exclude(df, includes=["a"], excludes=["a", "b"])
	3×1 DataFrame
	Row │ a
		│ Int64
	─────┼───────
	1 │     1
	2 │     2
	3 │     3

"""
function include_or_exclude(df; includes=nothing, excludes=nothing)
	if excludes === nothing && includes === nothing
		return df
	end

	if excludes === nothing
		# force vector to makes sure it returns a DataFrame
		return df[!, force_vector(includes)]
	end

	if includes === nothing
		return df[!, Not(excludes)]
	end
	# we have includes *and* excludes.
	# Includes take priority

	# makes sure they are both lists, and the same datatype
	includes = map(i->string(i), force_vector(includes))
	excludes = map(e->string(e), force_vector(excludes))

	return df[!, Not(filter(e->! (e in includes), excludes))]
end
"""
	sum_columns(df, group_by::Vector{String}=Vector{String}())

Sum the columns of the dataframe, optionally grouping by group_by

All the columns not in the grouping *must* have a `+` method

# Arguments

- `df` DataFrame to sum
- `group_by` Vector of String names of the columns to group by
- `replace_with` by default DataFrames takes the column names and appends _function. Instead use this string.
"""
sum_columns(df; group_by=Vector{String}(), replace_with="s") = @pipe map(String, group_by) |>
		groupby(df, _) |>
		combine(_,  [ c => c->sum(skipmissing(c)) for c in filter(n->!(n in group_by), names(df)) ]) |>
		rename(_, map(c-> c=>replace(c, "_function"=>replace_with), names(_)))

"""
	group_data_into_periods(df::DataFrame, date_column::Union{Symbol, AbstractString}, period::Union{Symbol, AbstractString}; andgrpby = Vector{String}())

Using the given data_column, group the dataframe into periods.

All columns except the grouping ones *must* have methods for `+``

# Arguments
- `df` DataFrame to group
- `date_column` the column to use as the grouping
- `period` Period to use, options are: :Quarter, :Month, :Year, :Week (or as strings)
- `andgrpby` optional keyword to use additional groupings

# Examples
	group_data_into_periods(df, :SaleDate, :Week)
	group_data_into_periods(df, "SaleDate", :Year, "BranchId")
	group_data_into_periods(df, :SaleDate, "Quarter", [:Area, :BranchId)]
"""
function group_data_into_periods(df, date_column, period; andgrpby=Vector{String}())
	if isa(period, String)
		period = Symbol(period)
	end
	if isa(andgrpby, String) || isa(andgrpby, Symbol)
		andgrpby = [andgrpby]
	end

	period_fns = Dict(:Quarter => pQuarter, :Month => pMonth, :Year => pYear, :Week => pWeek)

	@pipe transform(df, date_column => ByRow(dt -> period_fns[period](dt)) => "$(date_column)_function") |>
		select(_, Not(date_column)) |>
		sum_columns(_, group_by=vcat(map(String, andgrpby), ["$(date_column)_function"]))
end

# period functions
"""
	pQuarter(dt)

turn a Date / Datetime into its eqivalent YearQn representation e.g. 2001Q1
"""
pQuarter(dt)   = ismissing(dt) ? missing : string(Dates.year(dt)) * "Q" * string(Dates.quarterofyear(dt))
"""
pMonth(dt)

turn a Date / Datetime into its eqivalent YearMonth representation e.g. 2001-01
"""
pMonth(dt) = ismissing(dt) ? missing : string(Dates.year(dt)) * "-" * ("00" * string(Dates.month(dt)))[end-1:end]
"""
	pYear(dt)

turn a Date / Datetime into its eqivalent Year representation e.g. 2001
"""
pYear(dt)  = ismissing(dt) ? missing : Dates.year(dt)
"""
	pWeek(dt)

Turn a Date / Datetime into its eqivalent YearWeek representation e.g. 2001-01
Preserves 2000-01-01 becoming 1999-52
"""
function pWeek(dt) # this is the tricky one because 2000-01-01 is 1999-52
	if ismissing(dt)
		return  missing
	end
	m = Dates.month(dt)
	w = Dates.week(dt)
	y = Dates.year(dt)
	if m == 1 && w > 10
		y = y - 1
	end
	"$y-" * ("00" * string(w))[end-1:end] # right pad with 0
end
"""
	match_row(df, col, val)

Just a shortcut for `filter(row -> row[col] == val, df)`

"""
match_row(df, col, val) = filter(row -> row[col] == val, df)

"""
	to_json_var(io::IO, df::DataFrame, keys::Union{AbstractString, Vector{AbstractString}}, var="object")

Output the dataframe to json using to_json but wrap it inside a Javascript JSON.parse() for direct use in Javascript

# Arguments
- `io` IO handle to write to
- `df` DataFrame to write
- `keys` Keys to lift outside the object - see [`to_json`](@ref) for explanation of that
- `var` JavaScript object name it will be assign to with `var \$var = ....`

# Examples

	df = DataFrame([[1,2],[3,4],[4,5]],["a", "b", "c"]), "a")

	to_json_var(stdout, df, "a", "object")

	"var object = JSON.parse('{ \"1\" : {\"a\" : \"1\", \"b\" : \"3\", \"c\" : \"4\" } , \"2\" : {\"a\" : \"2\", \"b\" : \"4\", \"c\" : \"5\" }  }');"
"""
function to_json_var(io::IO, df::DataFrame, keys, var="object")
	print(io, "var $var = JSON.parse('")
	to_json(io, df, keys)
	println(io, "');");
end

"""
	to_js_value(io::IO, df::DataFrame, keys::Union{AbstractString, Vector{AbstractString}}, jsdeclaration="var object")

Output the dataframe to json using to_json but wrap it inside a Javascript JSON.parse() for direct use in Javascript

# Arguments
- `io` IO handle to write to
- `df` DataFrame to write
- `keys` Keys to lift outside the object - see [`to_json`](@ref) for explanation of that
- `var` JavaScript object name it will be assign to with `var \$var = ....`

# Examples

	df = DataFrame([[1,2],[3,4],[4,5]],["a", "b", "c"]), "a")

	to_js_value(stdout, df, "a", "export const vals")

	"export const vals = JSON.parse('{ \"1\" : {\"a\" : \"1\", \"b\" : \"3\", \"c\" : \"4\" } , \"2\" : {\"a\" : \"2\", \"b\" : \"4\", \"c\" : \"5\" }  }');"
"""
function to_js_value(io::IO, df::DataFrame, keys, jsdeclaration="var object")
	print(io, "$jsdeclaration = JSON.parse('")
	to_json(io, df, keys)
	println(io, "');");
end

"""
	to_json(io::IO, data::DataFrame, keys::Union{AbstractString, Vector{AbstractString}})

Output the dataframe to Json, grouping by the given keys

	{ "row[keys[1]]" : { "row[keys[2]]" : { "names(row)[1]" : "row[names(row[1])]", "names(row)[2]" : "row[names(row[2])]", ..., "names(row)[end]" : "row[names(row[end])]"}}}

The motivation for this is to enable referencing the objects via their nested keys in JavaScript

	julia> df = DataFrame([[10, 10, 11], [1, 2, 3], ["Fred", "Luke", "Alice"]], ["Area", "EmpId", "Name"])
	3×3 DataFrame
	 Row │ Area   EmpId  Name
		 │ Int64  Int64  String
	─────┼──────────────────────
	   1 │    10      1  Fred
	   2 │    10      2  Luke
	   3 │    11      1  Alice

	   julia> to_json(stdout, df, ["Area", "EmpId"])

	   { "10" : {"1" : {"Area" : "10", "EmpId" : "1", "Name" : "Fred" } , "2" : {"Area" : "10", "EmpId" : "2", "Name" : "Luke" }  }, "11" : {"3" : {"Area" : "11", "EmpId" : "3", "Name" : "Alice" }  } }

in Javascript

	let fred = object[10][1]
	let luke = object[10][2]
	let alice = object[11][1]

# Arguments
- `io` IO handle to write to
- `df` DataFrame to write
- `keys` Keys to lift outside the object

# Examples

	df = DataFrame([[1,2],[3,4],[4,5]],["a", "b", "c"]), "a")

	to_json(stdout, df, "a")

	"{ \"1\" : {\"a\" : \"1\", \"b\" : \"3\", \"c\" : \"4\" } , \"2\" : {\"a\" : \"2\", \"b\" : \"4\", \"c\" : \"5\" }  }"

	to_json(stdout, df, ["a", "b"])

	"{ \"1\" : {\"3\" : {\"a\" : \"1\", \"b\" : \"3\", \"c\" : \"4\" }  }, \"2\" : {\"4\" : {\"a\" : \"2\", \"b\" : \"4\", \"c\" : \"5\" }  } }")

"""
function to_json(io, df, keys)
	print(io, "{ ")
	if isa(keys, Array)
		_to_json(io, groupby(df, keys[1]), keys, 1)
	else
		_to_json(io, groupby(df, keys), [keys], 1)
	end
	print(io, " }")
end

# turn the first row of a DF into a DataFrameRow, a helper for _to_json
row(grp) = eachrow(grp)[1]

# helper for to_json
function _to_json(io, data, keys, depth)
	prefix = ""
	if depth == length(keys)
		print_data_row(io, row(data[1]), keys[depth], "")
		for grp in data[2:end]
			print_data_row(io, row(grp), keys[depth], ", ")
		end
	else
		for grp in data
			k = kesc(grp[!, keys[depth]][1])
			print(io, prefix, "\"$k\" : {")
			_to_json(io, groupby(grp, keys[depth+1]), keys, depth+1)
			print(io, " }");
			prefix = ", "
		end
	end
end

# key / value escapers
jesc(v) = replace(string(v), "'"=>"\\'")
kesc(k) = replace(jesc(k), " "=>"")

# print a key value pair
print_term(io, row, key, prefix="") = print(io, prefix, "\"$(kesc(key))\" : ", "\"$(jesc(row[key]))\"");

# print all key value pairs, with appropriate comma-ing
function print_terms(io, row)
	print_term(io, row, names(row)[1], "")
	foreach(term -> print_term(io, row, term, ", "), names(row)[2:end])
end

# print a compete row, with single pkey
function print_data_row(io, row, pkey, prefix)
	print(io, prefix, "\"$(kesc(row[pkey]))\" : {")
	print_terms(io, row)
	print(io, " } ");
end
"""
	nthrow(df)

Return the nth row of a dataframe as a DataFrameRow, if n is out of bounds, return nothing
"""
function nthrow(df, n)
	if size(df)[1] < n || n < 1
		return
	end
	eachrow(df)[n]
end

"""
	firstrow(df)

Return the first row as a DataFrameRow
"""
firstrow(df) = nthrow(df, 1)

"""
	lastrow(df)

	Return the last row of df as a DataFrameRow
"""
lastrow(df) = nthrow(df, size(df)[1])

"""
	exclude_rows(df, fn)

Exclude any row which has any column return true for the function

# Examples

	exclude_rows(df, ismissing)
	exclude_rows(df, isnan)

"""

function exclude_rows(df, fn)

	function pass(row)
		for n in names(df)
			if fn(row[n])
				return false
			end
		end
		return true
	end

	filter(pass, df)
end

"""
	de_miss_rows(df)

For a given DataFrame, remove any rows in which any column has a missing value

# Examples

	df = DataFrame([[1,missing,3], [10,20,30]], ["a", "b"])
	3×2 DataFrame
	 Row │ a        b
		 │ Int64?   Int64?
	─────┼─────────────────
	   1 │       1      10
	   2 │ missing      20
	   3 │       3      30

	julia> dm = de_miss_rows(df)
	2×2 DataFrame
	 Row │ a       b
		 │ Int64?  Int64?
	─────┼────────────────
	   1 │      1      10
	   2 │      3      30

"""
de_miss_rows(df) = exclude_rows(df, ismissing)

"""
	to_csv_text(df)

turn the dataframe into the string representation of the CSV

# Examples

	df = DataFrame(
	[[53.685335, 53.785335, 53.885335],
	[-0.416387,-0.426387,-0.436387],
	[53.479544, 53.479544,53.479544],
	[-2.954101,-2.954101,-2.954101]], ["lat1", "lon1", "lat2", "lon2"])

	to_csv_text(df)
	"lat1,lon1,lat2,lon2\\n53.685335,-0.416387,53.479544,-2.954101\\n53.785335,-0.426387,53.479544,-2.954101\\n53.885335,-0.436387,53.479544,-2.954101\\n"
"""
function to_csv_text(df)
	out = IOBuffer()
	CSV.write(out, df)
	seek(out, 0)
	read(out, String)
end

"""
	from_csv_text(csv_text)

Create a dataframe from the string represenation of a CSV

# Examples

	from_csv_text("lat1,lon1,lat2,lon2\\n53.685335,-0.416387,53.479544,-2.954101\\n53.785335,-0.426387,53.479544,-2.954101\\n53.885335,-0.436387,53.479544,-2.954101\\n")

	3×4 DataFrame
	Row │ lat1     lon1       lat2     lon2
		│ Float64  Float64    Float64  Float64
   ─────┼──────────────────────────────────────
	  1 │ 53.6853  -0.416387  53.4795  -2.9541
	  2 │ 53.7853  -0.426387  53.4795  -2.9541
	  3 │ 53.8853  -0.436387  53.4795  -2.9541

"""
from_csv_text(csv_text) = CSV.read(IOBuffer(csv_text), DataFrame)

"""
	concat!(df, args...)

All of the dataframes supplied as args, appended to df
"""
concat!(df, args...) = reduce((adf, df)->append!(adf, df), args, init=df)

"""
	leftjoiner(df, on, args...; kw...)

# Arguments
- `df` The initial DataFrame
- `on` The field(s) to join with
- `args...` The DataFrames to join to the df
- `kw...` KW args passed to leftjoin
"""
leftjoiner(df, on, args...; kw...) = reduce((accumdf, nextdf)->leftjoin(accumdf, nextdf, on=on, kw...), args, init=df)

"""
	tryRename(df, renamelist)
	tryRename!(df, renamelist)

Attempts to renames the given fields, however, unlike DataFrames.rename, doesn't complain if the column doesn't exist

"""
tryRename(df, renamelist::Pair) = tryRename(df, [renamelist])
tryRename(df, renamelist::Vector{Pair{T,T}}) where T <: Any = tryRename(df, map(p->Symbol(p[1])=>Symbol(p[2]), renamelist))
tryRename(df, renamelist::Vector{Pair{Symbol, Symbol}}) = tryRenameFn(df, filter(p->p[1] in map(Symbol, names(df)), renamelist), rename)

tryRename!(df, renamelist::Pair) = tryRename(df, [renamelist])
tryRename!(df, renamelist::Vector{Pair{T,T}}) where T <: Any = tryRename(df, map(p->Symbol(p[1])=>Symbol(p[2]), renamelist))
tryRename!(df, renamelist::Vector{Pair{Symbol, Symbol}}) = tryRenameFn(df, filter(p->p[1] in map(Symbol, names(df)), renamelist), rename!)
tryRenameFn(df, renamelist, fn) = fn(df, renamelist)

"""
	dropByName!(df, fn)
	dropByName(df, fn)

Drop the columns whose names return true for the given function
# Example
	dropByName!(orders, n->startswith(n, "Sale") || startswith(n, "Order"))
"""
dropByName!(df, fn) = select!(df, Not(map(fn, names(df))))
dropByName(df, fn) = select(df, Not(map(fn, names(df))))

"""
	colsToGrid(df, x, y, v, col1name)

For a given DataFrame, turn the values of one of the columns into the name of the output columns
and use a given column as the values. That's harder to describe than illustrate!

# Examples

	julia> df = DataFrame([["a", "b", "c"], ["d", "e", "f"], [1,2,3], [4,5,6]], ["c1", "c2", "c3", "v"])
	3×4 DataFrame
	Row │ c1      c2      c3     v
		│ String  String  Int64  Int64
   ─────┼──────────────────────────────
	  1 │ a       d           1      4
	  2 │ b       e           2      5
	  3 │ c       f           3      6

   julia> colsToGrid(df, :c1, :c2, :v, "see")

   3×4 DataFrame
   Row │ see     d        e        f
	   │ String  Int64?   Int64?   Int64?
  ─────┼───────────────────────────────────
	 1 │ a             4  missing  missing
	 2 │ b       missing        5  missing
	 3 │ c       missing  missing        6

	 #  result = DataFrame([["a", "b", "c"], [4, missing, missing], [missing, 5, missing], [missing, missing, 6]], ["see", "d", "e", "f"])
"""
function colsToGrid(df, x, y, v, col1name)
	xs = unique(df[!, x]) |> sort
	ys = unique(df[!, y]) |> sort
	xref = Dict(xx=>Dict([r[y]=>r[v] for r in eachrow(filter(row->row[x] == xx, df))]) for xx in xs)
	grid = DataFrame()
	grid[!, col1name] = xs
	for yy in ys
		grid[!, yy] = map(k->get(xref[k], yy, missing), xs)
	end
	grid
end

###
end


module TestingMayOptimizeMacros

using Test, LinearAlgebra
using MayOptimize
using MayOptimize: Standard, CodeChoice

struct Dummy <: OptimLevel end

function sum_inbounds(::Type{P},
                      x::AbstractVector{T}) where {T<:AbstractFloat,
                                                   P<:OptimLevel}
    s = zero(T)
    @maybe_inbounds P for i in eachindex(x)
        s += x[i]
    end
    return s
end

# Sub-module to check detection of errors.
module Bogus

abstract type Debug end
abstract type InBounds end
abstract type Vectorize end
import MayOptimize: OptimLevel, @maybe_inbounds, @maybe_vectorized
import ..TestingMayOptimizeMacros: sum_inbounds

# Correct versions.
sum0_inbounds(::Type{P}, x::AbstractVector) where {P<:OptimLevel} =
    sum_inbounds(P, x)
sum0_vectorize(::Type{P}, x::AbstractVector) where {P<:OptimLevel} =
    sum(P, x)

# Bogus versions based on sum_inbounds.
sum1_debug(x::AbstractVector)     = sum_inbounds(Debug, x)
sum1_inbounds(x::AbstractVector)  = sum_inbounds(InBounds, x)
sum1_vectorize(x::AbstractVector) = sum_inbounds(Vectorize, x)

# Bogus versions based on sum.
sum2_debug(x::AbstractVector)     = sum(Debug, x)
sum2_inbounds(x::AbstractVector)  = sum(InBounds, x)
sum2_vectorize(x::AbstractVector) = sum(Vectorize, x)

end # Bogus module

dims = (10_000,)

@testset "Macros" begin
    for T in (Float32, Float64)
        x = rand(T, dims)
        y = rand(T, dims)
        @test_throws ErrorException sum(Dummy, x) ≈ sum(x)
        @test_throws ErrorException sum_inbounds(Dummy, x) ≈ sum(x)
        @test sum_inbounds(Debug, x) ≈ sum(x)
        @test sum_inbounds(InBounds, x) ≈ sum(x)
        @test sum_inbounds(Vectorize, x) ≈ sum(x)
        @test sum(Standard, x) == sum(x)
        @test sum(Debug, x) ≈ sum(x)
        @test sum(InBounds, x) ≈ sum(x)
        @test sum(Vectorize, x) ≈ sum(x)
        @test dot(Standard, x, y) == dot(x,y)
        @test dot(Debug, x, y) ≈ dot(x,y)
        @test dot(InBounds, x, y) ≈ dot(x,y)
        @test dot(Vectorize, x, y) ≈ dot(x,y)
        # These ones should work.
        @test Bogus.sum0_inbounds(Debug, x) ≈ sum(x)
        @test Bogus.sum0_inbounds(InBounds, x) ≈ sum(x)
        @test Bogus.sum0_inbounds(Vectorize, x) ≈ sum(x)
        @test Bogus.sum0_vectorize(Debug, x) ≈ sum(x)
        @test Bogus.sum0_vectorize(InBounds, x) ≈ sum(x)
        @test Bogus.sum0_vectorize(Vectorize, x) ≈ sum(x)
        # These ones should fail, although for different reasons.
        @test_throws MethodError Bogus.sum0_inbounds(Bogus.Debug, x) ≈ sum(x)
        @test_throws MethodError Bogus.sum0_inbounds(Bogus.InBounds, x) ≈ sum(x)
        @test_throws MethodError Bogus.sum0_inbounds(Bogus.Vectorize, x) ≈ sum(x)
        @test_throws MethodError Bogus.sum0_vectorize(Bogus.Debug, x) ≈ sum(x)
        @test_throws MethodError Bogus.sum0_vectorize(Bogus.InBounds, x) ≈ sum(x)
        @test_throws MethodError Bogus.sum0_vectorize(Bogus.Vectorize, x) ≈ sum(x)
        @test_throws MethodError Bogus.sum1_debug(x) ≈ sum(x)
        @test_throws MethodError Bogus.sum1_inbounds(x) ≈ sum(x)
        @test_throws MethodError Bogus.sum1_vectorize(x) ≈ sum(x)
        @test_throws MethodError Bogus.sum2_debug(x) ≈ sum(x)
        @test_throws MethodError Bogus.sum2_inbounds(x) ≈ sum(x)
        @test_throws MethodError Bogus.sum2_vectorize(x) ≈ sum(x)
    end
end

end # module


include("components.jl")
using DataFrames

type SPNdata
  unit_size::Int
  num_unit::Int
  plaintexts::Array
  ciphertexts::Array
  traces::Array
  res_dir::String
end

type TA_TRAINdata
  unit_size::Int
  num_unit::Int
  plaintexts::Array
  ciphertexts::Array
  keys::Array
  traces::Array
  res_dir::String
end

type TA_TESTdata
  unit_size::Int
  num_unit::Int
  plaintexts::Array
  ciphertexts::Array
  traces::Array
  res_dir::String
end

function loadSPNtrace(path_to_csvtraces, unit_size)
  #creating directory to store the results
  part_names = split(path_to_csvtraces,"/")
  path_to_resdir = ""
  for a in part_names[1:end-1]
    path_to_resdir = join([path_to_resdir, a, "/"])
  end
  path_to_resdir = join([path_to_resdir, part_names[end], "results"])
  mkdir(path_to_resdir)

  pct = readtable(path_to_csvtraces,header=false)

  plaintexts = pct[1]; #all the plaintexts are suppose to be of the same length and in hex string
  ciphertexts = pct[2]; #all the ciphertexts are suppose to be of the same length and in hex string
  traces = Array(pct[:,3:end]);
  num_unit = div(length(plaintexts[1])*4, unit_size)

  #the HW creation
  for i in 0:2^unit_size-1
    HW[i] = hw_look_up_maker(bin(i,unit_size))
  end

  x = SPNdata(unit_size,num_unit,plaintexts,ciphertexts,traces,path_to_resdir)
end
#add more loading function for other types of cipher if needed

#this function will return the info (stored in array types) of the total number
#of templates that have to be built before classification of any new trace
function loadTA_TRAINtrace(path_to_csvtraces, unit_size)
  #creating the director to store the trained templates for later testing
  part_names = split(path_to_csvtraces,"/")
  path_to_resdir = ""
  for a in part_names[1:end-1]
    path_to_resdir = join([path_to_resdir, a, "/"])
  end
  path_to_resdir = join([path_to_resdir, part_names[end], "templates"])
  mkdir(path_to_resdir)

  pct = readtable(path_to_csvtraces,header=false)

  #HW creation may be needed in some kind of templates
  for i in 0:2^unit_size-1
    HW[i] = hw_look_up_maker(bin(i,unit_size))
  end

  plaintexts = pct[1]
  ciphertexts = pct[2]
  keys = pct[3]
  traces = Array(pct[:,4:end])
  println("dbg: plaintexts[1] is: ",plaintexts[1])
  println("dbg: length(plaintexts[1)] is: ",length(plaintexts[1]))
  num_unit = div(length(plaintexts[1])*4, unit_size)

  return TA_TRAINdata(unit_size, num_unit, plaintexts, ciphertexts, keys, traces, path_to_resdir)
end

function loadTA_TESTtrace(path_to_csvtraces, unit_size)
  #for the testing case the traces need not have plaintexts and ciphertexts
  #for some template models, In such casses user have to ensure that there are
  #plaintexts and ciphertexts cols are filled with 000...0 with apporpriate number of
  #zeros in so that num_unit can be calculated

  #creating the directory to store results
  part_names = split(path_to_csvtraces,"/")
  path_to_resdir = ""
  for a in part_names[1:end-1]
    path_to_resdir = join([path_to_resdir, a, "/"])
  end
  path_to_resdir = join([path_to_resdir, part_names[end], "results"])
  mkdir(path_to_resdir)

  pct = readtable(path_to_csvtraces,header=false)

  plaintexts = pct[1]; #all the plaintexts are suppose to be of the same length and in hex string
  ciphertexts = pct[2]; #all the ciphertexts are suppose to be of the same length and in hex string
  traces = Array(pct[:,3:end]);
  num_unit = div(length(plaintexts[1])*4, unit_size)

  #the HW creation
  for i in 0:2^unit_size-1
    HW[i] = hw_look_up_maker(bin(i,unit_size))
  end

  return TA_TESTdata(unit_size,num_unit,plaintexts,ciphertexts,traces,path_to_resdir)
end


module TestAccessors

using Test
using ..TestParsing: load_sample

function test_default()
    perf = load_sample("default")

    @test perf.context_switches == 124
    @test perf.cpu_migrations == 0
    @test perf.page_faults == 92541
    @test perf.cycles == 3410157225
    @test perf.stalled_cycles_frontend == 74012542
    @test perf.stalled_cycles_backend == 1498764411
    @test perf.instructions == 5107038786
    @test perf.instructions_per_cycle ≈ 1.5 rtol = 0.005
    @test perf.stalled_cycles_per_instructions ≈ 0.29 atol = 0.005

    @test perf.ratio.stalled_cycles_frontend == 74012542 / 3410157225
    @test perf.ratio.stalled_cycles_backend == 1498764411 / 3410157225
end

function test_d1_with_llc()
    perf = load_sample("d1-with-llc")

    @test perf.branches == 402774448
    @test perf.branch_misses == 6375125
    @test perf.l1_dcache_loads == 650213241
    @test perf.l1_dcache_load_misses == 19264237
    @test perf.l1_icache_loads === nothing
    @test perf.l1_icache_load_misses === nothing
    @test perf.llc_loads == 2029190
    @test perf.llc_load_misses == 1158158

    @test perf.ratio.branch_misses == 6375125 / 402774448
    @test perf.ratio.l1_dcache_load_misses == 19264237 / 650213241
    @test perf.ratio.l1_icache_load_misses === nothing
    @test perf.ratio.llc_load_misses == 1158158 / 2029190

    @test perf.percent.branch_misses == 6375125 / 402774448 * 100
    @test perf.percent.l1_dcache_load_misses == 19264237 / 650213241 * 100
    @test perf.percent.l1_icache_load_misses === nothing
    @test perf.percent.llc_load_misses == 1158158 / 2029190 * 100
end

end  # module


using CSV
using Dates
using DataFrames
using DelimitedFiles
using XLSX
using YAML

using SLiDE  # see src/SLiDE.jl

# DATA STREAM -- This will take awhile (~10 minutes, maximum), but will only be necessary
# the first time the SLiDE package is installed and when data is updated.
@time include(joinpath(SLIDE_DIR, "dev", "datastream", "auto_generate_maps.jl"))     # ~60 seconds
@time include(joinpath(SLIDE_DIR, "dev", "datastream", "auto_standardize_data.jl"))  # ~7.5 minutes

# BUILD STREAM -- Prepare data for the calibration scheme.
# This is where user customizations can be applied (future work).
(d, set) = build_data()


export dual
export truncate, ambo, bevel, expand, snub
export kiss, join, meta, ortho, gyro

function dual end

function truncate end
function ambo end
function bevel end
function expand end
function snub end

function kiss end
function join end
function meta end
function ortho end
function gyro end


module TestUpdateMethod

using MLJBase
using Test

import LightGBM

# Setup stuff
nrows = 3_000
nfeatures = 10

x = randn(nrows, nfeatures)
y = randn(nrows)
k = Dict(Symbol(i) => Int64(i) for i in 1:nfeatures)
X = MLJBase.Tables.MatrixTable(collect(keys(k)), k, x)
w = abs.(randn(nrows))
w = w / sum(w)

# setup MLJ machinery -- will be used in several testsets
r = LightGBM.MLJInterface.LGBMRegressor(num_iterations=1, min_data_in_leaf=20)
m = MLJBase.machine(r, X, y)

# initial fit
MLJBase.fit!(m; verbosity=0)

@testset "MLJ update" begin
    # machine keeps a field `fitresult` which is the the fitresult,
    # the first element of this tuple is the LGBMEstimator
    iteration_count = LightGBM.get_iter_number(first(m.fitresult))
    cached_iteration_count = sum(m.cache.num_boostings_done)
    @test iteration_count == 1
    @test iteration_count == cached_iteration_count
    @test m.cache.num_boostings_done == [1]

    # check an update chaining on from a fit works
    m.model.num_iterations += 10

    MLJBase.fit!(m; verbosity=0)

    iteration_count = LightGBM.get_iter_number(first(m.fitresult))
    cached_iteration_count = sum(m.cache.num_boostings_done)
    @test iteration_count == 11
    @test iteration_count == cached_iteration_count
    @test m.cache.num_boostings_done == [1, 10]
end

@testset "MLJ update chain" begin
    # check an update chaining on from an update works
    m.model.num_iterations += 5

    MLJBase.fit!(m; verbosity=0)

    iteration_count = LightGBM.get_iter_number(first(m.fitresult))
    cached_iteration_count = sum(m.cache.num_boostings_done)
    @test iteration_count == 16
    @test iteration_count == cached_iteration_count
    @test m.cache.num_boostings_done == [1, 10, 5]

end

@testset "MLJ update less iterations triggers refit" begin
    m.model.num_iterations -= 1
    MLJBase.fit!(m; verbosity=0)

    iteration_count = LightGBM.get_iter_number(first(m.fitresult))
    cached_iteration_count = sum(m.cache.num_boostings_done)
    @test iteration_count == 15
    @test iteration_count == cached_iteration_count
    @test m.cache.num_boostings_done == [15]
end

@testset "MLJ update non-updatable parameters" begin
    # next, check that  changing parameters other than num_iterations results in entirely refitting
    m.model.num_iterations += 1
    m.model.min_data_in_leaf += 5

    MLJBase.fit!(m; verbosity=0)

    iteration_count = LightGBM.get_iter_number(first(m.fitresult))
    cached_iteration_count = sum(m.cache.num_boostings_done)
    @test iteration_count == 16
    @test iteration_count == cached_iteration_count
    @test m.cache.num_boostings_done == [16]
end

@testset "MLJ User provided weights" begin
# check with user provided weights
    r = LightGBM.MLJInterface.LGBMRegressor(num_iterations=1, min_data_in_leaf=20)
    weights_machine = MLJBase.machine(r, X, y, w)

    MLJBase.fit!(weights_machine; verbosity=0)

    iteration_count = LightGBM.get_iter_number(first(weights_machine.fitresult))
    cached_iteration_count = sum(weights_machine.cache.num_boostings_done)
    @test iteration_count == 1
    @test iteration_count == cached_iteration_count
    @test weights_machine.cache.num_boostings_done == [1]

    weights_machine.model.num_iterations += 1
    MLJBase.fit!(weights_machine; verbosity=0)

    iteration_count = LightGBM.get_iter_number(first(weights_machine.fitresult))
    cached_iteration_count = sum(weights_machine.cache.num_boostings_done)
    @test iteration_count == 2
    @test iteration_count == cached_iteration_count
    @test weights_machine.cache.num_boostings_done == [1, 1]

end

end # module


export buildtrainingdata, loadtrainingdata, loaddemanddata, savetrainingdata

# Can call this with ssp5 or ssp32 (global constants)
# Since not all GADM level 0 regions are assigned to SSP regions, some regions are missed
# (mostly small island nations and the like). This captures 99.85% of global population.
function make_sspregionlookup(ssp)
    println("Lookup SSP R5.2 region names for each GADM level 0 region...")
    _, _, regionlist, _, _ = loadregions("Global_GADM0")
    sspregions = fill("", length(regionlist))
    for (sspregionname, gadmregionnames) in ssp
        for gname in gadmregionnames
            index = findfirst(isequal(Symbol(gname)), regionlist)
            if index == nothing
                error("Region $gname is not a GADM level 0 region.")
            end
            sspregions[index] = sspregionname
        end
    end
    return sspregions # Vector{String}(length numcountries, indexed by gadm country code)
end

function getnationalpopulation(scenarioyear)
    filename = in_datafolder("nationalpopulation_$scenarioyear.jld")
    natpop = isfile(filename) ? JLD.load(filename, "natpop") : savenationalpopulation(scenarioyear)
    return natpop
end

# load population dataset and sum globally by country
# Vector{Float64}(length numcountries, indexed by gadm country code)
function savenationalpopulation(scenarioyear)
    println("Calculating population in all GADM level 0 regions...")
    pop = JLD.load(in_datafolder("population_$scenarioyear.jld"), "population")   # persons per pixel
    regions, _, regionlist, lonrange, latrange = loadregions("Global_GADM0")
    natpop = zeros(length(regionlist))
    for j in latrange
        for i in lonrange
            reg = regions[i,j]
            if reg > 0
                natpop[reg] += pop[i,j]
            end
        end
    end
    JLD.save(in_datafolder("nationalpopulation_$scenarioyear.jld"), "natpop", natpop, compress=true)
    return natpop
end

# Get current national electricity demand from IEA online statistics (actually "domestic supply").
# https://www.iea.org/statistics/?country=MOROCCO&year=2016&category=Electricity&indicator=ElecGenByFuel&mode=table&dataTable=ELECTRICITYANDHEAT
function ieademand()
    println("Get current national electricity demand from IEA statistics...")
    iea = CSV.read(in_datafolder("ieademand_2016.csv"), DataFrame)      # GWh/year
    _, _, regionlist, _, _ = loadregions("Global_GADM0")
    nationaldemand = zeros(length(regionlist))
    for row in eachrow(iea)
        country, demand = row
        country == uppercase(country) && continue   # skip IEA aggregated regions in uppercase, e.g. "ASIA"
        f = findfirst(regionlist .== Symbol(country))
        f == nothing && error("IEA country $country missing in GADM regions.")
        nationaldemand[f] = demand/1000
    end
    return nationaldemand        # TWh/year
end

# Lookup electricity demand from a SSP database dataframe. Return a single dataframe row.
ssplookup(ssp, model, scen, region) =
    ssp[(ssp[:,:MODEL] .== model) .& (ssp[:,:SCENARIO] .== scen) .& (ssp[:,:REGION] .== region), :][1,:]

# Calculate regional multiplers for demand from 2016 to target year
function calcdemandmultipliers(sspscenario::String, year::Int)
    println("Calculate demand multipliers...")
    # First read electricity demand from the SSP database into a dataframe.
    ssp = CSV.read(in_datafolder("SSP v2 Final Energy - Electricity.csv"), DataFrame)

    # SSP scenarios also include radiative forcing targets, e.g. SSP2-34
    # (only for IAM energy system results, not underlying population & GDP scenario)
    # We'll hardcode the 3.4 W/m2 scenario variant for now and make it a configurable option later.
    scen = uppercase(sspscenario)   # e.g. "SSP2-34"

    # We'll take the average result from IMAGE and MESSAGE models.
    demandmult2050 = Dict{String, Float64}()
    for sspreg in keys(ssp5)
        reg = "R5.2$sspreg"
        image = ssplookup(ssp, "IMAGE", scen, reg)
        message = ssplookup(ssp, "MESSAGE-GLOBIOM", scen, reg)
        # Take the year 2020 to represent 2016, then adjust for this difference.
        image_mult = image[Symbol(year)] / image[Symbol(2020)]
        message_mult = message[Symbol(year)] / message[Symbol(2020)]
        demandmult2050[sspreg] = ((image_mult + message_mult)/2) ^ ((year-2016)/(year-2020))
    end
    return demandmult2050
end

function makeregionaldemanddata(gisregion, sspscenario::String, year::Int)
    res = 0.01          # resolution of auxiliary datasets [degrees per pixel]

    scenarioyear = "$(sspscenario[1:4])_$year"
    datasetinfo = Dict(:gisregion=>gisregion, :scenarioyear=>scenarioyear, :res=>res)
    regions, _, regionlist, _, pop, _, _, _, lonrange, latrange = read_datasets(datasetinfo)     # pop unit: people/grid cell

    gdp = JLD.load(in_datafolder("gdp_$(scenarioyear).jld"))["gdp"][lonrange,latrange]       # unit: USD(2010)/grid cell, PPP

    gadm0regions, _, gadm0regionlist, _, _ = loadregions("Global_GADM0")
    countrycodes = gadm0regions[lonrange,latrange]

    lats = (90-res/2:-res:-90+res/2)[latrange]
    cellarea = rastercellarea.(lats, res)
    popdens = pop ./ cellarea'

    natpop = getnationalpopulation(scenarioyear)            # people/GADMregion (vector, length numGADMcountries)
    demandpercapita = ieademand()./natpop * 1e6             # MWh/year/capita (vector, length numGADMcountries)
    ssp5region = make_sspregionlookup(ssp5)                 # SSP 5-region names (vector, length numGADMcountries)
    demandmult = calcdemandmultipliers(sspscenario, year)   # Dict: SSP region name => multiplier

    numreg = length(regionlist)
    regionaldemand = zeros(numreg)
    regionalpop = zeros(numreg)
    regionalgdp = zeros(numreg)

    println("Calculating annual electricity demand for model regions...")
    nlons, nlats = size(regions)
    updateprogress = Progress(nlats, 1)
    for j = 1:nlats
        for i = 1:nlons
            reg = regions[i,j]
            countrycode = countrycodes[i,j]
            if reg > 0 && reg != NOREGION && countrycode > 0
                sspreg = ssp5region[countrycode]
                if isempty(sspreg)
                    # error("Oops, no SSP region assigned to region $(gadm0regionlist[countrycode]).")
                    continue
                end
                regionaldemand[reg] += demandpercapita[countrycode] * pop[i,j]/1e6 * demandmult[sspreg]     # TWh/year
                regionalpop[reg] += pop[i,j]        # unit: people
                regionalgdp[reg] += gdp[i,j]        # unit: USD(2010)
            end
        end
        next!(updateprogress)
    end
    regionaldemandpercapita = regionaldemand ./ regionalpop * 1e6   # MWh/year/capita
    regionalgdppercapita = regionalgdp ./ regionalpop               # USD(2010)/capita

    return regionlist, regionaldemandpercapita, regionalgdppercapita
end

function buildtrainingdata(; gisregion="Europe8", sspscenario="ssp2-34", sspyear=2050, era_year=2018, numcenters=3, mindist=3.3)
    println("\nBuilding training data for $gisregion...")
    regionlist, demandpercapita, gdppercapita =
                    makeregionaldemanddata(gisregion, sspscenario, sspyear)
    scenarioyear = "$(sspscenario[1:4])_$sspyear"
    hours, temp_popcenters = GIStemp(gisregion, scenarioyear, era_year, numcenters, mindist)
    offsets, zone_maxpop, population = regional_timezone_offsets_Jan1(gisregion=gisregion, scenarioyear=scenarioyear, era_year=era_year)

    numreg, numhours = length(regionlist), length(hours)
    firsttime = ZonedDateTime.(hours[1], zone_maxpop)
    zonedtime = hcat(collect.([firsttime[i]:Hour(1):firsttime[i]+Hour(8759) for i = 1:numreg])...)[:]

    println("\nShifting hourly temperatures from UTC to local time...")
    temperature_top3_mean = dropdims(mean(temp_popcenters, dims=3), dims=3)
    quantiles = hcat([quantile(temp_popcenters[:,c,1], [0.05, 0.5, 0.95]) for c = 1:numreg]...)
    shifted_temperature_top3_mean = similar(temperature_top3_mean)
    shifted_temp1 = similar(temperature_top3_mean)
    for r = 1:numreg
        shifted_temperature_top3_mean[:,r] = circshift(temperature_top3_mean[:,r], round(Int, offsets[r]))
        shifted_temp1[:,r] = circshift(temp_popcenters[:,r,1], round(Int, offsets[r]))
    end

    # dataframe with hourly data
    df_time = DataFrame(
        localtime = DateTime.(zonedtime, Local),
        country = repeat(string.(regionlist), inner=numhours),
        temp_top3 = shifted_temperature_top3_mean[:],
        temp1 = shifted_temp1[:],
        localhour = hour.(zonedtime),
        month = month.(zonedtime),
        weekend01 = Int.(dayofweek.(zonedtime) .>= 6)
    )

    # sort by average monthly temperature (in popcenter 1), store rank in ranked_month
    df_monthlytemp = combine(groupby(df_time, [:country, :month]), :temp1 => mean => :temp_monthly) |>
            d -> sort!(d, [:country, :temp_monthly]) |>
            d -> insertcols!(d, 4, :ranked_month => repeat(1:12, outer=numreg))

    # dataframe with regional data
    df_reg = DataFrame(country=string.(regionlist), demandpercapita=demandpercapita, gdppercapita=gdppercapita,
                        temp1_qlow=quantiles[1,:], temp1_mean=quantiles[2,:], temp1_qhigh=quantiles[3,:])

    # join everything together
    df = innerjoin(df_time, df_monthlytemp, on=[:country, :month]) |>
                    d -> innerjoin(d, df_reg, on=:country)
    return df, offsets, population
end

function loadtrainingdata()
    filename = in_datafolder("syntheticdemand_trainingdata.csv")
    if !isfile(filename)
        savetrainingdata()
    end
    df_train = CSV.read(filename, DataFrame)
    offsets = CSV.read(in_datafolder("syntheticdemand_timezoneoffsets.csv"), DataFrame)[:,1]
    return df_train, offsets
end

function savetrainingdata(; numcenters=3, mindist=3.3)
    create_scenario_datasets("SSP2", 2020)
    println("\nCreating training dataset for synthetic demand...")
    println("(This requires ERA5 temperature data for the year 2015 and scenario datasets for SSP2 2020.)")
    df_train, offsets, _ = buildtrainingdata(gisregion="SyntheticDemandRegions",
                sspscenario="SSP2-34", sspyear=2020, era_year=2015,
                numcenters=numcenters, mindist=mindist)
    CSV.write(in_datafolder("syntheticdemand_timezoneoffsets.csv"), DataFrame(offsets=offsets))
    CSV.write(in_datafolder("syntheticdemand_trainingdata.csv"), df_train)
end

loaddemanddata() = CSV.read(in_datafolder("syntheticdemand_demanddata.csv"), DataFrame)

function regional_timezone_offsets_Jan1(; gisregion="Europe8", scenarioyear="ssp2_2050", era_year=2018)
    println("\nCalculating population-weighted regional time zone offsets...")
    regions, offshoreregions, regionlist, lonrange, latrange = loadregions(gisregion)
    tzindices, tznames = loadtimezones(lonrange, latrange)
    popscale = 1e5
    pop = JLD.load(in_datafolder("population_$scenarioyear.jld"), "population")[lonrange,latrange] ./ popscale   # scale down for better precision
    numreg = length(regionlist)
    numhours = 24*daysinyear(era_year)
    offsets = zeros(numreg)
    population = zeros(numreg)
    zone_maxpop = fill(TimeZone("Europe/London"), numreg)
    updateprogress = Progress(numreg, 1)
    for r = 1:numreg
        reg = (regions .== r)
        regindices = tzindices[reg]
        regpop = pop[reg]
        zoneindices = unique(regindices)
        weightedoffset = 0.0
        zones = TimeZone[]
        pops = Float64[]
        for idx in zoneindices
            tzname = tznames[idx]
            zone = tzname[1:3] == "Etc" ? TimeZone(tzname, TimeZones.Class(:LEGACY)) : TimeZone(tzname)
            push!(zones, zone)
            firsthour = ZonedDateTime(DateTime(era_year,1,1,0), zone)
            hours = firsthour : Hour(1) : firsthour + Hour(numhours-1)
            offset = tzoffset(hours[1])
            localpop = sum(regpop[regindices.==idx])
            push!(pops, localpop)
            weightedoffset += localpop * offset
        end
        _, i = findmax(pops)    # find time zone with the most population
        zone_maxpop[r] = zones[i]
        population[r] = sum(pops) * popscale    # scale up again
        offsets[r] = weightedoffset / sum(pops)
        next!(updateprogress)
    end

    return offsets, zone_maxpop, population
end

function saveVilledemand()
    vv = CSV.read(in_datafolder("syntheticdemand", "data", "df_model_features.csv"), DataFrame)
    hours = DateTime(2015, 1, 1, 0) : Hour(1) : DateTime(2015, 12, 31, 23)
    demand1D = hcat(DataFrame(hours=repeat(hours, outer=44)), select(vv, [:country, :demand_total_mwh]))
    demand2D = unstack(demand1D, :country, :demand_total_mwh)
    rename!(demand2D, Dict("Bosnia and Herz." => "Bosnia and Herzegovina",
                                "Czech Rep." => "Czech Republic",
                                "Korea" => "South Korea"))
    select!(demand2D, [:hours; sort(names(demand2D)[2:end])])

    dem2 = stack(demand2D, variable_name=:country, value_name=:demand_MW)
    select!(dem2, [3,1,2])
    meandemand = by(dem2, :country, meandemand = :demand_MW => mean)
    demand = join(dem2, meandemand, on=:country)

    insertcols!(demand, 5, normdemand=demand[:,:demand_MW]./demand[:,:meandemand])
    CSV.write(in_datafolder("syntheticdemand_demanddata.csv"), demand)
end

tzoffset(tz::FixedTimeZone) = tz.offset.std.value / 3600
tzoffset(tz::VariableTimeZone) = tzoffset(tz.transitions[end].zone)
tzoffset(dt::ZonedDateTime) = TimeZones.value(dt.zone.offset) / 3600

function timezonetest()
    # zones = TimeZone.(["Europe/London", "Europe/Stockholm", "Europe/Helsinki", "Europe/Minsk"])
    zones = TimeZone.(["Australia/Perth", "Australia/Adelaide", "Australia/Darwin", "Australia/Brisbane", "Australia/Sydney"])
    # all hours in 2015 for each zone in local time
    hours = hcat([ZonedDateTime(DateTime(2015,1,1,0), z) : Hour(1) : ZonedDateTime(DateTime(2015,12,31,23), z) for z in zones]...)
    # shift hours by the time zone offset to ensure simultaneous rows
    # take the offset of ZonedDateTime in hour 1 (in case hour 1 is DST), not the UTC time zone offset
    shifted = hcat([circshift(hours[:,i], round(Int, -tzoffset(hours[1,i]))) for i = 1:length(zones)]...)
    # convert to UTC and ensure all columns identical (except last 24 hours because of shifted hours from previous year)
    utctime = DateTime.(shifted, UTC)
    # @assert all([all(utctime[1:end-24,1] .== utctime[1:end-24,i]) for i=2:length(zones)])  # doesn't work for non-integer time offsets
    # show the local time in each zone around the shift to summer time (daylight savings) in the EU
    localtime = Time.(DateTime.(shifted, Local))
    # display(localtime[2085:2095,:])   # EU
    display(localtime[6615:6620,:])     # Australia
    return hours, shifted, utctime, localtime
end


module OutputSizes

using Flux
using LinearAlgebra

"""
    Nil <: Number

Nil is a singleton type with a single instance `nil`. Unlike
`Nothing` and `Missing` it subtypes `Number`.

"""
struct Nil <: Number end

const nil = Nil()

Nil(::T) where T<:Number = nil

Base.copy(::Nil) = nil

Base.:+(::Nil) = nil
Base.:-(::Nil) = nil

Base.:+(::Nil, ::Nil) = nil
Base.:+(::Nil, ::Number) = nil
Base.:+(::Number, ::Nil) = nil

Base.:-(::Nil, ::Nil) = nil
Base.:-(::Nil, ::Number) = nil
Base.:-(::Number, ::Nil) = nil

Base.:*(::Nil, ::Nil) = nil
Base.:*(::Nil, ::Number) = nil
Base.:*(::Number, ::Nil) = nil

Base.:/(::Nil, ::Nil) = nil
Base.:/(::Nil, ::Number) = nil
Base.:/(::Number, ::Nil) = nil

Base.inv(::Nil) = nil

Base.isless(::Nil, ::Nil) = true
Base.isless(::Nil, ::Number) = true
Base.isless(::Number, ::Nil) = true

Base.abs(::Nil) = nil
Base.exp(::Nil) = nil

Base.typemin(::Type{Nil}) = nil
Base.typemax(::Type{Nil}) = nil
Base.:^(::Nil, ::Nil) = nil

Base.promote(x::Nil, y::Nil) = (nil, nil)
Base.promote(x::Nil, y) = (nil, nil)
Base.promote(x, y::Nil) = (nil, nil)
Base.promote(x::Nil, y, z) = (nil, nil, nil)
Base.promote(x, y::Nil, z) = (nil, nil, nil)
Base.promote(x, y, z::Nil) = (nil, nil, nil)
Base.promote(x::Nil, y, z::Nil) = (nil, nil, nil)
Base.promote(x::Nil, y::Nil, z::Nil) = (nil, nil, nil)
Base.promote(x::Nil, y::Nil, z) = (nil, nil, nil)

LinearAlgebra.adjoint(::Nil) = nil
LinearAlgebra.transpose(::Nil) = nil

# Since some Flux layers don't work out of the box,
# we're "cheating" here by implementing the layers
# for `Nil` with the correct output size.

function (c::Flux.Conv)(x::AbstractArray{Nil})
    return fill(nil, Flux.outdims(c, size(x))..., size(c.weight, 4), size(x, 4))
end

function (c::Flux.DepthwiseConv)(x::AbstractArray{Nil})
    return fill(nil, Flux.outdims(c, size(x))..., size(c.weight, 4), size(x, 4))
end

function (c::Flux.ConvTranspose)(x::AbstractArray{Nil})
    return fill(nil, Flux.outdims(c, size(x))..., size(c.weight, 4), size(x, 4))
end

# Interface

function outputsize(model, sz; addbatch = true)
    return obssize(model(fill(nil, sz..., 1)))
end

# Utils

obssize(a::AbstractArray) = size(a)[1:end-1]
obssize(t::Union{Tuple, NamedTuple}) = map(obssize, t)

export nil, Nil, outputsize

end  # module


import StaticArrays.SVector
import Colors: RGB, distinguishable_colors
import GeometryBasics: Point

export Layer, PenPlot

"""
Represents a layer of a plot.

A "layer" is a full pass of one pen. A layer has an associated color which
is used for previewing the plot in the SVG export, but the color is not used
by the plotting software.
"""
struct Layer
    name::String
    paths::MultiPath
    color::RGB
end

mutable struct LayerMaker
    num_layers::Int
    default_colors::Array{RGB,1}
    current_index::Int

    function LayerMaker(num_layers::Int)
        default_colors = distinguishable_colors(num_layers, lchoices = 0:40)
        new(num_layers, default_colors, 1)
    end
end

function makelayer(layer_maker::LayerMaker, layer::Layer)
    layer_maker.current_index += 1
    layer
end

function makelayer(layer_maker::LayerMaker, paths::MultiPath)
    name = "$(layer_maker.current_index)-layer"
    color = layer_maker.default_colors[layer_maker.current_index]
    layer_maker.current_index += 1
    Layer(name, paths, color)
end

function makelayer(layer_maker::LayerMaker, name_paths_pair::Pair{String,MultiPath})
    color = layer_maker.default_colors[layer_maker.current_index]
    layer_maker.current_index += 1
    Layer(name_paths_pair.first, name_paths_pair.second, color)
end

function makenotch(corner::Point, prev::Point, next::Point; frac=0.05)
    p1 = prev * frac + corner * (1-frac)
    p2 = corner
    p3 = next * frac + corner * (1-frac)
    Path([p1, p2, p3])
end

"""
A plot, consisting of one or more layers.

Constructed with:

    PenPlot(layers...)

Construct a pen plot consisting of the given layers.

Layers may be:
- [`Layer`](@ref) objects
- [`MultiPath`](@ref) objects
- `Pair{String, MultiPath}` objects, where the string becomes the name of
  the layer.
"""
struct PenPlot
    layers::Array{Layer}

    function PenPlot(layers...; addoutline=false)
        new_layers = []
        layer_maker = LayerMaker(length(layers) + (addoutline ? 1 : 0))

        new_layers = map(collect(layers)) do layerdata
            makelayer(layer_maker, layerdata)
        end
        plot = new(new_layers)

        if addoutline
            plx = extent(plot)

            upperright = Point(plx.lowerright[1], plx.upperleft[2])
            lowerleft = Point(plx.upperleft[1], plx.lowerright[2])

            outline = [
                makenotch(plx.upperleft, lowerleft, upperright),
                makenotch(upperright, plx.upperleft, plx.lowerright),
                makenotch(plx.lowerright, upperright, lowerleft),
                makenotch(lowerleft, plx.lowerright, plx.upperleft),
            ]
            push!(plot.layers, makelayer(layer_maker, outline))
        end

        plot
    end
end


# Minimum segment length for ray tracing
const minimum_segment_length = 1e-4

# QuadraticSegment_2D
# ---------------------------------------------------------------------------------------------------
# Threshold beneath which the segment is treated as linear.
# Equal to 2|d⃗|θ, where d⃗ is the displacement of x⃗₃ from the midpoint of the linear segment (x⃗₁, x⃗₂)
# and θ is the angle between the intersecting line and d⃗
const QuadraticSegment_2D_1_intersection_ϵ = 5e-6

# UnstructuredMesh_2D
# ---------------------------------------------------------------------------------------------------
# Cell types are the same as VTK
const UnstructuredMesh_2D_linear_cell_types = UInt32[5, # Triangle
                                                     9  # Quadrilateral
                                                    ]
const UnstructuredMesh_2D_quadratic_cell_types = UInt32[22, # Triangle6
                                                        23  # Quadrilateral8
                                                       ]
const UnstructuredMesh_2D_cell_types = vcat(UnstructuredMesh_2D_linear_cell_types,
                                            UnstructuredMesh_2D_quadratic_cell_types)


#export

## experimental and not exported or documented!
function _adjacency_matrix(data::PopData)
    data_loci = groupby(data.genodata, :locus)
    out_vec = Vector{Array{Int8,2}}(undef, data.metadata.loci)
    for (j,i) in enumerate(data_loci)
        uniq = unique(skipmissing(i.genotype))
        adj_mat = fill(Int8(0), data.metadata.samples, length(uniq))
        for (j,k) in zip(i.genotype, eachrow(adj_mat))
            k .= Ref(j) .=== uniq
        end
        out_vec[j] = adj_mat
    end
    return out_vec
end

"""
    _p_adjust(pvals::Vector{T}, method::String) where T <: Union{Missing, <:AbstractFloat}
Modification to `MultipleTesting.adjust` to include `missing` values in the
returned array. See MultipleTesting.jl docs for full more detailed information.

**Example**
```
julia> _p_adjust([0.1, 0.01, 0.005, 0.3], "bh")

```
### `correction` methods (case insensitive)
- `"bonferroni"` : Bonferroni adjustment
- `"holm"` : Holm adjustment
- `"hochberg"` : Hochberg adjustment
- `"bh"` : Benjamini-Hochberg adjustment
- `"by"` : Benjamini-Yekutieli adjustment
- `"bl"` : Benjamini-Liu adjustment
- `"hommel"` : Hommel adjustment
- `"sidak"` : Šidák adjustment
- `"forwardstop"` or `"fs"` : Forward-Stop adjustment
- `"bc"` : Barber-Candès adjustment
"""
@inline function _p_adjust(pvals::Vector{T}, method::String) where T <: Union{Missing, <:AbstractFloat}
    # make a dict of all possible tests and their respective functions
    d = Dict(
        "bonferroni" => Bonferroni(),
        "holm" => Holm(),
        "hochberg" => Hochberg(),
        "bh" => BenjaminiHochberg(),
        "by" => BenjaminiYekutieli(),
        "bl" => BenjaminiLiu(),
        "hommel" => Hommel(),
        "sidak" => Sidak(),
        "forwardstop" => ForwardStop(),
        "fs" => ForwardStop(),
        "bc" => BarberCandes(),
    )
    p_copy = copy(pvals)
    p_copy[.!ismissing.(p_copy)] .= adjust(p_copy[.!ismissing.(p_copy)] |> Vector{Float64}, d[lowercase(method)])
    return p_copy
end

feature_req() = "\nPlease open an Issue or Pull Request on https://www.github.com/biojulia/PopGen.jl if you would like this feature implemented"


const ModelSolution{T,NPT_FULL,NP_SAVED} = Union{ODECompositeSolution{T,NPT_FULL,<:AbstractArray{<:AbstractArray{T,NP_SAVED}}},ODESolution{T,NPT_FULL,<:AbstractArray{<:AbstractArray{T,NP_SAVED}}}}#,RODESolution{T,NPT_FULL,<:AbstractArray{<:AbstractArray{T,NP_SAVED}}}}

# FIXME not real dispatch, since it's just an alias
@inline population(A::AbstractArray{T,N}, i) where {T,N} = view_slice_last(A, i)
function population_coordinates(coordinates::AbstractArray{<:CartesianIndex,N}, P) where N
    cat([[CartesianIndex(coord,i) for coord in coordinates] for i in 1:P]...; dims=N+1)
end
population_repeat(arr::AbstractArray{T,N}, P) where {T,N} = repeat(arr, outer=([1 for _ in 1:N]..., P))
"""
    population_timepoint(solution, pop_dx, time_dx)

Return spatial frame for a given population `pop_dx` and time `time_dx`.
"""
@generated function population_timepoint(solution::ModelSolution{T,NPT_FULL,NP_SAVED}, pop_dx::Int, time_dx::Int) where {T,NPT_FULL,NP_SAVED}
    N = NP_SAVED - 1 # N + pops
    colons = [:(:) for i in 1:N]
    :(solution(solution.t[time_dx])[$(colons...), pop_dx])
end

# Going forward, make macro to construct actions and interactions between
# desired numbers of populations

# Essentially: Slow, bad type inference at construction level, but good, strong type
# inference at the solver-level.
# e.g.: use length of vector input to choose which type to construct, but then when the type
# is given to the solver, it's set and so all the inference is set for the hard part.

type_name(T::DataType) = T.name.wrapper
type_name(T::UnionAll) = type_name(T.body)
function (ap::Type{<:AbstractParameter})(pop::POP) where {POP <: AbstractPopulationP}
	type_name(POP)((ap(param) for param in getfield.(Ref(pop), fieldnames(POP)))...)
end

abstract type AbstractPOneD{NP,AP} <: AbstractPopulationP{NP,AP} end
abstract type AbstractPTwoD{NP,AP} <: AbstractPopulationP{NP,AP} end
abstract type AbstractPopulationActionsParameters{NP,AP} <: AbstractPOneD{NP,AP} end
abstract type AbstractPopulationInteractionsParameters{NP,AP} <: AbstractPTwoD{NP,AP} end
abstract type AbstractPopulationActions{NP,AP} <: AbstractPOneD{NP,AP} end
abstract type AbstractPopulationInteractions{NP,AP} <: AbstractPTwoD{NP,AP} end

const PopActParam{P,T} = Union{Array{<:AbstractPopulationActionsParameters,1},AbstractPopulationActionsParameters{P,<:T}}
const PopInteractParam{P,T} = Union{Array{<:AbstractPopulationInteractionsParameters,1},AbstractPopulationInteractionsParameters{P,T}}
const PopAct{P,T} = Union{Array{<:AbstractPopulationActions,1},AbstractPopulationActions{P,T}}
const PopInteract{P,T} = Union{Array{<:AbstractPopulationInteractions,1},AbstractPopulationInteractions{P,T}}

function array(p1d::AbstractPOneD{2})
    [p1d.p1, p1d.p2]
end

function array(p2d::AbstractPTwoD{2})
    [p2d.p11 p2d.p12;
     p2d.p21 p2d.p22]
end

struct PopulationActionsParameters2{P,P1<:P,P2<:P} <: AbstractPopulationActionsParameters{2,P}
    p1::P1
    p2::P2
    PopulationActionsParameters2(p1::P1,p2::P2) where {P1,P2} = new{typejoin(P1,P2),P1,P2}(p1,p2)
    PopulationActionsParameters2{P,P1,P2}(p1::P1,p2::P2) where {P,P1<:P,P2<:P} = new(p1,p2)
end
(pp::PopulationActionsParameters2)(args...) = PopulationActions2(pp.p1(args...), pp.p2(args...))

struct PopulationActions2{P,P1<:P,P2<:P} <: AbstractPopulationActions{2,P}
    p1::P1
    p2::P2
    PopulationActions2(p1::P1,p2::P2) where {P1,P2} = new{typejoin(P1,P2),P1,P2}(p1,p2)
end
function (pop_actions::PopulationActions2)(inplace::AbstractArray{T,N}, source::AbstractArray{T,N}, t) where {T,N}
    pop_actions.p1(population(inplace, 1), population(source, 1), t)
    pop_actions.p2(population(inplace, 2), population(source, 2), t)
end

struct PopulationInteractionsParameters2{P,P11<:P,P12<:P,P21<:P,P22<:P} <: AbstractPopulationInteractionsParameters{2,P}
    p11::P11
	p21::P21
    p12::P12
    p22::P22
    PopulationInteractionsParameters2(p11::P11,p21::P21,p12::P12,p22::P22) where {P11,P12,P21,P22} = new{typejoin(P11,P12,P21,P22),P11,P12,P21,P22}(p11,p21,p12,p22)
end
struct PopulationInteractions2{P,P11<:P,P12<:P,P21<:P,P22<:P} <: AbstractPopulationInteractions{2,P}
    p11::P11
    p21::P21
    p12::P12
    p22::P22
    PopulationInteractions2(p11::P11,p21::P21,p12::P12,p22::P22) where {P11,P12,P21,P22} = new{typejoin(P11,P12,P21,P22),P11,P12,P21,P22}(p11,p21,p12,p22)
end
(pp::PopulationInteractionsParameters2)(args...) = PopulationInteractions2(pp.p11(args...), pp.p21(args...),
                                                                           pp.p12(args...), pp.p22(args...))
function (pop_interactions::PopulationInteractions2)(inplace::ARR1, source::ARR2, t) where {T,N,ARR1<:AbstractArray{T,N},ARR2<:AbstractArray{T,N}}
	pop_interactions.p11(population(inplace, 1), population(source, 1), t)
    pop_interactions.p12(population(inplace, 1), population(source, 2), t)
    pop_interactions.p21(population(inplace, 2), population(source, 1), t)
    pop_interactions.p22(population(inplace, 2), population(source, 2), t)
end
export PopulationInteractions2

function (arr::Array{<:AbstractPopulationP})(args...)
	[p(args...) for p in arr]
end

PopulationActionsParameters(p1::AbstractParameter, p2::AbstractParameter) = PopulationActionsParameters2(p1, p2)
PopulationActionsParameters(p1::AbstractAction, p2::AbstractAction) = PopulationActions2(p1, p2)
PopulationInteractionsParameters(p11::AbstractParameter,p21::AbstractParameter,p12::AbstractParameter,p22::AbstractParameter) = PopulationInteractionsParameters2(p11,p21,p12,p22)
PopulationInteractionsParameters(p11::AbstractAction,p21::AbstractAction,p12::AbstractAction,p22::AbstractAction) = PopulationInteractions2(p11,p21,p12,p22)

const MissingAP = Union{Missing, <:AbstractParameter}
const MissingAA = Union{Missing, <:AbstractAction}

PopulationActionsParameters(p1::MissingAP, p2::MissingAP) = missing
PopulationInteractionsParameters(p11::MissingAP, p21::MissingAP, p12::MissingAP, p22::MissingAP) = missing

function PopulationActionsParameters(p1::AbstractAction, p2::Missing)
    missing
end
function PopulationActionsParameters(p1::Missing, p2::AbstractAction)
    missing
end
function PopulationActionsParameters(p1::Missing, p2::Missing)
    missing
end
PopulationInteractionsParameters(p11::MissingAA, p21::MissingAA, p12::MissingAA, p22::MissingAA) = missing

function pops(t::Type{T}; kwargs...) where T
    syms = keys(kwargs)
    args = zip(values(kwargs)...)
    new_kwargs = (zip(syms, arg) for arg in args)
    params = map((single_kwargs) -> t(;single_kwargs...), new_kwargs)
	if ndims(params) == 1
		return PopulationActionsParameters(params...)
	elseif ndims(params) == 2
		return PopulationInteractionsParameters(params...)
	else
		error("Weirdly shaped parameters.")
	end
end

function pops(ps::Vector)# where {P<:AbstractParameter}
    PopulationActionsParameters(ps...)
end
function pops(ps::Matrix)# where {P<:AbstractParameter}
    PopulationInteractionsParameters(ps...)
end

function pops(ps::Vector{A}) where {A <: AbstractPopulationInteractionsParameters}
    PopulationInteractionsParameters([CompositeParameter([pop_params...]) for pop_params in zip(array.(ps)...)]...)
end


# To run this script, `cd` to the `./test/fixtures` directory and then, from the Julia terminal, `include("./runner.jl")`.

import JSON

# Negative small values:
x = linspace( -2.0^-29, -2.0^-54, 500 )

y = log1p( x )
println( y )

data = Dict([
	("x", x),
	("expected", y)
])

outfile = open( "./small_negative.json", "w" )
JSON.json( data )

write( outfile, JSON.json(data) )
close( outfile )

# Positive small values:
x = linspace( 2.0^-54, 2.0^-29, 500 )

y = log1p( x )
println( y )

data = Dict([
	("x", x),
	("expected", y)
])

outfile = open( "./small_positive.json", "w" )
JSON.json( data )

write( outfile, JSON.json(data) )
close( outfile )

# Negative tiny values:
x = linspace( -2.0^-54, 0, 500 )

y = log1p( x )
println( y )

data = Dict([
	("x", x),
	("expected", y)
])

outfile = open( "./tiny_negative.json", "w" )
JSON.json( data )

write( outfile, JSON.json(data) )
close( outfile )

# Positive tiny values:
x = linspace( 0, 2.0^-54, 500 )

y = log1p( x )
println( y )

data = Dict([
	("x", x),
	("expected", y)
])

outfile = open( "./tiny_positive.json", "w" )
JSON.json( data )

write( outfile, JSON.json(data) )
close( outfile )

# Negative medium values:
x = linspace( -0.41422, -0.2929, 500 )

y = log1p( x )
println( y )

data = Dict([
	("x", x),
	("expected", y)
])

outfile = open( "./medium_negative.json", "w" )
JSON.json( data )

write( outfile, JSON.json(data) )
close( outfile )

# Positive medium values:
x = linspace( 0.2929, 0.41422, 500 )

y = log1p( x )
println( y )

data = Dict([
	("x", x),
	("expected", y)
])

outfile = open( "./medium_positive.json", "w" )
JSON.json( data )

write( outfile, JSON.json(data) )
close( outfile )

# Negative large values:
x = linspace( -0.9999, -0.41422, 500 )

y = log1p( x )
println( y )

data = Dict([
	("x", x),
	("expected", y)
])

outfile = open( "./large_negative.json", "w" )
JSON.json( data )

write( outfile, JSON.json(data) )
close( outfile )

# Positive large values:
x = linspace( 0.41422, 100, 500 )

y = log1p( x )
println( y )

data = Dict([
	("x", x),
	("expected", y)
])

outfile = open( "./large_positive.json", "w" )
JSON.json( data )

write( outfile, JSON.json(data) )
close( outfile )

# Positive big values:
x = linspace( 2.0^30, 2.0^52, 500 )

y = log1p( x )
println( y )

data = Dict([
	("x", x),
	("expected", y)
])

outfile = open( "./big_positive.json", "w" )
JSON.json( data )

write( outfile, JSON.json(data) )
close( outfile )

# Positive huge values:
x = linspace( 1e200, 1e300, 500 )

y = log1p( x )
println( y )

data = Dict([
	("x", x),
	("expected", y)
])

outfile = open( "./huge_positive.json", "w" )
JSON.json( data )

write( outfile, JSON.json(data) )
close( outfile )


"""
    sparse_pflap(args; kwargs)

Find a transformation flow (complex displacment field), that transforms image `source` to closer to image `target`.
Returns the transformation a [`Flow`](@ref) and the registered `source` image.

# Arguments:
- `target::Image`: target/fixed grayscale image.
- `source::Image`: source/moving grayscale image.

# Keyword Arguments:
- `filter_count::Integer=3`: the number of basis filters used (so far only =3 implemented).
- `timer::TimerOutput=TimerOutput("sparse pflap")`: provide a timer which times certain blocks in the function.
- `display::Bool=false`: verbose and debug prints.
- `point_count::Int=500`: the number of points attempted to be found at the edges of the target image.
- `spacing::Int=10`: the minimal distance between two points.
- `match_source_histogram::Bool=true`: choose to match the `source` image histogram to the `target` image histogram.

## Note:
If `display=true` is chosen the function will also return an array of figures demonstrating the
work of the registration.

See also: [`sparse_lap`](@ref), [`pflap`](@ref), [`showflow`](@ref), [`single_lap`](@ref), [`sparse_pflap`](@ref).
"""
function sparse_pflap_psnr(target::Image,
                           source::Image;
                           filter_count::Integer=3,
                           max_repeats::Integer=3,
                           display::Bool=false,
                           point_count::Int=500,
                           spacing::Int=13,
                           timer::TimerOutput=TimerOutput("sparse pflap"),
                           match_source_histogram::Bool=true,
                           rescale_intensities::Bool=false)

    @timeit_debug timer "setup" begin
        # convert images to floats
        source = Float64.(source)
        target = Float64.(target)

        # rescale images to have the whole [0, 1] spectrum.
        if rescale_intensities
            target, source = rescale!(target, source)
        end

        # adjust the source histogram to be like target
        if match_source_histogram
            @timeit_debug timer "hist match" begin
                source = adjust_histogram(source, Matching(targetimg = target)) # takes 8ms on average
            end
        end

        # pad with zeros if sizes difer.
        target, source = pad_images(target, source)

        image_size = size(target)

        # set number of layers in the filter pyramid.
        level_count = floor(Int64, log2(minimum(size(target))/8)+1)+1

        @assert ((2^(level_count)+1) <= minimum(image_size)) "level number results in a filter larger than the size of the input images."

        # displacement init.
        u_est = zeros(ComplexF64, size(target))

        # filter half sizes array eg. [16, 8, 4, 2, 1]
        half_size_pyramid = Int64.(2 .^ range(level_count-1, stop=0, length=level_count))

        if display
            num_plots = 4
            figs = Array{Figure}(undef, level_count, max_repeats, num_plots)
        end

        psnr_threshold = 0.3
        psnr_before = 0
        source_reg = source

        #TODO edit
        # get edge points
        mask = falses(size(target))
        fhs = 3
        mask[fhs+1:end-fhs, fhs+1:end-fhs] .= true
        @timeit_debug timer "find edge points" begin
            inds = find_edge_points(target, spacing=spacing, point_count=point_count, mask=mask)
        end
        if display
            println("ind count: ", length(inds))
        end

    end # "setup"

    for level in 1:level_count

        @timeit_debug timer "single filter pyramid level" begin
            if display
                println("###################")
                println("ITERATION: ", level)
                println("filter_half_size: ", half_size_pyramid[level])
            end

            fhs = half_size_pyramid[level]
            window_size = Int64.(2 .* fhs .+ (1, 1))

            for iter_repeat in 1:max_repeats

                @timeit_debug timer "filter inds" begin
                    current_inds = filter_border_inds(inds, size(target), fhs)
                    if display
                        println("\tnon border inds: $(length(current_inds))/$(length(inds)) ")
                    end
                    if level != 1
                        current_inds = LAP_julia.mapped_out(current_inds, target)
                    end
                end
                if display
                    println("inner loop iter: $iter_repeat")
                    println("\tnon mapped out inds: $(length(current_inds))/$(length(inds)) ")
                end
                @timeit_debug timer "sparse lap" begin
                    new_estim_at_inds, current_inds = single_lap_at_points(target, source_reg, fhs, window_size, current_inds, timer=timer, display=display)
                end

                # interpolate flow
                if all(isnan, new_estim_at_inds)
                    @timeit_debug timer "interpolate flow" begin
                        if display
                            println("\tIMPORTANT: all new estim vectors are NaN.")
                        end
                    end
                    break
                else
                    @timeit_debug timer "flow interpolation" begin
                        # extract u_est_vecs from the previous estimate and add them to the newly estimated ones
                        # then interpolate a new flow.
                        if display
                            println("\tnon NaN new estim flow vector count: ", length(new_estim_at_inds))
                        end
                        u_est_at_inds = map(ind -> u_est[ind], current_inds) .+ new_estim_at_inds
                        # @assert any(.!isnan.(u_est_at_inds)) "new estim nans", count(isnan, u_est_at_inds), length(u_est_at_inds)
                        u_est_adept = interpolate_flow(u_est_at_inds, current_inds, size(u_est))
                    end
                end

                # if the interpolation failes and returns u_est_adept full of NaNs go next level.
                if any(isnan.(u_est_adept))
                    if display
                        println("\tIMPORTANT: u_est_adept full of NaNs, skipping")
                    end
                    break
                end

                # if the interpolation estimated a flow 20% larger than the fhs then go to next level
                # if any(vec_len.(u_est_adept) .> (fhs*1.3))
                #     if display
                #         println("\tIMPORTANT: u_est_adept is above size threshold, skipping (max: $(maximum(vec_len.(u_est_adept))), fhs: $fhs)")
                #     end
                #     break
                # end
                @assert any(.!isnan.(u_est_adept)) count(isnan, u_est_adept), length(u_est_adept)

                # linear interpolation
                @timeit_debug timer "image interpolation" begin
                    # source_reg = warp_img(source, -real(u_est_adept), -imag(u_est_adept), target)
                    source_reg = warp_img(source, -real(u_est_adept), -imag(u_est_adept))
                end

                # calculate the psnr after warping the source by u_est_adept
                psnr_after = assess_psnr(source_reg, target)

                # compare psnr before and after to see if there was an improvement larger than psnr_threshold
                if psnr_after - psnr_before > psnr_threshold
                    # nice, the improvement is large -> try to run again with the same filter size.
                    u_est = u_est_adept
                    psnr_before = psnr_after
                    if display
                        println("\tstatus: success, repeating filter size")
                    end
                else
                    # the improvement isn't larger than psnr_threshold -> next filter size
                    # check to see if it was even an improvement:
                    if psnr_after - psnr_before > 0
                        # it was an improvement, save the variables:
                        u_est = u_est_adept
                        psnr_before = psnr_after
                        if display
                            println("\tstatus: success, next filter size")
                            add_figs_sparse_pflap(figs, level, iter_repeat, u_est, new_estim_at_inds, source_reg, level_count, max_repeats, current_inds, fhs)
                        end
                    else
                        # it wasn't an improvement, skip
                        if display
                            println("\tstatus: failure, trashing")
                        end
                    end
                    break # go for next filter size.
                end

                if display
                    add_figs_sparse_pflap(figs, level, iter_repeat, u_est, new_estim_at_inds, source_reg, level_count, max_repeats, current_inds, fhs)
                end
            end
        end # "single filter pyramid level"
    end

    if display; return u_est, source_reg, figs; end
    return u_est, source_reg
end


module ArrayInterface

using ArrayInterfaceCore
import ArrayInterfaceCore: allowed_getindex, allowed_setindex!, aos_to_soa, buffer,
    parent_type, fast_matrix_colors, findstructralnz, has_sparsestruct,
    issingular, isstructured, matrix_colors, restructure, lu_instance,
    safevec, zeromatrix, ColoringAlgorithm, fast_scalar_indexing, parameterless_type,
    ndims_index, ndims_shape, is_splat_index, is_forwarding_wrapper, IndicesInfo

# ArrayIndex subtypes and methods
import ArrayInterfaceCore: ArrayIndex, MatrixIndex, VectorIndex, BidiagonalIndex, TridiagonalIndex
# managing immutables
import ArrayInterfaceCore: ismutable, can_change_size, can_setindex
# constants
import ArrayInterfaceCore: MatAdjTrans, VecAdjTrans, UpTri, LoTri
# device pieces
import ArrayInterfaceCore: AbstractDevice, AbstractCPU, CPUPointer, CPUTuple, CheckParent,
    CPUIndex, GPU, can_avx

import ArrayInterfaceCore: known_first, known_step, known_last

using Static
using Static: Zero, One, nstatic, eq, ne, gt, ge, lt, le, eachop, eachop_tuple,
    permute, invariant_permutation, field_type, reduce_tup, find_first_eq

using IfElse

using Base.Cartesian
using Base: @propagate_inbounds, tail, OneTo, LogicalIndex, Slice, ReinterpretArray,
    ReshapedArray, AbstractCartesianIndex

using Base.Iterators: Pairs
using LinearAlgebra

import Compat

n_of_x(::StaticInt{N}, x::X) where {N,X} = ntuple(Compat.Returns(x), Val{N}())

_add1(@nospecialize x) = x + oneunit(x)
_sub1(@nospecialize x) = x - oneunit(x)

@generated function merge_tuple_type(::Type{X}, ::Type{Y}) where {X<:Tuple,Y<:Tuple}
    Tuple{X.parameters...,Y.parameters...}
end

const CanonicalInt = Union{Int,StaticInt}
canonicalize(x::Integer) = Int(x)
canonicalize(@nospecialize(x::StaticInt)) = x

abstract type AbstractArray2{T,N} <: AbstractArray{T,N} end

Base.size(A::AbstractArray2) = map(Int, ArrayInterface.size(A))
Base.size(A::AbstractArray2, dim) = Int(ArrayInterface.size(A, dim))

function Base.axes(A::AbstractArray2)
    is_forwarding_wrapper(A) && return ArrayInterface.axes(parent(A))
    throw(ArgumentError("Subtypes of `AbstractArray2` must define an axes method"))
end
function Base.axes(A::AbstractArray2, dim::Union{Symbol,StaticSymbol})
    axes(A, to_dims(A, dim))
end

function Base.strides(A::AbstractArray2)
    defines_strides(A) && return map(Int, ArrayInterface.strides(A))
    throw(MethodError(Base.strides, (A,)))
end
Base.strides(A::AbstractArray2, dim) = Int(ArrayInterface.strides(A, dim))

function Base.IndexStyle(::Type{T}) where {T<:AbstractArray2}
    is_forwarding_wrapper(T) ? IndexStyle(parent_type(T)) : IndexCartesian()
end

function Base.length(A::AbstractArray2)
    len = known_length(A)
    if len === nothing
        return Int(prod(size(A)))
    else
        return Int(len)
    end
end

@propagate_inbounds Base.getindex(A::AbstractArray2, args...) = getindex(A, args...)
@propagate_inbounds Base.getindex(A::AbstractArray2; kwargs...) = getindex(A; kwargs...)

@propagate_inbounds function Base.setindex!(A::AbstractArray2, val, args...)
    return setindex!(A, val, args...)
end
@propagate_inbounds function Base.setindex!(A::AbstractArray2, val; kwargs...)
    return setindex!(A, val; kwargs...)
end

@inline static_first(x) = Static.maybe_static(known_first, first, x)
@inline static_last(x) = Static.maybe_static(known_last, last, x)
@inline static_step(x) = Static.maybe_static(known_step, step, x)

@inline function _to_cartesian(a, i::CanonicalInt)
    @inbounds(CartesianIndices(ntuple(dim -> indices(a, dim), Val(ndims(a))))[i])
end
@inline function _to_linear(a, i::Tuple{CanonicalInt,Vararg{CanonicalInt}})
    _strides2int(offsets(a), size_to_strides(size(a), static(1)), i) + static(1)
end

"""
    has_parent(::Type{T}) -> StaticBool

Returns `static(true)` if `parent_type(T)` a type unique to `T`.
"""
has_parent(x) = has_parent(typeof(x))
has_parent(::Type{T}) where {T} = _has_parent(parent_type(T), T)
_has_parent(::Type{T}, ::Type{T}) where {T} = False()
_has_parent(::Type{T1}, ::Type{T2}) where {T1,T2} = True()

"""
    device(::Type{T}) -> AbstractDevice

Indicates the most efficient way to access elements from the collection in low-level code.
For `GPUArrays`, will return `ArrayInterface.GPU()`.
For `AbstractArray` supporting a `pointer` method, returns `ArrayInterface.CPUPointer()`.
For other `AbstractArray`s and `Tuple`s, returns `ArrayInterface.CPUIndex()`.
Otherwise, returns `nothing`.
"""
device(A) = device(typeof(A))
device(::Type) = nothing
device(::Type{<:Tuple}) = CPUTuple()
device(::Type{T}) where {T<:Array} = CPUPointer()
device(::Type{T}) where {T<:AbstractArray} = _device(has_parent(T), T)
function _device(::True, ::Type{T}) where {T}
    if defines_strides(T)
        return device(parent_type(T))
    else
        return _not_pointer(device(parent_type(T)))
    end
end
_not_pointer(::CPUPointer) = CPUIndex()
_not_pointer(x) = x
_device(::False, ::Type{T}) where {T<:DenseArray} = CPUPointer()
_device(::False, ::Type{T}) where {T} = CPUIndex()

"""
    is_lazy_conjugate(::AbstractArray) -> Bool

Determine if a given array will lazyily take complex conjugates, such as with `Adjoint`. This will work with
nested wrappers, so long as there is no type in the chain of wrappers such that `parent_type(T) == T`

Examples

    julia> a = transpose([1 + im, 1-im]')
    2×1 transpose(adjoint(::Vector{Complex{Int64}})) with eltype Complex{Int64}:
     1 - 1im
     1 + 1im

    julia> ArrayInterface.is_lazy_conjugate(a)
    True()

    julia> b = a'
    1×2 adjoint(transpose(adjoint(::Vector{Complex{Int64}}))) with eltype Complex{Int64}:
     1+1im  1-1im

    julia> ArrayInterface.is_lazy_conjugate(b)
    False()

"""
is_lazy_conjugate(::T) where {T<:AbstractArray} = _is_lazy_conjugate(T, False())
is_lazy_conjugate(::AbstractArray{<:Real}) = False()

function _is_lazy_conjugate(::Type{T}, isconj) where {T<:AbstractArray}
    Tp = parent_type(T)
    if T !== Tp
        _is_lazy_conjugate(Tp, isconj)
    else
        isconj
    end
end

function _is_lazy_conjugate(::Type{T}, isconj) where {T<:Adjoint}
    Tp = parent_type(T)
    if T !== Tp
        _is_lazy_conjugate(Tp, !isconj)
    else
        !isconj
    end
end

"""
    insert(collection, index, item)

Returns a new instance of `collection` with `item` inserted into at the given `index`.
"""
Base.@propagate_inbounds function insert(collection, index, item)
    @boundscheck checkbounds(collection, index)
    ret = similar(collection, length(collection) + 1)
    @inbounds for i = firstindex(ret):(index-1)
        ret[i] = collection[i]
    end
    @inbounds ret[index] = item
    @inbounds for i = (index+1):lastindex(ret)
        ret[i] = collection[i-1]
    end
    return ret
end

function insert(x::Tuple{Vararg{Any,N}}, index, item) where {N}
    @boundscheck if !checkindex(Bool, StaticInt{1}():StaticInt{N}(), index)
        throw(BoundsError(x, index))
    end
    return unsafe_insert(x, Int(index), item)
end

@inline function unsafe_insert(x::Tuple, i::Int, item)
    if i === 1
        return (item, x...)
    else
        return (first(x), unsafe_insert(tail(x), i - 1, item)...)
    end
end

"""
    deleteat(collection, index)

Returns a new instance of `collection` with the item at the given `index` removed.
"""
Base.@propagate_inbounds function deleteat(collection::AbstractVector, index)
    @boundscheck if !checkindex(Bool, eachindex(collection), index)
        throw(BoundsError(collection, index))
    end
    return unsafe_deleteat(collection, index)
end
Base.@propagate_inbounds function deleteat(collection::Tuple{Vararg{Any,N}}, index) where {N}
    @boundscheck if !checkindex(Bool, StaticInt{1}():StaticInt{N}(), index)
        throw(BoundsError(collection, index))
    end
    return unsafe_deleteat(collection, index)
end

function unsafe_deleteat(src::AbstractVector, index)
    dst = similar(src, length(src) - 1)
    @inbounds for i in indices(dst)
        if i < index
            dst[i] = src[i]
        else
            dst[i] = src[i+1]
        end
    end
    return dst
end

@inline function unsafe_deleteat(src::AbstractVector, inds::AbstractVector)
    dst = similar(src, length(src) - length(inds))
    dst_index = firstindex(dst)
    @inbounds for src_index in indices(src)
        if !in(src_index, inds)
            dst[dst_index] = src[src_index]
            dst_index += one(dst_index)
        end
    end
    return dst
end

@inline function unsafe_deleteat(src::Tuple, inds::AbstractVector)
    dst = Vector{eltype(src)}(undef, length(src) - length(inds))
    dst_index = firstindex(dst)
    @inbounds for src_index in static(1):length(src)
        if !in(src_index, inds)
            dst[dst_index] = src[src_index]
            dst_index += one(dst_index)
        end
    end
    return Tuple(dst)
end

@inline unsafe_deleteat(x::Tuple{T}, i) where {T} = ()
@inline unsafe_deleteat(x::Tuple{T1,T2}, i) where {T1,T2} =
    isone(i) ? (x[2],) : (x[1],)
@inline function unsafe_deleteat(x::Tuple, i)
    if i === one(i)
        return tail(x)
    elseif i == length(x)
        return Base.front(x)
    else
        return (first(x), unsafe_deleteat(tail(x), i - one(i))...)
    end
end

include("array_index.jl")
include("ranges.jl")
include("axes.jl")
include("size.jl")
include("dimensions.jl")
include("indexing.jl")
include("stridelayout.jl")
include("broadcast.jl")

end