# WARNING this file is only loaded if DataFrames.jl is also active
# This all happens thanks to the Requires.jl package

import Base: getindex
import Base: setindex!
import SimpleSDMLayers: clip, latitudes, longitudes

"""
    Base.getindex(layer::T, df::DataFrames.DataFrame; latitude = :latitude, longitude = :longitude) where {T <: SimpleSDMLayer}

Returns the values of a layer at all occurrences in a `DataFrame`.
"""
function Base.getindex(layer::T, df::DataFrames.DataFrame; latitude = :latitude, longitude = :longitude) where {T <: SimpleSDMLayer}
    lats = df[:, latitude]
    lons = df[:, longitude]
    return [layer[lon, lat] for (lon, lat) in zip(lons, lats)]
end

"""
    clip(layer::T, df::DataFrames.DataFrame; latitude = :latitude, longitude = :longitude) where {T <: SimpleSDMLayer}

Returns a clipped version (with a 10% margin) around all occurences in a
`DataFrame`.
"""
function SimpleSDMLayers.clip(layer::T, df::DataFrames.DataFrame; latitude = :latitude, longitude = :longitude) where {T <: SimpleSDMLayer}
   occ_latitudes = filter(!ismissing, df[:, latitude])
   occ_longitudes = filter(!ismissing, df[:, longitude])

   lat_min = minimum(occ_latitudes)
   lat_max = maximum(occ_latitudes)
   lon_min = minimum(occ_longitudes)
   lon_max = maximum(occ_longitudes)

   lat_Δ = abs(lat_max - lat_min)
   lon_Δ = abs(lon_max - lon_min)

   scaling = 0.1
   lon_s = scaling*lon_Δ
   lat_s = scaling*lat_Δ

   lat_max = min(layer.top, lat_max+lat_s)
   lat_min = max(layer.bottom, lat_min-lat_s)
   lon_max = min(layer.right, lon_max+lon_s)
   lon_min = max(layer.left, lon_min-lon_s)

   return layer[left=lon_min, right=lon_max, bottom=lat_min, top=lat_max]
end

"""
    DataFrames.DataFrame(layer::T) where {T <: SimpleSDMLayer}

Returns a DataFrame from a `SimpleSDMLayer` element, with columns for latitudes,
longitudes and grid values.
"""
function DataFrames.DataFrame(layer::T; kw...) where {T <: SimpleSDMLayer}
    lats = repeat(latitudes(layer), outer = size(layer, 2))
    lons = repeat(longitudes(layer), inner = size(layer, 1))
    return DataFrames.DataFrame(latitude = lats, longitude = lons, values = vec(layer.grid); kw...)
end

"""
    DataFrames.DataFrame(layers::Array{SimpleSDMLayer})

Returns a single DataFrame from an `Array` of compatible`SimpleSDMLayer`
elements, with every layer as a column, as well as columns for latitudes and longitudes.
"""
function DataFrames.DataFrame(layers::Array{T}; kw...) where {T <: SimpleSDMLayer}
    l1 = layers[1]
    all(x -> SimpleSDMLayers._layers_are_compatible(x, l1), layers)

    lats = repeat(latitudes(l1), outer = size(l1, 2))
    lons = repeat(longitudes(l1), inner = size(l1, 1))
    values = mapreduce(x -> vec(x.grid), hcat, layers)

    df = DataFrames.DataFrame(values; kw...)
    DataFrames.insertcols!(df, 1, :latitude => lats)
    DataFrames.insertcols!(df, 1, :longitude => lons)
    return df
end

for ty in (:SimpleSDMResponse, :SimpleSDMPredictor)
    eval(
        quote
            """
                $($ty)(df::DataFrame, col::Symbol, layer::T; latitude::Symbol = :latitude, longitude::Symbol = :longitude) where {T <: SimpleSDMLayer}

            Returns a `$($ty)` from a `DataFrame`.
            """
            function SimpleSDMLayers.$ty(df::DataFrames.DataFrame, col::Symbol, layer::SimpleSDMLayer; latitude::Symbol = :latitude, longitude::Symbol = :longitude)
                lats = df[:, latitude]
                lons = df[:, longitude]

                uniquelats = unique(lats)
                uniquelons = unique(lons)

                grid = Array{Any}(nothing, size(layer))

                if uniquelats == latitudes(layer) && uniquelons == longitudes(layer)
                    grid[:] = df[:, col]
                else
                    lats_idx = [SimpleSDMLayers._match_latitude(layer, lat) for lat in lats]
                    lons_idx = [SimpleSDMLayers._match_longitude(layer, lon) for lon in lons]
                    for (lat, lon, value) in zip(lats_idx, lons_idx, df[:, col])
                        grid[lat, lon] = value
                    end
                end

                internal_types = unique(typeof.(grid))
                return SimpleSDMLayers.$ty(Array{Union{internal_types...}}(grid), layer)
            end
        end,
    )
end


"""
## Extract epoch data from SSR

#### Arguments
* `a`: A SSR object
* `valid_triggers`: Trigger numbers that are considered valid ([1,2])
* `remove_first`: Remove the first n triggers (0)
* `remove_last`: Remove the last n triggers (0)

#### Example

```julia
epochs = extract_epochs(SSR, valid_triggers=[1,2])
```
"""
function extract_epochs(a::SSR; valid_triggers::Union{AbstractArray, Int}=[1,2], remove_first::Int=0, remove_last::Int=0, kwargs...)

    merge!(a.processing, Dict("epochs" => extract_epochs(a.data, a.triggers, valid_triggers, remove_first, remove_last)))

    return a
end

function epoch_rejection(a::SSR; retain_percentage::Number=0.95, kwargs...)

    a.processing["epochs"] = epoch_rejection(a.processing["epochs"], retain_percentage; kwargs...)

    return a
end

function create_sweeps(a::SSR; epochsPerSweep::Int=64, kwargs...)

    if epochsPerSweep > size(a.processing["epochs"], 2)
        error("Sweep length is longer than number of epochs will allow")
    end

    merge!(a.processing, Dict("sweeps" => create_sweeps(a.processing["epochs"], epochsPerSweep)))

    return a
end

#######################################
#
# Add triggers for more epochs
#
#######################################

function add_triggers(a::SSR; kwargs...)

    @debug("Adding triggers to reduce SSR. Using SSR modulation frequency")

    add_triggers(a, modulationrate(a); kwargs...)
end

function add_triggers(a::SSR, mod_freq::Number; kwargs...)

    @debug("Adding triggers to reduce SSR. Using $(mod_freq)Hz")

    epochIndex = DataFrame(Code = a.triggers["Code"], Index = [Int(i) for i in a.triggers["Index"]]);
    epochIndex[:Code] = epochIndex[:Code] - 252

    add_triggers(a, mod_freq, epochIndex; kwargs...)
end

function add_triggers(a::SSR, mod_freq::Number, epochIndex; cycle_per_epoch::Int=1, kwargs...)

    @info("Adding triggers to reduce SSR. Reducing $(mod_freq)Hz to $cycle_per_epoch cycle(s).")

    # Existing epochs
    existing_epoch_length   = median(diff(epochIndex[:Index]))     # samples
    existing_epoch_length_s = existing_epoch_length / samplingrate(a)
    @debug("Existing epoch length: $(existing_epoch_length_s)s")

    # New epochs
    new_epoch_length_s = cycle_per_epoch / mod_freq
    new_epochs_num     = round.(existing_epoch_length_s / new_epoch_length_s) - 2
    new_epoch_times    = collect(1:new_epochs_num)*new_epoch_length_s
    new_epoch_indx     = [0; round.(new_epoch_times * samplingrate(a))]
    @debug("New epoch length = $new_epoch_length_s")
    @debug("New # epochs     = $new_epochs_num")

    # Place new epoch indices
    @debug("Was $(length(epochIndex[:Index])) indices")
    new_indx = epochIndex[:Index][1:end-1] .+ new_epoch_indx'
    new_indx = reshape(new_indx', length(new_indx), 1)[1:end-1]
    @debug("Now $(length(new_indx)) indices")

    # Place in dict
    new_code = round.(Int, ones(1, length(new_indx))) .+ 252
    a.triggers = Dict("Index" => vec((new_indx)'), "Code" => vec(new_code), "Duration" => ones(length(new_code), 1)')
    #TODO Possible the trigger duration of one is not long enough

    return a
end

#######################################
#
# Rejection channels
#
#######################################

function channel_rejection(a::SSR; threshold_abs::Number=1000, threshold_var::Number=2, kwargs...)

    # Run on epochs if available, else run on the raw data
    if haskey(a.processing, "epochs")
        data = reshape(a.processing["epochs"],
                size(a.processing["epochs"], 1) * size(a.processing["epochs"], 2), size(a.processing["epochs"],3))
    else
        data = a.data
    end

    valid = vec(channel_rejection(data, threshold_abs, threshold_var))

    @info("Rejected $(sum(.!valid)) channels $(join(channelnames(a)[find(.!valid)], " "))")

    remove_channel!(a, channelnames(a)[.!valid])

    return a
end


push!(LOAD_PATH,".")
using CodeGenerator

buffer=CodeGenerator.read_grammar()
g=CodeGenerator.Grammar(buffer)
println(g)

CodeGenerator.prettyshow(stdout,g)

G=CodeGenerator.Generator(g)

CodeGenerator.bind(G,Symbol("<Number>"),()->rand(1:9))

res=CodeGenerator.generate(G,Symbol("Expr");max_depth=6)

for each in res
      println(each)
end

res=CodeGenerator.generate(G,Symbol("Expr");max_depth=2,generator=CodeGenerator.generate_bfs)

for each in res
      println(each)
end


# synthetic_L1_include.jl
include("../tcm_code/tcm_transactionsToCounts.jl")

include("../tcm_code/tcm_estimateMNL.jl")
include("../tcm_code/tcm_predictMNL.jl")

include("../tcm_code/tcm_estimateMMNL.jl")
include("../tcm_code/tcm_estimateMMNL_kcv.jl")
include("../tcm_code/tcm_predictMMNL.jl")

include("../tcm_code/tcm_estimateHALOMNL.jl")
include("../tcm_code/tcm_predictHALOMNL.jl")

include("../tcm_code/tcm_estimateRanking.jl")
include("../tcm_code/tcm_predictRanking.jl")
include("../tcm_code/tcm_estimateRanking_L1.jl")

include("../tcm_code/tcm_estimateForest.jl")
include("../tcm_code/tcm_estimateForest_kcv.jl")
include("../tcm_code/tcm_estimateForest_L1.jl")
include("../tcm_code/tcm_predictForest.jl")

include("../tcm_code/tcm_createGrandAssortments.jl")

include("../tcm_code/tcm_evaluateKL.jl")

include("synthetic_L1_helperfn.jl")


function createdataranges(start::Point{Int}, data, incols::Bool=true)
    tbldata = Matrix[]
    tblrngs = CellRange{Int}[]
    nelements = incols ? size(data[1],2) : size(data[1],1)
    kd = length(data)

    if incols
        numRows = size(data[1],1)
        for col=1:nelements
            for k=1:kd
                el = data[k]
                j = (col-1)*kd + k
                statrow = CellRange(
                    Box{Int}(Point(start.x + j - 1, start.y),
                             Point(start.x + j, start.y + numRows)),
                j+5,k+5)
                push!(tblrngs, statrow)
                push!(tbldata, reshape(el[:, col], numRows, 1))
            end
        end
    else
        numCols = size(data[1],2)
        for row=1:nelements
            for k=1:kd
                el = data[k]
                i = (row-1)*kd + k
                statrow = CellRange(
                    Box{Int}(Point(start.x, start.y + i - 1),
                             Point(start.x + numCols, start.y + i)),
                    i+5,k+5)
                push!(tblrngs, statrow)
                push!(tbldata, el[row,:])
            end
        end
    end
    return tbldata, tblrngs
end

const EMPTY = 1
const COLNAME = 2
const ROWNAME = 3
const COLLEVELS = 4
const ROWLEVELS = 5

checkstylelen(styles, p::Pivot) = (length(p.data) + 5 == length(styles)) ||
  error("length of data + 3 != length of styles")

function checkdatasize(p::Pivot)
    res = [size(el) for el in p.data]
    n = length(res)
    if n > 0
        el = res[1]
        for i=2:n
            if el != res[i]
                error("data sizes are not all the same\n$res")
            end
        end
    end
    return true
end

function totable(p::Pivot)
    styles = pivotstyles(p)
    checkstylelen(styles, p)
    checkdatasize(p)

    pm = p.metadata
    collevels = CartesianDataElement(colLevels(pm), true)
    rowlevels = CartesianDataElement(rowLevels(pm), false)
    rownames = rowNames(pm)
    rownames = reshape(rownames, 1, length(rownames))
    colnames = colNames(pm)

    # figure out where to place the tables
    rowRibbonHeight, rowRibbonWidth  = size(rowlevels)
    colRibbonHeight, colRibbonWidth = size(collevels)
    numCols = rowRibbonWidth + colRibbonWidth
    numRows = rowRibbonHeight + colRibbonHeight + 1

    emptydata = ExpandDataElement("", colRibbonHeight, rowRibbonWidth)
    colnames =  ExpandDataElement(join(colnames, " / "), 1, colRibbonWidth)

    emptyrng = CellRange(
        Box{Int}(Point(0,0), Point(rowRibbonWidth, colRibbonHeight)),
        EMPTY,EMPTY)
    colnamerng = CellRange(
        Box{Int}(Point(rowRibbonWidth, 0), Point(numCols, 1)),
        COLNAME,COLNAME)
    rownamerng = CellRange(
        Box{Int}(Point(0, colRibbonHeight), Point(rowRibbonWidth, colRibbonHeight + 1)),
        ROWNAME,ROWNAME)
    collevelrng = CellRange(
        Box{Int}(Point(rowRibbonWidth, 1), Point(numCols, 1+colRibbonHeight)),
        COLLEVELS,COLLEVELS)
    rowlevelrng = CellRange(
        Box{Int}(Point(0, 1+colRibbonHeight), Point(rowRibbonWidth, numRows)),
        ROWLEVELS,ROWLEVELS)

    data = Any[emptydata, colnames, rownames, collevels, rowlevels]
    rngs = CellRange{Int}[emptyrng; colnamerng; rownamerng; collevelrng; rowlevelrng]

    ndata, nrngs = createdataranges(Point(rowRibbonWidth, colRibbonHeight + 1),
                                    p.data, incols(p.metadata))
    append!(data, ndata)
    append!(rngs, nrngs)

    return Table(data, Layout(rngs), styles)
end

function strings(xs)
    n = length(xs)
    res = fill(String[], n)
    for i=1:n
        res[i] = map(string, xs[i])
    end
    return res
end

function levellength(xs)
    if length(xs) > 0
        return prod([length(x) for x in xs])
    else
        return 1
    end
end

@doc """
Args:
  df:           DataFrame to summarize
  rowvars:      row variables in table
  colvars:      column variables in table
  stats:        aggregate statistics to decribe each combination
                of row and column variables
  statnames:    statistic names used for table
  varorders:    makes the levels of the row and column variables
                follow the order given
  stat_in_cols: if true then the statistics are considered column
                levels of the "stats" variable. Otherwise they are
                row levels.

Usage:

```
using RDatasets
using Pivots
using Tables.HTMLFormat
const tb = Tables.HTMLFormat

house = RDatasets.dataset("Ecdat", "Housing")
p = summarize(
    house,
    rowvars      = [:Bedrooms],
    colvars      = Symbol[],
    statnames    = String["mean(price)", "std(price)", "count"],
    stats        = [df->mean(df[:Price]), df->std(df[:Price]), df->length(df[:Price])],
    varorders    = [:Bedrooms => Int32[1:6], :Driveway => ["no", "yes"]],
    stat_in_cols = true)

tbl = totable(p)
tb.HTML(tbl)
```
""" ->
function summarize{T <: String}(
    df::AbstractDataFrame;
    rowvars::Vector{Symbol} = Symbol[],
    colvars::Vector{Symbol} = Symbol[],
    stats::Vector{Function} = Function[],
    statnames::Vector{T} = fill("", length(stats)),
    varorders=Dict{Symbol,Any}(),
    stat_in_cols=true)

    @assert length(statnames) == length(stats)

    levels_of_rowvars = [haskey(varorders, rowvar) ? varorders[rowvar] :
                           unique(df[rowvar]) for rowvar in rowvars]
    levels_of_colvars = [haskey(varorders, colvar) ? varorders[colvar] :
                           unique(df[colvar]) for colvar in colvars]

    rownames = String[string(rowvar) for rowvar in rowvars]
    colnames = String[string(colvar) for colvar in colvars]

    pm = PivotMetadata(rownames, colnames,
                       strings(levels_of_rowvars),
                       strings(levels_of_colvars), statnames, stat_in_cols)

    statlen = length(stats)

    # determine how big the statistic matrices are
    n = levellength(levels_of_rowvars)
    m = levellength(levels_of_colvars)

    # create the global statistics
    frsts = [stat(df[1,:]) for stat in stats]
    types = [typeof(frst) for frst in frsts]

    # make the statistic matrices
    results = [fill(Nullable{tp}(), n, m) for tp in types]

    allvars = [colvars; rowvars]
    if length(allvars) > 0
        for (i,grp) in groupby(df, allvars, varorders)
            for k=1:statlen
                results[k][i] = Nullable{types[k]}(stats[k](grp))
            end
        end
    else
        for k=1:length(types)
            results[k][1,1] = Nullable{types[k]}(statistics[k](df))
        end
    end

    p = Pivot(results, pm)

    return p
end


"""
    RectRegion(
        RegID :: AbstractString,
        ParID :: AbstractString,
        name  :: AbstractString,
        bound :: Vector{<:Real},
        savegeo :: Bool = true,
        ST = String,
        FT = Float64
    ) -> RectRegion{ST,FT}

Creates a rectilinear GeoRegion `RegID`.

Arguments
=========

- `RegID` : The keyword ID that will be used to identify the GeoRegion.
            If the ID is already in use, then an error will be thrown.

- `ParID` : The ID of the parent GeoRegion where information can be extracted from
- `name`  : A name for the GeoRegion (meta information, can be used in Logging)
- `bound` : The [N,S,E,W] coordinates defining the region
- `savegeo` : Save the GeoRegion into the master list?
"""
function RectRegion(
    RegID :: AbstractString,
    ParID :: AbstractString,
    name  :: AbstractString,
    bound :: Vector{<:Real};
    savegeo :: Bool = true,
    ST = String,
    FT = Float64
)

    if isGeoRegion(RegID,throw=false)
        error("$(modulelog()) - The GeoRegion $(RegID) has already been defined.  Please use another identifier.")
    else
        @info "$(modulelog()) - Adding the GeoRegion $(RegID) to the list."
    end

    if !isGeoRegion(ParID,throw=false)
        error("$(modulelog()) - The GeoRegion $(ParID) was defined to be the parent GeoRegion of $(RegID), but the GeoRegion $(ParID) is not defined.  Please define the GeoRegion $(ParID) and its properties.")
    end

    N,S,E,W = bound; is180,is360 = checkbounds(N,S,E,W)
    geo  = RectRegion{ST,FT}(RegID,ParID,name,N,S,E,W,is180,is360)
    par  = GeoRegion(ParID); isinGeoRegion(geo,par)
    name = replace(name," "=>"-")

    if savegeo
        open(joinpath(DEPOT_PATH[1],"files","GeoRegions","rectlist.txt"),"a") do io
            write(io,"$RegID, $ParID, $N, $W, $S, $E, $name\n")
        end
    end

    return geo

end

"""
    PolyRegion(
        RegID :: AbstractString,
        ParID :: AbstractString,
        name  :: AbstractString,
        lonpt :: Vector{<:Real},
        latpt :: Vector{<:Real};
        savegeo :: Bool = true,
        ST = String,
        FT = Float64
    ) -> PolyRegion{ST,FT}

Creates a rectilinear GeoRegion `RegID`.

Arguments
=========

- `RegID` : The keyword ID that will be used to identify the GeoRegion.
            If the ID is already in use, then an error will be thrown.

- `ParID` : The ID of the parent GeoRegion where information can be extracted from
- `name`  : A name for the GeoRegion (meta information, can be used in Logging)
- `lonpt` : A vector containing the longitude points
- `latpt` : A vector containing the latitude points
- `savegeo` : Save the GeoRegion into the master list?

!!! info "Start and End Points"
    The 1st and last elements of `lonpt` and `latpt` must be equal.
"""
function PolyRegion(
    RegID :: AbstractString,
    ParID :: AbstractString,
    name  :: AbstractString,
    lonpt :: Vector{<:Real},
    latpt :: Vector{<:Real},;
    savegeo :: Bool = true,
    ST = String,
    FT = Float64
)

    if isGeoRegion(RegID,throw=false)
        error("$(modulelog()) - The GeoRegion $(RegID) has already been defined.  Please use another identifier.")
    else
        @info "$(modulelog()) - Adding the GeoRegion $(RegID) to the list."
    end

    if !isGeoRegion(ParID)
        error("$(modulelog()) - The GeoRegion $(ParID) was defined to be the parent GeoRegion of $(RegID), but the GeoRegion $(ParID) is not defined.  Please define the GeoRegion $(ParID) and its properties.")
    end

    if lonpt[1] != lonpt[end]
        error("$(modulelog()) - The first and last points of the vector of longitude coordinates must be the same")
    end

    if latpt[1] != latpt[end]
        error("$(modulelog()) - The first and last points of the vector of latitude coordinates must be the same")
    end

    N = maximum(latpt); S = minimum(latpt)
    E = maximum(lonpt); W = minimum(lonpt)
    is180,is360 = checkbounds(N,S,E,W)
    shape = Point2.(lonpt,latpt)
    geo   = PolyRegion{ST,FT}(RegID,ParID,name,N,S,E,W,shape,is180,is360)
    par   = GeoRegion(ParID); isinGeoRegion(geo,par)
    name  = replace(name," "=>"-")

    npt = length(lonpt)
    if savegeo
        open(joinpath(DEPOT_PATH[1],"files","GeoRegions","polylist.txt"),"a") do io
            write(io,"\n  RegID $(RegID), $(ParID), $name\n")
            write(io,"  RegX  $(lonpt[1])")
            for ipt = 2 : npt; write(io,", $(lonpt[ipt])") end
            write(io,"\n")
            write(io,"  RegY  $(latpt[1])")
            for ipt = 2 : npt; write(io,", $(latpt[ipt])") end
            write(io,"\n")
        end
    end

    return geo

end

"""
    removeGeoRegion(RegID::AbstractString)

Creates the GeoRegion associated with the ID `RegID`.

Arguments
=========

- `RegID` : The keyword ID that will be used to identify the GeoRegion.
            If the ID is not valid (i.e. not being used), then an error will be thrown.
"""
function removeGeoRegion(
    RegID :: AbstractString
)

    if RegID == "GLB"
        error("$(modulelog()) - The Global GeoRegion \"GLB\" is an integral part of the GeoRegions.jl package and cannot be removed.")
    else
        @info "$(modulelog()) - Removing the GeoRegion $(RegID) ..."
    end

    regvec,filevec,typevec = listGeoRegions(); isgeoregion(RegID,regvec)
    ind = findall(RegID.==regvec)[1]
    geo = getgeoregion(
        RegID,
        joinpath(DEPOT_PATH[1],"files","GeoRegions",filevec[ind]),
        typevec[ind]
    )
    removegeoregion(geo,filevec[ind])

end

function removegeoregion(
    geo  :: PolyRegion,
    fgeo :: AbstractString
)

    rvec = listpolyregions(joinpath(DEPOT_PATH[1],"files","GeoRegions",fgeo))
    ind  = findall(rvec.==geo.regID)[1]
    ind  = (ind) * 4 .+ (0:3)

    flines = readlines(joinpath(DEPOT_PATH[1],"files","GeoRegions",fgeo))
    nlines = length(flines)

    open("tmp.txt","w") do io
        for iline = 1 : nlines
            if !any(iline.==ind)
                write(io,"$(flines[iline])\n")
            end
        end
    end

    mv("tmp.txt",joinpath(DEPOT_PATH[1],"files","GeoRegions",fgeo),force=true)

    return nothing

end

function removegeoregion(
    geo  :: RectRegion,
    fgeo :: AbstractString
)

    flines = readlines(joinpath(DEPOT_PATH[1],"files","GeoRegions",fgeo))
    nlines = length(flines)

    open("tmp.txt","w") do io
        for iline = 1 : nlines
            if !occursin(geo.regID,flines[iline])
                write(io,"$(flines[iline])\n")
            end
        end
    end

    mv("tmp.txt",joinpath(DEPOT_PATH[1],"files","GeoRegions",fgeo),force=true)

    return nothing

end


# implementation of MLJ measure interface for LossFunctions.jl

function naked(T::Type)
    without_module_name = split(string(T), '.') |> last
    without_type_parameters = split(without_module_name, '{') |> first
    return Symbol(without_type_parameters)
end

const WITHOUT_PARAMETERS =
    setdiff(LOSS_FUNCTIONS, WITH_PARAMETERS)

## WRAPPER

abstract type SupervisedLoss <: Unaggregated end

struct MarginLoss{L<:LossFunctions.MarginLoss} <: SupervisedLoss
    loss::L
end

struct DistanceLoss{L<:LossFunctions.DistanceLoss} <: SupervisedLoss
    loss::L
end

# INTERFACE FOR EXTRACTING PARAMETERS

# LossFunctions.jl does not have a uniform interface for extacting
# parameters, and hence:

_parameter(loss::LossFunctions.DWDMarginLoss) = loss.q
_parameter(loss::LossFunctions.SmoothedL1HingeLoss) = loss.gamma
_parameter(loss::LossFunctions.HuberLoss) = loss.d
_parameter(loss::LossFunctions.L1EpsilonInsLoss) = loss.ε
_parameter(loss::LossFunctions.L2EpsilonInsLoss) = loss.ε
_parameter(::LossFunctions.LPDistLoss{P}) where P = P
_parameter(::LossFunctions.L1DistLoss) = 1
_parameter(::LossFunctions.L2DistLoss) = 2
_parameter(loss::LossFunctions.QuantileLoss) = loss.τ

## CONSTRUCTORS AND CALLING BEHAVIOUR

err_wrap(n) = ArgumentError("Bad @wrap syntax: $n. ")

# We define amacro to wrap a concrete `LossFunctions.SupervisedLoss`
# type and define its constructor, and to define property access in
# case of paramters; the macro also defined calling behaviour:
macro wrap_loss(ex)
    ex.head == :call || throw(err_wrap(1))
    Loss_ex = ex.args[1]
    Loss_str = string(Loss_ex)
    if Loss_ex in MARGIN_LOSSES
        T = :MarginLoss
    else
        T = :DistanceLoss
    end

    # bind name to wrapped version of LossFunctions loss:
    program = quote
        const $Loss_ex = $T{<:LossFunctions.$Loss_ex}
        name(M::Type{<:$Loss_ex}) = $Loss_str
    end

    # defined instances
    alias = snakecase(string(Loss_ex))
    push!(program.args, quote
          instances(::Type{<:$Loss_ex}) = [$alias, ]
          end)

    # define kw constructor and expose any parameter as a property:
    if length(ex.args) == 1
        push!(program.args, quote
              $Loss_ex() = $T(LossFunctions.$Loss_ex())
              Base.propertynames(::$Loss_ex) = ()
              end)
    elseif length(ex.args) > 1
        sub_ex = ex.args[2]
        sub_ex.head == :parameters || throw(err_wrap(2))
        length(sub_ex.args) == 1 || throw(err_wrap("Only 1 kwarg supported"))
        sub_ex.args[1].head == :kw || throw(err_wrap(3))
        var_ex = sub_ex.args[1].args[1]
        var_str = string(var_ex)
        val_ex = sub_ex.args[1].args[2]
        push!(program.args, quote
              $Loss_ex(; $var_ex=$val_ex) =
                  $T(LossFunctions.$Loss_ex($var_ex))
              $Loss_ex(p) = $Loss_ex($var_ex=p)
              Base.propertynames(::$Loss_ex) = (Symbol($var_str), )
              function Base.getproperty(wrapper::$Loss_ex, name::Symbol)
                  if name === Symbol($var_str)
                      return _parameter(getfield(wrapper, :loss)) # see below
                  end
                  error("type $($Loss_ex) has no property $name")
              end
              end)
    else
        throw(err_wrap(4))
    end

    esc(program)
end

for Loss in WITHOUT_PARAMETERS
    eval(:(@wrap_loss $Loss()))
end

@wrap_loss DWDMarginLoss(; q=1.0)
@wrap_loss SmoothedL1HingeLoss(; gamma=1.0)
@wrap_loss HuberLoss(; d=1.0)
@wrap_loss L1EpsilonInsLoss(; ε=1.0)
@wrap_loss L2EpsilonInsLoss(; ε=1.0)
@wrap_loss LPDistLoss(; P=2)
@wrap_loss QuantileLoss(; τ=0.7)

## GENERIC TRAITS

const LossFunctions = LossFunctions
is_measure_type(::Type{<:SupervisedLoss})          = true
orientation(::Type{<:SupervisedLoss})              = :loss
reports_each_observation(::Type{<:SupervisedLoss}) = true
is_feature_dependent(::Type{<:SupervisedLoss})     = false
supports_weights(::Type{<:SupervisedLoss}) = true
docstring(M::Type{<:SupervisedLoss})       = name(M)

## CALLING - DISTANCE BASED LOSS FUNCTIONS

MMI.prediction_type(::Type{<:DistanceLoss}) = :deterministic
MMI.target_scitype(::Type{<:DistanceLoss}) = Union{Vec{Continuous},Vec{Count}}

call(measure::DistanceLoss, yhat, y) =
    LossFunctions.value(getfield(measure, :loss), y, yhat)

function call(measure::DistanceLoss, yhat, y, w::ArrMissing{Real})
    return w .* call(measure, yhat, y)
end

## CALLING - MARGIN BASED LOSS FUNCTIONS

MMI.prediction_type(::Type{<:MarginLoss}) = :probabilistic
MMI.target_scitype(::Type{<:MarginLoss})  = AbstractArray{<:Finite{2}}

# rescale [0, 1] -> [-1, 1]:
_scale(p) = 2p - 1

function call(measure::MarginLoss, yhat, y)
    probs_of_observed = broadcast(pdf, yhat, y)
    return (LossFunctions.value).(getfield(measure, :loss),
                                  1, _scale.(probs_of_observed))
end

call(measure::MarginLoss, yhat, y, w::ArrMissing{Real}) =
    w .* call(measure, yhat, y)

## ADJUSTMENTS

human_name(::Type{<:L1EpsilonInsLoss}) = "l1 ϵ-insensitive loss"
human_name(::Type{<:L2EpsilonInsLoss}) = "l2 ϵ-insensitive loss"
human_name(::Type{<:DWDMarginLoss}) = "distance weighted discrimination loss"

_signature(::Any) = ""
_signature(::Type{<:HuberLoss}) = "`HuberLoss(; d=1.0)`"
_signature(::Type{<:DWDMarginLoss}) = "`DWDMarginLoss(; q=1.0)`"
_signature(::Type{<:SmoothedL1HingeLoss}) = "`SmoothedL1HingeLoss(; gamma=1.0)`"
_signature(::Type{<:L1EpsilonInsLoss}) = "`L1EpsilonInsLoss(; ε=1.0)`"
_signature(::Type{<:L2EpsilonInsLoss}) = "`L2EpsilonInsLoss(; ε=1.0)`"
_signature(::Type{<:LPDistLoss}) = "`LPDistLoss(; P=2)`"
_signature(::Type{<:QuantileLoss}) = "`QuantileLoss(; τ=0.7)`"

## ALIASES AND DOCSTRINGS

const DOC_LOSS_FUNCTIONS =
"""
For more detail, see the original LossFunctions.jl documentation *but
note differences in the signature.*

Losses from LossFunctions.jl do not support `missing` values. To use
with `missing` values, replace `(ŷ, y)` with `skipinvalid(ŷ, y))`.
"""

for Loss_ex in DISTANCE_LOSSES
    eval(quote
         sig = _signature($Loss_ex)
         isempty(sig) || (sig = "Constructor signature: "*sig)
         @create_aliases $Loss_ex
         @create_docs($Loss_ex,
                      typename = name($Loss_ex),
                      body=DOC_LOSS_FUNCTIONS,
                      footer=sig)
         end)
end

for Loss_ex in MARGIN_LOSSES
    eval(quote
         sig = _signature($Loss_ex)
         isempty(sig) || (sig = "Constructor signature: "*sig)
         @create_aliases $Loss_ex
         @create_docs($Loss_ex,
                      typename = name($Loss_ex),
                      body=DOC_LOSS_FUNCTIONS,
                      scitype=DOC_FINITE_BINARY,
                      footer= sig)
         end)
end


using Metal

@show devices()
dev = MtlDevice(1)

vecC=rand(Float32, 128)

bufferSize = 128
bufferA = MtlBuffer(Float32, dev, bufferSize, storage=Shared)
bufferB = MtlBuffer(Float32, dev, bufferSize, storage=Shared)
bufferC = MtlBuffer(Float32, dev, bufferSize, storage=Shared)

vecA = unsafe_wrap(Vector{Float32}, bufferA, (bufferSize,))
vecB = unsafe_wrap(Vector{Float32}, bufferB, (bufferSize,))
vecC = unsafe_wrap(Vector{Float32}, bufferC, (bufferSize,))

using Random
rand!(vecA)

queue = global_queue(dev)

vecB .= 0
cmdBuffer = commit!(queue) do buffer
    MTL.MtlBlitCommandEncoder(buffer) do enc
        MTL.append_copy!(enc, bufferA, 1, bufferC, 1, 128*4)
    end
end

Base.unsafe_copyto!(dev, bufferA, 1, bufferB, 1, 128)

vecD = rand(Float32, 128)
ptrD = pointer(vecD)
Base.unsafe_copyto!(dev, ptrD, bufferB, 0, 128)

###
arr2 = MTL.MtlArray{Float32,1}(undef, (bufferSize,), storage=MTL.MtResourceStorageModeShared)
arrptr = MTL.content(arr2.buffer)
arrvec = unsafe_wrap(Vector{Float32}, arrptr, bufferSize)

Base.unsafe_copyto!(dev, arr2.buffer, 1, bufferB, 1, 128)

MTL.waitUntilCompleted(cmdBuffer)


include("../functions/analysisPULSE.jl")

####################################################################
step_max_perturb_MTiXsens=step_max_perturb

Energysignal_MTiXsens,
rms_MTiXsens,
vdv_MTiXsens,
crest_factor_MTiXsens,
Energysignal_MTiXsens_polar,
rms_MTiXsens_polar,
vdv_MTiXsens_polar=analysisPULSE_MTiXsens(AccFree_MTiXsens_sync_perturb,
                                                    filenameMTiXsens,
                                                    sensorXsensNames,
                                                    Ts_MTiXsens,
                                                    step_max_perturb_MTiXsens)

splitName,
plot_order,
perturbLabel,
perturbIndices,
wheelType=splitFileName(filenameMTiXsens,0.05)

include("../functions/analysisPULSE.jl")
plotAnalysisPULSE_MTiXsens(
        Energysignal_MTiXsens,
        rms_MTiXsens,
        vdv_MTiXsens,
        crest_factor_MTiXsens,
        filenameMTiXsens,
        sensorXsensNames,
        ["x-y";"z"],
        perturbLabel_,
        plot_order,
        perturbIndices,
        [:blue :red :green :orange :yellow],
        [:rect :rect :rect :rect :circle])

plotAnalysisPULSE_MTiXsens(
        Energysignal_MTiXsens_polar,
        rms_MTiXsens_polar,
        vdv_MTiXsens_polar,
        crest_factor_MTiXsens,
        filenameMTiXsens,
        sensorXsensNames,
        ["r";"\u03B8"],
        perturbLabel_,
        plot_order,
        perturbIndices,
        [:blue :red :green :orange :yellow],
        [:rect :rect :rect :rect :circle])

# using JLD
# save("results/PULSE_analysis_gel.jld",
#         "filenameMTiXsens",filenameMTiXsens,
#         "sensorXsensNames",sensorXsensNames,
#         "step_max_perturb_MTiXsens",step_max_perturb_MTiXsens,
#         "Energysignal_MTiXsens",Energysignal_MTiXsens,
#         "rms_MTiXsens",rms_MTiXsens,
#         "vdv_MTiXsens",vdv_MTiXsens,
#         "crest_factor_MTiXsens",crest_factor_MTiXsens,
#         "Energysignal_MTiXsens_polar",Energysignal_MTiXsens_polar,
#         "rms_MTiXsens_polar",rms_MTiXsens_polar,
#         "vdv_MTiXsens_polar",vdv_MTiXsens_polar)


module PlasmoSolverInterface

using Plasmo
using Libdl
using MPI
using Distributed

import Plasmo: solve, AbstractPlasmoSolver

export PipsSolver,pipsnlp_solve,dsp_solve

include("wrapped_solvers.jl")

include("plasmoPipsNlpInterface.jl")

include("DspCInterface.jl")

include("plasmoDspInterface.jl")

using .PlasmoPipsNlpInterface

using .PlasmoDspInterface

end # module


#InspectDR: Defaults
#-------------------------------------------------------------------------------

#==Constants: Initial default values
===============================================================================#
#Default values for data area dimensions (used to save single plot):
const DEFAULT_DATA_WIDTH = 500.0
const DEFAULT_DATA_HEIGHT = DEFAULT_DATA_WIDTH / φ #Use golden ratio

#Default values for plot dimensions (used to save multi-plot):
const DEFAULT_PLOT_WIDTH = 600.0
const DEFAULT_PLOT_HEIGHT = DEFAULT_PLOT_WIDTH / φ #Use golden ratio

#Default font:
const DEFAULT_FONTNAME = @windows? "Cambria": "Serif"
#Cairo "built-in": Serif, Sans, Serif, Fantasy, Monospace

#==Types
===============================================================================#
type Defaults
	rendersvg::Bool #Might want to dissalow SVG renderings for performance reasons
	showtimestamp::Bool
	fontname::AbstractString
	fontscale::Float64

	#Default values for data-area dimensions (saving single plot):
	wdata::Float64
	hdata::Float64

	#Default values for plot dimensions (saving multi-plot):
	wplot::Float64
	hplot::Float64
end
Defaults() =
	Defaults(true, false, DEFAULT_FONTNAME, 1,
		DEFAULT_DATA_WIDTH, DEFAULT_DATA_HEIGHT,
		DEFAULT_PLOT_WIDTH, DEFAULT_PLOT_HEIGHT,
	)

#==Data
===============================================================================#
function __init__()
	global const defaults = Defaults()
	local userdefaults

	try; userdefaults = Main.DEFAULTS_INSPECTDR
	catch; return;	end

	for (k, v) in userdefaults
		try
			setfield!(defaults, k, v)
		catch
			warn("Cannot set InspectDR.defaults.$k = $v.")
		end
	end
end

#==Functions
===============================================================================#
#Forces a function call to read *current* settings during parameter initialization:
#=Not needed:
function getdefaults(field::Symbol)
	global defaults
	return getfield(defaults, field)
end
=#

#Last line


# targets for phase in / phase out of policies

pinft = 0.4*pinf_o # infection prob at meeting

#socialmaxyyt = 2 # max no. people met outside firm young_young
#socialmaxoyt = 1 # max no. people met outside firm old_young
#socialmaxoot = 0.5 # max no. people met outside firm old_old

#phomeofficet = 1

#pshopt = [[0.85,0.5,1],[0.85,0.5,1]]

#workmeett = [4,5,8,2] # max number of people met at work: man / ser / food /public
#shopmeett = [5,14,10] # max number of people met at each shopping instant  man / ser / food

# direct change in variable through policy
tauadj = 0

#genhomeoffice = true

ptravel = 0

#bailoutprogram = true # initially no bailouts

#shorttimeprogram = true # initially no short term program


module TestExtras
    using Base.Test
    using DataArrays
    using StatsBase
    using Compat

    ##########
    ## countmap
    ##########

    d = @data [NA,3,3]
    w = weights([1.1,2.2,3.3])
    cm = Dict{(@compat Union{Int, NAtype}), Int}([(NA, 1), (3, 2)])
    cmw = Dict{(@compat Union{Int, NAtype}), Real}([(NA, 1.1), (3, 5.5)])
    @assert isequal(countmap(d), cm)
    @assert isequal(countmap(d, w), cmw)

    ##########
    ## cut
    ##########

    @assert isequal(cut([2, 3, 5], [1, 3, 6]), PooledDataArray(["(1,3]", "(1,3]", "(3,6]"]))
    @assert isequal(cut([2, 3, 5], [3, 6]), PooledDataArray(["[2,3]", "[2,3]", "(3,6]"]))
    @assert isequal(cut([2, 3, 5, 6], [3, 6]), PooledDataArray(["[2,3]", "[2,3]", "(3,6]", "(3,6]"]))
    @assert isequal(cut([1, 2, 4], [1, 3, 6]), PooledDataArray(["[1,3]", "[1,3]", "(3,6]"]))
    @assert isequal(cut([1, 2, 4], [3, 6]), PooledDataArray(["[1,3]", "[1,3]", "(3,6]"]))
    @assert isequal(cut([1, 2, 4], [3]), PooledDataArray(["[1,3]", "[1,3]", "(3,4]"]))
    @assert isequal(cut([1, 5, 7], [3, 6]), PooledDataArray(["[1,3]", "(3,6]", "(6,7]"]))

    ages = [20, 22, 25, 27, 21, 23, 37, 31, 61, 45, 41, 32]
    bins = [18, 25, 35, 60, 100]
    cats = cut(ages, bins)
    pdv = PooledDataArray(["(18,25]", "(18,25]", "(18,25]",
                           "(25,35]", "(18,25]", "(18,25]",
                           "(35,60]", "(25,35]", "(60,100]",
                           "(35,60]", "(35,60]", "(25,35]"])
    @assert isequal(cats, pdv)

    ##########
    ## repeat
    ##########

    @test isequal(repeat(@data [3.0, 2.0, NA]; inner = 2, outer = 1),
                  @data [3.0, 3.0, 2.0, 2.0, NA, NA])
    @test isequal(repeat(@pdata ["a", "b", NA]; inner = 2, outer = 1),
                  @pdata ["a", "a", "b", "b", NA, NA])
    @test isequal(repeat(@data [1 2; 3 NA]; inner = [1, 2], outer = [2, 1]),
                  @data [1 1 2 2; 3 3 NA NA; 1 1 2 2; 3 3 NA NA])
    @test isequal(repeat(@pdata [:a :b NA]; inner = [2,1], outer = [1,3]),
                  @pdata [:a :b NA :a :b NA :a :b NA;
                          :a :b NA :a :b NA :a :b NA])
end


function _next_10_multiple(x::Int)
	while x % 10 != 0
		x = x + 1
	end
	return x
end

function _is_working_end_time(time::DateTime)
	return Dates.hour(time) >= WORK_END
end

function _is_school_end_time(time::DateTime)
	return Dates.hour(time) >= SCHOOL_END
end

function _is_morning(time::DateTime)
	return DAY_START <= Dates.hour(time) < DAY_MEDIAN
end

function _is_afternoon(time::DateTime)
	return DAY_MEDIAN <= Dates.hour(time) < WORK_END
end

function _is_evening(time::DateTime)
	return WORK_END <= Dates.hour(time) < DAY_END
end

function _is_holiday(time::DateTime)
	return Dates.issaturday(time) || Dates.issunday(time)
end

function _get_day_phase(time::DateTime)
	if _is_morning(time)
		return MORNING
	elseif _is_afternoon(time)
		return AFTERNOON
	elseif _is_evening(time)
		return EVENING
	end
end

"""
	Return exact age, or the Integer mean between min and max age
"""
function calculate_age(exact,min,max)
	return exact != -1 ? exact : (min == -1 || max == -1 ? -1 : floor(Int,(min+max)/2) )
end

function is_student(p)
	return p.part_occupation == SCHOOL
end

function is_worker(p)
	return p.part_occupation == WORKING
end

function is_home(p)
	return p.part_occupation == UNEMPLOYED
end

# function divide_students_by_age(students)
# 	pre_school_students = []
# 	primary_students = []
# 	secondary_students = []
# 	higher_students = []
# 	for student in students
# 		if student.age <= PRE_SCHOOL_LIMIT
# 			push!(pre_school_students,student)
# 		elseif student.age <= PRIMARY_SCHOOL_LIMIT
# 			push!(primary_students,student)
# 		elseif student.age <= SECONDARY_SCHOOL_LIMIT
# 			push!(secondary_students,student)
# 		else
# 			push!(higher_students,student)
# 		end
# 	end

# 	return pre_school_students,primary_students, secondary_students, higher_students
# end


mutable struct Mesures

    deb_mesure::Array{Int32,1} ;
    temps_mesure::Array{Int32,1} ;

    function Mesures(nb_pop::Int32)

        deb_mesure = zeros(Int32,nb_pop) ;
        temps_mesure = zeros(Int32,nb_pop) ;

        if(mesures.tracage)
            deb_mesure .= Trace.delai_trace ;
            temps_mesure .= Trace.jours_conf ;
        end

        return new(deb_mesure,temps_mesure)

    end

end


@shorthands meanplot
@shorthands autocorplot
@shorthands mixeddensity
@shorthands pooleddensity
@shorthands traceplot
@shorthands corner

struct _TracePlot; c; val; end
struct _MeanPlot; c; val;  end
struct _DensityPlot; c; val;  end
struct _HistogramPlot; c; val;  end
struct _AutocorPlot; lags; val;  end

# define alias functions for old syntax
const translationdict = Dict(
                        :traceplot => _TracePlot,
                        :meanplot => _MeanPlot,
                        :density => _DensityPlot,
                        :histogram => _HistogramPlot,
                        :autocorplot => _AutocorPlot,
                        :pooleddensity => _DensityPlot
                      )

const supportedplots = push!(collect(keys(translationdict)), :mixeddensity, :corner)

@recipe f(c::AbstractChains, s::Symbol) = c, [s]

@recipe function f(c::AbstractChains, i::Int;
    colordim = :chain,
    barbounds = (-Inf, Inf),
    maxlag = nothing,
    section = :parameters,
    append_chains = false)
    st = get(plotattributes, :seriestype, :traceplot)
    c = append_chains || st == :pooleddensity ? pool_chain(c) : c

    if colordim == :parameter
        title --> "Chain $(chains(c)[i])"
        label --> names(c)
        val = c.value[:, :, i]
    elseif colordim == :chain
        title --> names(c)[i]
        label --> map(k -> "Chain $(chains(c)[k])", 1:size(c)[3])
        val = c.value[:, i, :]
    else
        throw(ArgumentError("`colordim` must be one of `:chain` or `:parameter`"))
    end

    if st == :mixeddensity || st == :pooleddensity
        discrete = MCMCChains.indiscretesupport(c, barbounds)
        st = if colordim == :chain
            discrete[i] ? :histogram : :density
        else
            # NOTE: It might make sense to overlay histograms and density plots here.
            :density
        end
        seriestype := st
    end

    if st == :autocorplot
        lags = 0:(maxlag === nothing ? round(Int, 10 * log10(length(range(c)))) : maxlag)
        ac = MCMCChains.autocor(c, lags=collect(lags); showall=true)
        ac_mat = convert(Matrix, ac)
        val = colordim == :parameter ? ac_mat[:, :, i]' : ac_mat[i, :, :]
        _AutocorPlot(lags, val)
    elseif st ∈ supportedplots
        translationdict[st](c, val)
    else
        range(c), val
    end
end

@recipe function f(p::_DensityPlot)
    xaxis --> "Sample value"
    yaxis --> "Density"
    [collect(skipmissing(p.val[:,k])) for k in 1:size(p.val, 2)]
end

@recipe function f(p::_HistogramPlot)
    xaxis --> "Sample value"
    yaxis --> "Frequency"
    fillalpha --> 0.7
    bins --> 25
    [collect(skipmissing(p.val[:,k])) for k in 1:size(p.val, 2)]
end

@recipe function f(p::_MeanPlot)
    seriestype := :path
    xaxis --> "Iteration"
    yaxis --> "Mean"
    range(p.c), MCMCChains.cummean(p.val)
end

@recipe function f(p::_AutocorPlot)
    seriestype := :path
    xaxis --> "Lag"
    yaxis --> "Autocorrelation"
    p.lags, p.val
end

@recipe function f(p::_TracePlot)
    seriestype := :path
    xaxis --> "Iteration"
    yaxis --> "Sample value"
    range(p.c), p.val
end

@recipe function f(chn::MCMCChains.AbstractChains, parameters::AbstractVector{Symbol};
        colordim = :chain, section = :parameters, append_chains = false)
    c = Chains(chn, section)
    c = append_chains ? pool_chain(c) : c
    colordim != :chain && error("Symbol names are interpreted as parameter names, only compatible with `colordim = :chain`")
    ret = indexin(parameters, Symbol.(keys(c)))
    any(y -> y == nothing, ret) && error("Parameter not found")
    c, Int.(ret)
end

@recipe function f(chn::MCMCChains.AbstractChains,
                   parameters::AbstractVector{<:Integer} = Int[];
                   width = 500,
                   height = 250,
                   colordim = :chain,
                   section = :parameters,
                   append_chains = false
                  )
    c = isempty(parameters) ? Chains(chn, section; sorted=true) : sort(chn)
    c = append_chains ? pool_chain(c) : c
    ptypes = get(plotattributes, :seriestype, (:traceplot, :mixeddensity))
    ptypes = ptypes isa AbstractVector || ptypes isa Tuple ? ptypes : (ptypes,)
    @assert all(map(ptype -> ptype ∈ supportedplots, ptypes))
    ntypes = length(ptypes)
    nrows, nvars, nchains = size(c)
    isempty(parameters) && (parameters = colordim == :chain ? (1:nvars) : (1:nchains))
    N = length(parameters)

    if !(:corner ∈ ptypes)
        layout := (N, ntypes)
        size --> (ntypes*width, N*height)
        indices = reshape(1:N*ntypes, ntypes, N)'
        legend --> false
        for (j, ptype) in enumerate(ptypes)
            for (i, par) in enumerate(parameters)
                @series begin
                    subplot := indices[i, j]
                    colordim := colordim
                    seriestype := ptype
                    c, par
                end
            end
        end
    else
        length(ptypes) > 1 && error(":corner is not compatible with multiple seriestypes")
        Corner(c, names(c)[parameters])
    end
end

struct Corner
    c
    parameters
end

@recipe function f(corner::Corner)
    label --> permutedims(corner.parameters)
    compact --> true
    size --> (600, 600)
    ar = collect(Array(corner.c.value[:, corner.parameters,i]) for i in chains(corner.c))
    RecipesBase.recipetype(:cornerplot, vcat(ar...))
end


#OnDieDecap.jl: Package to generate plots for on-die decoupling models
#-------------------------------------------------------------------------------

module OnDieDecap

using Colors
import DelimitedFiles: readdlm
using MDDatasets
import CMDimData
using CMDimData.EasyPlot #set, cons
import CMDimCircuits
import InspectDR
CMDimData.@includepkg EasyPlotInspect

const j = im
include("plotbuilders.jl")
include("plots/supplyimpedance.jl")
include("plots/supplybounce.jl")

run_app() = nothing

end # module

#Last line


### Fetch required packages and reaction networks ###
using Catalyst, Test

### Tests various functions for retrieving properties of a reaction system ###


using IntensityScans
using IntensityScans.ThorlabsLTStage
using IntensityScans.TcpInstruments
using IntensityScans.Unitful

using Test

include("emulate_scan.jl")

@testset "IntensityScans.jl" begin
    include("test_scans.jl")
    include("test_metrics.jl")
    include("test_plotting.jl")
    include("test_loading_scans.jl")
end


# Multi dimensional array broadcast
# this kernel can handle all arrays with different dimensions and broadcasting multiple dimensions
# x and y expected to be an array
#in stride_x and stride_y, values corresponding to broadcasting dim should be zero
#in the one that is being broadcasted
# N_z is the size of the z array
# dimlen_z is the number of the dimensions in z

#performance limitations
#stride values and coordinates kept as arrays, causing them to replaced in heap,shared memory
# (slow: http://stackoverflow.com/a/13485322)
# TODO-enis understand heap memory and dynamic allocation for possible bug cause with arrays of >100 dims
# http://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#heap-memory-allocation

fp = open("cuda17.cu","w")
#using Knet: broadcast_ops

function cuda17src(f, j=f, ex="$f(xi,yi)")
  sprint() do s

    for (T,F) in [("float","$(f)_32"),("double","$(f)_64")]
        print(s,

"""
__global__ void _$(F)_17($T *x,$T *y, $T *z,
                                int *stride_x, int *stride_y,
                                int *stride_z,int N_z,int dimlen_z) {

    int index_z = threadIdx.x + (blockIdx.x * blockDim.x);
    int index_x,index_y,coords;
    int temp_index;

    while (index_z < N_z) {
        temp_index = index_z;

        // TODO replace (i/n) == (i>>log2(n)) also %
        index_x =0;
        index_y = 0;
        for (int i=dimlen_z-1; i>0; i--)
        {
            coords = temp_index / stride_z[i];
            index_x+= stride_x[i]*coords;
            index_y+= stride_y[i]*coords;
            temp_index = temp_index % stride_z[i];
        }
        index_x+= temp_index;
        index_y+= temp_index;

        $T xi = x[index_x];
        $T yi = y[index_y];
        z[index_z]=$ex;;
        index_z+=(blockDim.x * gridDim.x);
    }
}

extern "C" {
  $DLLEXPORT void $(F)_17($T *x,$T *y,$T *z, int *stride_x, int *stride_y,int *stride_z,int N_z,int dimlen_z) {

    _$(F)_17<<<256,256>>>(x,y,z,stride_x,stride_y,stride_z,N_z,dimlen_z);

  }
}
""")
    end
  end
end

for a in broadcast_ops
    if !isa(a,Tuple); a=(a,); end
    print(fp,cuda17src(a...))
end

close(fp)


#  models created by contextualizing Human1 using data from Hart et al., “High-Resolution CRISPR Screens Reveal Fitness Genes and Genotype-Specific Cancer Liabilities.”

const HART_GEM_PREFIX = "HartGEM"

function load_Hart_raw_dat()
    hart_file = rawdir(Human1, [ "Human1_Publication_Data_Scripts/tINIT_GEMs/run_tINIT_outputs/Hart2015" ], "tINIT_Hart2015_HumanGEM_outputs.mat")
    return MAT.matread(hart_file)["INIT_output"]
end

function load_Hart_mat_model(;tissue = "GBM")
    dat = load_Hart_raw_dat()
    target_modeli = findfirst(isequal(tissue), vec(dat["tissues"]))
    return vec(dat["model"])[target_modeli]
end

function load_Hart_raw_model(; uncompress = false, tissue = "GBM")
    model_dict = load_Hart_mat_model(;tissue)
    model = MetNets.MetNet(model_dict; reshape = true)
    uncompress ? model : MetNets.compressed_model(model)
end

save_model(model::MetNets.MetNet; verbose = true, model_params...) =
    sdat(HartGEMs, MetNets.compressed_model(model),
        HART_GEM_PREFIX, model_params, ".jls";  verbose
    )

function load_model(;uncompress = false, verbose = false,
        model_params...
    )

    modelid = get(model_params, :modelid, "") |> string
    tissue = get(model_params, :tissue, "") |> string
    stst = get(model_params, :stst, "") |> string

    if modelid == "mat"
        return load_Hart_mat_model(;tissue)
    elseif modelid == "raw"
        return load_Hart_raw_model(;tissue, uncompress)
    elseif modelid == "base" || modelid == "fva_base" || modelid == "scaled" || modelid == "fva_scaled"
        model = ldat(HartGEMs, HART_GEM_PREFIX, (;modelid, tissue), ".jls"; verbose)
        return uncompress ? MetNets.uncompressed_model(model) : model
    elseif modelid == "stst_base" || modelid == "stst_scaled"
        model = ldat(HartGEMs, HART_GEM_PREFIX, (;modelid, tissue, stst), ".jls"; verbose)
        return uncompress ? MetNets.uncompressed_model(model) : model
    else
        error("Unknown '$(modelid)' modelid")
    end
end

function check_modelfile(;params...)
    exist = isfile(datdir(HartGEMs, HART_GEM_PREFIX, params, ".jls"))
    exist && @info("Model cached ", params...)
    return exist
end


lattice = load_lattice(joinpath(@__DIR__, "CrFe2Ga_qe.in"))
atoms   = load_atoms(  joinpath(@__DIR__, "CrFe2Ga_qe.in"))
atoms   = attach_pseudos(atoms, Cr="hgh/pbe/cr-q14.hgh",
                                Fe="hgh/pbe/fe-q16.hgh",
                                Ga="hgh/pbe/ga-q3.hgh")
magnetic_moments = build_magnetic_moments(atoms; Ga=0.0, Fe=5.0, Cr=5.0)

cases = Dict(
    "standard_02" => (damping=DFTK.FixedDamping(0.3), ),
)

config[@__DIR__] = (
    directory        =  @__DIR__,
    lattice          = lattice,
    atoms            = atoms,
    magnetic_moments = magnetic_moments,
    #
    kwargs_model     = (smearing=Smearing.Gaussian(), temperature=0.01),
    Ecut             = 45,
    kgrid            = [13, 13, 13],
    supersampling    = 2.0,
    potential        = nothing,
    kwargs_scf       = (acceleration=DFTK.AndersonAcceleration(; maxcond=1e2), ),
    mixing           = KerkerMixing(),
    cases            = cases,
)


import TurbulenceConvection
const TC = TurbulenceConvection

const tc_dir = dirname(dirname(pathof(TurbulenceConvection)))
include(joinpath(tc_dir, "driver", "generate_namelist.jl"))
include(joinpath(tc_dir, "driver", "Cases.jl"))
include(joinpath(tc_dir, "driver", "parameter_set.jl"))
include(joinpath(tc_dir, "driver", "dycore.jl"))
include(joinpath(tc_dir, "driver", "main.jl"))
import .NameList

TurbulenceConvection.initialize_io(sim::Simulation1d) = nothing
TurbulenceConvection.io(sim::Simulation1d) = nothing

update_n(sim, tendencies, N::Int) = update_n(sim, tendencies, Val(N))

function update_n(sim, tendencies, ::Val{N}) where {N}
    grid = sim.grid
    TS = sim.TS
    prog = sim.state.prog
    aux = sim.state.aux
    params = (; edmf = sim.edmf, grid = grid, gm = sim.gm, case = sim.Case, TS = TS, aux = aux)
    for i in 1:N
        TC.∑tendencies!(tendencies, prog, params, TS.t)
    end
    return nothing
end

function init_sim(case_name)
    @info "Initializing $case_name for single timestep, with no IO."
    @info "call update_n(sim, tendencies, n) to run update n-times"
    namelist = NameList.default_namelist(case_name)
    namelist["time_stepping"]["t_max"] = namelist["time_stepping"]["dt_max"]
    namelist["stats_io"]["frequency"] = 10.0e10
    namelist["stats_io"]["skip"] = true
    namelist["meta"]["uuid"] = "01"
    sim = Simulation1d(namelist)

    Cases.initialize_profiles(sim.Case, sim.grid, sim.gm, sim.state)
    satadjust(sim.gm, sim.grid, sim.state)

    Cases.initialize_surface(sim.Case, sim.grid, sim.state, sim.param_set)
    Cases.initialize_forcing(sim.Case, sim.grid, sim.state, sim.gm, sim.param_set)
    Cases.initialize_radiation(sim.Case, sim.grid, sim.state, sim.gm, sim.param_set)

    initialize_edmf(sim.edmf, sim.grid, sim.state, sim.Case, sim.gm, sim.TS)
    return sim
end


# Run the benchmarks for Julia (run.jl, and boid/run.jl) and Python (run.py and boid/run.py). Copy the results below and plot

using VegaLite
using DataFrames

## Agents.jl benchmark results

# forest fire
jlresults = [0.0639249, 0.142405601, 0.257741999, 0.371782101]

# boid
jlboid = [0.101076999, 0.270401899, 0.5469489, 0.8079563]

# Mesa benchmark results

#forest fire
pyresults = [0.8553307999998196, 2.0069307999999637, 3.3087123000000247, 4.781681599999956]

# boid
pyboid = [0.8770560000000387, 2.4145003999999517, 4.189664500000049, 6.677871200000027]

dd = DataFrame(runtime=vcat(pyresults./jlresults, pyboid./jlboid),
              model = vcat(fill("Forest fire", 4), fill("Boid flocking", 4)))

p = @vlplot(
  width=300, height=90,
  data = dd,
  mark = :circle,
  y = {"model:n", axis={title=""}},
  x = {:runtime, axis={title="Mesa/Agents run time"}},
  color = {"model:n", legend=false}
)
save("benchmark_mesa.svg", p)


using X86Assembly
reload("X86Assembly")

mutable struct GenerativeIOTree <: IO
  stack::Vector{UInt8}
  masks::Vector{UInt8}
  idx::Int
  hadImmediate::Bool
end
GenerativeIOTree() = GenerativeIOTree(UInt8[], UInt8[], 1, false)

struct AbstractImmediate
  size::Int
end
Base.rem(x::AbstractImmediate, ::Type{T}) where {T} = T(0)

struct AbstractBit
  # (-1%UInt8) indicates concrete
  byte_pos::UInt8
  bit_pos::UInt8
  set::Bool
end

struct AbstractByte
  tree::Union{GenerativeIOTree, Void}
  # 1 = low bit
  abstract_bits::NTuple{8, AbstractBit}
end

concrete_bit(x::Integer) = (@assert x == 0 || x == 1; AbstractBit(-1%UInt8,-1%UInt8,x!=0))
isconcrete(x::AbstractBit) = x.byte_pos == -1%UInt8 && x.bit_pos == -1%UInt8
function Base.convert(::Type{AbstractByte}, x::UInt8)
  AbstractByte(nothing, tuple((concrete_bit((x >> i) & 1) for i = 0:7)...))
end
Base.convert(::Type{AbstractByte}, x::Integer) = AbstractByte(convert(UInt8, x))

X86Assembly.representative(x::Integer) = x
function X86Assembly.representative(x::AbstractByte)
  val::UInt8 = 0
  for i = 1:8
    val |= (UInt8(x.abstract_bits[i].set) << (i-1))
  end
  val
end
X86Assembly.representative(x::X86Assembly.Operand) =
  X86Assembly.Operand{UInt8, UInt32}(x.deref, X86Assembly.representative(x.base), X86Assembly.representative(x.scale),
                                              X86Assembly.representative(x.index), X86Assembly.representative(x.disp))
X86Assembly.representative(x::X86Assembly.AVX512Operand) =
  X86Assembly.AVX512Operand(
                X86Assembly.representative(x.mask),
                X86Assembly.representative(x.zero_combining),
                X86Assembly.representative(x.sae),
                X86Assembly.representative(x.er),
                X86Assembly.representative(x.rounding_mode))

function _step!(tree::GenerativeIOTree)
  # addition with carries over ignored bits
  mask = UInt16(tree.masks[end]) | UInt16(0xff) << 8
  val = UInt16(tree.stack[end] + 1)
  while (val & mask) != 0
      if (val & (UInt16(0xff) << 8)) != 0
          pop!(tree.stack)
          pop!(tree.masks)
          !isempty(tree.stack) && _step!(tree)
          return
      end
      carry = val & mask
      val &= ~mask
      val += carry << 1
  end
  tree.stack[end] = val
end

function step!(tree::GenerativeIOTree)
  tree.idx = 1
  tree.hadImmediate = false
  _step!(tree)
end

function Base.read(tree::GenerativeIOTree, ::Type{UInt8})
  @assert !tree.hadImmediate
  tree.idx += 1
  if tree.idx - 1 <= length(tree.stack)
    return AbstractByte(tree, tuple((AbstractBit(tree.idx - 1, i, ((tree.stack[tree.idx - 1] >> (i-1))&1) != 0) for i = 1:8)...))
  end
  push!(tree.stack, 1)
  push!(tree.masks, 0xff)
  bits = tuple((AbstractBit(tree.idx - 1, i, ((tree.stack[end] >> (i-1))&1) != 0) for i = 1:8)...)
  return AbstractByte(tree, bits)
end

import Base: &, <<, |, >>, ==, ~

function (&)(x::AbstractByte, mask::UInt8)
  AbstractByte(x.tree, tuple((((mask >> (i-1)) & 1) == 1 ? x.abstract_bits[i] : concrete_bit(0) for i = 1:8)...))
end
(&)(mask::UInt8, x::AbstractByte) = x & mask
function select_bit(x, y, i)
    xbit = x.abstract_bits[i]; ybit = y.abstract_bits[i]
    (isconcrete(xbit) && xbit.set) && return xbit
    (isconcrete(ybit) && ybit.set) && return ybit
    @assert !(!isconcrete(xbit) && !isconcrete(ybit))
    isconcrete(xbit) ? ybit : xbit
end
function (|)(x::AbstractByte, y::AbstractByte)
  xtree = x.tree
  ytree = y.tree
  (xtree === nothing) && (xtree = ytree)
  (ytree === nothing) && (ytree = xtree)
  @assert xtree === ytree
  AbstractByte(xtree, tuple((select_bit(x, y, i) for i = 1:8)...))
end
function (|)(x::AbstractByte, mask::UInt8)
  AbstractByte(x.tree, tuple((((mask >> (i-1)) & 1) == 1 ? concrete_bit(1) : x.abstract_bits[i] for i = 1:8)...))
end
function (<<)(x::AbstractByte, shift::Integer)
  AbstractByte(x.tree, tuple(((i - shift >= 1) ? x.abstract_bits[i - shift] : concrete_bit(0) for i = 1:8)...))
end
function (>>)(x::AbstractByte, shift::Integer)
  AbstractByte(x.tree, tuple(((i + shift <= 8) ? x.abstract_bits[i + shift] : concrete_bit(0) for i = 1:8)...))
end
~(x::AbstractBit) = AbstractBit(x.byte_pos, x.bit_pos, !x.set)
~(x::AbstractByte) = AbstractByte(x.tree, tuple((~x.abstract_bits[i] for i = 1:8)...))

const IntOrAbstract = Union{Integer, AbstractByte}
function X86Assembly.Operand(x::Bool, base::IntOrAbstract, scale::IntOrAbstract, index::IntOrAbstract, disp::T) where T
    X86Assembly.Operand{AbstractByte, T}(x, base, scale, index, disp)
end

function concretize!(x::AbstractByte)
  for bit in x.abstract_bits
    isconcrete(bit) && continue
    x.tree.masks[bit.byte_pos] &= ~(1 << (bit.bit_pos-1))
  end
  X86Assembly.representative(x)
end
X86Assembly.concretize!(x::AbstractByte) = concretize!(x)

function Base.isless(a::AbstractByte, b::Integer)
  Base.isless(concretize!(a), b)
end
function Base.isless(a::Integer, b::AbstractByte)
  Base.isless(a, concretize!(b))
end
function Base.getindex(a::Dict, b::AbstractByte)
  Base.getindex(a::Dict, concretize!(b))
end
function Base.haskey(a::Dict, b::AbstractByte)
  Base.haskey(a::Dict, concretize!(b))
end
function (==)(x::AbstractByte, y::Integer)
  concretize!(x) == y
end
(==)(x::Integer, y::AbstractByte) = y == x
(::Type{UInt16})(x::AbstractByte) = UInt16(concretize!(x))

function X86Assembly.readImmediate(tree::GenerativeIOTree, T)
  tree.hadImmediate = true
  AbstractImmediate(sizeof(T))
end

Base.eof(tree::GenerativeIOTree) = false

isdone(tree::GenerativeIOTree) = length(tree.stack) >= 1 && tree.stack[1] == 0xff

struct prefix_status{T}
    nprefix_bytes::UInt8
    first_f2f3::UInt8
    last_f2f3::UInt8
    last_valid_seg_prefix::UInt8
    has_lock::Bool
    has_66h::Bool
    has_67h::Bool
    rex::T
    first_non_prefix_byte::T
end

using ProgressMeter

f2f3_choices = UInt8[0x0, 0xF2, 0xF3]
function run!()
  mode = X86Assembly.processor_modes(true, true)
  n = 0
  p = Progress(1616850)
  for last_f2f3 in f2f3_choices
      for first_f2f3 in (last_f2f3 == 0 ? (UInt8(0),) : UInt8[0xF2, 0xF3])
          for last_valid_seg_prefix in UInt8[0x0, 0x64, 0x65]
              for has_lock in (true, false),
                  has_66h in (true, false),
                  has_67h in (true, false),
                  rex in [collect(0x40:0x48); 0]
                    prefix_bytes = UInt8[]
                    first_f2f3 != 0 && push!(prefix_bytes, first_f2f3)
                    (last_f2f3 != first_f2f3) && push!(prefix_bytes, last_f2f3)
                    (last_valid_seg_prefix != 0) && push!(prefix_bytes, last_valid_seg_prefix)
                    has_lock && push!(prefix_bytes, 0xF0)
                    has_66h && push!(prefix_bytes, 0x66)
                    has_67h && push!(prefix_bytes, 0x67)
                    rex != 0 && push!(prefix_bytes, rex)
                    tree = GenerativeIOTree()
                    while !isdone(tree)
                      first_non_prefix_byte = read(tree, UInt8)
                      prefix = X86Assembly.prefix_status(0x0, first_f2f3,
                        last_f2f3, last_valid_seg_prefix, has_lock, has_66h,
                        has_67h, AbstractByte(rex), first_non_prefix_byte)
                      data = X86Assembly._parseOpc(tree, prefix, mode)
                      # Invalid encoding
                      hadImmediate = tree.hadImmediate ? "..." : ""
                      if data == nothing
                          println("$([prefix_bytes; tree.stack])$hadImmediate: Invalid")
                      else
                          print("$([prefix_bytes; tree.stack])$hadImmediate: $data")
                      end
                      step!(tree)
                      next!(p)
                      n += 1
                    end
              end
          end
      end
  end
  println("Number of instruction variants: $n")
end
run!()


using Statistics
using Oceananigans.Architectures

using OffsetArrays: OffsetArray
using Oceananigans.ImmersedBoundaries: ImmersedBoundaryGrid

import Base: getindex, size, show, minimum, maximum
import Statistics: mean

import Oceananigans.Fields: AbstractField, AbstractDataField, AbstractReducedField, Field, ReducedField, minimum, maximum, mean, location, short_show, KernelComputedField
import Oceananigans.Grids: new_data
import Oceananigans.BoundaryConditions: FieldBoundaryConditions

struct CubedSphereFaces{E, F}
    faces :: F
end

function CubedSphereFaces(faces::F) where F
    E = typeof(faces[1])
    return CubedSphereFaces{E, F}(faces)
end

@inline Base.getindex(f::CubedSphereFaces, i::Int) = @inbounds f.faces[i]

#####
##### Dispatch the world / insane type unions
#####

const CubedSphereData = CubedSphereFaces{<:OffsetArray}

# Some dispatch foo to make a type union for CubedSphereFaceField...
#
# Conformal cubed sphere grid wrapped in ImmersedBoundaryGrid:
const ImmersedConformalCubedSphereFaceGrid = ImmersedBoundaryGrid{FT, TX, TY, TZ, <:ConformalCubedSphereFaceGrid} where {FT, TX, TY, TZ}

# CubedSphereFaceField:
const NonImmersedCubedSphereFaceField = AbstractField{X, Y, Z, A, <:ConformalCubedSphereFaceGrid} where {X, Y, Z, A}
const ImmersedCubedSphereFaceField    = AbstractField{X, Y, Z, A, <:ImmersedConformalCubedSphereFaceGrid} where {X, Y, Z, A}

const CubedSphereFaceField = Union{NonImmersedCubedSphereFaceField{X, Y, Z, A},
                                      ImmersedCubedSphereFaceField{X, Y, Z, A}} where {X, Y, Z, A}

const NonImmersedCubedSphereAbstractReducedFaceField = AbstractReducedField{X, Y, Z, A, D, <:ConformalCubedSphereFaceGrid} where {X, Y, Z, A, D}
const ImmersedCubedSphereAbstractReducedFaceField = AbstractReducedField{X, Y, Z, A, D, <:ImmersedConformalCubedSphereFaceGrid} where {X, Y, Z, A, D}

const CubedSphereAbstractReducedFaceField = Union{NonImmersedCubedSphereAbstractReducedFaceField{X, Y, Z, A},
                                                     ImmersedCubedSphereAbstractReducedFaceField{X, Y, Z, A}} where {X, Y, Z, A}

# CubedSphereField

# Flavors of CubedSphereField
const CubedSphereField                     = Field{X, Y, Z, A, <:CubedSphereData} where {X, Y, Z, A}
const CubedSphereReducedField              = ReducedField{X, Y, Z, A, <:CubedSphereData} where {X, Y, Z, A}
const CubedSphereAbstractField             = AbstractField{X, Y, Z, A, <:ConformalCubedSphereGrid} where {X, Y, Z, A}
const CubedSphereAbstractDataField         = AbstractDataField{X, Y, Z, A, <:ConformalCubedSphereGrid} where {X, Y, Z, A}

const AbstractCubedSphereField{X, Y, Z, A} = Union{    CubedSphereAbstractField{X, Y, Z, A},
                                                   CubedSphereAbstractDataField{X, Y, Z, A},
                                                        CubedSphereReducedField{X, Y, Z, A},
                                                               CubedSphereField{X, Y, Z, A}} where {X, Y, Z, A}

#####
##### new data
#####

function new_data(FT, arch::CPU, grid::ConformalCubedSphereGrid, (X, Y, Z))
    faces = Tuple(new_data(FT, arch, face_grid, (X, Y, Z)) for face_grid in grid.faces)
    return CubedSphereFaces{typeof(faces[1]), typeof(faces)}(faces)
end

function new_data(FT, arch::GPU, grid::ConformalCubedSphereGrid, (X, Y, Z))
    faces = Tuple(new_data(FT, arch, face_grid, (X, Y, Z)) for face_grid in grid.faces)
    return CubedSphereFaces{typeof(faces[1]), typeof(faces)}(faces)
end

#####
##### FieldBoundaryConditions
#####

function FieldBoundaryConditions(grid::ConformalCubedSphereGrid, (X, Y, Z); user_defined_bcs...)

    faces = Tuple(
        inject_cubed_sphere_exchange_boundary_conditions(
            FieldBoundaryConditions(face_grid, (X, Y, Z); user_defined_bcs...),
            face_index,
            grid.face_connectivity
        )
        for (face_index, face_grid) in enumerate(grid.faces)
    )

    return CubedSphereFaces{typeof(faces[1]), typeof(faces)}(faces)
end

#####
##### Utils
#####

function Base.show(io::IO, field::AbstractCubedSphereField)
    LX, LY, LZ = location(field)
    arch = architecture(field)
    A = typeof(arch)
    return print(io, "$(typeof(field).name.wrapper) at ($LX, $LY, $LZ)\n",
          "├── architecture: $A\n",
          "└── grid: $(short_show(field.grid))")
end

@inline function interior(field::AbstractCubedSphereField)
    faces = Tuple(interior(face_field_face) for face_field in faces(field))
    return CubedSphereFaces(faces)
end

Base.size(data::CubedSphereData) = (size(data.faces[1])..., length(data.faces))

@inline function get_face(field::CubedSphereField, face_index)
    X, Y, Z = location(field)

    # Should we define a new lower-level constructor for Field that doesn't call validate_field_data?
    return Field{X, Y, Z}(get_face(field.data, face_index),
                          field.architecture,
                          get_face(field.grid, face_index),
                          get_face(field.boundary_conditions, face_index))
end

@inline function get_face(reduced_field::CubedSphereReducedField, face_index)
    X, Y, Z = location(reduced_field)

    return ReducedField{X, Y, Z}(get_face(reduced_field.data, face_index),
                                 reduced_field.architecture,
                                 get_face(reduced_field.grid, face_index),
                                 reduced_field.dims,
                                 get_face(reduced_field.boundary_conditions, face_index))
end

@inline function get_face(kernel_field::CubedSphereAbstractDataField, face_index)
    X, Y, Z = location(kernel_field)

    return Field{X, Y, Z}(get_face(kernel_field.data, face_index),
                                 kernel_field.architecture,
                                 get_face(kernel_field.grid, face_index),
                                 get_face(kernel_field.boundary_conditions, face_index))
end

faces(field::AbstractCubedSphereField) = Tuple(get_face(field, face_index) for face_index in 1:length(field.data.faces))

minimum(field::AbstractCubedSphereField; dims=:) = minimum(minimum(face_field; dims) for face_field in faces(field))
maximum(field::AbstractCubedSphereField; dims=:) = maximum(maximum(face_field; dims) for face_field in faces(field))
mean(field::AbstractCubedSphereField; dims=:) = mean(mean(face_field; dims) for face_field in faces(field))

minimum(f::Function, field::AbstractCubedSphereField; dims=:) = minimum(minimum(f, face_field; dims) for face_field in faces(field))
maximum(f::Function, field::AbstractCubedSphereField; dims=:) = maximum(maximum(f, face_field; dims) for face_field in faces(field))
mean(f::Function, field::AbstractCubedSphereField; dims=:) = mean(mean(f, face_field; dims) for face_field in faces(field))

λnodes(field::CubedSphereFaceField{LX, LY, LZ}) where {LX, LY, LZ} = λnodes(LX(), LY(), LZ(), field.grid)
φnodes(field::CubedSphereFaceField{LX, LY, LZ}) where {LX, LY, LZ} = φnodes(LX(), LY(), LZ(), field.grid)


# Transit structures
abstract type TransitOutput{T} <: AbstractOutput{T} end

"""
    TransitTiming{T<:AbstractFloat} <: TransitOutput{T}

Transit times and derivatives.

# (User-facing) Fields
- `tt::Matrix{T}` : The transit times of each body.
- `dtdq0::Array{T,4}` : Derivatives of the transit times with respect to the initial Cartesian coordinates and masses.
- `dtdelements::Array{T,4}` : Derivatives of the transit times with respect to the initial orbital elements and masses.
"""
struct TransitTiming{T<:AbstractFloat} <: TransitOutput{T}
    tt::Matrix{T}
    dtdq0::Array{T,4}
    dtdelements::Array{T,4}

    # Internal
    count::Vector{Int64}
    ntt::Int64
    ti::Int64
    occs::Vector{Int64}
    dtdq::Array{T,3}
    gsave::Vector{T}
    s_prior::State{T}
    s_transit::State{T}
end

"""
    TransitTiming(tmax, ic; ti)

Constructor for [`TransitTiming`](@ref) type.

# Arguments
- `tmax::T` : Expected total elapsed integration time. (Allocates arrays accordingly)
- `ic::ElementsIC{T}` : Initial conditions for the system

## Optional
- `ti::Int64=1` : Index of the body with respect to which transits are measured. (Default is the central body)
"""
function TransitTiming(tmax::T,ic::ElementsIC{T},ti::Int64=1) where T<:AbstractFloat
    n = ic.nbody
    ind = isfinite.(tmax./ic.elements[:,2])
    ntt = maximum(ceil.(Int64,abs.(tmax./ic.elements[ind,2])).+3)
    tt = zeros(T,n,ntt)
    dtdq0 = zeros(T,n,ntt,7,n)
    dtdelements = zeros(T,n,ntt,7,n)
    count = zeros(Int64,n)
    occs = setdiff(collect(1:n),ti)
    dtdq = zeros(T,1,7,n)
    gsave = zeros(T,n)
    s_prior = State(ic)
    s_transit = State(ic)
    return TransitTiming(tt,dtdq0,dtdelements,count,ntt,ti,occs,dtdq,gsave,s_prior,s_transit)
end

"""
    TransitParameters{T<:AbstractFloat} <: TransitOutput{T}

Transit times, impact parameters, sky-velocities, and derivatives.

# (User-facing) Fields
- `ttbv::Matrix{T}` : The transit times, impact parameter, and sky-velocity of each body.
- `dtbvdq0::Array{T,5}` : Derivatives of the transit times, impact parameters, and sky-velocities with respect to the initial Cartesian coordinates and masses.
- `dtbvdelements::Array{T,5}` : Derivatives of the transit times, impact parameters, and sky-velocities with respect to the initial orbital elements and masses.
"""
struct TransitParameters{T<:AbstractFloat} <: TransitOutput{T}
    ttbv::Array{T,3}
    dtbvdq0::Array{T,5}
    dtbvdelements::Array{T,5}

    # Internal
    count::Vector{Int64}
    ntt::Int64
    ti::Int64
    occs::Vector{Int64}
    dtbvdq::Array{T,3}
    gsave::Vector{T}
    s_prior::State{T}
    s_transit::State{T}
end

"""
    TransitParameters(tmax, ic; ti)

Constructor for [`TransitParameters`](@ref) type.

# Arguments
- `tmax::T` : Expected total elapsed integration time. (Allocates arrays accordingly)
- `ic::ElementsIC{T}` : Initial conditions for the system

## Optional
- `ti::Int64=1` : Index of the body with respect to which transits are measured. (Default is the central body)
"""
function TransitParameters(tmax::T,ic::ElementsIC{T},ti::Int64=1) where T<:AbstractFloat
    n = ic.nbody
    ind = isfinite.(tmax./ic.elements[:,2])
    ntt = maximum(ceil.(Int64,abs.(tmax./ic.elements[ind,2])).+3)
    ttbv = zeros(T,3,n,ntt)
    dtbvdq0 = zeros(T,3,n,ntt,7,n)
    dtbvdelements = zeros(T,3,n,ntt,7,n)
    count = zeros(Int64,n)
    occs = setdiff(collect(1:n),ti)
    dtbvdq = zeros(T,3,7,n)
    gsave = zeros(T,n)
    s_prior = State(ic)
    s_transit = State(ic)
    return TransitParameters(ttbv,dtbvdq0,dtbvdelements,count,ntt,ti,occs,dtbvdq,gsave,s_prior,s_transit)
end

struct TransitSnapshot{T<:AbstractFloat} <: TransitOutput{T}
    nt::Int64
    times::Vector{T}
    bsky2::Matrix{T}
    vsky::Matrix{T}
    dbvdq0::Array{T,5}
    dbvdelements::Array{T,5}
end

function TransitSnapshot(times::Vector{T},ic::ElementsIC{T}) where T<:AbstractFloat
    n = ic.nbody
    nt = length(times)
    return TransitSeries(nt,times,zeros(T,n,nt),zeros(T,n,nt),zeros(T,2,n,nt,7,n),zeros(T,2,n,nt,7,n))
end

function zero_out!(tt::TransitOutput{T}) where T
    for i in 1:length(fieldnames(typeof(tt)))
        if typeof(getfield(tt,i)) <: Array{T}
            getfield(tt,i) .= zero(T)
        end
    end
    tt.count .= 0
end

"""

Main integrator method for Transit calculations.
"""
function (intr::Integrator)(s::State{T}, tt::TransitOutput{T}, d::Derivatives{T}; grad::Bool=true) where T<:AbstractFloat

    t0 = s.t[1] # Initial time
    nsteps = abs(round(Int64,intr.tmax/intr.h))
    h = intr.h * check_step(t0, intr.tmax+t0) # get direction of integration

    for i in tt.occs
        # Compute the relative sky velocity dotted with position:
        tt.gsave[i] = g!(i,tt.ti,s.x,s.v)
    end

    istep = 0
    for _ in 1:nsteps

        # Take an integration step
        if grad
            intr.scheme(s,d,h)
        else
            intr.scheme(s,h)
        end
        istep += 1
        s.t[1] = t0 + (istep * h)

        # Check if a transit occured; record time.
        detect_transits!(s,d,tt,intr,grad=grad)
    end
    # Calculate derivatives
    if grad
        calc_dtdelements!(s,tt)
    end
end

"""Wrapper so the user doesn't need to create a `Derivatives` type."""
function (intr::Integrator)(s::State{T}, tt::TransitOutput{T}; grad::Bool=true, return_arrays::Bool=false) where T<:AbstractFloat
    # Preallocate arrays
    d = Derivatives(T, s.n)

    intr(s, tt, d, grad=grad)
    if return_arrays; return d; end # Return preallocated arrays
    return
end

"""

Integrator method for outputting `TransitSnapshot`.
"""
function (intr::Integrator)(s::State{T},ts::TransitSnapshot{T};grad::Bool=true) where T<:AbstractFloat
    # Integrate to, and output b and v_sky for each body, for a list of times.
    # Should be sorted times.
    # NOTE: Need to fix so that if initial time is a 0, s.dqdt isn't 0s.
    if grad; dbvdq = zeros(T,2,7,s.n); end

    # Integrate to each time, using intr.h, and output b and vsky (only want primary transits for now)
    t0 = s.t[1]
    for (j,ti) in enumerate(ts.times)
        intr(s,ti;grad=grad)
        for i in 2:s.n
            if grad
                ts.vsky[i,j],ts.bsky2[i,j] = calc_dbvdq!(s,dbvdq,1,i)
                for ibv=1:2, k=1:7, p=1:s.n
                    ts.dbvdq0[ibv,i,j,k,p] = dbvdq[ibv,k,p]
                end
            else
                ts.vsky[i,j],ts.bsky2[i,j] = calc_bv(s,1,i)
            end
        end
        # Step to the next full time step, as measured from t0.
        #steps = round(Int64 ,(s.t[1] - t0) / intr.h, RoundUp)
        #intr(s,t0 + (steps * intr.h); grad=grad)
    end
    calc_dbvdelements!(s,ts)
    return
end

# Includes for source
files = ["timing.jl","snapshot.jl"]
include.(files)


"""
    vel_copy!(vel::NamedTuple, u, v, w, g::AbstractGrid{TX, TY, TZ}) where {TX, TY, TZ}
Copy external velocities into `PlanktonModel`
"""
function vel_copy!(vel::NamedTuple, u, v, w, g::AbstractGrid{TX, TY, TZ}) where {TX, TY, TZ}
    copy_interior_u!(vel.u.data, u, g, TX())
    copy_interior_v!(vel.v.data, v, g, TY())
    copy_interior_w!(vel.w.data, w, g, TZ())

    fill_halo_vel!(vel, g)
end

@inline function copy_interior!(c, t, g::AbstractGrid)
    copyto!(view(c, g.Hx+1:g.Hx+g.Nx, g.Hy+1:g.Hy+g.Ny, g.Hz+1:g.Hz+g.Nz), t)
end
@inline function copy_interior_u!(c, t, g::AbstractGrid, ::Periodic)
    copyto!(view(c, g.Hx+1:g.Hx+g.Nx, g.Hy+1:g.Hy+g.Ny, g.Hz+1:g.Hz+g.Nz), t)
end
@inline function copy_interior_v!(c, t, g::AbstractGrid, ::Periodic)
    copyto!(view(c, g.Hx+1:g.Hx+g.Nx, g.Hy+1:g.Hy+g.Ny, g.Hz+1:g.Hz+g.Nz), t)
end
@inline function copy_interior_w!(c, t, g::AbstractGrid, ::Periodic)
    copyto!(view(c, g.Hx+1:g.Hx+g.Nx, g.Hy+1:g.Hy+g.Ny, g.Hz+1:g.Hz+g.Nz), t)
end

@inline function copy_interior_u!(c, t, g::AbstractGrid, ::Bounded)
    copyto!(view(c, g.Hx+1:g.Hx+g.Nx+1, g.Hy+1:g.Hy+g.Ny, g.Hz+1:g.Hz+g.Nz), t)
end
@inline function copy_interior_v!(c, t, g::AbstractGrid, ::Bounded)
    copyto!(view(c, g.Hx+1:g.Hx+g.Nx, g.Hy+1:g.Hy+g.Ny+1, g.Hz+1:g.Hz+g.Nz), t)
end
@inline function copy_interior_w!(c, t, g::AbstractGrid, ::Bounded)
    copyto!(view(c, g.Hx+1:g.Hx+g.Nx, g.Hy+1:g.Hy+g.Ny, g.Hz+1:g.Hz+g.Nz+1), t)
end


"""
    Script to generate boxplot graphs associated with:
    - Experiment 1 (randV_randE.jl)
    - Experiment 2 (randV_propE05.jl)
"""
using Pkg
Pkg.activate(".")
using LTMSim
using SimpleHypergraphs
using PyPlot
using Serialization
using LaTeXStrings

project_path = dirname(pathof(LTMSim))

fname = "randV_propE05.data" #"randV_randE.data"
pname = "randV_propE05.png" #"randV_randE.png"
data_path = joinpath(project_path, "..", "res", "journal", fname)

hg_files = readdir(joinpath(project_path, "..", "data", "hgs"))
hg_names = [split(file, ".")[1] for file in hg_files]

hg_names[8] = "music-rev"
hg_names[10] = "rest.-rev"
hg_names[11] = "bars-rev"

#
# DATA
#
data = deserialize(data_path)
hgs = [prune_hypergraph!(hg_load(joinpath(project_path, "..", "data", "hgs", hg_file))) for hg_file in hg_files]

#
# PLOTTING INFO
#
algorithms = [
    "BinarySearch(H)", "Greedy([H]₂)", "Greedy(H)", "SubTSS(H)"
]

labels = Dict{String, String}(
    "BinarySearch(H)" => "StaticGreedy",
    "Greedy(H)" => "DynamicGreedy",
    "Greedy([H]₂)" => L"DynamicGreedy_{[H]_2}",
	"SubTSS(H)" => "SubTSS"
)

#
# BOXPLOTS
#
ticks = [1]

function set_box_color(bp, color)
    plt.setp(bp["boxes"], color=color)
    plt.setp(bp["whiskers"], color=color)
    plt.setp(bp["caps"], color=color)
    plt.setp(bp["medians"], color=color)
end

clf()
plt.figure(figsize=(20,8))

val = -0.7
c = 1
pos = 1.5

colorz=["#2C7BB6", "#D7191C", "#FF8900", "#33CC33", "#cc0099", "#4d4dff", "#008080", "#2C7BB6"]

for algo in algorithms
    global val, c

    println(algo)

    ndata = [elem / nhv(hgs[index]) for (index, elem) in enumerate(data[algo])]

    b = plt.boxplot(
        ndata,
        #positions=collect(range(0, stop=length(data[algo])-1)).*2.0.+val,
        positions=collect(range(0, stop=length(data[algo])-1)).*2.5.+val,
        #positions = [pos+val],
        sym="",
        widths=0.2
    )

    set_box_color(b, colorz[c])
    plt.plot([], c=colorz[c], label=labels[algo])

    c+=1
    val+=0.3
end

plt.legend(fontsize="20")

plt.xticks(range(0, (length(hg_names))*2.4, step=2.5), hg_names, fontsize="20", rotation=0)
plt.yticks(fontsize="xx-large")

title("randV - propE05", fontstyle = "italic", fontsize="xx-large")

plt.ylim(0, 0.7)
ylabel("Seed set size / n", fontstyle = "italic", fontsize="20", labelpad=10)

plt.tight_layout()
gcf()

PyPlot.savefig(joinpath(project_path, "..", "res", "journal", "plot", pname))

#
# some distribution data
#
amazon = hgs[2]
amazon_he_dist = [length(getvertices(amazon, he)) for he in 1:nhe(amazon)]
amazon_v_dist = [length(gethyperedges(amazon, v)) for v in 1:nhv(amazon)]

sum(amazon_v_dist[amazon_v_dist .== 1]) * 100 / length(amazon_v_dist)

clf()
hist(amazon_v_dist)
gcf()

dblp = hgs[3]
dblp_he_dist = [length(getvertices(dblp, he)) for he in 1:nhe(dblp)]
dblp_v_dist = [length(gethyperedges(dblp, v)) for v in 1:nhv(dblp)]

sum(dblp_v_dist[dblp_v_dist .== 1]) * 100 / length(dblp_v_dist)

clf()
hist(dblp_v_dist)
gcf()


############################################################################################
function init(
    ::Type{NUTS},
    config::ConfigNUTS,
    objective::Objective,
    difftune::AbstractDifferentiableTune,
)
    return NUTS(
        ModelWrappers.log_density_and_gradient(objective, difftune), difftune, config.energy
    )
end

function infer(
    _rng::Random.AbstractRNG,
    diagnostics::Type{AbstractDiagnostics},
    kernel::NUTS,
    model::ModelWrapper,
    data::D,
) where {D}
    return DiagnosticsNUTS{model.info.reconstruct.default.output}
end

############################################################################################
# Export
export init, infer


# Functional implementation of https://egraphs-good.github.io/
# https://dl.acm.org/doi/10.1145/3434304

using DataStructures

"""
Abstract type representing an [`EGraph`](@ref) analysis,
attaching values from a join semi-lattice domain to
an EGraph
"""
abstract type AbstractAnalysis end

const ClassMem = Dict{Int64,Vector{Any}}
const HashCons = Dict{Any,Int64}
const Parent = Tuple{Any,Int64} # parent enodes and eclasses
const ParentMem = Dict{Int64,Vector{Parent}}
const AnalysisData = Dict{Int64,Any}
const Analyses = Vector{AbstractAnalysis}

"""
A concrete type representing an [`EGraph`].
See the [egg paper](https://dl.acm.org/doi/pdf/10.1145/3434304)
for implementation details
"""
mutable struct EGraph
    """stores the equality relations over e-class ids"""
    U::IntDisjointSets
    """map from eclass id to eclasses"""
    M::ClassMem             #
    H::HashCons             # hashcons
    parents::ParentMem
    """worklist for ammortized upwards merging"""
    dirty::Vector{Int64}
    root::Int64
    """A vector of analyses associated to the EGraph"""
    analyses::Analyses
end

EGraph() = EGraph(
    IntDisjointSets(0),
    ClassMem(),
    HashCons(),
    ParentMem(),
    Vector{Int64}(),
    0,
    Analyses(),
)

function EGraph(e)
    G = EGraph()
    rootclass = addexpr!(G, e)
    G.root = rootclass.id
    G
end

function addparent!(G::EGraph, a::Int64, parent::Parent)
    @assert isenode(parent[1])
    if !haskey(G.parents, a)
        G.parents[a] = [parent]
    else
        union!(G.parents[a], [parent])
    end
end

"""
Inserts an e-node in an [`EGraph`](@ref)
"""
function add!(G::EGraph, n)::EClass
    @debug("adding ", n)
    canonicalize!(G.U, n)
    if haskey(G.H, n)
        return find(G, G.H[n]) |> EClass
    end
    @debug(n, " not found in H")
    id = push!(G.U) # create new singleton eclass
    !haskey(G.parents, id) && (G.parents[id] = [])
    getfunargs(n) .|> x -> addparent!(G, x.id, (n, id))

    G.H[n] = id
    G.M[id] = [n]

    # make analyses for new enode
    for analysis ∈ G.analyses
        if !islazy(analysis)
            analysis[id] = make(analysis, n)
            modify!(analysis, id)
        end
    end

    return EClass(id)
end
add!(g::EGraph, c::EClass)::EClass = EClass(find(g, c))

"""
Recursively traverse an [`Expr`](@ref) and insert terms into an
[`EGraph`](@ref). If `e` is not an [`Expr`](@ref), then directly
insert the literal into the [`EGraph`](@ref).
"""
function addexpr!(G::EGraph, e)::EClass
    e = cleanast(e)
    df_walk((x -> add!(G, x)), e; skip_call = true)
end

function mergeparents!(G::EGraph, from::Int64, to::Int64)
    !haskey(G.parents, from) && (G.parents[from] = []; return)
    !haskey(G.parents, to) && (G.parents[to] = [])

    union!(G.parents[to], G.parents[from])
    delete!(G.parents, from)
end

"""
Given an [`EGraph`](@ref) and two e-class ids, set
the two e-classes as equal.
"""
function Base.merge!(G::EGraph, a::Int64, b::Int64)::Int64
    id_a = find(G, a)
    id_b = find(G, b)
    id_a == id_b && return id_a
    id_u = union!(G.U, id_a, id_b)

    @debug "merging" id_a id_b

    from, to = if (id_u == id_a)
        id_b, id_a
    elseif (id_u == id_b)
        id_a, id_b
    else
        error("egraph invariant maintenance error")
    end

    push!(G.dirty, id_u)

    clean(t) = begin
        delete!(G.H, t)
        canonicalize!(G.U, t)
        G.H[t] = to
        t
    end

    G.M[from] = map(clean, G.M[from])
    G.M[to] = map(clean, G.M[to])
    G.M[to] = G.M[from] ∪ G.M[to]

    if from == G.root
        G.root = to
    end

    delete!(G.M, from)
    mergeparents!(G, from, to)
    for analysis ∈ G.analyses
        if haskey(analysis, from) && haskey(analysis, to)
            analysis[to] = join(analysis, analysis[from], analysis[to])
            delete!(analysis, from)
        end
    end

    return id_u
end

"""
Returns the canonical e-class id for a given e-class.
"""
find(G::EGraph, a::Int64)::Int64 = find_root!(G.U, a)
find(G::EGraph, a::EClass)::Int64 = find_root!(G.U, a.id)

"""
This function restores invariants and executes
upwards merging in an [`EGraph`](@ref). See
the [egg paper](https://dl.acm.org/doi/pdf/10.1145/3434304)
for more details.
"""
function rebuild!(G::EGraph)
    while !isempty(G.dirty)
        todo = unique([find(G, id) for id ∈ G.dirty])
        empty!(G.dirty)
        foreach(todo) do x
            repair!(G, x)
        end
    end

    if G.root != 0
        G.root = find(G, G.root)
    end
end

function repair!(G::EGraph, id::Int64)
    id = find(G, id)
    @debug "repairing " id

    for (p_enode, p_eclass) ∈ G.parents[id]
        #old_id = G.H[p_enode]
        #delete!(G.M, old_id)
        delete!(G.H, p_enode)
        @debug "deleted from H " p_enode
        n = canonicalize(G.U, p_enode)
        n_id = find(G, p_eclass)
        G.H[n] = n_id
    end

    new_parents = OrderedDict{Any,Int64}()

    for (p_enode, p_eclass) ∈ G.parents[id]
        canonicalize!(G.U, p_enode)
        # deduplicate parents
        if haskey(new_parents, p_enode)
            @debug "merging classes" p_eclass (new_parents[p_enode])
            merge!(G, p_eclass, new_parents[p_enode])
        end
        new_parents[p_enode] = find(G, p_eclass)
    end
    G.parents[id] = collect(new_parents) .|> Tuple
    @debug "updated parents " id G.parents[id]

    # Analysis invariant maintenance
    for an ∈ G.analyses
        haskey(an, id) && modify!(an, id)
        # modify!(an, id)
        id = find(G, id)
        for (p_enode, p_eclass) ∈ G.parents[id]
            # p_eclass = find(G, p_eclass)
            if !islazy(an) && !haskey(an, p_eclass)
                an[p_eclass] = make(an, p_enode)
            end
            if haskey(an, p_eclass)
                new_data = join(an, an[p_eclass], make(an, p_enode))
                if new_data != an[p_eclass]
                    an[p_eclass] = new_data
                    push!(G.dirty, p_eclass)
                end
            end
        end
    end
end

"""
Recursive function that traverses an [`EGraph`](@ref) and
returns a vector of all reachable e-classes from a given e-class id.
"""
function reachable(g::EGraph, id::Int64; hist=Int64[])
    id = find(g, id)
    hist = hist ∪ [id]
    for n ∈ g.M[id]
        if n isa Expr
            for child_eclass ∈ getfunargs(n)
                c_id = child_eclass.id
                if c_id ∉ hist
                    hist = hist ∪ reachable(g, c_id; hist=hist)
                end
            end
        end
    end

    return hist
end


using Augmentor, ISICArchive, Plots
pyplot(reuse=true);
mkpath(fname) = joinpath(dirname(@__FILE__()), fname)

srand(123)

img = get(ImageThumbnailRequest(id = "5592ac599fc3c13155a57a85"))

pipeline =
    Either(1=>FlipX(), 1=>FlipY(), 2=>NoOp()) |>
    Rotate(0:360) |>
    ShearX(-5:5) * ShearY(-5:5) |>
    CropSize(165, 165) |>
    Zoom(1:0.05:1.2) |>
    Resize(64, 64)

# Create image that shows the input
plot(img, size=(256,169), xlim=(1,255), ylim=(1,168), grid=false, ticks=true)
Plots.png(mkpath("readme/isic_in.png"))

# create animate gif that shows 10 outputs
anim = @animate for i=1:10
    plot(augment(img, pipeline), size=(169,169), xlim=(1,63), ylim=(1,63), grid=false, ticks=true);
end
Plots.gif(anim, mkpath("readme/isic_out.gif"), fps = 2)


# MIT License
#
# Copyright (c) 2018 Martin Biel
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

struct RawCor{T <: AbstractFloat}
    name::String
    n::Int
    m₁::Int
    m₂::Int
    vars::OrderedDict{Col, Int}
    rows::OrderedDict{Row, Tuple{Int,Int,Symbol}}
    cols::OrderedDict{Col, Vector{Pair{Symbol,T}}}
    rhs::Dict{Row, T}
    ranges::Dict{Row, T}
    bounds::OrderedDict{Col, Vector{Pair{Symbol,T}}}
    objgiven::Bool
    objname::Symbol
    rhsname::Symbol

    function RawCor(n::Integer, m₁::Integer, m₂::Integer,
                    vars::OrderedDict{Col, Int}, rows::OrderedDict{Row, Tuple{Int,Int,Symbol}},
                    cols::OrderedDict{Col, Vector{Pair{Symbol,T}}}, rhs::Dict{Row, T},
                    ranges::Dict{Row, T}, bounds::OrderedDict{Col, Vector{Pair{Symbol,T}}};
                    objgiven::Bool = false,
                    objname::Symbol = OBJ,
                    rhsname::Symbol = RHS,
                    name::String = "LP") where T <: AbstractFloat
        return new{T}(name,
                      n, m₁, m₂,
                      vars, rows, cols,
                      rhs, ranges, bounds,
                      objgiven, objname, rhsname)
    end
end

function parse_cor(::Type{T}, filename::AbstractString) where T <: AbstractFloat
    # Initialize auxiliary variables
    name       = "SLP"
    mode       = NAME
    rowidx     = 1
    eqrowidx   = 0
    ineqrowidx = 0
    varidx     = 1
    objgiven   = false
    objname    = OBJ
    rhsname    = RHS
    # Define sections
    vars    = OrderedDict{Col, Int}()
    rows    = OrderedDict{Row, Tuple{Int,Int,Symbol}}()
    cols    = OrderedDict{Col, Vector{Pair{Symbol,T}}}()
    rhs     = Dict{Row, T}()
    ranges  = Dict{Row, T}()
    bounds  = OrderedDict{Col, Vector{Pair{Symbol,T}}}()
    # Parse the file
    open(filename) do io
        firstline = split(readline(io))
        if Symbol(firstline[1]) == NAME
            name = join(firstline[2:end], " ")
        else
            throw(ArgumentError("`NAME` field is expected on the first line."))
        end
        for line in eachline(io)
            if mode == END
                # Parse finished
                break
            end
            words = split(line)
            first_word = Symbol(words[1])
            if first_word in COR_MODES && first_word != mode
                mode = first_word
                continue
            end
            if mode == ROWS
                rowsym = Symbol(words[2])
                rows[rowsym] =
                    words[1] == "N" ? (objgiven = true; objname = rowsym; (rowidx, 0, OBJ)) :
                    words[1] == "L" ? (ineqrowidx += 1; (rowidx, ineqrowidx, LEQ))            :
                    words[1] == "G" ? (ineqrowidx += 1; (rowidx, ineqrowidx, GEQ))            :
                    (eqrowidx   += 1; (rowidx, eqrowidx, EQ))
                rowidx += 1
            elseif mode == COLUMNS
                var = Symbol(words[1])
                if get!(vars, var, 0) == 0
                    vars[var] = varidx
                    varidx    += 1
                end
                for idx = 2:2:length(words)
                    push!(get!(cols, var, Pair{Symbol,T}[]),
                          Pair(Symbol(words[idx]), convert(T, parse(Float64, words[idx+1]))))
                end
            elseif mode == RHS
                _rhsname = Symbol(words[1])
                if _rhsname  != RHS && rhsname == RHS
                    rhsname = _rhsname
                end
                rhsname == _rhsname || error("Multiple RHS names given.")
                for idx = 2:2:length(words)
                    rhs[Symbol(words[idx])] = convert(T, parse(Float64, words[idx+1]))
                end
            elseif mode == RANGES
                for idx = 2:2:length(words)
                    ranges[Symbol(words[idx])] = convert(T, parse(Float64, words[idx+1]))
                end
            elseif mode == BOUNDS
                var = Symbol(words[3])
                bnd = words[1] == "LO" ? LOWER :
                      words[1] == "UP" ? UPPER :
                      words[1] == "BV" ? BINARY :
                      words[1] == "I"  ? INTEGER :
                      words[1] == "LI" ? INTEGER_LOWER :
                      words[1] == "UI" ? INTEGER_UPPER :
                      words[1] == "FR" ? FREE  : FIXED
                push!(get!(bounds, var, Pair{Symbol,T}[]),
                      Pair(bnd, convert(T, parse(Float64, bnd in [FREE, BINARY, INTEGER] ? "0" : words[4]))))
            else
                throw(ArgumentError("$(mode) is not a valid cor file mode."))
            end
        end
    end
    # Return raw data
    return RawCor(varidx - 1,
                  eqrowidx,
                  ineqrowidx,
                  vars,
                  rows,
                  cols,
                  rhs,
                  ranges,
                  bounds;
                  objgiven = objgiven,
                  objname = objname,
                  rhsname = rhsname,
                  name = name)
end
parse_cor(filename::AbstractString) = parse_cor(Float64, filename)

function sparsity(cor::RawCor)
    return 1 - length(cor.cols) / (cor.n * (cor.m₁ + cor.m₂))
end


#-------------------------------------------------------------------------------
# File: constants.jl
# Description:
# Date: October 31, 2019
# Author: Jonathan Fontaine, Killian Fretaud, Rémi Garcia,
#         Boualem Lamraoui, Benoît Le Badezet, Benoit Loger
#-------------------------------------------------------------------------------

const WEIGHTS_OBJECTIVE_FUNCTION = [1000, 1, 1000000]

# TIME
const TIME_LIMIT = 60.0
const TIME_PART_ILS = 0.9
const TIME_PART_VNS_LPRC = 0.96
const TIME_PART_VNS_PCC = 1 # Unused for now

# ILS-HPRC
const ALPHA_ILS = 25
const BETA_ILS = 50
const STOPPING_CRITERIA_ILS_HPRC = 3
const NBCAR_PERTURBATION = 5
const NBCAR_DIVERSIFICATION = 50

# VNS-LPRC
const VNS_LPRC_MAX_NON_IMPROVEMENT = 150
const VNS_LPRC_MIN_INSERT = 3
const VNS_LPRC_MAX_INSERT = 8
const VNS_LPRC_MIN_EXCHANGE = 5
const VNS_LPRC_MAX_EXCHANGE = 12

# VNS-PCC
const VNS_PCC_MIN_INSERT = 2
const VNS_PCC_MAX_INSERT = 8
const VNS_PCC_MIN_EXCHANGE = 2
const VNS_PCC_MAX_EXCHANGE = 10
const VNS_PCC_MINMAX = [(VNS_PCC_MIN_INSERT, VNS_PCC_MAX_INSERT), (VNS_PCC_MIN_EXCHANGE, VNS_PCC_MAX_EXCHANGE)]

# Files names
const INSTANCES = Dict{String, Array{String, 1}}(
    "A" => [
        "022_3_4_EP_RAF_ENP",
        "022_3_4_RAF_EP_ENP",
        "024_38_3_EP_ENP_RAF",
        "024_38_3_EP_RAF_ENP",
        "024_38_5_EP_ENP_RAF",
        "024_38_5_EP_RAF_ENP",
        "025_38_1_EP_ENP_RAF",
        "025_38_1_EP_RAF_ENP",
        "039_38_4_EP_RAF_ch1",
        "039_38_4_RAF_EP_ch1",
        "048_39_1_EP_ENP_RAF",
        "048_39_1_EP_RAF_ENP",
        "064_38_2_EP_RAF_ENP_ch1",
        "064_38_2_EP_RAF_ENP_ch2",
        "064_38_2_RAF_EP_ENP_ch1",
        "064_38_2_RAF_EP_ENP_ch2"
    ],
    "B" => [
        "022_EP_ENP_RAF_S22_J1",
        "022_EP_RAF_ENP_S22_J1",
        "022_RAF_EP_ENP_S22_J1",
        "023_EP_ENP_RAF_S23_J3",
        "023_EP_RAF_ENP_S23_J3",
        "023_RAF_EP_ENP_S23_J3",
        "024_V2_EP_ENP_RAF_S22_J1",
        "024_V2_EP_RAF_ENP_S22_J1",
        "024_V2_RAF_EP_ENP_S22_J1",
        "025_EP_ENP_RAF_S22_J3",
        "025_EP_RAF_ENP_S22_J3",
        "025_RAF_EP_ENP_S22_J3",
        "028_ch1_EP_ENP_RAF_S22_J2",
        "028_ch1_EP_RAF_ENP_S22_J2",
        "028_ch1_RAF_EP_ENP_S22_J2",
        "028_ch2_EP_ENP_RAF_S23_J3",
        "028_ch2_EP_RAF_ENP_S23_J3",
        "028_ch2_RAF_EP_ENP_S23_J3",
        "029_EP_ENP_RAF_S21_J6",
        "029_EP_RAF_ENP_S21_J6",
        "029_RAF_EP_ENP_S21_J6",
        "035_ch1_EP_ENP_RAF_S22_J3",
        "035_ch1_EP_RAF_ENP_S22_J3",
        "035_ch1_RAF_EP_ENP_S22_J3",
        "035_ch2_EP_ENP_RAF_S22_J3",
        "035_ch2_EP_RAF_ENP_S22_J3",
        "035_ch2_RAF_EP_ENP_S22_J3",
        "039_ch1_EP_ENP_RAF_S22_J4",
        "039_ch1_EP_RAF_ENP_S22_J4",
        "039_ch1_RAF_EP_ENP_S22_J4",
        "039_ch3_EP_ENP_RAF_S22_J4",
        "039_ch3_EP_RAF_ENP_S22_J4",
        "039_ch3_RAF_EP_ENP_S22_J4",
        "048_ch1_EP_ENP_RAF_S22_J3",
        "048_ch1_EP_RAF_ENP_S22_J3",
        "048_ch1_RAF_EP_ENP_S22_J3",
        "048_ch2_EP_ENP_RAF_S22_J3",
        "048_ch2_EP_RAF_ENP_S22_J3",
        "048_ch2_RAF_EP_ENP_S22_J3",
        "064_ch1_EP_ENP_RAF_S22_J3",
        "064_ch1_EP_RAF_ENP_S22_J3",
        "064_ch1_RAF_EP_ENP_S22_J3",
        "064_ch2_EP_ENP_RAF_S22_J4",
        "064_ch2_EP_RAF_ENP_S22_J4",
        "064_ch2_RAF_EP_ENP_S22_J4"
    ],
    "X" => [
        "022_RAF_EP_ENP_S49_J2",
        "023_EP_RAF_ENP_S49_J2",
        "024_EP_RAF_ENP_S49_J2",
        "025_EP_ENP_RAF_S49_J1",
        "028_CH1_EP_ENP_RAF_S50_J4",
        "028_CH2_EP_ENP_RAF_S51_J1",
        "029_EP_RAF_ENP_S49_J5",
        "034_VP_EP_RAF_ENP_S51_J1_J2_J3",
        "034_VU_EP_RAF_ENP_S51_J1_J2_J3",
        "035_CH1_RAF_EP_S50_J4",
        "035_CH2_RAF_EP_S50_J4",
        "039_CH1_EP_RAF_ENP_S49_J1",
        "039_CH3_EP_RAF_ENP_S49_J1",
        "048_CH1_EP_RAF_ENP_S50_J4",
        "048_CH2_EP_RAF_ENP_S49_J5",
        "064_CH1_EP_RAF_ENP_S49_J1",
        "064_CH2_EP_RAF_ENP_S49_J4",
        "655_CH1_EP_RAF_ENP_S51_J2_J3_J4",
        "655_CH2_EP_RAF_ENP_S52_J1_J2_S01_J1"
    ]
)