struct CacheKey{C <: AbstractCache, D <: PSY.Device}
    cache_type::Type{C}
    device_type::Type{D}
end

function CacheKey(cache::C) where {C <: AbstractCache}
    return CacheKey(C, cache.device_type)
end
# The current implementation will require all custom caches to have the same data template
"""
Tracks the last time status of a device changed in a simulation
"""
mutable struct TimeStatusChange <: AbstractCache
    device_type::Type{<:PSY.Device}
    value::JuMP.Containers.DenseAxisArray{Dict{Symbol, Float64}}
    units::Dates.TimePeriod
    ref::UpdateRef

    function TimeStatusChange(
        device_type::Type{<:PSY.Device},
        value::JuMP.Containers.DenseAxisArray{Dict{Symbol, Float64}},
        ref::UpdateRef,
        units::Dates.TimePeriod = Dates.Hour(1),
    )
        units = IS.time_period_conversion(units)
        new(device_type, value, units, ref)
    end
end

function TimeStatusChange(::Type{T}, name::AbstractString) where {T <: PSY.Device}
    value_array = JuMP.Containers.DenseAxisArray{Dict{Symbol, Float64}}(undef, 1)
    return TimeStatusChange(T, value_array, UpdateRef{JuMP.VariableRef}(T, name))
end

mutable struct StoredEnergy <: AbstractCache
    device_type::Type{<:PSY.Device}
    value::JuMP.Containers.DenseAxisArray{Float64}
    ref::UpdateRef
end

function StoredEnergy(::Type{T}, name::AbstractString) where {T <: PSY.Device}
    value_array = JuMP.Containers.DenseAxisArray{Float64}(undef, 1)
    return StoredEnergy(T, value_array, UpdateRef{JuMP.VariableRef}(T, name))
end

cache_value(cache::AbstractCache, key) = cache.value[key]


function PhaseIonosphericDelayModifier(arg0::IonosphericModel, arg1::jdouble)
    return PhaseIonosphericDelayModifier((IonosphericModel, jdouble), arg0, arg1)
end

function get_parameters_drivers(obj::PhaseIonosphericDelayModifier)
    return jcall(obj, "getParametersDrivers", List, ())
end

function modify(obj::PhaseIonosphericDelayModifier, arg0::EstimatedMeasurement)
    return jcall(obj, "modify", void, (EstimatedMeasurement,), arg0)
end


module PlantBiophysics

# For model parameters (efficient and still mutable!)
using MutableNamedTuples

# For IO:
using YAML
using CSV
using OrderedCollections

using Dates

import DataFrames.DataFrame # For convenience transformations
import DataFrames.Not
import DataFrames.rename!
import DataFrames: select!, select
import DataFrames.dropmissing!
import Base.show
import Base.getindex
import Impute.locf # For filling missing values in comments in Walz to identify curves
import LsqFit: curve_fit
using RecipesBase
import Statistics: mean

# Generic structures:
include("structs/Abstract_model_structs.jl")
include("structs/concrete_component_structs.jl")

# Physical constants:
include("structs/constants.jl")

# Atmosphere computations (vapor pressure...)
include("structs/atmosphere.jl")

# Models helpers:
include("models_helpers.jl")

# Conversions
include("conversions.jl")

# Light interception
include("light_interception/generic_structs.jl")
include("light_interception/Ignore.jl")
include("light_interception/Translucent.jl")

# Photosynthesis related files:
include("photosynthesis/photosynthesis.jl")
include("photosynthesis/constantA.jl")
include("photosynthesis/FvCB.jl")
include("photosynthesis/FvCBIter.jl")
include("photosynthesis/FvCBRaw.jl")
include("photosynthesis/temperature-dependence.jl")

# Stomatal conductance related files:
include("conductances/stomatal/stomatal_conductance.jl")
include("conductances/stomatal/constantGs.jl")
include("conductances/stomatal/medlyn.jl")

# Boundary layer conductance:
include("conductances/boundary/gb.jl")

# Energy balance
include("energy/longwave_energy.jl")
include("energy/energy_balance.jl")
include("energy/Missing.jl")
include("energy/Monteith.jl")

# File IO
include("io/read_model.jl")
include("io/read_weather.jl")
include("io/read_walz.jl")

# Parameters optimization
include("fitting/fit.jl")
include("fitting/fit_FvCB.jl")

# Model evaluation
include("evaluation/statistics.jl")

# Depreciations
include("depreciations/models.jl")

# File IO:
export read_model
export is_model
export read_weather
export read_walz

# Physical constants
export Constants

# Status
export init_status!

# Get models informations
export variables, variables_typed
export inputs, outputs, defaults
export to_initialise, is_initialised

# Atmosphere
export vapor_pressure
export e_sat
export e_sat_slope
export air_density
export Atmosphere
export Weather

# Conversions
export rh_from_vpd
export rh_from_e
export ms_to_mol
export mol_to_ms
export gbh_to_gbw
export gbw_to_gbh
export gsw_to_gsc
export gsc_to_gsw

# Energy balance
export AbstractEnergyModel
export black_body
export grey_body
export psychrometer_constant
export net_longwave_radiation
export energy_balance # main interface to user
export energy_balance! # main interface to user
export Monteith       # a struct to hold the values for the model of Monteith and Unsworth (2013)
export latent_heat
export sensible_heat
export γ_star
export latent_heat_vaporization

# structure for light interception
export Translucent
export Ignore
export OpticalProperties
export σ
export AbstractInterceptionModel

# Photosynthesis
export AbstractAModel
export ConstantA
export Fvcb # Parameters for the coupled Farquhar et al. (1980) model
export FvcbIter # To update...
export FvcbRaw # Parameters for the original Farquhar et al. (1980) model
export Constants
export photosynthesis!
export photosynthesis
export photosynthesis!_

# Conductances
export AbstractGsModel
export gbh_to_gbw
export gbₕ_free
export gbₕ_forced
export gs, gs!
export Medlyn
export ConstantGs

# Temporary structures (to move to another package)
export Translucent
export Ignore
export get_km, Γ_star, arrhenius, get_J, gs_closure, get_Cᵢⱼ, get_Cᵢᵥ, get_Dₕ
export init_variables_manual, init_variables, Fvcb_net_assimiliation
export get_component_type, get_process, get_model, instantiate, get_component_type

export AbstractModel

# Components (structures that hold models)
export AbstractComponentModel
export LeafModels

# Parameters optimization
export fit

# Convenience functions
export DataFrame, copy, getindex

# Model evaluation
export EF, RMSE, dr

end


using Revise
using VCFTools
using MendelImpute
using GeneticVariation
using Random
using StatsBase
using CodecZlib
using ProgressMeter
using JLD2, FileIO, JLSO
using BenchmarkTools
using GroupSlices
using LinearAlgebra
using ThreadPools

function compress()
        max_d = 1000
	chr = 10
	minwidth = 0
	overlap = 0.0

        tgtfile = "target.chr$chr.typedOnly.masked.vcf.gz"
        reffile = "ref.chr$chr.excludeTarget.vcf.gz"
        outfile = "ref.chr$chr.excludeTarget.jlso"
        @time compress_haplotypes(reffile, tgtfile, outfile, max_d, minwidth, overlap)
end
compress()


using Oceananigans.Grids: AbstractGrid

""" Store source terms for `u`, `v`, and `w`. """
@kernel function store_field_tendencies!(G⁻, grid::AbstractGrid{FT}, G⁰) where FT
    i, j, k = @index(Global, NTuple)
    G⁻[i, j, k] = G⁰[i, j, k]
end

""" Store previous source terms before updating them. """
function store_tendencies!(model)

    barrier = Event(device(model.architecture))

    workgroup, worksize = work_layout(model.grid, :xyz)

    model_fields = fields(model)

    store_field_tendencies_kernel! = store_field_tendencies!(device(model.architecture), workgroup, worksize)

    events = []

    for (i, field) in enumerate(model_fields)

        field_event = store_field_tendencies_kernel!(model.timestepper.G⁻[i],
                                                     model.grid,
                                                     model.timestepper.Gⁿ[i],
                                                     dependencies=barrier)

        push!(events, field_event)
    end

    wait(device(model.architecture), MultiEvent(Tuple(events)))

    return nothing
end


function device_outage!(
    optimization_container::PSI.OptimizationContainer,
    constraint_info::Vector{DeviceOutageConstraintInfo},
    cons_name::Symbol,
    var_names::Tuple{Symbol, Symbol},
)
    time_steps = PSI.model_time_steps(optimization_container)
    varstart = PSI.get_variable(optimization_container, var_names[1])
    varstop = PSI.get_variable(optimization_container, var_names[2])

    name_start = PSI.middle_rename(cons_name, PSI.PSI_NAME_DELIMITER, "start")
    name_stop = PSI.middle_rename(cons_name, PSI.PSI_NAME_DELIMITER, "stop")

    set_names = [PSI.get_component_name(ic) for ic in constraint_info]
    con_start =
        PSI.add_cons_container!(optimization_container, name_start, set_names, time_steps)
    con_stop =
        PSI.add_cons_container!(optimization_container, name_stop, set_names, time_steps)

    for cont in constraint_info
        name = PSI.get_component_name(cont)
        con_start[name, 1] = JuMP.@constraint(
            optimization_container.JuMPmodel,
            varstart[name, 1] >= cont.initial_condition.value - cont.timeseries[1]
        )
        con_stop[name, 1] = JuMP.@constraint(
            optimization_container.JuMPmodel,
            varstop[name, 1] >= cont.timeseries[1] - cont.initial_condition.value
        )

        for t in time_steps[2:end]
            con_start[name, t] = JuMP.@constraint(
                optimization_container.JuMPmodel,
                varstart[name, t] >= cont.timeseries[t - 1] - cont.timeseries[t]
            )
            con_stop[name, t] = JuMP.@constraint(
                optimization_container.JuMPmodel,
                varstop[name, t] >= cont.timeseries[t] - cont.timeseries[t - 1]
            )
        end
    end
    return
end

function device_outage_parameter!(
    optimization_container::PSI.OptimizationContainer,
    constraint_info::Vector{DeviceOutageConstraintInfo},
    cons_name::Symbol,
    var_names::Tuple{Symbol, Symbol},
    param_reference::PSI.UpdateRef,
)
    time_steps = PSI.model_time_steps(optimization_container)
    varstart = PSI.get_variable(optimization_container, var_names[1])
    varstop = PSI.get_variable(optimization_container, var_names[2])

    name_start = PSI.middle_rename(cons_name, PSI.PSI_NAME_DELIMITER, "start")
    name_stop = PSI.middle_rename(cons_name, PSI.PSI_NAME_DELIMITER, "stop")

    set_names = [PSI.get_component_name(ic) for ic in constraint_info]
    con_start =
        PSI.add_cons_container!(optimization_container, name_start, set_names, time_steps)
    con_stop =
        PSI.add_cons_container!(optimization_container, name_stop, set_names, time_steps)

    container_outage = PSI.add_param_container!(
        optimization_container,
        param_reference,
        set_names,
        time_steps,
    )
    param = PSI.get_parameter_array(container_outage)
    multiplier = PSI.get_multiplier_array(container_outage)

    for cont in constraint_info
        name = PSI.get_component_name(cont)
        param[name, 1] =
            PJ.add_parameter(optimization_container.JuMPmodel, cont.timeseries[1])
        multiplier[name, 1] = cont.multiplier

        con_start[name, 1] = JuMP.@constraint(
            optimization_container.JuMPmodel,
            varstart[name, 1] >= cont.initial_condition.value - param[name, 1]
        )
        con_stop[name, 1] = JuMP.@constraint(
            optimization_container.JuMPmodel,
            varstop[name, 1] >= param[name, 1] - cont.initial_condition.value
        )

        for t in time_steps[2:end]
            param[name, t] =
                PJ.add_parameter(optimization_container.JuMPmodel, cont.timeseries[t])
            multiplier[name, t] = cont.multiplier

            con_start[name, t] = JuMP.@constraint(
                optimization_container.JuMPmodel,
                varstart[name, t] >= param[name, t - 1] - param[name, t]
            )
            con_stop[name, t] = JuMP.@constraint(
                optimization_container.JuMPmodel,
                varstop[name, t] >= param[name, t] - param[name, t - 1]
            )
        end
    end
    return
end

function device_outage_ub!(
    optimization_container::PSI.OptimizationContainer,
    constraint_info::Vector{DeviceOutageConstraintInfo},
    cons_name::Symbol,
    var_name::Symbol,
)
    time_steps = PSI.model_time_steps(optimization_container)
    varp = PSI.get_variable(optimization_container, var_name)

    set_names = [PSI.get_component_name(ic) for ic in constraint_info]
    constraint =
        PSI.add_cons_container!(optimization_container, cons_name, set_names, time_steps)

    for cont in constraint_info, t in time_steps
        name = PSI.get_component_name(cont)
        constraint[name, t] = JuMP.@constraint(
            optimization_container.JuMPmodel,
            varp[name, t] <= (1 - cont.timeseries[t]) * PSI.M_VALUE
        )
    end
    return
end

function device_outage_ub_parameter!(
    optimization_container::PSI.OptimizationContainer,
    constraint_info::Vector{DeviceOutageConstraintInfo},
    cons_name::Symbol,
    var_name::Symbol,
    param_reference::PSI.UpdateRef,
)
    time_steps = PSI.model_time_steps(optimization_container)
    varp = PSI.get_variable(optimization_container, var_name)

    set_names = [PSI.get_component_name(ic) for ic in constraint_info]
    constraint =
        PSI.add_cons_container!(optimization_container, cons_name, set_names, time_steps)

    container_outage = PSI.add_param_container!(
        optimization_container,
        param_reference,
        set_names,
        time_steps,
    )
    param = PSI.get_parameter_array(container_outage)
    multiplier = PSI.get_multiplier_array(container_outage)

    for cont in constraint_info, t in time_steps
        name = PSI.get_component_name(cont)
        param[name, t] =
            PJ.add_parameter(optimization_container.JuMPmodel, cont.timeseries[t])
        multiplier[name, t] = cont.multiplier
        constraint[name, t] = JuMP.@constraint(
            optimization_container.JuMPmodel,
            varp[name, t] <= (1 - param[name, t]) * PSI.M_VALUE
        )
    end
    return
end


function tests_invert_bkwds()
    model = MetNets.toy_model()

    bkwd_idxs = MetNets.bkwds_bounded(model)
    rename = (rxn) -> string(rxn, "_REV")
    new_model = MetNets.invert_bkwds(model; rename)

    for rxn_i in bkwd_idxs
        @test all(new_model.S[:,rxn_i] .== -model.S[:,rxn_i])
        @test new_model.lb[rxn_i] == -model.ub[rxn_i]
        @test new_model.ub[rxn_i] == -model.lb[rxn_i]
        @test new_model.rxns[rxn_i] == rename(model.rxns[rxn_i])
        @test new_model.rxnNames[rxn_i] == model.rxnNames[rxn_i]
    end
end
tests_invert_bkwds()


"""
    SubsampledDataCausalityTest

A causality test where the data is subsampled in some manner before
applying the test.
"""
abstract type SubsampledDataCausalityTest{CT} <: MetaCausalityTest{CT} end

get_ηs(x::SubsampledDataCausalityTest) = get_ηs(x.test)

export SubsampledDataCausalityTest, get_ηs

include("RandomSequencesTest.jl")
include("causality_RandomSequencesTest.jl")


addprocs() #Can be turned on for faster calculations
using Base.Test
using ThreePhotons

function test_file(filename)
  print_with_color(:green, "Testing: $filename\n")
  include(filename)
end

#General test parameters
K = 15
L = 12
N = 16
KMAX = 38
LMAX = 25

test_file("test_structure.jl")
# test_file("test_cubic.jl")
# test_file("test_spherical_harmonics.jl")
# test_file("test_datagen.jl")
# test_file("test_correlations.jl")
test_file("test_cuda.jl")
# test_file("test_determination.jl")
# test_file("test_phases.jl")
# test_file("test_data_processing.jl")
# test_file("test_utilities.jl")

print_with_color(:green, "Testing complete!\n")


# # Magnetic field direction
#
# In this example we'll look at the influence of the magnetic field direction on
# a nighttime and daytime ionosphere propagation path.
# Results will be compared to LWPC.
#
# ## Generate scenarios
#
# We'll use the JSON I/O functionality of LongwaveModePropagator to create
# the different magnetic field direction scenarios as a `BatchInput{ExponentialInput}`.
# We're doing this to ensure identical inputs are passed to LWPC.
# Although commented out in the `generate` function below, we locally saved the
# `BatchInput` as a JSON file and used it to feed a custom Python code
# which generates LWPC input files, runs LWPC, and saves the results in an HDF5 file.
#
# First load the packages we need.

using Dates, Printf
using HDF5
using Plots

using LongwaveModePropagator
using LongwaveModePropagator: buildrun, Progress, next!

# For convenience, we'll define global `OUTPUT_RANGES` at which the electric field
# will be computed, and the magnetic field dip and azimuth angles `B_DIPS` and `B_AZS`
# (converted below from degrees to radians).

const OUTPUT_RANGES = 0:5e3:3000e3
const B_DIPS = [90.0, 60, 60, 60, 60, 60, 60, 60, 60]
const B_AZS = [0.0, 0, 45, 90, 135, 180, 225, 270, 315]  ## vertical, N, E, S, W
nothing  #hide

# The `generate` function produces the [`BatchInput`](@ref) of the scenarios.
# If we wanted, we could comment out the JSON lines near the bottom of the function
# to also write the `BatchInput` to a file.
# That's not necessary here.

function generate(hp, β)
    batch = BatchInput{ExponentialInput}()
    batch.name = "Magnetic field tests"
    batch.description = "Varying magnetic field directions: vertical, N, E, S, W."
    batch.datetime = Dates.now()

    ## Constants
    frequency = 24e3
    output_ranges = collect(OUTPUT_RANGES)
    segment_ranges = [0.0]
    hprime = [hp]
    beta = [β]

    ## Ocean
    ground_epsr = [81]
    ground_sigma = [4.0]

    b_mag = [50e-6]
    b_dips = deg2rad.(B_DIPS)
    b_azs = deg2rad.(B_AZS)

    N = length(b_azs)
    inputs = Vector{ExponentialInput}(undef, N)
    for i in 1:N
        input = ExponentialInput()

        input.name = @sprintf("%d_%.0f_%.0f", i, b_dips[i], b_azs[i])
        input.description = "Wait ionosphere with ocean ground at 24 kHz."
        input.datetime = Dates.now()
        input.segment_ranges = segment_ranges
        input.hprimes = hprime
        input.betas = beta
        input.b_mags = b_mag
        input.b_dips = [b_dips[i]]
        input.b_azs = [b_azs[i]]
        input.ground_sigmas = ground_sigma
        input.ground_epsrs = ground_epsr
        input.frequency = frequency
        input.output_ranges = output_ranges

        inputs[i] = input
    end
    batch.inputs = inputs

    ## json_str = JSON3.write(batch)
    ##
    ## open("bfields.json","w") do f
    ##     write(f, json_str)
    ## end

    return batch
end

batch = generate(82.0, 0.6);

# ## Run the model
#
# Results from my custom LWPC script are saved in an
# [HDF5 file](@__REPO_ROOT_URL__/examples/bfields_lwpc.h5).
# For easy comparison, we will also save the results from the
# Longwave Mode Propagator to an HDF5 file.
# Although not strictly necessary for a small number inputs like we have here,
# using an HDF5 file rather than the JSON output file that is generated automatically by
# LongwaveModePropagator when a file is passed to `propagate` is more robust
# for thousands or tens of thousands of input scenarios.

function runlmp(inputs, outfile)

    h5open(outfile, "cw") do fid
        ## Create Batch attributes if they don't already exist
        fid_attrs = attributes(fid)
        haskey(fid_attrs, "name") || (fid_attrs["name"] = inputs.name)
        haskey(fid_attrs, "description") || (fid_attrs["description"] = inputs.description)
        haskey(fid_attrs, "datetime") || (fid_attrs["datetime"] = string(Dates.now()))

        if haskey(fid, "outputs")
            g = fid["outputs"]
        else
            g = create_group(fid, "outputs")
        end
    end

    PM = Progress(length(inputs.inputs), 5)
    for i in eachindex(inputs.inputs)
        ## If we've already run this, we can skip ahead
        complete = h5open(outfile, "r") do fid
            g = open_group(fid, "outputs")
            haskey(g, string(i)) ? true : false
        end
        complete && (next!(PM); continue)

        output = buildrun(inputs.inputs[i])

        h5open(outfile, "r+") do fid
            g = open_group(fid, "outputs")
            o = create_group(g, string(i))
            attributes(o)["name"] = output.name
            attributes(o)["description"] = output.description
            attributes(o)["datetime"] = string(Dates.now())
            o["output_ranges"] = output.output_ranges
            o["amplitude"] = output.amplitude
            o["phase"] = output.phase
        end
        next!(PM)
    end
end
nothing  #hide

#
# `examples_dir` below will need to be modified to the directory where
# `bfields_lmp.h5` is located and where you would like `bfields_lmp.h5` to be saved.

root_dir = dirname(dirname(pathof(LongwaveModePropagator)))
examples_dir = joinpath(root_dir, "examples")
lmpfile = joinpath(examples_dir, "bfields_lmp.h5")
runlmp(batch, lmpfile)

# ## Plots
#
# To more easily compare the amplitude and phase curves, we define a function
# to process the HDF5 files into arrays of amplitude and phase
# versus range and magnetic field azimuth.

function process(outputs)
    dist = read(outputs["outputs"]["1"]["output_ranges"])
    mask = indexin(OUTPUT_RANGES, dist)

    agrid = Array{Float64}(undef, length(OUTPUT_RANGES), length(B_AZS))
    pgrid = similar(agrid)

    for i in eachindex(B_AZS)
        o = outputs["outputs"][string(i)]

        amp = read(o["amplitude"])
        phase = read(o["phase"])

        agrid[:,i] = amp[mask]
        pgrid[:,i] = rad2deg.(phase[mask])
    end

    return agrid, pgrid
end

agrid, pgrid = h5open(lmpfile, "r") do o
    agrid, pgrid = process(o)
end

nothing  #hide

# Here is the amplitude plot itself.

labels = string.(trunc.(Int, B_AZS), "°")
labels[1] = "90°, "*labels[1]
labels[2] = "60°, "*labels[2]
labels[3:end] .= "      ".*labels[3:end]

colors = [palette(:phase, length(B_AZS))...]
pushfirst!(colors, RGB(0.0, 0, 0))

plot(OUTPUT_RANGES/1000, agrid;
     linewidth=1.5, palette=colors, colorbar=false,
     xlabel="range (km)", ylabel="amplitude (dB)",
     labels=permutedims(labels), legendtitle="  dip, az", legend=true)
#md savefig("magneticfield_amplitude.png"); nothing # hide
#md # ![](magneticfield_amplitude.png)

# The amplitude corresponding to each magnetic field azimuth
# (where 0° is along the propagation direction)
# is color-coded in the plot above, but a couple of the colors appear to be missing.
# This is because 0° and 180° are identical. There is no "north/south" dependence.
# Therefore 45°/135° and 315°/225° are also identical.
#
# On the other hand, there is a significant difference between propagation
# "east" (pinks) versus "west" (greens).
# This effect can be [observed in real life](https://doi.org/10.1016/0021-9169(58)90081-3).
#
# Now we'll plot the difference from LWPC.

lwpcfile = joinpath(examples_dir, "bfields_lwpc.h5")

lagrid, lpgrid = h5open(lwpcfile, "r") do o
    lagrid, bpgrid = process(o)
end

adifference = agrid - lagrid

plot(OUTPUT_RANGES/1000, adifference;
     linewidth=1.5, palette=colors, colorbar=false,
     xlabel="range (km)", ylabel="amplitude difference (dB)",
     labels=permutedims(labels), legendtitle="  dip, az", legend=true)
#md savefig("magneticfield_diff.png"); nothing # hide
#md # ![](magneticfield_diff.png)

# The two models are a very close match.
# Where there are large differences they occur because of slight misalignment of
# nulls with respect to range from the transmitter.
# The 315°/225° line has the worst match, although it is not clear if there is
# a particular cause for this outside of the general differences between the
# two models.

# ## Daytime ionosphere
#
# Does magnetic field direction have a different level of influence with daytime ionospheres?

lmpfile = joinpath(examples_dir, "bfields_daytime_lmp.h5")
runlmp(generate(72.0, 0.3), lmpfile)

agrid, pgrid = h5open(lmpfile, "r") do o
    agrid, pgrid = process(o)
end

plot(OUTPUT_RANGES/1000, agrid;
     linewidth=1.5, palette=colors, colorbar=false,
     xlabel="range (km)", ylabel="amplitude (dB)",
     labels=permutedims(labels), legendtitle="  dip, az", legend=true)
#md savefig("magneticfield_daytime_amplitude.png"); nothing # hide
#md # ![](magneticfield_daytime_amplitude.png)

# At these lower reflection altitudes the effect of Earth's magnetic field is decreased.


module Day17

using AdventOfCode2019

function day17(input::String = readInput(joinpath(@__DIR__, "..", "data", "day17.txt")))
    program = parse.(Int, split(input, ','))

    view = read_camera!(copy(program))
    intersections = find_intersections(view)
    part1 = ((x[1]-1) * (x[2]-1) for x in intersections) |> sum

    instructions = full_instruction(view)
    compressed = split_instructions(instructions)
    part2 = control_robot!(compressed, program)
    return [part1, part2]
end

function read_camera!(program::Array{Int,1})
    view = Array{Int,1}()
    out = Channel{Int}(5)
    t = @async AdventOfCode2019.IntCode.run_program!(program, nothing, out, nothing)
    while !istaskdone(t)
        push!(view, take!(out))
    end
    close(out)

    rows = split((strip(join(Char.(view)))),'\n')
    mat = Array{Char,2}(undef, length(rows), length(rows[1]))
    for (i, row) in enumerate(rows)
        for (j, char) in enumerate(row)
            mat[i, j] = char
        end
    end
    return mat
end

function find_intersections(view)
    intersections = Array{CartesianIndex{2},1}()
    for i = 2:size(view, 1) - 1
        for j = 2:size(view, 2) - 1
            if view[i,j] == '#'
                if view[i-1,j] == '#' && view[i+1,j] == '#' && view[i,j-1] == '#' && view[i,j+1] == '#'
                    push!(intersections, CartesianIndex(i, j))
                end
            end
        end
    end
    return intersections
end

function full_instruction(view)
    instructions = Array{Union{Char,Int},1}()
    asciiDirs = ['^', 'v', '<', '>']
    start = findfirst(x -> x in asciiDirs, view)

    direction = 0
    for i = 1:4
        if view[start] == asciiDirs[i]
            direction = i
            break
        end
    end

    neighbours = []
    d = direction
    for i = 1:4
        push!(neighbours, view[(start.I .+ dir_1d_to_2d(d))...])
        d = turn_left(d)
    end
    nLeftTurns = findfirst(x -> x == '#', neighbours) - 1
    if nLeftTurns == 3
        push!(instructions, 'R')
    elseif nLeftTurns == 2
        throw(AssertionError("not solved for this case"))
        push!(instructions, 'L')
        push!(instructions, 'L')
    elseif nLeftTurns == 1
        push!(instructions, 'L')
    end
    for i = 1:nLeftTurns
        direction = turn_left(direction)
    end

    i, j = start.I
    while true
        k, l = (i, j) .+ dir_1d_to_2d(direction)
        if inbounds(k, l, view) && view[k, l] == '#'
            push!(instructions, 1)
            i, j = k, l
            continue
        end
        k, l = (i, j) .+ dir_1d_to_2d(turn_left(direction))
        if inbounds(k, l, view) && view[k, l] == '#'
            push!(instructions, 'L')
            direction = turn_left(direction)
            continue
        end
        k, l = (i, j) .+ dir_1d_to_2d(turn_right(direction))
        if inbounds(k, l, view) && view[k, l] == '#'
            push!(instructions, 'R')
            direction = turn_right(direction)
            continue
        end
        break
    end

    # simplify instructions
    simplifiedInstructions = Array{Union{Char,Int},1}()
    count = 0
    for (i, inst) in enumerate(instructions)
        if inst in ('L', 'R')
            push!(simplifiedInstructions, inst)
        else
            count += inst
            if i + 1 > length(instructions) || isa(instructions[i+1], Char)
                push!(simplifiedInstructions, count)
                count = 0
            end
        end
    end

    instructionPairs = Array{Pair{Char,Int},1}()
    for i = 1:2:length(simplifiedInstructions) - 1
        push!(instructionPairs, Pair(simplifiedInstructions[i], simplifiedInstructions[i+1]))
    end

    return instructionPairs
end

function split_instructions(pairs::Array{Pair{Char,Int},1})
    main, a, b, c = "", "", "", ""
    for lenA = 2:10
        s = join(join.(pairs, ","), ",")
        rega = Regex("^([RL],[0-9]+,?){$lenA}")
        ra = findfirst(rega, s)
        isa(ra, Nothing) && continue
        a = strip(s[ra], ',')
        sa = replace(s, a => "A")
        for lenB = 2:10
            regb = Regex("([RL],[0-9]+,?){$lenB}")
            rb = findall(regb, sa)
            for r in rb
                b = strip(sa[r], ',')
                sb = replace(sa, b => "B")
                for lenC = 2:10
                    regc = Regex("([RL],[0-9]+,?){$lenC}")
                    rc = findall(regc, sb)
                    for t in rc
                        c = strip(sb[t], ',')
                        sc = replace(sb, c => "C")
                        finalreg = r"^([ABC],?){2,10}+$"
                        if !isa(findfirst(finalreg, sc), Nothing)
                            main = sc
                            @goto found
                        end
                    end
                end
            end
        end
    end
    throw(AssertionError("Unable to compress instructions"))
    @label found
    return Dict(
        "Main" => main * "\n",
        "A" => a * "\n",
        "B" => b * "\n",
        "C" => c * "\n",
    )
end

function control_robot!(compressedInstruction, program)
    inp = Channel{Int}(100)
    out = Channel{Int}(1)
    program[1] == 1 || throw(AssertionError("Invalid program"))
    program[1] = 2

    main = [Int(compressedInstruction["Main"][i]) for i in eachindex(compressedInstruction["Main"])]
    a = [Int(compressedInstruction["A"][i]) for i in eachindex(compressedInstruction["A"])]
    b = [Int(compressedInstruction["B"][i]) for i in eachindex(compressedInstruction["B"])]
    c = [Int(compressedInstruction["C"][i]) for i in eachindex(compressedInstruction["C"])]
    vidS = "n\n"
    video = [Int(vidS[i]) for i in eachindex(vidS)]

    t = @async AdventOfCode2019.IntCode.run_program!(program, inp, out, nothing)

    for val in main
        put!(inp, val)
    end
    for val in a
        put!(inp, val)
    end
    for val in b
        put!(inp, val)
    end
    for val in c
        put!(inp, val)
    end
    for val in video
        put!(inp, val)
    end

    while true
        ret = take!(out)
        if istaskdone(t)
            close(inp)
            close(out)
            return ret
        end
    end
end

function inbounds(i::Int, j::Int, view)
    return i >= 1 && j >= 1 && i <= size(view, 1) && j <= size(view, 2)
end

function turn_left(d::Int)
    d == 1 && return 3
    d == 2 && return 4
    d == 3 && return 2
    d == 4 && return 1
end

function turn_right(d::Int)
    d == 1 && return 4
    d == 2 && return 3
    d == 3 && return 1
    d == 4 && return 2
end

function dir_1d_to_2d(d::Int)
    d == 1 && return (-1, 0)
    d == 2 && return (1, 0)
    d == 3 && return (0, -1)
    d == 4 && return (0, 1)
end

function dir_2d_to_1d(d::Tuple{Int,Int})
    d == (-1, 0) && return 1
    d == (1, 0) && return 2
    d == (0, -1) && return 3
    d == (0, 1) && return 4
end

end # module


abstract type AbstractSymmetry end
struct EvenSymmetry <: AbstractSymmetry end
struct OddSymmetry <: AbstractSymmetry end
struct NoSymmetry <: AbstractSymmetry end
const Even = EvenSymmetry()
const Odd = OddSymmetry()


export create_new_pwq, generate_PiecewiseQuadratic, insert, delete_next
export get_right_endpoint, evalPwq, get_vals, find_minimum, find_minimum_value

"""
    PiecewiseQuadratic{T}

Represents a piecewise-quadratic as a linked list of QuadraticPolynomial{T} and left endpoints.

Note that the first element of the list is sometimes assumed to be a list head with `NaN` polynomial and `left_endpoint=NaN`.

The list/element `x` is the last element in the list when `x.next === x` or `x.next.left_endpoint==Inf`.
"""
mutable struct PiecewiseQuadratic{T}
    p::QuadraticPolynomial{T}
    left_endpoint::T
    next::PiecewiseQuadratic{T}

    PiecewiseQuadratic{T}()  where T = (x=new(); x.left_endpoint=Inf; x.next = x)
    PiecewiseQuadratic{T}(p, left_endpoint, next) where T = new(p, left_endpoint, next)
end

"""
Constructs piece of quadratic polynomial poiting to NULL, i.e.
a rightmost segment, or one that has not been inserted into a piecewise quadratic
"""
function PiecewiseQuadratic(p::QuadraticPolynomial{T}, left_endpoint) where T
    PiecewiseQuadratic{T}(p, left_endpoint, PiecewiseQuadratic{T}())
end
function PiecewiseQuadratic(p::QuadraticPolynomial{T}, left_endpoint, next::PiecewiseQuadratic{T}) where T
    PiecewiseQuadratic{T}(p, left_endpoint, next)
end

islast(λ::PiecewiseQuadratic) = λ.next.left_endpoint == Inf

"""
Creates empty piecewise-quadratic polynomial consisting only of the list head
"""
function create_new_pwq(T)
    return PiecewiseQuadratic(QuadraticPolynomial(T[NaN,NaN,NaN]), T(NaN), PiecewiseQuadratic{T}())
end

"""
Creates piecewise-quadratic polynomial containing one element
"""
function create_new_pwq(p::QuadraticPolynomial{T}) where T
    pwq = PiecewiseQuadratic(p, T(-Inf), PiecewiseQuadratic{T}())
    return PiecewiseQuadratic(QuadraticPolynomial(T[NaN,NaN,NaN]), T(NaN), pwq)
end

function generate_PiecewiseQuadratic(args::Tuple{Vector{T},T}...) where T
    return PiecewiseQuadratic(QuadraticPolynomial(T[NaN,NaN,NaN]), T(NaN), _generate_PiecewiseQuadratic_helper(args...))
end

function _generate_PiecewiseQuadratic_helper(arg)
    return PiecewiseQuadratic(QuadraticPolynomial(arg[1]), arg[2])
end

function _generate_PiecewiseQuadratic_helper(arg, args...)
    return PiecewiseQuadratic(QuadraticPolynomial(arg[1]), arg[2],  _generate_PiecewiseQuadratic_helper(args...))
end

function Base.iterate(pwq::PiecewiseQuadratic{T}, iterstate::PiecewiseQuadratic{T}=pwq.next) where T
    if (iterstate.left_endpoint == Inf)
        return nothing
    end

    return (iterstate, iterstate.next)
end

# For trouble-shooting etc.
function getindex(Λ::PiecewiseQuadratic, n::Integer)
    if n <= 0
        throw(BoundsError(Λ, n))
    end
    # If first element has NaN left_endpoint, assume it to be empty list-head
    λ = isnan(Λ.left_endpoint) ? Λ.next : Λ
    for k=1:n-1
        if islast(λ)
            throw(BoundsError(Λ, n))
        end
        λ = λ.next
    end
    return λ
end

function insert(pwq::PiecewiseQuadratic{T}, p::QuadraticPolynomial, left_endpoint) where T
    pwq.next = PiecewiseQuadratic(p, left_endpoint, pwq.next)
    return pwq.next
end

# Delete the node after pwq and return the node that will follow after
# pwq after the deletion
#OBS This function is unsafe if pwq.next does not exist
function delete_next(pwq::PiecewiseQuadratic{T}) where T
    pwq.next = pwq.next.next
    return pwq.next
end

function get_right_endpoint(λ::PiecewiseQuadratic)
    return λ.next.left_endpoint
end

function length(pwq::PiecewiseQuadratic)
    n = 0
    for x in pwq
        n += 1
    end
    return n
end

function show(io::IO, Λ::PiecewiseQuadratic{T}) where T
    print(io, "PiecewiseQuadratic{$T} with $(length(Λ)) elements:\n")

    for λ in Λ
        if λ.left_endpoint == -Inf
            print(io, "[  -∞ ,")
        else
            @printf(io, "[%3.2f,", λ.left_endpoint)
        end

        if get_right_endpoint(λ) == Inf
            print(io, " ∞  ]")
        else
            @printf(io, " %3.2f]", get_right_endpoint(λ))
        end

        print(io, "\t  :   ")
        show(io, λ.p)
        print(io, "\n")
    end
    return
end

function (Λ::PiecewiseQuadratic)(x::Number)
    for λ in Λ
        if λ.left_endpoint <= x < get_right_endpoint(λ)
            return λ.p(x)
        end
    end
    return NaN
end

function (Λ::PiecewiseQuadratic{T1})(x::AbstractArray) where T1
    y = fill(zero(promote_type(T1,eltype(x))), length(x))
    for λ in Λ
        inds = λ.left_endpoint .<= x .< get_right_endpoint(λ)
        y[inds] .= λ.p.(x[inds])
    end
    return y
end

"""
    x, y, x_all, y_all = get_vals(Λ::PiecewiseQuadratic)
Evaluate `Λ` so that `y .= Λ.(x)` for a set of gridpoints `x`.
Also returns lists x_all, y_all so that `y_all[i] .= Λ[i].(x_all[i])`
where `Λ[i]` is the `i`th segment of the piecewise quadratic.

The gridpoints in x_all[i] cointains points in the interval `[xmin[i]-1,xmax[i]+1]` where
`xmin[i],xmax[i]` are the extrema of the interval for which `Λ[i]` defines `Λ`.

The gridpoints at the ends default to `xmin[1] := xmax[1] - 1` and `xmax[end] := xmin[end] + 1`.
"""
function get_vals(Λ::PiecewiseQuadratic)
    x = Float64[]
    y = Float64[]
    x_all = Array{Float64,1}[]
    y_all = Array{Float64,1}[]
    for λ in Λ
        left_endpoint = λ.left_endpoint
        right_endpoint = get_right_endpoint(λ)

        if left_endpoint != -Inf || right_endpoint != Inf
            if left_endpoint == -Inf
                left_endpoint = right_endpoint - 2.0
            end

            if right_endpoint == Inf
                right_endpoint = left_endpoint + 2.0
            end
        else
            left_endpoint = -10.
            right_endpoint = 10.
        end

        y_grid = range(left_endpoint, stop=right_endpoint, length=50)
        vals =  λ.p.(y_grid)
        append!(x, y_grid)
        append!(y, vals)
        push!(x_all, y_grid)
        push!(y_all, vals)
    end
    return x, y, x_all, y_all
end

#
# function plot_pwq(Λ::PiecewiseQuadratic)
#     x, y, x_all, y_all = get_vals(Λ)
#     p = plot(x,y, l =(3,:red), lab="minimum", size=(1200,800))
#     for i in eachindex(x_all)
#         plot!(p, x_all[i][[1,end]], y_all[i][[1,end]], l = 0, m=(1,:cross,:orange))
#         plot!(p, x_all[i], y_all[i], l=(1,:blue), lab="")
#     end
#     p
#     return p
# end

"""
    p_opt, x_opt, f_opt = find_minimum(Λ::PiecewiseQuadratic)

Find the point `x_opt = argmin Λ(x)` and return `f_opt = Λ(x_opt)` and the quadratic
function `p_opt` which defines `Λ` around the point `x_opt`.
"""
function find_minimum(Λ::PiecewiseQuadratic)
    # May assume that the minimum is at the staionary point of the polynomial
    # TODO: True?

    f_opt = Inf
    p_opt = typeof(Λ.p)()
    x_opt = NaN

    for λ in Λ
        x, f = find_minimum(λ.p)
        if f < f_opt
            f_opt = f
            x_opt = x
            p_opt = λ.p
        end
    end
    return p_opt, x_opt, f_opt
end

function find_minimum_value(Λ::PiecewiseQuadratic)
    f_opt = Inf

    for λ in Λ
        _, f = find_minimum(λ.p)
        if f < f_opt
            f_opt = f
        end
    end
    if isnan(f_opt)
        println("find_minimum_value failed to find a minimum, no minimum exists")
    end
    return f_opt
end


using SAMTools
using AvailablePotentialEnergyFramework
using DelimitedFiles

file_control = "/global/homes/a/aramreye/SAM_model/SAM6.10.6/exp/f5e-4_2km_1000km_control/init_snd/snd300"
file_ensemble1 = "/global/homes/a/aramreye/SAM_model/SAM6.10.6/exp/f5e-4_2km_1000km_control_ensemble1/init_snd/snd300"
file_ensemble2 = "/global/homes/a/aramreye/SAM_model/SAM6.10.6/exp/f5e-4_2km_1000km_control_ensemble2/init_snd/snd300"

profile_control = readdlm(file_control;header=true,skipstart=1)[1][1:56,2:4]
profile_ensemble1= readdlm(file_ensemble1;header=true,skipstart=1)[1][1:56,2:4]
profile_ensemble2 = readdlm(file_ensemble2;header=true,skipstart=1)[1][1:56,2:4]

profile_enseble1_modified = copy(profile_ensemble1)


using Test
using ForestBiometrics

@test limiting_distance(10,12.4,34.0) == "The tree is in"


"""
    macro bu_str(unit)

String macro to easily recall Batsrus units located in the `UnitfulBatsrus` package.
Although all unit symbols in that package are suffixed with `_bu`, the suffix should not be
used when using this macro.
Note that what goes inside must be parsable as a valid Julia expression.
Examples:
```
julia> 1.0bu"u"
1.0 u
julia> 1.0bu"amucc"
1.0 amu/cc
```
"""
macro bu_str(unit)
   ex = Meta.parse(unit)
   esc(replace_value(ex))
end

const allowed_funcs = [:*, :/, :^, :sqrt, :√, :+, :-, ://]
function replace_value(ex::Expr)
   if ex.head == :call
      ex.args[1] in allowed_funcs ||
         error("""$(ex.args[1]) is not a valid function call when parsing a unit.
          Only the following functions are allowed: $allowed_funcs""")
      for i=2:length(ex.args)
         if typeof(ex.args[i])==Symbol || typeof(ex.args[i])==Expr
            ex.args[i] = replace_value(ex.args[i])
         end
      end
      return Core.eval(@__MODULE__, ex)
   elseif ex.head == :tuple
      for i=1:length(ex.args)
         if typeof(ex.args[i]) == Symbol
            ex.args[i] = replace_value(ex.args[i])
         else
            error("only use symbols inside the tuple.")
         end
      end
      return Core.eval(@__MODULE__, ex)
   else
      error("Expr head $(ex.head) must equal :call or :tuple")
   end
end

dottify(s, t, u...) = dottify(Expr(:(.), s, QuoteNode(t)), u...)
dottify(s) = s

function replace_value(sym::Symbol)
   s = Symbol(sym, :_bu)
   if !(isdefined(UnitfulBatsrus, s) && typecheck_bool(getfield(UnitfulBatsrus, s)))
      error("Symbol $s could not be found in UnitfulBatsrus.")
   end

   return getfield(UnitfulBatsrus, s)
end

replace_value(literal::Number) = literal

typecheck_bool(x::Unitful.Number) = true
typecheck_bool(x::Unitful.Unitlike) = true
typecheck_bool(x::Unitful.Quantity) = true
typecheck_bool(x::Unitful.MixedUnits) = true
typecheck_bool(x) = false


function englishiness(ptarray::Array{UInt8, 1})
    EnglishFrequencies = Dict( # from http://scottbryce.com/cryptograms/stats.htm
      'e'=>12.51,
      't'=>9.25,
      'a'=>8.04,
      'o'=>7.6,
      'i'=>7.26,
      'n'=>7.09,
      's'=>6.54,
      'r'=>6.12,
      'h'=>5.49,
      'l'=>4.14,
      'd'=>3.99,
      'c'=>3.06,
      'u'=>2.71,
      'm'=>2.53,
      'f'=>2.3,
      'p'=>2.0,
      'g'=>1.96,
      'w'=>1.92,
      'y'=>1.73,
      'b'=>1.54,
      'v'=>0.99,
      'k'=>0.67,
      'x'=>0.19,
      'j'=>0.16,
      'q'=>0.11,
      'z'=>0.09,
    )
    s=0
    for ll in ptarray
        llchar = Char(ll)
        if isalpha(llchar)
            s+=get(EnglishFrequencies, lowercase(llchar), -5)
        else
            s+=-30
        end
    end
    return s
end

function genBestSingleByteXORDecrypt(ctbytes::Array{UInt8, 1})
    bestdecrypt = []
    bestscore = -Inf
    bestkey = 0x00
    for xx in 0x00:0xff
        if isascii(Char(xx))
            ptbytes = ctbytes $ xx
            ptscore = englishiness(ptbytes)
            if ptscore > bestscore
                bestscore = ptscore
                bestdecrypt = ptbytes
                bestkey = xx
            end
        end
    end
    return (bestscore, bestdecrypt, bestkey)
end

# from challenge 5
function stringToBytes(X::String)
    return [UInt8(X[ii]) for ii in 1:length(X)]
end;

function repeatToLength(a::Array{UInt8,1}, len::Int64)
    if len < length(a)
        return a[1:len]
    else
        b = repeat(a, outer=convert(Int64, ceil(len / length(a))))
        return b[1:len]
    end
end

function repeatingKeyXOR(pt::Array{UInt8,1}, key::Array{UInt8,1})
    return pt $ repeatToLength(key, length(pt))
end

function repeatingKeyXOR(pt::String, key::String)
    return repeatingKeyXOR(stringToBytes(pt), stringToBytes(key))
end

# from challenge 6
function hammingDistance(a::Array{UInt8,1}, b::Array{UInt8,1})
    differentbits = a $ b
    d = 0
    for ii in 1:length(differentbits)
        c = bits(differentbits[ii])
        for jj in 1:8
            if c[jj] == '1'
                d+=1
            end
        end
    end
    return d
end

function hammingDistance(a::String, b::String)
    return hammingDistance(stringToBytes(a), stringToBytes(b))
end;

# from challenge 7
function decryptAES128ECBFileWithKey(fname, key)
    decrypt = chomp(readstring(`openssl enc -d -a -aes-128-ecb -K $key -in $fname`))
    return decrypt
end

# from challenge 9
  function padToNBytes(a::Array{UInt8,1}, nbytes)
      newa = Array{UInt8,1}(nbytes)
      ntopad = nbytes - length(a)
      newa[1:length(a)] = a
      newa[(length(a)+1):nbytes] = UInt8(ntopad)
      return newa
  end

  function padToNBytes(a::String, nbytes)
      return String(padToNBytes(stringToBytes(a), nbytes))
  end;

# assumes ciphertxt is base64encoded, and key is in hex
function decryptAES128ECBWithKey(ciphertext::String, key::String)
    decrypt = chomp(
        readstring(
            pipeline(`echo -n $ciphertext`,
                     `base64 --decode`,
                     `openssl enc -nopad -d -aes-128-ecb -K $key`,
                     `base64`)))
    return decrypt
end

# takes bytes and returns bytes
function decryptAES128ECBWithKey(cipherbytes::Array{UInt8,1}, keybytes::Array{UInt8,1})
    ciphertext = base64encode(cipherbytes)
    keytext = bytes2hex(keybytes)
    return base64decode(decryptAES128ECBWithKey(ciphertext, keytext))
end

function encryptAES128ECBWithKey(plaintext::String, key::String)
    encrypt = chomp(
        readstring(
            pipeline(`echo -n $plaintext`,
                     `base64 --decode`,
                     `openssl enc -nopad -e -aes-128-ecb -K $key`,
                     `base64`)))
    return encrypt
end

function encryptAES128ECBWithKey(plaintextbytes::Array{UInt8,1}, keybytes::Array{UInt8,1})
    plaintext = base64encode(plaintextbytes)
    keytext = bytes2hex(keybytes)
    return base64decode(encryptAES128ECBWithKey(plaintext, keytext))
end

function encryptCBCByHand(ptbytes::Array{UInt8,1}, blocksize, ivbytes::Array{UInt8,1}, keybytes::Array{UInt8,1})
    cipherbytes = Array{UInt8,1}()
    nblocks = convert(Int64, ceil(length(ptbytes)/blocksize))
    ptbytes = padToNBytes(ptbytes, blocksize*nblocks)
    lastblock = ivbytes
    for ii in 1:nblocks
        bytestoencrypt = ptbytes[((ii-1)*blocksize+1):(ii*blocksize)] $ lastblock
        encryptedbytes = encryptAES128ECBWithKey(bytestoencrypt, keybytes)
        append!(cipherbytes, encryptedbytes)
        lastblock = encryptedbytes
    end
    return cipherbytes
end;

function decryptCBCByHand(ctbytes::Array{UInt8,1}, blocksize, ivbytes::Array{UInt8,1}, keybytes::Array{UInt8,1})
    ptbytes = Array{UInt8,1}()
    nblocks = convert(Int64, length(ctbytes)/blocksize)
    lastblock = ivbytes
    for ii in 1:nblocks
        bytestodecrypt = ctbytes[((ii-1)*blocksize+1):(ii*blocksize)]
        decryptedbytes = decryptAES128ECBWithKey(bytestodecrypt, keybytes) $ lastblock
        append!(ptbytes, decryptedbytes)
        lastblock = bytestodecrypt
    end
    return ptbytes
end;

function isThisECBOrCBC(ctbytes::Array{UInt8,1}, blocksize=16)
    nblocks = convert(Int64, length(ctbytes)/blocksize)
    interblocksimilarity = []
    for ii in 1:(nblocks-1)
        for jj in (ii+1):nblocks
            blockii = ctbytes[((ii-1)*blocksize+1):(ii*blocksize)]
            blockjj = ctbytes[((jj-1)*blocksize+1):(jj*blocksize)]
            samebits = ~(blockii $ blockjj)
            similarity = 0
            for cc in samebits
                ccbits = bits(cc)
                for dd in 1:length(ccbits)
                    if ccbits[dd]=='1'
                        similarity+=1
                    end
                end
            end
            push!(interblocksimilarity, similarity/(8*blocksize))
        end
    end
    pexactsame = sum(interblocksimilarity.==1)/length(interblocksimilarity)
    cipher = "cbc"
    if pexactsame > 0.5
        cipher = "ecb"
    end
    return (cipher, pexactsame, interblocksimilarity)
end;


# Copyright (c) 2018: Matthew Wilhelm & Matthew Stuber.
# This code is licensed under MIT license (see LICENSE.md for full details)
#############################################################################
# EAGO
# A development environment for robust and global optimization
# See https://github.com/PSORLab/EAGO.jl
#############################################################################
# src/eago_optimizer/logging/log_iteration.jl
# Defines all routine to store information at each iteration.
#############################################################################

"""
    log_iteration!(x::GlobalOptimizer)

If 'logging_on' is true, the 'global_lower_bound', 'global_upper_bound',
'run_time', and 'node_count' are stored every 'log_interval'. If
'log_subproblem_info' then the lower bound, feasibility and run times of the
subproblems are logged every 'log_interval'.
"""
function log_iteration!(x::GlobalOptimizer{N,T,S}) where {N,T<:AbstractFloat,S}

    if x.log_on
        log = x._log
        if (mod(x._iteration_count, x.log_interval) == 0 || x._iteration_count == 1)
            if x.log_subproblem_info
                if x._input_problem._optimization_sense === MOI.MIN_SENSE
                    push!(log.current_lower_bound, x._lower_objective_value)
                    push!(log.current_upper_bound, x._upper_objective_value)
                else
                    push!(log.current_lower_bound, -x._upper_objective_value)
                    push!(log.current_upper_bound, -x._lower_objective_value)
                end

                push!(log.preprocessing_time, x._last_preprocess_time)
                push!(log.lower_problem_time, x._last_lower_problem_time)
                push!(log.upper_problem_time, x._last_upper_problem_time)
                push!(log.postprocessing_time, x._last_postprocessing_time)

                push!(log.preprocess_feasibility, x._preprocess_feasibility)
                push!(log.lower_problem_feasibility, x._lower_feasibility)
                push!(log.upper_problem_feasibility, x._upper_feasibility)
                push!(log.postprocess_feasibility, x._postprocess_feasibility)
            end

            if x._input_problem._optimization_sense === MOI.MIN_SENSE
                push!(log.global_lower_bound, x._global_lower_bound)
                push!(log.global_upper_bound, x._global_upper_bound)
            else
                push!(log.global_lower_bound, -x._global_upper_bound)
                push!(log.global_upper_bound, -x._global_lower_bound)
            end
            push!(log.run_time, x._run_time)
            push!(log.node_count, x._node_count)
        end
    end
    return
end


using Makie
using Observables

function cleanaxes()
    scene, layout = layoutscene()
    ax = layout[1, 1] = Axis(scene)
    axbox = pixelarea(ax.scene)[]
    lim = ax.finallimits[]
    e = events(ax.scene)
    return ax, axbox, lim, e
end

@testset "zoom Axis" begin
    ax, axbox, lim, e = cleanaxes()
    # Put the mouse in the center
    e.mouseposition[] = Tuple(axbox.origin + axbox.widths/2)
    # zoom in
    e.scroll[] = (0.0, -1.0)
    newlim = ax.finallimits[]
    @test newlim.widths ≈ 0.9 * lim.widths
    @test newlim.origin ≈ lim.origin + (lim.widths - newlim.widths)/2
    # zoom out restores original position
    e.scroll[] = (0.0, 1.0)
    newlim = ax.finallimits[]
    @test newlim.widths ≈ lim.widths
    @test all(abs.(newlim.origin - lim.origin) .< 1e-7*lim.widths)
    ax.finallimits[] = lim
    # Put mouse in corner
    e.mouseposition[] = Tuple(axbox.origin)
    # zoom in
    e.scroll[] = (0.0, -1.0)
    newlim = ax.finallimits[]
    @test newlim.widths ≈ 0.9 * lim.widths
    @test all(abs.(newlim.origin - lim.origin) .< 1e-7*lim.widths)
    # zoom out
    e.scroll[] = (0.0, 1.0)
    newlim = ax.finallimits[]
    @test newlim.widths ≈ lim.widths
    @test all(abs.(newlim.origin - lim.origin) .< 1e-7*lim.widths)
    ax.finallimits[] = lim

    # Zoom only x or y
    for (lockname, idx, zoomkey) in ((:xzoomlock, 1, :yzoomkey), (:yzoomlock, 2, :xzoomkey))
        ax, axbox, lim, e = cleanaxes()
        lock = getproperty(ax, lockname)
        @test !lock[]
        lock[] = true
        e.mouseposition[] = Tuple(axbox.origin + axbox.widths/2)
        e.scroll[] = (0.0, -1.0)
        newlim = ax.finallimits[]
        @test newlim.widths[idx] == lim.widths[idx]
        @test newlim.widths[3-idx] ≈ 0.9 * lim.widths[3-idx]
        @test newlim.origin[idx] == lim.origin[idx]
        @test newlim.origin[3-idx] ≈ lim.origin[3-idx] + (lim.widths[3-idx] - newlim.widths[3-idx])/2
        e.scroll[] = (0.0, 1.0)
        newlim = ax.finallimits[]
        @test newlim.widths ≈ lim.widths
        @test all(abs.(newlim.origin - lim.origin) .< 1e-7*lim.widths)
        ax.finallimits[] = lim
        lock[] = false
        # Simulate pressing the keys
        key = getproperty(ax, zoomkey)[]
        keypresses = getfield(e, Makie.button_key(key))[]
        @test isempty(keypresses)
        push!(keypresses, key)
        e.scroll[] = (0.0, -1.0)
        newlim = ax.finallimits[]
        @test newlim.widths[idx] == lim.widths[idx]
        @test newlim.widths[3-idx] ≈ 0.9 * lim.widths[3-idx]
        @test newlim.origin[idx] == lim.origin[idx]
        @test newlim.origin[3-idx] ≈ lim.origin[3-idx] + (lim.widths[3-idx] - newlim.widths[3-idx])/2
        e.scroll[] = (0.0, 1.0)
        newlim = ax.finallimits[]
        @test newlim.widths ≈ lim.widths
        @test all(abs.(newlim.origin - lim.origin) .< 1e-7*lim.widths)
    end

    # Rubber band selection
    scene, layout = layoutscene()
    ax = layout[1, 1] = Axis(scene)
    plot!(ax, [10, 15, 20])
    axbox = pixelarea(ax.scene)[]
    lim = ax.finallimits[]
    e = events(ax.scene)

    e.mouseposition[] = Tuple(axbox.origin)
    e.mousebuttons[] = Set([Mouse.left])
    e.mousedrag[] = Mouse.down
    e.mouseposition[] = Tuple(axbox.origin + axbox.widths ./ Vec2(2, 3))
    e.mousedrag[] = Mouse.pressed
    e.mousebuttons[] = Set{typeof(Mouse.left)}()
    e.mousedrag[] = Mouse.up
    newlim = ax.finallimits[]
    @test newlim.origin ≈ lim.origin
    @test newlim.widths ≈ lim.widths ./ Vec2(2, 3)
    # Ctrl-click to restore
    key = Makie.Keyboard.left_control
    keypresses = e.keyboardbuttons[]
    @test isempty(keypresses)
    push!(keypresses, key)
    buttons = e.mousebuttons[]
    @test isempty(buttons)
    e.mousebuttons[] = Set([Mouse.left])
    empty!(e.mousebuttons[])
    empty!(keypresses)
    newlim = ax.finallimits[]
    @test all(lim.origin .>= newlim.origin) && all(lim.origin+lim.widths .<= newlim.origin+newlim.widths)
end

@testset "pan Axis" begin
    ax, axbox, lim, e = cleanaxes()
    e.mouseposition[] = Tuple(axbox.origin + axbox.widths/2)
    e.scroll[] = (0.0, -1.0)
    newlim = ax.finallimits[]
    e.mouseposition[] = Tuple(axbox.origin)
    panbtn = ax.panbutton[]
    e.mousebuttons[] = Set([panbtn])
    e.mousedrag[] = Mouse.down
    e.mouseposition[] = Tuple(axbox.origin + axbox.widths/10)
    e.mousedrag[] = Mouse.pressed
    e.mousebuttons[] = Set{typeof(panbtn)}()
    e.mousedrag[] = Mouse.up
    panlim = ax.finallimits[]
    @test panlim.widths == newlim.widths
    @test (5/4)*panlim.origin ≈ -newlim.origin
end


__precompile__(true)
module JuML
export importcsv,
       Seq,
       DataFrame,
       Factor,
       Covariate,
       BoolVariate,
       factor,
       covariate,
       getname,
       getlevels,
       isordinal,
       xgblogit,
       cvxgblogit,
       predict,
       getstats,
       getauc,
       getlogloss,
       CovariateStats,
       FactorStats,
       LevelStats

include("const.jl")
include("DataStructures/seq.jl")
include("DataStructures/list.jl")
include("DataStructures/tree.jl")
include("util.jl")
include("DataImport/dataimport.jl")
include("DataFrame/dataframe.jl")

include("Factors/factor.jl")
include("Factors/constfactor.jl")
include("Factors/filefactor.jl")
include("Factors/ordinalfactor.jl")
include("Factors/cachedfactor.jl")
include("Factors/bincovfactor.jl")
include("Factors/boolvarfactor.jl")
include("Factors/maplevelfactor.jl")
include("Factors/permutefactor.jl")
include("Factors/widerfactor.jl")

include("Covariates/covariate.jl")
include("Covariates/constcovariate.jl")
include("Covariates/filecovariate.jl")
include("Covariates/cachedcovariate.jl")
include("Covariates/transcovariate.jl")
include("Covariates/trans2covariate.jl")
include("Covariates/parsefactorcovariate.jl")
include("Covariates/ifelsecovariate.jl")

include("BoolVariates/boolvariate.jl")
include("BoolVariates/transboolvariate.jl")
include("BoolVariates/trans2boolvariate.jl")
include("BoolVariates/transcovboolvariate.jl")
include("BoolVariates/trans2covboolvariate.jl")

include("XGBoost/tree.jl")
include("XGBoost/split.jl")
include("XGBoost/logistic.jl")

end


"""
Read a text file consistent entirely of floating point numbers separated
by single spaces or newlines.

I saw that this was about 1.5-2x faster than CSV.jl, which was the fastest
of the existing read methods.
"""

@inline mytryparse(::Type{Float64}, s::Vector{UInt8}, pos::Int64, len::Int64) =
            ccall(:jl_try_substrtod, Nullable{Float64}, (Ptr{UInt8},Csize_t,Csize_t), s, pos, len)

@inline function myparse(::Type{Float64}, s::Vector{UInt8}, pos::Int64, last::Int64)
    result = mytryparse(Float64, s, pos-1, last-pos+1)
    if isnull(result)
        throw(ArgumentError("cannot parse $(repr(s)) as $Float64"))
    end
    if VERSION < v"0.6"
      # when introduced, the function of unsafe_get is
      # unsafe_get(x::Nullable) = x.value
      # unsafe_get(x) = x
      return result.value
    else
      return unsafe_get(result)
    end
end

@inline function myparse(::Type{Int32}, s::Vector{UInt8}, pos::Int64, last::Int64)
    val = myparse(Float64, s, pos, last)
    return convert(Int32, val)
end

"""
Find the next space in the buffer a, starting the search at position = pos
and continuing until we find a space or hit len. If we can't find a space,
return -1, otherwise, return the index of the space.
"""
@inline function next_space(a::Vector{UInt8}, pos, len)
    @inbounds for i = pos:len
        if a[i] == UInt8(' ') || a[i] == UInt8('\n') || a[i] == UInt8('\t')
            return i
        end
    end
    return -1
end

@inline function next_non_space(a::Vector{UInt8}, pos, len)
    @inbounds for i = pos:len
        if a[i] != UInt8(' ') && a[i] != UInt8('\n') && a[i] != UInt8('\t')
            return i
        end
    end
    return -1
end

@inline function next_space_with_leading(a::Vector{UInt8}, pos, len)
    skip = next_non_space(a, pos, len)
    skip != -1 && return next_space(a, skip, len)
    return skip
end

@inline function all_spaces(a::Vector{UInt8}, pos, len)
    @inbounds for i = pos:len
        if a[i] != UInt8(' ') && a[i] != UInt8('\n') && a[i] != UInt8('\t')
            return false
        end
    end
    return true
end

mydebug = false

function tseq2(io::IO, a; maxbuf::Int=2^10, ntok::Int=-1)
    buf = Vector{UInt8}(maxbuf) # 64k bytes
    curtok = 0
    curbuf = 1

    tokinc = 1
    if ntok == -1
        tokinc = 0
        ntok = 1
    end

    while !eof(io) && curtok < ntok
        x = read(io, UInt8)
        if curbuf > 1
            # then we are processing a token
            if x == UInt8(' ') || x == UInt8('\n') || x == UInt8('\t')
                # there is a token from buf[1:curbuf]
                #@show "token", String(buf[1:curbuf-1])
                push!(a, myparse(Float64, buf, 1, curbuf-1))
                curtok += tokinc
                curbuf = 1
            elseif curbuf > maxbuf
                throw(ArgumentError("current token exceeded maxbuf=$(maxbuf)"))
            else
                buf[curbuf] = x
                curbuf += 1
            end
        else
            # otherwise we are handling spaces
            if x == UInt8(' ') || x == UInt8('\n') || x == UInt8('\t')
                continue
            else
                buf[curbuf] = x
                curbuf += 1 # we are in a token now!
            end
        end
    end
    if eof(io) && curtok < ntok && curbuf > 1
        # @show "token", String(buf[1:curbuf-1])
        push!(a, myparse(Float64, buf, 1, curbuf-1))
    end
    return a
end

function tseq(io::IO, a; maxbuf::Int=2^10)
    buf = Vector{UInt8}(maxbuf) # 64k bytes
    buf2 = Vector{UInt8}(maxbuf) # 64k bytes
    eof = false
    nb = readbytes!(io, buf)
    if nb == 0
        return a # just return
    end
    (nb < maxbuf) && (eof = true)

    curspace = next_non_space(buf, 1, nb)

    @inbounds while nb >= 0 && curspace >= 0
        nextspace = next_space_with_leading(buf, curspace+1, nb)
        if nextspace >= 0
            mydebug && @show "parsing", String(buf[curspace:nextspace])
            push!(a, myparse(Float64, buf, curspace, nextspace))
        elseif curspace == 1 && eof == true
            push!(a, myparse(Float64, buf, curspace, nb))
            # We are at eof and just processed the last of the buffer
        elseif curspace == 1
            @show curspace, nextspace
            throw(ArgumentError("current value exceeded maxbuf=$(maxbuf)"))
        else
            # we didn't see a space, that means we need to read more buffer
            # move things to beginning
            mydebug && print("Refilling: ", replace(String(buf),"\n","@"), " ", curspace, " ", String(buf[curspace:nb]), "\n")
            copy!(buf, 1, buf, curspace, nb - curspace + 1)
            #print("     Move: ", replace(String(buf),"\n","@"), "\n")
            bufstart = nb - curspace + 2
            #buffree = maxbuf - bufstart + 1
            curspace = 1

            # Try reading
            #nread = readbytes!(io, @view(buf[bufstart:end]))
            nread = readbytes!(io, buf2, maxbuf-bufstart+1)
            copy!(buf, bufstart, buf2, 1, maxbuf-bufstart+1)
            (nread < maxbuf-bufstart+1) && (eof = true)

            mydebug && print("   Reload: ", replace(String(buf),"\n","@"), "\n")

            mydebug && @show nread

            if nread == 0
                # We couldn't read any more, that means we are at the end!
                #print("    Final: ", replace(String(buf),"\n","@"), "\n")
                mydebug && @show "final", String(buf[curspace:bufstart-1])
                if bufstart > 2 && !all_spaces(buf, curspace, bufstart-1)
                    push!(a, myparse(Float64, buf, curspace, bufstart-1))
                    return a
                end
            else
                nb = bufstart + nread - 1
                mydebug && @show String(buf[curspace:nb])
                continue
            end

        end
        curspace = nextspace
    end
    return a
end

function read_file_to_float64_array(filename::AbstractString)
	a = zeros(0)
	open(filename, "r") do fh
		return tseq(fh, a)
	end
end

function readarrayt(io; kwargs...)
    line = readline(io) # read the first line
    a = zeros(0) # allocate the output
    tseq(IOBuffer(line), a; kwargs...) # parse the first line
    ncols = length(a) # this is the number of columns
    tseq(io, a; kwargs...) # read the rest of the array
    nrows, nrem = divrem(length(a),ncols)
    if nrem != 0
        throw(ArgumentError("first line of file had $(ncols) entries," *
            " but file has $(length(a)) for $(nrows) rows and $(nrem) left"))
    else
        return reshape(a, ncols, nrows)
    end
end
readarray(io; kwargs...) = readarrayt(io; kwargs...)'

#=
function read_number_stream(io, record::Tuple{Type}, nrec; all=true)

end
=#


export parse

#
# Parse given `ms` into a Dict().
#

function parse (ms::Int)
  qr = divrem(ms, 1000)
  ms = qr[2]

  qr = divrem(qr[1], 60)
  s = qr[2]

  qr = divrem(qr[1], 60)
  m = qr[2]

  qr = divrem(qr[1], 24)
  h = qr[2]
  d = qr[1]

  return [
    "days" => d,
    "hours" => h,
    "minutes" => m,
    "seconds" => s,
    "milliseconds" => ms
  ]
end


#
# Functions to convert between body names and NAIF ID codes,
# and vice versa. This functionality is enabled by the
# CSPICE library, CSPICE_jll, and SPICE.jl!
#
# Mass parameter and radii values are also provided through
# functions. This functionality is enabled by JPL HORIZONS
# and the IAU!
#

export naifcode, naifname

"""
Return the NAIF ID for the provided celestial body name. If no ID for
the provided body is found, a `KeyError` exception is thrown.

!!! note
    This is a simple wrapper around `SPICE.bodn2c`, which is itself
    a wrapper around the CSPICE library's `bod2nc` function! This
    integer code lookup is robust to different capitalizations.
    For example, `"Earth"`, `"earth"`, and `"eArTH"` should all return
    integer code `399`.
"""
function naifcode(name::AbstractString)
    code = SPICE.bodn2c(name)

    if code isa Integer
        return code
    else
        throw(KeyError("No NAIF ID for $name could be found!"))
    end
end

"""
Return the celestial body name for the provided NANIF ID. If no name for
the provided ID is found, a `KeyError` exception is thrown.

!!! note
    This is a simple wrapper around `SPICE.bodc2n`, which is itself
    a wrapper around the CSPICE library's `bodc2n` function!
"""
function naifname(code::Int)
    name = SPICE.bodc2n(code)

    if name isa AbstractString
        return name
    else
        throw(KeyError("No body name for $code could be found!"))
    end
end


# --------------------------------------------------------------------------------
# Liquidity traits
# --------------------------------------------------------------------------------

abstract type LiquidityStyle end
struct IsLiquid <: LiquidityStyle end
struct IsIlliquid <: LiquidityStyle end

# Default behavior is illiquid
LiquidityStyle(::Type{<:Asset}) = IsIlliquid()

# Cash is always liquid
LiquidityStyle(::Type{<:Cash}) = IsLiquid()

# Any subtype of Investment is liquid
LiquidityStyle(::Type{<:Investment}) = IsLiquid()

# --- Trait behavior ---

# The thing is tradable if it is liquid
tradable(x::T) where {T} = tradable(LiquidityStyle(T), x)
tradable(::IsLiquid, x) = true
tradable(::IsIlliquid, x) = false

# The thing has a market price if it is liquid
marketprice(x::T) where {T} = marketprice(LiquidityStyle(T), x)
marketprice(::IsLiquid, x) = error("Please implement pricing function for ", typeof(x))
marketprice(::IsIlliquid, x) = error("Price for illiquid asset $x is not available.")

# Sample pricing functions for Money and Stock
marketprice(x::Money) = x.amount
marketprice(x::Stock) = rand(200:250)


using Graphene
using Test

function tests()
    @testset "Subset of tests" begin
        include("test_fileio.jl")
        include("test_atoms.jl")
        include("test_bonds.jl")
        include("test_polygons.jl")
        include("test_defects.jl")
  end
end

tests()
@time tests()


using Documenter
using Latexify
using Markdown
using Markdown: MD, Paragraph, LineBreak
using Plots

using GeometricIntegrators
using GeometricIntegrators.Integrators.VPRK

using GeometricProblems.Diagnostics
using GeometricProblems.PlotRecipes

function Integrators.integrate!(int::Union{IntegratorVPRK{DT,TT,D,S},IntegratorFIRKimplicit{DT,TT,D,S},IntegratorSRKimplicit{DT,TT,D,S}}, sol::Solution{DT,TT}) where {DT,TT,D,S}
    asol = AtomicSolution(int)

    # loop over initial conditions showing progress bar
    for m in eachsample(sol)
        get_initial_conditions!(sol, asol, m, 1)
        initialize!(int, asol)

        n = 0
        try
            for outer n in eachtimestep(sol)
                integrate!(int, sol, asol, m, n)
            end
        catch ex
            tstr = " in time step " * string(n)

            if nsamples(sol) > 1
                tstr *= " for initial condition " * string(m)
            end

            tstr *= "."

            if isa(ex, DomainError)
                show(stdout, "text/markdown", Markdown.parse("DOMAIN ERROR: Simulation crashed" * tstr))
                @warn("DOMAIN ERROR: Simulation crashed" * tstr)
            elseif isa(ex, NonlinearSolverException)
                show(stdout, "text/markdown", Markdown.parse("SOLVER ERROR: Simulation crashed" * tstr))
                show(stdout, "text/markdown", Markdown.parse(ex.msg))
                @warn("SOLVER ERROR: Simulation crashed" * tstr)
                @warn(ex.msg)
            else
                show(stdout, "text/markdown", Markdown.parse("ERROR: Simulation crashed with $(typeof(ex)) " * tstr))
                show(stdout, "text/markdown", Markdown.parse(ex.msg))
                @warn("ERROR: Simulation crashed with $(typeof(ex)) " * tstr)
                @warn(ex.msg)
                # showerror(stdout, ex, catch_backtrace())
            end
        end
    end

    nothing
end

_linebreak(io) = show(io, "text/markdown", MD(Paragraph([LineBreak()])))

function _plot_figure_md(file, name, filename)
    # if isfile(filename)
        show(file, "text/markdown", Markdown.parse("![$name]($filename)"))
        _linebreak(file)
    # else
    #     show(stdout, "text/markdown", Markdown.parse("ERROR: Plot output $filename does not exist!"))
    #     @warn("Plot output $filename does not exist!")
    # end
end

function write_plots(dir, file, name, fig_suff)

    plot_file = file * ".md"

    open(plot_file, "w") do f
        show(f, "text/markdown", Markdown.parse("# $name"))
        _linebreak(f)

        _plot_figure_md(f, name, "$(dir)/$(file)_solution$(fig_suff)")
        _plot_figure_md(f, name, "$(dir)/$(file)_traces$(fig_suff)")

        show(f, "text/markdown", Markdown.parse("## Energy Error"))
        _linebreak(f)

        _plot_figure_md(f, name, "$(dir)/$(file)_energy_error$(fig_suff)")
        _plot_figure_md(f, name, "$(dir)/$(file)_energy_drift$(fig_suff)")

        show(f, "text/markdown", Markdown.parse("## Constraint"))
        _linebreak(f)

        _plot_figure_md(f, name, "$(dir)/$(file)_constraint_error$(fig_suff)")
    end
end

function plot(sol, equ, dir, file, fig_suff)
    if !isdir(dir)
        mkdir(dir)
    end

    try
        plot_lotka_volterra_2d(sol, equ; nt=lastentry(sol), fmt=:png)
        savefig(dir * "/" * file * fig_suff)

        plot_lotka_volterra_2d_solution(sol, equ; nt=lastentry(sol), fmt=:png)
        savefig(dir * "/" * file * "_solution" * fig_suff)

        plot_lotka_volterra_2d_traces(sol, equ; nt=lastentry(sol), fmt=:png)
        savefig(dir * "/" * file * "_traces" * fig_suff)

        H, ΔH = compute_energy_error(sol.t, sol.q, equ.parameters)
        plotenergyerror(sol.t, ΔH; nt=lastentry(sol), fmt=:png)
        savefig(dir * "/" * file * "_energy_error" * fig_suff)

        plotenergydrift(compute_error_drift(sol.t, ΔH, div(ntime(sol), 10))...; nt=lastentry(sol))
        savefig(dir * "/" * file * "_energy_drift" * fig_suff)

        plotconstrainterror(sol.t, compute_momentum_error(sol.t, sol.q, sol.p, (t,q,k)->ϑ(t,q,equ.parameters,k)); nt=lastentry(sol), fmt=:png)
        savefig(dir * "/" * file * "_constraint_error" * fig_suff)
    catch ex
        if isa(ex, DomainError)
            show(stdout, "text/markdown", Markdown.parse("DOMAIN ERROR: Diagnostics crashed."))
            @warn("DOMAIN ERROR: Diagnostics crashed.")
        else
            show(stdout, "text/markdown", Markdown.parse("ERROR: Plot crashed with $(typeof(ex))"))
            show(stdout, "text/markdown", Markdown.parse(ex.msg))
            @warn("ERROR: Plot crashed with $(typeof(ex))")
            @warn(ex.msg)
            showerror(stdout, ex, catch_backtrace())
        end
    end
end

function run_integrator(iode, tab, integrator, nt, dir, file, fig_suff)
    sol = Solution(iode, Δt, nt)
    int = integrator(iode, tab, Δt)

    integrate!(int, sol)

    plot(sol, iode, dir, file, fig_suff)
end

function run_list(iode, name, list, plot_dir = PLOT_DIR, symp_dir = SYMP_DIR;
                    fig_suff = ".png")

    for run in list
        tab, file = run

        if length(run) ≥ 3
            integrator = run[3]
        else
            integrator = Integrator
        end

        write_plots(plot_dir, file, name, fig_suff)

        show(stdout, "text/markdown", Markdown.parse("## $(tab.name)"))
        _linebreak(stdout)

        show(stdout, "text/markdown", Markdown.parse("[Plots]($file.md)"))
        _linebreak(stdout)

        run_integrator(iode, tab, integrator, nt, plot_dir, file, fig_suff)
        show(stdout, "text/markdown", Markdown.parse("![$name]($plot_dir/$file$fig_suff)"))
    end

    nothing
end


using DimensionalData, Test
using DimensionalData: _set, AutoSampling

a = [1 2; 3 4]
dimz = (X((143.0, 145.0); mode=Sampled(order=Ordered()), metadata=Metadata(Dict(:meta => "X"))),
        Y((-38.0, -36.0); mode=Sampled(order=Ordered()), metadata=Metadata(Dict(:meta => "Y"))))
da = DimArray(a, dimz; name=:test)

a2 = [1 2 3 4
      3 4 5 6
      4 5 6 7]
dimz2 = (Dim{:row}(10.0:10.0:30.0), Dim{:column}(-2:1.0:1.0))
da2 = DimArray(a2, dimz2; name=:test2)

s = DimStack(da2, DimArray(2a2, dimz2; name=:test3))

@testset " Array fields" begin
    @test name(set(da2, :newname)) == :newname
    @test name(set(da2, Name(:newname))) == Name{:newname}()
    @test name(set(da2, NoName())) == NoName()
    @test metadata(set(da2, Metadata(Dict(:testa => "test")))).val == Dict(:testa => "test")
    @test parent(set(da2, fill(9, 3, 4))) == fill(9, 3, 4)
    # A differently sized array can't be set
    @test_throws ArgumentError parent(set(da2, [9 9; 9 9])) == [9 9; 9 9]
end

@testset "DimStack fields" begin
    @test_throws ArgumentError set(s, (x=a2, y=3a2))
    @test_throws ArgumentError set(s, (test2=a2, test3=hcat(a2, a2)))
    s2 = set(s, (test2=a2, test3=3a2))
    @test keys(s2) == (:test2, :test3)
    @test values(s2) == (a2, 3a2)
    @test metadata(set(s, Metadata(Dict(:testa => "test")))).val == Dict(:testa => "test")
end

@testset "DimStack Dimension" begin
    @test typeof(dims(set(s, row=X, column=Z))) <: Tuple{<:X,<:Z}
    @test typeof(dims(set(s, row=X(), column=Z()))) <: Tuple{<:X,<:Z}
    @test typeof(dims(set(s, row=:row2, column=:column2))) <: Tuple{<:Dim{:row2},<:Dim{:column2}}
    @test typeof(dims(set(s, :column => Ti(), :row => Z))) <: Tuple{<:Z,<:Ti}
    @test typeof(dims(set(s, Dim{:row}(Y()), Dim{:column}(X())))) <: Tuple{<:Y,<:X}
    @test typeof(dims(set(s, (Dim{:row}(Y), Dim{:column}(X))))) <: Tuple{<:Y,<:X}
    @test index(set(s, Dim{:row}([:x, :y, :z])), :row) == [:x, :y, :z]
end

@testset "DimArray dim Dimension" begin
    @test typeof(dims(set(da, X=:a, Y=:b))) <: Tuple{<:Dim{:a},<:Dim{:b}}
    @test typeof(dims(set(da2, Dim{:row}(Y()), Dim{:column}(X())))) <: Tuple{<:Y,<:X}
    @test typeof(dims(set(da, X => Ti(), Y => Z()))) <: Tuple{<:Ti,<:Z}
    @test typeof(dims(set(da2, :column => Ti(), :row => Z()))) <: Tuple{<:Z,<:Ti}
    @test typeof(dims(set(da2, (Dim{:row}(Y), Dim{:column}(X))))) <: Tuple{<:Y,<:X}
    @test typeof(dims(set(da2, row=X, column=Z))) <: Tuple{<:X,<:Z}
    @test typeof(dims(set(da2, row=X(), column=Z()))) <: Tuple{<:X,<:Z}
    @test typeof(dims(set(da2, row=:row2, column=:column2))) <: Tuple{<:Dim{:row2},<:Dim{:column2}}
    @test index(set(da2, Dim{:row}([:x, :y, :z])), :row) == [:x, :y, :z]
end

@testset "Dimension index" begin
    @test index(set(da2, :column => [:a, :b, :c, :d], :row => 4:6)) ==
        (4:6, [:a, :b, :c, :d])
    @test index(set(da2, column=Val((:a, :b, :c, :d)), row=Val((4:6...,)))) ==
        ((4:6...,), (:a, :b, :c, :d))
    @test index(set(s, :column => 10:5:20, :row => 4:6)) == (4:6, 10:5:20)
    @test step.(span(set(da2, :column => 10:5:20, :row => 4:6))) == (1, 5)
end

@testset "dim mode" begin
    interval_da = set(da, X=Intervals(), Y=Intervals())
    @test mode(set(da2, :column => NoIndex(), :row => Sampled(sampling=Intervals(Center())))) ==
        (Sampled(Ordered(), Regular(10.0), Intervals(Center())), NoIndex())
    @test mode(set(da2, column=NoIndex())) ==
        (Sampled(Ordered(), Regular(10.0), Points()), NoIndex())
    @test mode(set(da2, :column => NoIndex(), :row => Sampled())) ==
        (Sampled(Ordered(), Regular(10.0), Points()), NoIndex())
    @test mode(set(da, X=NoIndex(), Y=Categorical())) == (NoIndex(), Categorical())

    @testset "span" begin
        # TODO: should this error? the span step doesn't match the index step
        @test span(set(da2, row=Irregular(10, 12), column=Regular(9.9))) ==
            (Irregular(10, 12), Regular(9.9))
        @test _set(Sampled(), AutoSpan()) == Sampled()
        @test _set(Sampled(), Irregular()) == Sampled(AutoOrder(), Irregular(), AutoSampling())
    end

    @testset "locus" begin
        @test_throws ArgumentError set(da2, (End(), Center()))
        @test locus(set(interval_da, X(End()), Y(Center()))) == (End(), Center())
        @test locus(set(interval_da, X=>End(), Y=>Center())) == (End(), Center())
        @test locus(set(interval_da, X=End, Y=Center)) == (End(), Center())
        @test locus(set(da, Y=Center())) == (Center(), Center())
        @test _set(Intervals(Center()), Start()) == Intervals(Start())
        @test _set(Intervals(Center()), AutoLocus()) == Intervals(Center())
        @test _set(Points(), Center()) == Points()
        @test_throws ArgumentError _set(Points(), Start())
        @test_throws ArgumentError _set(Points(), End())
    end

    @testset "sampling" begin
        @test sampling(interval_da) == (Intervals(), Intervals())
        @test sampling(set(da, (X(Intervals(End())), Y(Intervals(Start()))))) ==
            (Intervals(End()), Intervals(Start()))
        @test _set(Sampled(), AutoSampling()) == Sampled()
        @test _set(Sampled(), Intervals()) == Sampled(AutoOrder(), AutoSpan(), Intervals())
        @test _set(Points(), AutoSampling()) == Points()
        @test _set(AutoSampling(), Intervals()) == Intervals()
        @test _set(AutoSampling(), AutoSampling()) == AutoSampling()
    end

    @testset "order" begin
        uda = set(da, Y(Unordered()))
        @test order(uda) == (Ordered(), Unordered(ForwardRelation()))
        @test order(set(uda, Y=ReverseRelation())) == (Ordered(), Unordered(ReverseRelation()))
        @test order(ArrayOrder, set(da, X=ReverseArray)) == (ReverseArray(), ForwardArray())
        @test relation(set(da, Y(ReverseRelation())), Y) == ReverseRelation()
        @test order(set(X(), AutoOrder())) == AutoOrder()
        @test indexorder(set(X(; mode=Sampled(order=Ordered())), ReverseIndex())) == ReverseIndex()
        @test arrayorder(set(X(; mode=Sampled(order=Ordered())), ReverseArray())) == ReverseArray()
        @test arrayorder(set(X(; mode=Sampled(order=Ordered())), ReverseArray())) == ReverseArray()
        @test indexorder(_set(Ordered(), ReverseIndex())) == ReverseIndex()
        @test arrayorder(_set(Ordered(), ReverseArray())) == ReverseArray()
        @test relation(_set(Ordered(), ReverseRelation())) == ReverseRelation()
        @test relation(_set(Unordered(), ReverseRelation())) == ReverseRelation()
        @test_throws ArgumentError _set(Unordered(), ReverseIndex())
        @test_throws ArgumentError _set(Unordered(), ReverseArray())
        @test _set(Sampled(), Ordered()) == Sampled(Ordered(), AutoSpan(), AutoSampling())
        @test _set(Sampled(; order=Unordered()), AutoOrder()) == Sampled(Unordered(), AutoSpan(), AutoSampling())
    end
end

@testset "metadata" begin
    @test metadata(set(X(), Metadata(Dict(:a=>1, :b=>2)))).val == Dict(:a=>1, :b=>2)
    dax = set(da, X(Metadata(Dict(:a=>1, :b=>2))))
    @test metadata(dims(dax), X).val == Dict(:a=>1, :b=>2)
    @test metadata(dims(dax), Y).val == Dict(:meta => "Y")
    dax = set(da, X(; metadata=Metadata(Dict(:a=>1, :b=>2))))
    @test metadata(dims(dax, X)).val == Dict(:a=>1, :b=>2)
    dax = set(da2, row=Metadata(Dict(:a=>1, :b=>2)))
    @test metadata(dims(dax, :row)).val == Dict(:a=>1, :b=>2)
    dax = set(da2, column=Metadata(Dict(:a=>1, :b=>2)))
    @test metadata(dims(dax, :column)).val == Dict(:a=>1, :b=>2)
    @test metadata(set(da, NoMetadata())) == NoMetadata()
    @test metadata(set(da, NoMetadata)) == NoMetadata()
end

@testset "all dim fields" begin
    dax = set(da, X(20:-10:10; mode=Sampled(), metadata=Metadata(Dict(:a=>1, :b=>2))))
    x = dims(dax, X)
    @test val(x) == 20:-10:10
    @test order(x) == Ordered(ReverseIndex(), ForwardArray(), ForwardRelation())
    @test span(x) == Regular(-10)
    @test mode(x) == Sampled(Ordered(ReverseIndex(), ForwardArray(), ForwardRelation()), Regular(-10), Points())
    @test metadata(x).val == Dict(:a=>1, :b=>2)
end

@testset "errors with set" begin
    @test_throws ArgumentError set(da, Sampled())
    @test_throws ArgumentError set(da, X=7)
    @test_throws ArgumentError _set(dims(da, X), X(7))
    @test_throws ArgumentError set(da, notadimname=Sampled())
end

@testset "_set nothing" begin
    @test _set(nothing, nothing) == nothing
    @test _set(1, nothing) == 1
    @test _set(nothing, 2) == 2
end


# This file is part of FlagSets.jl project, which is licensed under BDS-3-Clause.
# See file LICENSE.md for more information.

function flagset_flags end
function flag_bit_map end
function flags end
function flagkeys end

"""
    BitMask(bitmask::Integer)

Structure used internally by `FlagSets` for default constructor of subtypes of `FlagSet`.
"""
struct BitMask{T<:Integer}
    bitmask::T
end

basetype(::Type{<:FlagSet{F,B}}) where {F,B<:Integer} = B

Base.isvalid(::Type{T}, x::Integer) where {T<:FlagSet} = (x & basetype(T)(typemax(T)) == x)
Base.isvalid(::Type{T}, x::BitMask) where {T<:FlagSet} = (x.bitmask & basetype(T)(typemax(T)) == x.bitmask)
Base.isvalid(::Type{T}, x::T) where {T<:FlagSet} = true
Base.isvalid(x::T) where {T<:FlagSet} = true

function Base.isvalid(::Type{T}, x::S) where {S,T<:FlagSet}
    Base.isiterable(S) && all(elt ∈ flags(T) for elt ∈ x)
end

function Base.isvalid(::Type{T}, x::Symbol) where {T<:FlagSet{Symbol}}
    x ∈ flags(T)
end

(::Type{I})(x::FlagSet) where {I<:Integer} = I(x.bitflags)::I
Base.cconvert(::Type{I}, x::FlagSet) where {I<:Integer} = I(x.bitflags)
Base.write(io::IO, x::T) where {T<:FlagSet} = write(io, x.bitflags)
Base.read(io::IO, ::Type{T}) where {T<:FlagSet} = T(BitMask(read(io, basetype(T))))
#Base.isless(x::FlagSet{T}, y::FlagSet{T}) where {T<:Integer} = isless(T(x), T(y))
Base.:|(x::T, y::T) where {T<:FlagSet} = T(BitMask(x.bitflags | y.bitflags))
Base.:&(x::T, y::T) where {T<:FlagSet} = T(BitMask(x.bitflags & y.bitflags))
Base.:⊻(x::T, y::T) where {T<:FlagSet} = T(BitMask(x.bitflags ⊻ y.bitflags))
Base.:~(x::T) where {T<:FlagSet} = setdiff(typemax(T), x)
Base.iszero(x::FlagSet) = isempty(x)

# Iterator interface

function Base.iterate(x::T, xi::Integer = x.bitflags) where {T<:FlagSet}
    iszero(xi) && (return nothing)
    fs = flagset_flags(T)
    nbit = trailing_zeros(xi)
    xi ⊻= typeof(xi)(1) << nbit
    return (fs[nbit+1], xi)
end

Base.length(x::FlagSet) = count_ones(x.bitflags)
Base.isempty(x::FlagSet) = iszero(x.bitflags)

"""
    flags(x::FlagSet)

Return the set of flags in `x` as a `Tuple`.

# Examples
```julia
julia> @symbol_flagset FontFlags bold=1 italic=2 large=4

julia> flags(FontFlags(3))
(:bold, :italic)
```
"""
flags(x::T) where {T<:FlagSet} = NTuple{length(x),eltype(T)}(x)

"""
    flags(T)

Return the set of flags of type `T` as a `Tuple`.

# Examples
```julia
julia> @flagset RoundingFlags RoundDown RoundUp RoundNearest

julia> flags(RoundingFlags)
(RoundingMode{:Down}(), RoundingMode{:Up}(), RoundingMode{:Nearest}())
```
"""
flags(::Type{<:FlagSet})

# Set manipulation
Base.union(x::T, y::T) where {T<:FlagSet} = T(BitMask(x.bitflags | y.bitflags))
Base.intersect(x::T, y::T) where {T<:FlagSet} = T(BitMask(x.bitflags & y.bitflags))
Base.setdiff(x::T, y::T) where {T<:FlagSet} = T(BitMask(x.bitflags & ~y.bitflags))
Base.issubset(x::T, y::T) where {T<:FlagSet} = (x.bitflags & y.bitflags) == x.bitflags
Base.in(elt, x::T) where {T<:FlagSet} = !iszero(get_flag_bit(T, elt, zero(basetype(T))) & x.bitflags)
Base.:⊊(x::T, y::T) where {T<:FlagSet} = x != y && (x.bitflags & y.bitflags) == x.bitflags

Base.empty(s::FlagSet{B,F}, ::Type{F}) where {B,F} = typeof(s)()
Base.empty(s::FlagSet) = typeof(s)()

# Printing
function Base.show(io::IO, x::T) where {T<:FlagSet}
    compact = get(io, :compact, false)::Bool
    type_p = (get(io, :typeinfo, false) == T)::Bool
    malformed = !isvalid(x)
    if compact || malformed
        xi = Integer(x)
        type_p &= malformed
        type_p || show(io, T)
        type_p || print(io, "(")
        show(io, xi)
        type_p || print(io, malformed ? " #= Invalid code =#)" : ")")
    else
        show(io, T)
        print(io, "([")
        join(io, repr.(x), ", ")
        print(io, "])")
    end
end

function Base.show(io::IO, mime::MIME"text/plain", x::FlagSet)
    if get(io, :compact, false)
        show(io, x)
    else
        invoke(show, Tuple{typeof(io),typeof(mime),AbstractSet}, io, mime,  x)
    end
end

# Printing FlagSet type
function Base.show(io::IO, mime::MIME"text/plain", type::Type{<:FlagSet})
    if isconcretetype(type) && !(get(io, :compact, false))
        print(io, "FlagSet ")
        Base.show_datatype(io, type)
        print(io, ":")
        keys = map(key -> isnothing(key) ? "" : String(key), flagkeys(type))
        if !all(isempty, keys)
            klen = maximum(length, keys)
            keys = map(k -> isempty(k) ? ' '^(klen+2) : k * ' '^(klen-length(k)) * " = ", keys)
        end
        for (flag, key) ∈ zip(flags(type), keys)
            bit = get_flag_bit(type, flag)
            print(io, "\n ", key, repr(bit), " --> ", repr(flag))
        end
    else
        invoke(show, Tuple{typeof(io),typeof(mime),Type}, io, mime, type)
    end
end

function flagset_argument_error(typename, x)
    throw(ArgumentError(string("invalid value for FlagSet $(typename): ", repr(x))))
end

function get_flag_bit(::Type{T}, flag, default) where {T<:FlagSet}
    get(flag_bit_map(T), flag, default)
end

function get_flag_bit(::Type{T}, flag) where {T<:FlagSet}
    not_found = basetype(T)(0x7f)
    val = get(flag_bit_map(T), flag, not_found)
    val == not_found && flagset_argument_error(T, flag)
    val
end

# Default constructors

function (::Type{T})(flag::Symbol, flags::Symbol...) where {T<:FlagSet{Symbol}}
    T((flag, flags...))
end

function (::Type{T})(itr) where {T<:FlagSet}
    Base.isiterable(typeof(itr)) || flagset_argument_error(T, itr)
    bitmask::basetype(T) = 0
    for flag ∈ itr
        bitmask |= get_flag_bit(T, flag)
    end
    T(BitMask(bitmask))
end

function (::Type{T})(bitmask::Integer) where {T<:FlagSet}
    T(BitMask(bitmask))
end


"""
"""
struct DiscreteModelPortion{Dc,Dp} <: DiscreteModel{Dc,Dc}
  model::DiscreteModel{Dc,Dp}
  oldmodel::DiscreteModel{Dc,Dp}
  d_to_dface_to_old_dface::Vector{Vector{Int}}
end

get_grid(model::DiscreteModelPortion) = get_grid(model.model)

get_grid_topology(model::DiscreteModelPortion) = get_grid_topology(model.model)

get_face_labeling(model::DiscreteModelPortion) = get_face_labeling(model.model)

get_face_to_oldface(model::DiscreteModelPortion,d::Integer) = model.d_to_dface_to_old_dface[d+1]

get_cell_to_oldcell(model::DiscreteModelPortion) = get_face_to_oldface(model,num_cell_dims(model))

get_oldmodel(model::DiscreteModelPortion) = model.oldmodel

"""
"""
function DiscreteModelPortion(model::DiscreteModel, cell_to_oldcell::Vector{Int})
  grid_p =  GridPortion(get_grid(model),cell_to_oldcell)
  topo_p = GridTopology(grid_p)
  labels_p, d_to_dface_to_old_dface = _setup_labels_p(model,topo_p,cell_to_oldcell)
  model_p = DiscreteModel(grid_p,topo_p,labels_p)
  DiscreteModelPortion(model_p,model,d_to_dface_to_old_dface)
end

function DiscreteModelPortion(model::DiscreteModel, cell_to_is_in::Vector{Bool})
  cell_to_oldcell = findall(cell_to_is_in)
  DiscreteModelPortion(model,cell_to_oldcell)
end

function _setup_labels_p(model,topo_p,cell_to_oldcell)

  topo = get_grid_topology(model)
  labels = get_face_labeling(model)
  D = num_cell_dims(model)

  T = eltype(cell_to_oldcell)
  d_to_dface_to_old_dface = Vector{Vector{T}}(undef,D+1)
  for d in 0:(D-1)
    cell_lface_to_face = Table(get_faces(topo_p,D,d))
    ocell_lface_to_oface = Table(get_faces(topo,D,d))
    nfaces = num_faces(topo_p,d)
    d_to_dface_to_old_dface[d+1] = _find_face_to_oldface(
      cell_lface_to_face,
      ocell_lface_to_oface,
      cell_to_oldcell,
      nfaces)
  end
  d_to_dface_to_old_dface[D+1] = cell_to_oldcell

  d_to_dface_to_entity = [
    zeros(eltype(labels.d_to_dface_to_entity[d+1]),num_faces(topo_p,d)) for  d in 0:D]
  for d in 0:D
    face_to_label = d_to_dface_to_entity[d+1]
    oface_to_label = labels.d_to_dface_to_entity[d+1]
    face_to_oface = d_to_dface_to_old_dface[d+1]
    _update_labels_dim!(face_to_label,oface_to_label,face_to_oface)
  end

  labels_p = FaceLabeling(d_to_dface_to_entity,labels.tag_to_entities,labels.tag_to_name)

  labels_p, d_to_dface_to_old_dface
end

function _find_face_to_oldface(
  cell_lface_to_face::Table, ocell_lface_to_oface::Table, cell_to_ocell, nfaces)

  # Assumes that the local faces have the same order in new and old

  T = eltype(cell_to_ocell)
  face_to_oface = zeros(T,nfaces)

  for cell in 1:length(cell_lface_to_face)
    ocell = cell_to_ocell[cell]
    a = cell_lface_to_face.ptrs[cell]-1
    b = cell_lface_to_face.ptrs[cell+1]
    nlfaces = b - (a+1)
    c = ocell_lface_to_oface.ptrs[ocell]-1
    for lface in 1:nlfaces
      face = cell_lface_to_face.data[a+lface]
      oface = ocell_lface_to_oface.data[c+lface]
      face_to_oface[face] = oface
    end
  end

  face_to_oface
end

function _update_labels_dim!(face_to_label,oface_to_label,face_to_oface)
  for face in 1:length(face_to_label)
    oface = face_to_oface[face]
    label = oface_to_label[oface]
    face_to_label[face] = label
  end
end


abstract type TimeEvolution end

const UNDEF_SAMPLEPETIME = -1 # For handling promotion of Matrix to LTISystem

struct Discrete{T} <: TimeEvolution
    Ts::T
    function Discrete{T}(Ts::T) where T
        if Ts <= 0 && Ts != UNDEF_SAMPLEPETIME
            throw(ErrorException("Creating a continuous time system by setting sample time to 0 is no longer supported."))
        end
        new{T}(Ts)
    end
end
Discrete{T}(x) where T = Discrete{T}(T(x))

struct Continuous <: TimeEvolution end
# Basic Constructors
Discrete(x::T) where T<:Number = Discrete{T}(x)
Continuous(x::Continuous) = x
# Simple converseion
Discrete{T}(x::Discrete) where T = Discrete{T}(x.Ts)

undef_sampletime(::Type{Discrete{T}}) where T = Discrete{T}(UNDEF_SAMPLEPETIME)
undef_sampletime(::Type{Continuous}) where T = Continuous()

# Promotion
# Fallback to give useful error
Base.promote_rule(::Type{<:Discrete}, ::Type{Continuous}) = throw(ErrorException("Sampling time mismatch"))
Base.promote_rule(::Type{Discrete{T1}}, ::Type{Discrete{T2}}) where {T1,T2}= Discrete{promote_type(T1,T2)}

Base.convert(::Type{Discrete{T1}}, x::Discrete{T2}) where {T1,T2} = Discrete{T1}(x.Ts)

# Promoting two or more systems systems should promote sample times
common_timeevol(x::TimeEvolution) = x
common_timeevol(x::TimeEvolution, y::TimeEvolution) = throw(ErrorException("Sampling time mismatch"))
common_timeevol(x::TimeEvolution, y::TimeEvolution, z...) = common_timeevol(common_timeevol(x, y), z...)
common_timeevol(a::Base.Generator) = reduce(common_timeevol, a)

function common_timeevol(x::Discrete{T1}, y::Discrete{T2}) where {T1,T2}
    if x != y && x.Ts != UNDEF_SAMPLEPETIME && y.Ts != UNDEF_SAMPLEPETIME
         throw(ErrorException("Sampling time mismatch"))
    end

    if x.Ts == UNDEF_SAMPLEPETIME
        return Discrete{promote_type(T1,T2)}(y)
    else
        return Discrete{promote_type(T1,T2)}(x)
    end
end

common_timeevol(x::Continuous, ys::Continuous...) = Continuous()

# Check equality
==(x::TimeEvolution, y::TimeEvolution) = false
==(x::Discrete, y::Discrete) = (x.Ts == y.Ts)
==(::Continuous, ::Continuous) = true

isapprox(x::TimeEvolution, y::TimeEvolution, args...; kwargs...) = false
isapprox(x::Discrete, y::Discrete, args...; kwargs...) = isapprox(x.Ts, y.Ts, args...; kwargs...)
isapprox(::Continuous, ::Continuous, args...; kwargs...) = true


s = read_SSR(   joinpath(dirname(@__FILE__), "..", "data", "test_Hz19.5-testing.bdf"))
s = read_evt(s, joinpath(dirname(@__FILE__), "..", "data", "test.evt"))


using Base: Float64
using Dates: string
println("---> START <---")
# Set up environment
using Pkg
Pkg.activate(".")
using JSON
using Glob
using CSV
using Random
using DataFrames

using Statistics
using StatsBase

using Distributions
using DataStructures
using Parameters

using Plots
using Colors
using ColorSchemes
using Dates
using Plotly

include("post_simulation_plots_functions.jl")

# SETUP
outputs_path = "/Users/kingasiuta/Desktop/results/results_5_07"
vars = ["p0"] # what has to be constant
vars_heatmap = ["N_agents", "p0"] # what has to be constant
points_in_time = [[1*24*3600, 2*24*3600, 3*24*3600]]
points_in_time_sing = [1*24*3600, 2*24*3600, 3*24*3600]

# reading possible parameter values

params_to_plot = CSV.read("$outputs_path/sim_params_comb_df.csv", DataFrame)

file_list = glob("simul_res_parset_id_*.json", outputs_path)
    for id in params_to_plot[:,:parset_id]
        if string(outputs_path,"/simul_res_parset_id_",id,".json") ∉ file_list
            filter!(r -> !in(r.parset_id, [id]), params_to_plot)
        end
    end
CSV.write("$outputs_path/sim_params_comb_df.csv", params_to_plot)

values_to_loop = Dict()
for var in vars
    values_to_loop[var] = unique(params_to_plot[:,var])
end

# actual loop
for n_ag in values_to_loop[vars[1]], #  THIS HAS TO BE CHANGED IF VAR NUMBER IS NOT 3
    speed in values_to_loop[vars[2]],
    p0 in values_to_loop[vars[3]]

    full_output, params_sets = read_results(outputs_path,
        control_vars = vars,
        control_val_equal = [n_ag, speed, p0],
        vars_to_check = ["TTC_car_freq"])

    develplot_by_var("total_infected", "TTC_car_freq", full_output,
    with_std = false,
    total_indic = "N_agents",
    div = 60,
    var_unit = "min",
    max_it = nothing,
    x_lab = "iteration",
    y_lab = "% infected",
    title = "% infected vs freq when ag = $n_ag, speed = $speed, p0 = $p0", #COSMETIC
    out_name = "total_infected_S/ag_$n_ag&_sp_$speed&_p0_$p0", #COSMETIC
    with_export = true)

    develplot_by_var("TTC_infected", "TTC_car_freq", full_output,
    with_std = false,
    div = 60,
    var_unit = "min",
    max_it = nothing,
    x_lab = "iteration",
    y_lab = "people infected",
    title = "TTC infected vs freq when ag = $n_ag, speed = $speed, p0 = $p0", #COSMETIC
    out_name = "TTC_infected_S/ag_$n_ag&_sp_$speed&_p0_$p0", #COSMETIC
    with_export = true)

    develplot_by_var("TTC_infected", "TTC_car_freq", full_output,
    with_std = false,
    div = 60,
    var_unit = "min",
    max_it = nothing,
    x_lab = "iteration",
    y_lab = "people infected",
    total_indic = "prct_of_agents_used_TTC",
    title = "% TTC infected vs freq when ag = $n_ag, speed = $speed, p0 = $p0", #COSMETIC
    out_name = "perc_used/ag_$n_ag&_sp_$speed&_p0_$p0", #COSMETIC
    with_export = true)

    for (index, value) in enumerate(points_in_time)

        point_in_time_by_var(["total_infected"], "TTC_car_freq", full_output,
            points_in_time = value,
            total_indic = "N_agents",
            div = 60,
            time_div = 3600,
            time_unit = "h",
            x_lab = "Public Transport Frequency",
            y_lab = "% infected",
            title = "Optimal Public Transport Frequency = 9 min", #COSMETIC
            out_name = "do_networks", #COSMETIC
            with_export = true)
        point_in_time_by_var(["total_infected"], "TTC_car_freq", full_output,
            points_in_time = value,
            total_indic = "N_agents",
            div = 60,
            time_div = 3600,
            time_unit = "h",
            x_lab = nothing,
            y_lab = "% infected",
            title = "%infected when ag = $n_ag, speed = $speed, p0 = $p0", #COSMETIC
            out_name = "points_in_time/ag_$n_ag&_sp_$speed&_p0_$p0&_$index", #COSMETIC
            with_export = true)

        point_in_time_by_var(["TTC_infected"], "TTC_car_freq", full_output,
            points_in_time = value,
            total_indic = "N_agents",
            div = 60,
            time_div = 3600,
            time_unit = "h",
            x_lab = nothing,
            y_lab = "% infected",
            title = "%infected when ag = $n_ag, speed = $speed, p0 = $p0", #COSMETIC
            out_name = "TTC_points_in_time/ag_$n_ag&_sp_$speed&_p0_$p0&_$index", #COSMETIC
            with_export = true)

        point_in_time_by_var(["TTC_infected"], "TTC_car_freq", full_output,
            points_in_time = value,
            total_indic = "prct_of_agents_used_TTC",
            div = 60,
            time_div = 3600,
            time_unit = "h",
            x_lab = nothing,
            y_lab = "% infected",
            title = "%infected when ag = $n_ag, speed = $speed, p0 = $p0", #COSMETIC
            out_name = "perc_used/ag_$n_ag&_sp_$speed&_p0_$p0&_$index", #COSMETIC
            with_export = true)
    end

    for (index, value) in enumerate(points_in_time_sing)
        point_in_time_by_var(["total_infected", "TTC_infected", "street_infected"],
        "TTC_car_freq", full_output,
        points_in_time = [value],
        total_indic = "N_agents",
        div = 60,
        time_div = 3600,
        time_unit = "h",
        x_lab = "TTC frequency",
        y_lab = "infected",
        title = "Odsetek zarażonych vs. częstotliwość kursowania TTC, EOD $index", #COSMETIC
        out_name = "multiple_variables/ag_$n_ag&_sp_$speed&_p0_$p0&_EOD$index", #COSMETIC
        with_export = true)

        plot_XY("prct_of_agents_used_TTC", "total_infected", full_output,
            point_in_time = value,
            total_indic = "N_agents",
            x_lab = "TTC users",
            y_lab = "infected",
            title = "Odsetek zarażonych vs. odsetek osób korzystających z TTC, EOD $index", #COSMETIC
            out_name = "XY_plots/ag_$n_ag&_sp_$speed&_p0_$p0&_EOD$index", #COSMETIC
            with_export = true)

        plot_XYY("prct_of_agents_used_TTC", "total_infected", "TTC_car_freq", full_output,
                point_in_time = value,
                divx = 60,
                total_indic = "N_agents",
                x_lab = "TTC frequency",
                y1_lab = "TTC_users",
                y2_lab = "infected",
                title = "Infected population & TTC usage by TTC frequency, EOD $index",
                out_name = "XYY_plots/ag_$n_ag&_sp_$speed&_p0_$p0&_EOD$index",
                with_export = true)
    end

    # full_output, params_sets = read_results(outputs_path,
    #     control_vars = vars_heatmap,
    #     control_val_equal = [n_ag, p0],
    #     vars_to_check = ["TTC_car_freq", "agents_speed_in_sec_per_m"])

    # make_heatmap("prct_of_agents_used_TTC", "TTC_car_freq",
    #     "agents_speed_in_sec_per_m", full_output, params_sets,
    #     div1 = 60,
    #     div2 = 1,
    #     out_name = "heatmaps/ag_$n_ag&_p0_$p0",
    #     with_export = true)

    println("parameters $n_ag agents, $speed speed, $p0 probability - finished" ) #COSMETIC
end

println("plotting finished")

#### plotting scenarios for p0

total_indic = "N_agents"
response_vars= ["total_infected"]
variable = "TTC_car_freq"
points_in_time = [1*24*3600, 2*24*3600, 3*24*3600, 4*24*3600, 5*24*3600]
div = 60

for p0 in values_to_loop["p0"]
    full_output, params_sets = read_results(outputs_path,
        control_vars = vars,
        control_val_equal = [p0],
        vars_to_check = ["TTC_car_freq"])

    point_in_time_by_var(["total_infected"], "TTC_car_freq", full_output,
    points_in_time = points_in_time,
    total_indic = "N_agents",
    div = div,
    time_div = 3600,
    time_unit = "h",
    x_lab = nothing,
    y_lab = "% infected",
    title = "%infected", #COSMETIC
    out_name = "more_iters_$p0", #COSMETIC
    with_export = true)
end