"""
Routines for single-node inference that aren't specific
to any particular method of inference (e.g MCMC, DeterministicVI),
yet are also not about the statistical model (i.e., the Model
module).
Currently what's in here are routines that effectively
truncate the Gaussians in the model.
TODO: rename this module to something more meaningful
"""
module Infer

import WCS
using StaticArrays

import ..Config
using ..Model
import ..Log

"""
Computes the nearby light sources in the catalog for each of the target
sources.

Arguments:
    target_sources: indexes of astronomical objects in the catalog to infer
    catalog: astronomical objects appearing the images
    images: astronomical images
"""
function find_neighbors(target_sources::Vector{Int64},
                        catalog::Vector{CatalogEntry},
                        images::Vector{Image})
    psf_width_ub = zeros(NUM_BANDS)
    for img in images
        psf_width = Model.get_psf_width(img.psf)
        psf_width_ub[img.b] = max(psf_width_ub[img.b], psf_width)
    end

    epsilon_lb = fill(Inf, NUM_BANDS)
    for img in images
        epsilon = img.sky[div(img.H, 2), div(img.W, 2)]
        epsilon_lb[img.b] = min(epsilon_lb[img.b], epsilon)
    end

    radii_map = zeros(length(catalog))
    for s in 1:length(catalog)
        ce = catalog[s]
        for img in images
            radius_pix = Model.choose_patch_radius(ce, img, width_scale=1.2)
            radii_map[s] = max(radii_map[s], radius_pix)
        end
        @assert radii_map[s] <= 25
    end

    dist(ra1, dec1, ra2, dec2) = (3600 / 0.396) * max(abs(dec2 - dec1), abs(ra2 - ra1))

    neighbor_map = Vector{Int64}[Int64[] for s in target_sources]

    # If this loop isn't super fast in pratice, we can tile (the sky, not the
    # images) or build a spatial index with a library before distributing
    for ts in 1:length(target_sources)
        s = target_sources[ts]
        ce = catalog[s]

        for s2 in 1:length(catalog)
            ce2 = catalog[s2]
            ctrs_dist = dist(ce.pos[1], ce.pos[2], ce2.pos[1], ce2.pos[2])

            if s2 != s && ctrs_dist < radii_map[s] + radii_map[s2]
                push!(neighbor_map[ts], s2)
            end
        end
    end

    neighbor_map
end

function get_sky_patches(images::Vector{Image},
                         catalog::Vector{CatalogEntry};
                         radius_override_pix=NaN)
    N = length(images)
    S = length(catalog)
    patches = Matrix{SkyPatch}(S, N)

    for n = 1:N
        img = images[n]

        for s=1:S
            world_center = catalog[s].pos
            pixel_center = WCS.world_to_pix(img.wcs, world_center)
            wcs_jacobian = Model.pixel_world_jacobian(img.wcs, pixel_center)
            radius_pix = Model.choose_patch_radius(catalog[s], img, width_scale=1.2)
            @assert radius_pix <= 25
            if !isnan(radius_override_pix)
                radius_pix = radius_override_pix
            end

            hmin = max(0, floor(Int, pixel_center[1] - radius_pix - 1))
            hmax = min(img.H - 1, ceil(Int, pixel_center[1] + radius_pix - 1))
            wmin = max(0, floor(Int, pixel_center[2] - radius_pix - 1))
            wmax = min(img.W - 1, ceil(Int, pixel_center[2] + radius_pix - 1))

            # some light sources are so far from some images that they don't
            # overlap at all
            H2 = max(0, hmax - hmin + 1)
            W2 = max(0, wmax - wmin + 1)

            # all pixels are active by default
            active_pixel_bitmap = trues(H2, W2)

            patches[s, n] = SkyPatch(world_center,
                                     radius_pix,
                                     img.psf,
                                     wcs_jacobian,
                                     pixel_center,
                                     SVector(hmin, wmin),
                                     active_pixel_bitmap)
        end
    end

    patches
end

"""
Record which pixels in each patch will be considered when computing the
objective function.

Non-standard arguments:
  noise_fraction: The proportion of the noise below which we will remove pixels.
"""
function load_active_pixels!(config::Config,
                             images::Vector{Image},
                             patches::Matrix{SkyPatch};
                             exclude_nan=true,
                             noise_fraction=0.5)
    S, N = size(patches)

    for n = 1:N, s=1:S
        img = images[n]
        p = patches[s,n]

        # (h2, w2) index the local patch, while (h, w) index the image
        H2, W2 = size(p.active_pixel_bitmap)
        for w2 in 1:W2, h2 in 1:H2
            h = p.bitmap_offset[1] + h2
            w = p.bitmap_offset[2] + w2

            # skip masked pixels
            if isnan(img.pixels[h, w]) && exclude_nan
                p.active_pixel_bitmap[h2, w2] = false
                continue
            end

            # include pixels that are close, even if they aren't bright
            sq_dist = (h - p.pixel_center[1])^2 + (w - p.pixel_center[2])^2
            if sq_dist < config.min_radius_pix^2
                p.active_pixel_bitmap[h2, w2] = true
                continue
            end

            # if this pixel is bright, let's include it
            # (in the future we may want to do something fancier, like
            # fitting an elipse, so we don't include nearby sources' pixels,
            # or adjusting active pixels during the optimization)
            # Note: This is risky because bright pixels are disproportionately likely
            # to get included, even if it's because of noise. Therefore it's important
            # to keep the noise fraction pretty low.
            threshold = img.nelec_per_nmgy[h] * img.sky[h, w] * (1. + noise_fraction)
            p.active_pixel_bitmap[h2, w2] = img.pixels[h, w] > threshold
        end
    end
end

# legacy wrapper
function load_active_pixels!(images::Vector{Image},
                             patches::Matrix{SkyPatch};
                             exclude_nan=true,
                             noise_fraction=0.5)
    load_active_pixels!(
        Config(),
        images,
        patches,
        exclude_nan=exclude_nan,
        noise_fraction=noise_fraction,
    )
end

# The range of image pixels in a vector of patches
function get_active_pixel_range(
    patches::Matrix{SkyPatch}, sources::Vector{Int}, n::Int)
    H_min = minimum([ p.bitmap_offset[1] + 1 for p in patches[sources, n] ])
    W_min = minimum([ p.bitmap_offset[2] + 1 for p in patches[sources, n] ])
    H_max = maximum([ p.bitmap_offset[1] + size(p.active_pixel_bitmap, 1)
                      for p in patches[sources, n] ])
    W_max = maximum([ p.bitmap_offset[2] + size(p.active_pixel_bitmap, 2)
                      for p in patches[sources, n] ])
    H_min, W_min, H_max, W_max
end

# Is a pixel (h, w) in whole-image coordinates an active pixel in the patch p?
function is_pixel_in_patch(h::Int, w::Int, p::SkyPatch)
    hp = h - p.bitmap_offset[1]
    wp = w - p.bitmap_offset[2]
    in_patch =
        (hp > 0) &
        (wp > 0) &
        (hp <= size(p.active_pixel_bitmap, 1)) &
        (wp <= size(p.active_pixel_bitmap, 2))
    if !in_patch
        return false
    else
        return p.active_pixel_bitmap[hp, wp]
    end
end

end  # module


# See **[`ProjectiveTransforms`](#).**

"""
    ProjectiveTransforms(size, [augmentations])

Pipeline step that resizes images and keypoints to `size`.

In context [`Training`](#), applies `augmentations`.
"""
@with_kw_noshow struct ProjectiveTransforms <: PipelineStep
    traintfm
    validtfm
    inferencetfm
end

function ProjectiveTransforms(
        size;
        augmentations = Identity(),
        inferencefactor = 1,
        buffered = true)
    tfms = (
        augmentations |> RandomResizeCrop(size),
        CenterResizeCrop(size),
        ResizePadDivisible(size, inferencefactor),
    )

    if buffered
        tfms = (
            BufferedThreadsafe(tfms[1]),
            BufferedThreadsafe(tfms[2]),
            # Inference transform is not buffered since it can have
            # varying sizes
            tfms[3]
        )
    end

    return ProjectiveTransforms(tfms...)
end

function Base.show(io::IO, spatial::ProjectiveTransforms)
    outsize = _parenttfm(spatial.validtfm).transforms[1].crop.size
    print(io, "ProjectiveTransforms($(outsize))")
end

function run(spatial::ProjectiveTransforms, context, datas::Tuple)
    items = makespatialitems(datas)
    tfm = _gettfm(spatial, context)
    tdatas = itemdata.(DataAugmentation.apply(tfm, items))
    return tdatas
end

function run!(bufs, spatial::ProjectiveTransforms, context, datas::Tuple)
    items = makespatialitems(datas)
    tfm = _gettfm(spatial, context)
    tdatas = DataAugmentation.apply(tfm, items) |> itemdata
    _copyrec!(bufs, tdatas)
    return bufs
end

run(spatial::ProjectiveTransforms, context, data) = run(spatial, context, (data,)) |> only
run!(buf, spatial::ProjectiveTransforms, context, data) = run!((buf,), spatial, context, (data,)) |> only

## Utils

_gettfm(spatial::ProjectiveTransforms, context::Training) = spatial.traintfm
_gettfm(spatial::ProjectiveTransforms, context::Validation) = spatial.validtfm
_gettfm(spatial::ProjectiveTransforms, context::Inference) = spatial.inferencetfm

function makespatialitems(datas::Tuple)
    if datas[begin] isa Item
        return makespatialitems(datas, getbounds(datas[begin]))
    else
        return makespatialitems(datas, makebounds(size(datas[begin])))
    end
end
function makespatialitems(datas::Tuple, bounds)
    return Tuple(makeitem(data, bounds) for data in datas)
end

"""
    makeitem(data, args...)

Tries to assign a `DataAugmentation.Item` from `data` based on its type.
`args` are passed to the chosen  `Item` constructor.

- `AbstractMatrix{<:Colorant}` -> `Image`
- `Vector{<:Union{Nothing, SVector}}` -> `Keypoints`
"""
makeitem(data, args...) = itemtype(data)(data, args...)
makeitem(item::Item, args...) = item
makeitem(datas::Tuple, args...) = Tuple(makeitem(data, args...) for data in datas)

itemtype(data::AbstractMatrix{<:Colorant}) = DataAugmentation.Image
itemtype(data::Vector{<:Union{Nothing, SVector}}) = DataAugmentation.Keypoints


module Flux

# Zero Flux Given

using Base: tail
using Zygote, MacroTools, Juno, Reexport, Statistics, Random
using MacroTools: @forward
@reexport using NNlib
using Zygote: Params, @adjoint, gradient, pullback
export gradient

export Chain, Dense, Maxout, RNN, LSTM, GRU, Conv, CrossCor, ConvTranspose, MaxPool, MeanPool,
       DepthwiseConv, Dropout, AlphaDropout, LayerNorm, BatchNorm, InstanceNorm, GroupNorm,
       SkipConnection, params, fmap, cpu, gpu, f32, f64

include("optimise/Optimise.jl")
using .Optimise
using .Optimise: @epochs
export SGD, Descent, ADAM, Momentum, Nesterov, RMSProp,
  ADAGrad, AdaMax, ADADelta, AMSGrad, NADAM,
  ADAMW, RADAM, InvDecay, ExpDecay, WeightDecay

using CUDAapi
if has_cuda()
  try
    using CuArrays
    @eval has_cuarrays() = true
  catch ex
    @warn "CUDA is installed, but CuArrays.jl fails to load" exception=(ex,catch_backtrace())
    @eval has_cuarrays() = false
  end
else
  has_cuarrays() = false
end

include("utils.jl")
include("onehot.jl")
include("functor.jl")

include("layers/stateless.jl")
include("layers/basic.jl")
include("layers/conv.jl")
include("layers/recurrent.jl")
include("layers/normalise.jl")

include("data/Data.jl")

include("deprecations.jl")

if has_cuarrays()
  include("cuda/cuda.jl")
end

end # module


@doc """
A RollCall object represents a complete roll-call data set along with metadata
about the encoding used by the dataset. This encoding is stored in the form
of a RollCallEncoding object.
""" ->
immutable RollCall{T1 <: Tuple, T2 <: Tuple, T3 <: Tuple, T4 <: Tuple}
    votes::Matrix{Int}
    encoding::RollCallEncoding{T1, T2, T3, T4}
end


#=

  "ESSE: Environmental and State Dependent Diversification" submodule package

=#

module ESSE

using Random: randexp
using DelimitedFiles: readdlm, writedlm
using ProgressMeter: Progress, next!
using DifferentialEquations: ODEProblem, init, reinit!, solve!, Tsit5, DiffEqBase
using LinearAlgebra: BLAS.gemv!, rank, mul!, ldiv!, qr!
using Distributed: @sync, @distributed
using SharedArrays: SharedArray

# other submodules dependencies
using ..Utils

# files
include("esse/musse_eqs.jl")
include("esse/esse_eqs.jl")
include("esse/states_handling.jl")
include("esse/loglik_mu.jl")
include("esse/simulate_esse.jl")
include("esse/approx_fun.jl")
include("esse/model_definitions.jl")
include("esse/simulate_geoesse.jl")
include("esse/sse_wrapper.jl")
include("esse/slice_sampler_utils.jl")
include("esse/slice_sampler.jl")
include("esse/ode_solve.jl")
include("esse/egeohisse_eqs.jl")
include("esse/prepare_data.jl")
include("esse/prepare_ll.jl")
include("esse/loglik_geo.jl")
include("esse/loglik_flow.jl")

end # module ESSE


# Affine decision function #
# ========================== #
mutable struct AffineDecisionConstraintBridge{T, S} <: MOIB.Constraint.AbstractBridge
    constraint::CI{MOI.ScalarAffineFunction{T}, S}
    decision_function::AffineDecisionFunction{T}
    set::S
end

function MOIB.Constraint.bridge_constraint(::Type{AffineDecisionConstraintBridge{T, S}},
                                           model,
                                           f::AffineDecisionFunction{T},
                                           set::S) where {T, S}
    # All decisions have been mapped to either the decision part constant or the variable part terms
    # at this point.
    F = MOI.ScalarAffineFunction{T}
    # Calculate total constant
    constant = f.variable_part.constant +
        f.known_part.constant
    # Add the bridged constraint
    constraint = MOI.add_constraint(model,
                                    MOI.ScalarAffineFunction(f.variable_part.terms, zero(T)),
                                    MOIU.shift_constant(set, -constant))
    # Save the constraint index, the decision function, and the set, to allow modifications
    return AffineDecisionConstraintBridge{T, S}(constraint, f, set)
end

function MOI.supports_constraint(::Type{<:AffineDecisionConstraintBridge{T}},
                                 ::Type{<:AffineDecisionFunction{T}},
                                 ::Type{<:MOI.AbstractScalarSet}) where {T}
    return true
end
function MOIB.added_constrained_variable_types(::Type{<:AffineDecisionConstraintBridge})
    return Tuple{DataType}[]
end
function MOIB.added_constraint_types(::Type{<:AffineDecisionConstraintBridge{T, S}}) where {T, S}
    return [(MOI.ScalarAffineFunction{T}, S)]
end
function MOIB.Constraint.concrete_bridge_type(::Type{<:AffineDecisionConstraintBridge{T}},
                                              ::Type{<:AffineDecisionFunction{T}},
                                              S::Type{<:MOI.AbstractScalarSet}) where T
    return AffineDecisionConstraintBridge{T, S}
end

MOI.get(b::AffineDecisionConstraintBridge{T, S}, ::MOI.NumberOfConstraints{MOI.ScalarAffineFunction{T}, S}) where {T, S} = 1
MOI.get(b::AffineDecisionConstraintBridge{T, S}, ::MOI.ListOfConstraintIndices{MOI.ScalarAffineFunction{T}, S}) where {T, S} = [b.constraint]

function MOI.get(model::MOI.ModelLike, attr::MOI.ConstraintFunction,
                 bridge::AffineDecisionConstraintBridge{T}) where T
    f = bridge.decision_function
    # Remove mapped variables to properly unbridge
    from_decision(v) = begin
        result = any(t -> t.variable_index == v, f.decision_part.terms)
    end
    g = AffineDecisionFunction(
        MOIU.filter_variables(v -> !from_decision(v), f.variable_part),
        copy(f.decision_part),
        copy(f.known_part))
    return g
end

function MOI.get(model::MOI.ModelLike, attr::MOI.ConstraintSet,
                 bridge::AffineDecisionConstraintBridge{T}) where T
    return bridge.set
end

function MOI.get(model::MOI.ModelLike, attr::MOI.ConstraintPrimal,
                 bridge::AffineDecisionConstraintBridge{T}) where T
    return MOI.get(model, attr, bridge.constraint)
end

function MOI.get(model::MOI.ModelLike, attr::MOI.ConstraintDual,
                 bridge::AffineDecisionConstraintBridge{T}) where T
    return MOI.get(model, attr, bridge.constraint)
end

function MOI.delete(model::MOI.ModelLike, bridge::AffineDecisionConstraintBridge)
    MOI.delete(model, bridge.constraint)
    return nothing
end

function MOI.set(model::MOI.ModelLike, ::MOI.ConstraintFunction,
                 bridge::AffineDecisionConstraintBridge{T,S},
                 f::AffineDecisionFunction{T}) where {T,S}
    # Update bridge functions and function constant
    bridge.decision_function = f
    # Change the function of the bridged constraints
    MOI.set(model, MOI.ConstraintFunction(), bridge.constraint,
            MOI.ScalarAffineFunction(f.variable_part.terms, zero(T)))
    # Recalculate total constant and shift constraint set
    constant = f.variable_part.constant +
        f.known_part.constant
    MOI.set(model, MOI.ConstraintSet(), bridge.constraint,
            MOIU.shift_constant(bridge.set, -constant))
    return nothing
end

function MOI.set(model::MOI.ModelLike, ::MOI.ConstraintSet,
                 bridge::AffineDecisionConstraintBridge{T,S}, change::S) where {T,S}
    f = bridge.decision_function
    bridge.set = change
    # Recalculate total constant and shift constraint set
    constant = f.variable_part.constant +
        f.known_part.constant
    MOI.set(model, MOI.ConstraintSet(), bridge.constraint,
            MOIU.shift_constant(change, -constant))
    return nothing
end

function MOI.modify(model::MOI.ModelLike, bridge::AffineDecisionConstraintBridge{T,S}, change::MOI.ScalarConstantChange) where {T,S}
    f = bridge.decision_function
    # Modify variable part of decision function
    f.variable_part.constant = change.new_constant
    # Recalculate total constant and shift constraint set
    constant = f.variable_part.constant +
        f.known_part.constant
    MOI.set(model, MOI.ConstraintSet(), bridge.constraint,
            MOIU.shift_constant(bridge.set, -constant))
    return nothing
end

function MOI.modify(model::MOI.ModelLike, bridge::AffineDecisionConstraintBridge{T,S}, change::MOI.ScalarCoefficientChange) where {T,S}
    f = bridge.decision_function
    # Update coefficient in variable part
    modify_coefficient!(f.variable_part.terms, change.variable, change.new_coefficient)
    # Modify the variable part of the constraint as usual
    MOI.modify(model, bridge.constraint, change)
    return nothing
end

function MOI.modify(model::MOI.ModelLike, bridge::AffineDecisionConstraintBridge{T,S}, change::DecisionCoefficientChange) where {T,S}
    f = bridge.decision_function
    # Update coefficient in decision part
    modify_coefficient!(f.decision_part.terms, change.decision, change.new_coefficient)
    # Update mapped variable through ScalarCoefficientChange
    MOI.modify(model, bridge, MOI.ScalarCoefficientChange(change.decision, change.new_coefficient))
    return nothing
end

function MOI.modify(model::MOI.ModelLike, bridge::AffineDecisionConstraintBridge{T,S}, change::KnownCoefficientChange) where {T,S}
    f = bridge.decision_function
    i = something(findfirst(t -> t.variable_index == change.known,
                            f.known_part.terms), 0)
    # Update known part of constraint constant
    coefficient = iszero(i) ? zero(T) : f.known_part.terms[i].coefficient
    known_value = MOI.get(model, MOI.VariablePrimal(), change.known)
    f.known_part.constant +=
        (change.new_coefficient - coefficient) * known_value
    # Update coefficient in known part
    modify_coefficient!(f.known_part.terms, change.known, change.new_coefficient)
    # Recalculate total constant and shift constraint set
    constant = f.variable_part.constant +
        f.known_part.constant
    MOI.set(model, MOI.ConstraintSet(), bridge.constraint,
            MOIU.shift_constant(bridge.set, -constant))
    return nothing
end

function MOI.modify(model::MOI.ModelLike, bridge::AffineDecisionConstraintBridge{T,S}, change::KnownValueChange) where {T,S}
    f = bridge.decision_function
    i = something(findfirst(t -> t.variable_index == change.known,
                            f.known_part.terms), 0)
    if iszero(i)
        # Known value not in objective, nothing to do
        return nothing
    end
    # Update known part of constraint constant
    coefficient = f.known_part.terms[i].coefficient
    f.known_part.constant +=
        coefficient * change.value_difference
    # Recalculate total constant and shift constraint set
    constant = f.variable_part.constant +
        f.known_part.constant
    MOI.set(model, MOI.ConstraintSet(), bridge.constraint,
            MOIU.shift_constant(bridge.set, -constant))
    return nothing
end

function MOI.modify(model::MOI.ModelLike, bridge::AffineDecisionConstraintBridge{T,S}, change::KnownValuesChange) where {T,S}
    f = bridge.decision_function
    known_val = zero(T)
    for term in f.known_part.terms
        known_val += term.coefficient * MOI.get(model, MOI.VariablePrimal(), term.variable_index)
    end
    # Update known part of constraint constant
    f.known_part.constant = known_val
    # Recalculate total constant and shift constraint set
    constant = f.variable_part.constant +
        f.known_part.constant
    MOI.set(model, MOI.ConstraintSet(), bridge.constraint,
            MOIU.shift_constant(bridge.set, -constant))
    return nothing
end


using Gurobi

const GRB_OPTIMIZER = Gurobi.Optimizer(OutputFlag=0)
MOI.set(GRB_OPTIMIZER, MOI.Silent(), true)
const GRB_CACHE = MOIU.UniversalFallback(MOIU.Model{Float64}())
const GRB_CACHED = MOIU.CachingOptimizer(GRB_CACHE, GRB_OPTIMIZER)
const GRB_BRIDGED = MOI.Bridges.full_bridge_optimizer(GRB_CACHED, Float64)

push!(solvers, (opt = GRB_BRIDGED, mode = BilevelJuMP.SOS1Mode()))
push!(solvers_quad, (opt = GRB_BRIDGED, mode = BilevelJuMP.SOS1Mode()))


"""
$(DocStringExtensions.README)
"""
module GalacticOptim

using DocStringExtensions
using Reexport
@reexport using SciMLBase
using Requires
using DiffResults
using Logging, ProgressLogging, Printf, ConsoleProgressMonitor, TerminalLoggers, LoggingExtras
using ArrayInterface, Base.Iterators
using Pkg

import SciMLBase: OptimizationProblem, OptimizationFunction, AbstractADType, __solve

@enum ObjSense MinSense MaxSense

include("utils.jl")
include("function/function.jl")

function __init__()
    # Optimization backends
    @require BlackBoxOptim="a134a8b2-14d6-55f6-9291-3336d3ab0209" include("solve/blackboxoptim.jl")
    @require CMAEvolutionStrategy="8d3b24bd-414e-49e0-94fb-163cc3a3e411" include("solve/cmaevolutionstrategy.jl")
    @require Evolutionary="86b6b26d-c046-49b6-aa0b-5f0f74682bd6" include("solve/evolutionary.jl")
    @require Flux="587475ba-b771-5e3f-ad9e-33799f191a9c" include("solve/flux.jl")
    @require GCMAES="4aa9d100-eb0f-11e8-15f1-25748831eb3b" include("solve/gcmaes.jl")
    @require MathOptInterface="b8f27783-ece8-5eb3-8dc8-9495eed66fee" include("solve/moi.jl")
    @require MultistartOptimization="3933049c-43be-478e-a8bb-6e0f7fd53575" include("solve/multistartoptimization.jl")
    @require NLopt="76087f3c-5699-56af-9a33-bf431cd00edd" include("solve/nlopt.jl")
    @require QuadDIRECT="dae52e8d-d666-5120-a592-9e15c33b8d7a" include("solve/quaddirect.jl")
    @require Optim="429524aa-4258-5aef-a3af-852621145aeb" include("solve/optim.jl")
    @require Nonconvex="01bcebdf-4d21-426d-b5c4-6132c1619978" begin
        @require ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" begin
            @require NonconvexBayesian="fb352abc-de7b-48de-9ebd-665b54b5d9b3" include("solve/nonconvex/nonconvex_bayesian.jl")
            @require NonconvexIpopt="bf347577-a06d-49ad-a669-8c0e005493b8" include("solve/nonconvex/nonconvex_ipopt.jl")
            @require NonconvexJuniper="611adb69-ebe7-45d0-83f5-90aabba2c123" include("solve/nonconvex/nonconvex_juniper.jl")
            @require NonconvexMMA="d3d89cbb-4ecd-4604-818d-8d1ff343e4da" include("solve/nonconvex/nonconvex_mma.jl")
            @require NonconvexMultistart="11b12826-7e46-4acf-9706-be0a67f2add7" include("solve/nonconvex/nonconvex_multistart.jl")
            @require NonconvexNLopt="b43a31b8-ff9b-442d-8e31-c163daa8ab75" include("solve/nonconvex/nonconvex_nlopt.jl")
            @require NonconvexPavito="75d5b151-dcdf-4236-8ef5-9c4e63ef33e2" include("solve/nonconvex/nonconvex_pavito.jl")
            @require NonconvexPercival="4296f080-e499-44ba-a02c-ae40015c44d5" include("solve/nonconvex/nonconvex_percival.jl")
            @require NonconvexSearch="75732972-a7cd-4375-b200-958e0814350d" include("solve/nonconvex/nonconvex_search.jl")
        end
    end
    @require Metaheuristics="bcdb8e00-2c21-11e9-3065-2b553b22f898" include("solve/metaheuristics.jl")
    @require NOMAD="02130f1c-4665-5b79-af82-ff1385104aa0" include("solve/nomad.jl")
    @require SpeedMapping="f1835b91-879b-4a3f-a438-e4baacf14412" include("solve/speedmapping.jl")

    # AD backends
    @require FiniteDiff="6a86dc24-6348-571c-b903-95158fe2bd41" include("function/finitediff.jl")
    @require ForwardDiff="f6369f11-7733-5829-9624-2563aa707210" include("function/forwarddiff.jl")
    @require ReverseDiff="37e2e3b7-166d-5795-8a7a-e32c996b4267" include("function/reversediff.jl")
    @require Tracker="9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c" include("function/tracker.jl")
    @require Zygote="e88e6eb3-aa80-5325-afca-941959d7151f" include("function/zygote.jl")
end

export solve

end # module


import RecipesBase

"""
    plot(s::SeisSplit.Result) -> ::Plots.Plot

Plot the results of shear wave splitting analysis from the `SeisSplit.splitting` function.
"""
plot

RecipesBase.@recipe function f(s::Result)
    grid --> false
    framestyle --> :box
    colorbar := false
    layout := 8
    title := hcat("Input", "Source pol. pre-corr", "Source pol. post-corr",
                  "Fast-slow pre-corr", "Fast-slow corr", "Small eigenvalue",
                  "PM pre-corr", "PM post-corr", "")
    titlefontsize --> 11
    legendfontsize --> 8

    # Input traces in N-E orientation
    n_orig, e_orig = Seis.rotate_through(s.trace1, s.trace2, -s.trace1.sta.azi)

    ## Input traces in original orientation with analysis window highlighted
    # Analysis window in colour behind traces
    # RecipesBase.@series begin
    #     subplot := 1
    #     w1, w2 =s.window_start, s.window_end
    #     []
    # end
    # Traces
    RecipesBase.@series begin
        subplot := 1
        label := [s.trace1.sta.cha s.trace2.sta.cha]
        Seis.times(s.trace1), [Seis.trace(s.trace1) Seis.trace(s.trace2)]
    end
    # Window limits
    RecipesBase.@series begin
        subplot := 1
        seriestype := :vline
        label := ""
        linecolor := :red
        linewidth := 1
        [s.window_start, s.window_end]
    end

    ## Rotated to spol
    RecipesBase.@series begin
        subplot := 2
        spol, spol_90 = Seis.cut.(Seis.rotate_through(n_orig, e_orig, 180-s.spol),
                                 s.window_start, s.window_end)
        label := ""
        linecolor := [:blue :red]
        Seis.times(spol), [Seis.trace(spol), Seis.trace(spol_90)]
    end

    ## Corrected and rotated to spol
    RecipesBase.@series begin
        subplot := 3
        label := ""
        linecolor := [:blue :red]
        n, e = deepcopy.((n_orig, e_orig))
        SeisSplit.apply_split!(Seis.cut!.((n, e), s.window_start, s.window_end)...,
                               s.phi_best, -s.dt_best)
        fast, slow = Seis.rotate_through(n, e, 180-s.spol)
        Seis.times(fast), [Seis.trace(fast), Seis.trace(slow)]
    end

    ## Fast and slow before dt shift
    sfast, sslow = Seis.cut.(Seis.rotate_through(n_orig, e_orig, s.phi_best),
                             s.window_start, s.window_end)
    RecipesBase.@series begin
        subplot := 4
        label := ""
        legend := false
        linecolor := [:blue :red]
        Seis.times(sfast), [Seis.trace(sfast), Seis.trace(sslow)]
    end

    ## Fast and slow after dt shift
    RecipesBase.@series begin
        subplot := 5
        legend := false
        linecolor := [:blue :red]
        xlims := (s.window_start, s.window_end)
        # Fake the shift by changing sample times
        [Seis.times(sfast), Seis.times(sslow) .- s.dt_best], [Seis.trace(sfast), Seis.trace(sslow)]
    end

    ## λ₂ surface
    # 10σ - 100σ contours
    RecipesBase.@series begin
        subplot := 6
        seriestype := :contour
        levels := 10:10:100
        linewidth := 1
        linecolor := :gray
        linealpha := 0.6
        s.dt, s.phi, s.lam2
    end

    # 1σ – 10σ contours
    RecipesBase.@series begin
        subplot := 6
        seriestype := :contour
        levels := 1:10
        linewidth := 1
        linecolor := :black
        s.dt, s.phi, s.lam2
    end
    # 1σ contour
    RecipesBase.@series begin
        subplot := 6
        seriestype := :contour
        levels := [1]
        linewidth := 3
        linecolor := :black
        s.dt, s.phi, s.lam2
    end
    # Best value
    RecipesBase.@series begin
        subplot := 6
        seriestype := :scatter
        xguide --> "dt / s"
        yguide --> "phi / deg"
        xlims := extrema(s.dt)
        ylims := extrema(s.phi)
        label := ""
        [s.dt_best], [s.phi_best]
    end

    ## Particle motion before correction
    amax = 1.2*maximum(x -> sqrt(x[1]^2 + x[2]^2), zip(Seis.trace(e_orig), Seis.trace(n_orig)))
    RecipesBase.@series begin
        subplot := 7
        aspect_ratio := :equal
        label := ""
        linecolor --> :black
        xguide := "East"
        yguide := "North"
        xticks --> nothing
        yticks --> nothing
        xlims := (-amax, amax)
        ylims := (-amax, amax)
        Seis.trace(Seis.cut(e_orig, s.window_start, s.window_end)),
            Seis.trace(Seis.cut(n_orig, s.window_start, s.window_end))
    end

    ## Particle motion after correction
    RecipesBase.@series begin
        subplot := 8
        aspect_ratio := :equal
        label := ""
        linecolor --> :black
        xguide := "East"
        yguide := "North"
        xticks --> nothing
        yticks --> nothing
        xlims := (-amax, amax)
        ylims := (-amax, amax)
        n, e = deepcopy.((n_orig, e_orig))
        SeisSplit.apply_split!(n, e, s.phi_best, -s.dt_best)
        Seis.cut!.((n, e), s.window_start, s.window_end)
        Seis.trace(e), Seis.trace(n)
    end

    ## Empty plot at bottom right
    # RecipesBase.@series begin
    #     subplot := 9
    #     framestyle := :empty
    #     nothing
    # end
end


#! /usr/bin/julia

# Rosetta Code, SHA-1

using Nettle

function sha1sum(s::String)
    bytes2hex(sha1_hash(s))
end

println("Testing SHA-1 function against FIPS 180-1")

s = sha1sum("abc")
t = lowercase("A9993E364706816ABA3E25717850C26C9CD0D89D")
print("    \"abc\" should yield ", t)
if s == t
    println(", and it does.")
else
    println(", but it yields ", s)
end

s = sha1sum("abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq")
t = lowercase("84983E441C3BD26EBAAE4AA1F95129E5E54670F1")
print("    \"abcdbc...\" should yield ", t)
if s == t
    println(", and it does.")
else
    println(", but it yields ", s)
end

s = sha1sum("a"^10^6)
t = lowercase("34AA973CD4C4DAA4F61EEB2BDBAD27316534016F")
print("    a million \"a\"s should yield ", t)
if s == t
    println(", and it does.")
else
    println(", but it yields ", s)
end

println("\nAlso")
s = "Rosetta Code"
h = sha1sum(s)
println("    ", s, " => ", h)


using BayesNets
using Discretizers
using JLD

basepath = "./results/ByIntersection/"
doingPeach = false
testnums = ["111","011"]#"000","001","010","011", "100"]
intersections = collect(1:9)
test_intersections = collect(1:9)
doSubtest = false #2500 features only

function makePaths(basepath, testnums, intersections)
    paths = []
    for testnum in testnums
        for intersection in intersections
            push!(paths, string(basepath, testnum, "/", intersection))
        end
    end
    return paths
end

paths = makePaths(basepath, testnums, intersections)

typeToEdges = Dict(
    "V"=>[0.0,0.5,20,40,60],
    "A"=>[-5000,-20,-5,-0.5,0.5,5,20,5000],
    "dist" => [0.0, 5, 25, 100, 500, 10000],
    "hdwy" => [0.0, 5, 25, 100, 250, 10000],)
idToType = Dict(
    3=>"V",
    4=>"A",
    5=> "yaw",
    #6 is indicator
    7=> "hdwy",
    8=> "dist",
)

#returns allFeatures, allTargets dictionaries wheren "testnum/intersection" are the keys
function loadAllFeaturesTargets(paths)
    allFeatures = Dict()
    allTargets = Dict()
    allFids = Dict()
    nFids = 0
    numFeatures = 0
    for path in paths #path is result/ByIntersection/yyy/x
        i = path[end-4:end]
        f = open("$path/featuresAndTargets")
        allFeatures[i] = readdlm(f)
        allTargets[i] = allFeatures[i][:,end]  #nextMove is the last thing only
        allFeatures[i] = allFeatures[i][:,1:end-3]

        if doSubtest
            allFeatures[i] = allFeatures[i][1:2500,:]
            allTargets[i] = allTargets[i][1:2500,:]
        end
        close(f)

        if numFeatures == 0
            numFeatures = length(allFeatures[i][1,:])
        end
    end
    return allFeatures, allTargets, numFeatures
end

function makeDiscretizedThings(trainlines, testlines, trainactuals, testactuals, numFeatures,
                                           indexToID, numYawBins, moveIndex)
  allLines = vcat(trainlines, testlines)
  allactuals = vcat(trainactuals, testactuals)
  discAllLines = zeros(Int64, (length(trainlines[:,1])+length(testlines[:,1]), numFeatures+1))
  discTrainlines = zeros(Int64, (length(trainlines[:,1]), numFeatures+1))
  discTestlines = zeros(Int64, (length(testlines[:,1]), numFeatures+1))
  for k in 1:numFeatures
    id = indexToID[k]
    if haskey(idToType, id) #if not, already discretized
      F_type = idToType[id]
      if haskey(typeToEdges, F_type)
        Disc = LinearDiscretizer(typeToEdges[F_type])
      else
        Disc = LinearDiscretizer(binedges(DiscretizeUniformWidth(numYawBins),allLines[:,k]))
      end
      DiscretizedAll = encode(Disc, allLines[:,k])
      DiscretizedTrain = encode(Disc, trainlines[:,k])
      DiscretizedTest = encode(Disc, testlines[:,k])
    else
      #println("index ", k, " has no id, aka already disc")
      DiscretizedAll = allLines[:,k]
      DiscretizedTrain = trainlines[:,k]
      DiscretizedTest = testlines[:,k]
    end
    discTrainlines[:,k] = encode(CategoricalDiscretizer(DiscretizedAll), DiscretizedTrain)
    discTestlines[:,k] = encode(CategoricalDiscretizer(DiscretizedAll), DiscretizedTest)
    discAllLines[:,k] = encode(CategoricalDiscretizer(DiscretizedAll), DiscretizedAll)
  end
  moveDiscretizer = CategoricalDiscretizer(trainactuals)
  discTrainlines[:,moveIndex] = encode(moveDiscretizer,trainactuals)
  discTestlines[:,moveIndex] = encode(moveDiscretizer,testactuals)
  discAllLines[:,moveIndex] = encode(moveDiscretizer,allactuals)

  alldata = convert(DataFrame, discAllLines) #used to find num bins
  traindata = convert(DataFrame, discTrainlines)
  testdata = convert(DataFrame, discTestlines)
  return alldata, traindata, testdata, moveDiscretizer
end

function makeIndexToID(numFeatures, testnum, laneTypeEncodedLen)
    useLaneType = (testnum[1] == '1')
    useHist = (testnum[2] == '1')
    useTraffic = (testnum[3] == '1')
    println(testnum, useLaneType, useHist, useTraffic)
    indexToID = zeros(Int64, (1, numFeatures+1+laneTypeEncodedLen))   #+1 for when no lane type, shift needs buffer
    coreLen = 8
    if useLaneType
        coreLen += laneTypeEncodedLen
    end
    histLen = 9 #indicator is first
    numHist = 4
    totHist = histLen * numHist
    traffLen = 8
    #base = lane, lane, v, a, yaw, 1{hdwy}, hdwy, dist
    #hist is the same, 4 times
    #traffic is dx, dy, v, a, yaw, 1{hdwy}, hdwy
    for index in 1:numFeatures
        ID = index
        if index <= coreLen
          if useLaneType
            if index > 4
              ID -= laneTypeEncodedLen
            else #laneTypeEncoding features are 0 ID
              ID = 0
            end
          end
          ID += 1 #because all others get shifted, limits amount of confusion i think
        elseif useHist == false
          ID = ((index - coreLen) % traffLen)
          if ID == 0
            ID = traffLen
          elseif ID == 2 || ID == 3
            ID = 9 #distance + 1
          end
        elseif useHist && index <= coreLen + totHist
            ID = ((index - coreLen) % histLen)
            if ID == 0
                ID = histLen
            end
        else  #use hist and in traffic section
          ID = ((index - (coreLen + totHist)) % traffLen)
          if ID == 0
            ID = traffLen
          elseif ID == 2 || ID == 3
            ID = 9 #distance + 1
          end
        end
        ID -= 1 #shift for indicator at front of each traff
        indexToID[index] = ID
    end
    #for i in 1:numFeatures
    #    id = indexToID[i]
    #    println("index: ",i, "id: ", id)
    #    if haskey(idToType, id)
    #        println("key:", idToType[id])
    #    end
    #end
    return indexToID
end

# fit function --- modified from https://github.com/sisl/BayesNets.jl/blob/master/src/DiscreteBayesNet/greedy_hill_climbing.jl
function Distributions.fit(::Type{DiscreteBayesNet}, data::DataFrame, params::GreedyHillClimbing, indexOfOut::Int;
    ncategories::Vector{Int} = map!(i->infer_number_of_instantiations(data[i]), Array(Int, ncol(data)), 1:ncol(data)),
    )

    n = ncol(data)
    parent_list = map!(i->Int[], Array(Vector{Int}, n), 1:n)
    datamat = convert(Matrix{Int}, data)'
    score_components = bayesian_score_components(parent_list, ncategories, datamat, params.prior, params.cache)

    while true
      best_diff = 0.0
      best_parent_list = parent_list
      i = indexOfOut
      parent_list[i] = collect(1:n)
      break
      #for i in 1:n #this is added for full fitting
        # 1) add an edge (j->i)
        if length(parent_list[i]) < params.max_n_parents
            for j in deleteat!(collect(1:n), parent_list[i])
                    if adding_edge_preserves_acyclicity(parent_list, j, i)
                        new_parents = sort!(push!(copy(parent_list[i]), j))
                        new_component_score = bayesian_score_component(i, new_parents, ncategories, datamat, params.prior, params.cache)
                    if new_component_score - score_components[i] > best_diff
                        best_diff = new_component_score - score_components[i]
                        best_parent_list = deepcopy(parent_list)
                        best_parent_list[i] = new_parents
                    end
                end
             end
        end

        # 2) remove an edge
        for (idx, j) in enumerate(parent_list[i])

            new_parents = deleteat!(copy(parent_list[i]), idx)
            new_component_score = bayesian_score_component(i, new_parents, ncategories, datamat, params.prior, params.cache)
            if new_component_score - score_components[i] > best_diff
                best_diff = new_component_score - score_components[i]
                best_parent_list = deepcopy(parent_list)
                best_parent_list[i] = new_parents
            end
            # 3) flip an edge
            new_parent_list = deepcopy(parent_list) # TODO: make this more efficient
            deleteat!(new_parent_list[i], idx)

            if adding_edge_preserves_acyclicity(new_parent_list, i, j)
                sort!(push!(new_parent_list[j], i))
                new_diff = bayesian_score_component(i, new_parent_list[i], ncategories, datamat, params.prior,
                                                       params.cache) - score_components[i]
                new_diff += bayesian_score_component(j, new_parent_list[j], ncategories, datamat, params.prior,
                                                       params.cache) - score_components[j]
                if new_diff > best_diff
                    best_diff = new_diff
                    best_parent_list = new_parent_list
                end
            end
        end

        if best_diff > 0.0
            parent_list = best_parent_list
            score_components = bayesian_score_components(parent_list, ncategories, datamat, params.prior, params.cache)
        else
            break
        end
      #end   #this is for the "for i in 1:n"
    end
    # construct the BayesNet
    cpds = Array(DiscreteCPD, n)
    varnames = names(data)
    for j in 1:n
        name = varnames[j]
        parents = varnames[parent_list[j]]
        cpds[j] = Distributions.fit(DiscreteCPD, data, name, parents, params.prior,
                      parental_ncategories=ncategories[parent_list[j]],
                      target_ncategories=ncategories[j])
    end
    #println(cpds)
    println("Parent List: ", parent_list)
    BayesNet(cpds)
end

#big for loop
println("Loading features/targets for BN")
allFeatures, allTargets, numFeatures = loadAllFeaturesTargets(paths)
println("Done loading features/targets for BN")
println("NumFeatures == ", numFeatures)
for inter in test_intersections
  for testnum in testnums
      i = string(testnum, "/", inter)
      scorename = "Ypred_BN"
      scorePath = string("$basepath",testnum, "/TestOn$inter","/$scorename")
      f = open(scorePath, "w")
      #truncate score file
      close(f)
      moveCats = [1,2,3]
      numNextMoves = length(moveCats)
      useLaneType = false
      if testnum[1] == "1"
        useLaneType = true
      end
      laneTypeIndex = 3
      laneTypeEncodedLen = length([0,0,0,0])
      numYawBins = 7
      #truncate the log file
      numFeatures = length(allFeatures[i][1,:])
      indexToID = makeIndexToID(numFeatures, testnum, laneTypeEncodedLen)

      println("index to ID dict: \n ",indexToID)

      moveIndex = numFeatures+1

      println("Done making discretizer dictionaries")
      println("Starting feature/target formatting for CV #: $i")
      testlines = allFeatures[i]
      testactuals = allTargets[i]

      trainlines = zeros(Float64, (1,numFeatures))
      trainactuals = zeros(Float64, (1,numFeatures))
      first = true
      for other_inter in intersections
        if other_inter != inter  #the train lines are all that are not test
          j = string(testnum, "/", other_inter)
          if first == true
            trainlines = allFeatures[j]
            trainactuals = allTargets[j]
            first = false
          else
            trainlines = vcat(trainlines, allFeatures[j])
            trainactuals = vcat(trainactuals, allTargets[j])
          end
        end
      end
      println("Done feature/target formatting for CV #: $i")

      println("Starting to discretize for CV #: $i")
      alldata, traindata, testdata, moveDiscretizer = makeDiscretizedThings(trainlines, testlines, trainactuals, testactuals,
                                                           numFeatures, indexToID, numYawBins, moveIndex)

      println("Done discretizing for CV #: $i")
      println("Starting to fit BN for CV #: $i")
      max_parents = numFeatures
      #max_parents = min(numFeatures, max_parents)
      println("Max parents: ", max_parents)

      params = GreedyHillClimbing(ScoreComponentCache(traindata), max_n_parents=max_parents, prior=UniformPrior())
      println("Done fitting params, starting to fit BN")

      num_bins_all = map!(i->infer_number_of_instantiations(alldata[i]), Array(Int, ncol(alldata)), 1:ncol(alldata))
      bnDis = @time fit(DiscreteBayesNet, traindata, params, moveIndex; ncategories=num_bins_all)
      println("Done fitting BN for CV #: $i")
      save(string("$basepath$i","/BN_model.jld"), "bnDis", bnDis)
      score = 0
      numNaN = 0
      nrows = length(testlines[:,1])
      moveCats = unique(collect(testdata[:,moveIndex]))
      numNextMoves = 3#length(moveCats)
      numRight = 0
      for index in 1:nrows
          actualNextMove = convert(Int64,decode(moveDiscretizer,testdata[index,moveIndex]))
          p_dists = zeros(Float64, (1, numNextMoves))
          for move in moveCats
            featureline = testdata[index,:]
            featureline[moveIndex] = encode(moveDiscretizer, move)
            p_dists[move] = pdf(bnDis, featureline)
          end
          for x in p_dists
            if !(x > 1) && !(x < 1) && !(x == 1)
              p_dists = [1.0/numNextMoves for i in 1:numNextMoves]
              numNaN += 1
              break
            end
          end
          p_dists = p_dists ./ sum(p_dists)
          p_right = p_dists[actualNextMove]
          score += (1 - p_right)
          if p_right == maximum(p_dists)
              numRight += 1
          end
          scoref = open(scorePath, "a")
          writedlm(scoref, p_dists)
          close(scoref)
      end
      println("Score:", score)
      println("Num Right:", numRight)
      println("Num Predicitons:", nrows)
      println("Num NaN:", numNaN)
      scoref = open("$basepath$i/$scorename", "a")
      #writedlm(scoref, score)
      #writedlm(scoref, numNaN)
      close(scoref)
      println("Done scoring BN for CV #: $i")
    end #end over testnums
    println("Done with intersection", inter)
end  #end over intersections


"""
This is a sample input file for the CloudScat2 code.  To use it run julia
and type

julia> include("scan_depths.jl")

The output file will be based on this file's name so to run different simulations
you can just copy this file, change the configuration and run it as above with
the new file name.  You can also run from the command line as e.g.

> julia scan_depths.jl

- - -
2019 Alejandro Luque IAA-CSIC (aluque@iaa.es)

"""

using CloudScat
using StaticArrays
using Printf
using Logging
using Dates
using Formatting

const co = CloudScat.constants

function runone(lmbd, radius_um, h_km)
    params = Params(
        # NUmber of simulated photons
        N = 5e6,

        # The wavelength of the scattered light
        λ = lmbd * co.nano,

        # The source is defined as a straight line running from source_a
        # to source_b.  Within this line photons are distributed uniformly with
        # initially random, isotropic directions.  To use a point source set
        # source_b = source_a
        source_a = @SVector([0, 0, h_km * co.kilo]),
        source_b = @SVector([0, 0, h_km * co.kilo]),

        zmin = 13 * co.kilo)

    # World geometry
    # Describe the cloud geometry here.  It may consist in combinations of
    # geometrical figures (cylinders, spheres, cones, semi-planes)
    cloud = Cylinder(7 * co.kilo, 15 * co.kilo, 0, 0, 10 * co.kilo)

    # Geometry of the full domain.  Photons outside the domain are
    # immediately discarded.
    domain = Cylinder(7 * co.kilo, 60 * co.kilo, 0, 0, 100 * co.kilo)

    # Set a homogeneous cloud droplet density and radius.
    # See variable_droplet_radius.jl for how to deal with inhomogeneous clouds.
    # Set a homogeneous cloud droplet density and radius.
    radius = 10e-6

    # Density of the scattering centers.
    nscat = 100 * co.centi^-3

    composition = Homogeneous(params.λ, nscat, radius)

    # Define the full simulation world
    world = World(cloud, domain, composition)

    # Observers record a time curve and an image of received light.  You can
    # set up as many observers as you wish but they have a significant impact
    # on performance.
    observers = [Observer(
        # Location of the observer
        position = [0, 0, 400 * co.kilo],

        # Sampling interval
        tsample = 1e-5,

        # Collected samples
        nsamples = 10000,

        # fov of the camera (half of the diagonal fov).
        fov = 40,

        # camera pixels
        pixels = 1024)]

    basename = splitext(splitdir(@__FILE__)[2])[1]

    # The saveto option specifies the output file.  Leave it as it is to base
    # the filename on the name of this file specifying the parameters.
    CloudScat.main(params, world, observers,
                    saveto=@sprintf("%s_%d_%02dum_%02dkm.h5",
                                    basename, lmbd, radius_um, h_km))
end

function run()
    for lmbd in [337, 777]
        for radius_um in [10, 20]
            for h_km in 8:14
                runone(lmbd, radius_um, h_km)
            end
        end
    end
end

isinteractive() || run()


module CudaNNUtils

using CUDArt
using Quaternions
using QuaternionArgs
import Base: transpose, ctranspose

const ptxdict = Dict()
const mdlist = Array(CuModule, 0)
const block_default = 128

function init(devlist)
    global ptxdict
    global mdlist
    isempty(mdlist) || return
    for dev in devlist
        device(dev)
        mod_dir = joinpath(dirname(@__FILE__), "kernels")
        mod_path = joinpath(mod_dir, "kernels.ptx")
        md = CuModule(mod_path, false)
        ptxdict[(dev, "equiv", Complex{Float32}, Quaternion{Float32})] = CuFunction(md, "q2c_float")
        ptxdict[(dev, "equiv", Complex{Float64}, Quaternion{Float64})] = CuFunction(md, "q2c_double")
        ptxdict[(dev, "equiv", Quaternion{Float32}, Complex{Float32})] = CuFunction(md, "c2q_float")
        ptxdict[(dev, "equiv", Quaternion{Float64}, Complex{Float64})] = CuFunction(md, "c2q_double")
        ptxdict[(dev, "actfunc", Float32)] = CuFunction(md, "actfunc_real_float")
        ptxdict[(dev, "actfunc", Float64)] = CuFunction(md, "actfunc_real_double")
        ptxdict[(dev, "actfunc", Complex{Float32})] = CuFunction(md, "actfunc_comp_float")
        ptxdict[(dev, "actfunc", Complex{Float64})] = CuFunction(md, "actfunc_comp_double")
        ptxdict[(dev, "actfunc", Quaternion{Float32})] = CuFunction(md, "actfunc_quat_float")
        ptxdict[(dev, "actfunc", Quaternion{Float64})] = CuFunction(md, "actfunc_quat_double")
        ptxdict[(dev, "actfunc_multi", Complex{Float32})] = CuFunction(md, "actfunc_comp_multi_float")
        ptxdict[(dev, "actfunc_multi", Complex{Float64})] = CuFunction(md, "actfunc_comp_multi_double")
        ptxdict[(dev, "actfunc_multi", Quaternion{Float32})] = CuFunction(md, "actfunc_quat_multi_float")
        ptxdict[(dev, "actfunc_multi", Quaternion{Float64})] = CuFunction(md, "actfunc_quat_multi_double")
        ptxdict[(dev, "fill", Float32)] = CuFunction(md, "fill_real_float")
        ptxdict[(dev, "fill", Float64)] = CuFunction(md, "fill_real_double")
        ptxdict[(dev, "fill", Complex{Float32})] = CuFunction(md, "fill_comp_float")
        ptxdict[(dev, "fill", Complex{Float64})] = CuFunction(md, "fill_comp_double")
        ptxdict[(dev, "fill", Quaternion{Float32})] = CuFunction(md, "fill_quat_float")
        ptxdict[(dev, "fill", Quaternion{Float64})] = CuFunction(md, "fill_quat_double")
        ptxdict[(dev, "compare", Int32)] = CuFunction(md, "compare_real_int")
        ptxdict[(dev, "compare", Float32)] = CuFunction(md, "compare_real_float")
        ptxdict[(dev, "compare", Float64)] = CuFunction(md, "compare_real_double")
        ptxdict[(dev, "compare", Complex{Float32})] = CuFunction(md, "compare_comp_float")
        ptxdict[(dev, "compare", Complex{Float64})] = CuFunction(md, "compare_comp_double")
        ptxdict[(dev, "compare", Quaternion{Float32})] = CuFunction(md, "compare_quat_float")
        ptxdict[(dev, "compare", Quaternion{Float64})] = CuFunction(md, "compare_quat_double")
        ptxdict[(dev, "transpose_pre", Complex{Float32})] = CuFunction(md, "transpose_pre_float")
        ptxdict[(dev, "transpose_pre", Complex{Float64})] = CuFunction(md, "transpose_pre_double")
        ptxdict[(dev, "transpose", Quaternion{Float32})] = CuFunction(md, "transpose_float")
        ptxdict[(dev, "transpose", Quaternion{Float64})] = CuFunction(md, "transpose_double")
        ptxdict[(dev, "ctranspose", Quaternion{Float32})] = CuFunction(md, "ctranspose_float")
        ptxdict[(dev, "ctranspose", Quaternion{Float64})] = CuFunction(md, "ctranspose_double")
        push!(mdlist, md)
    end
end

function close()
    for md in mdlist
        unload(md)
    end
    empty!(mdlist)
    empty!(ptxdict)
end

function equiv!{T}(dst::CudaArray{Complex{T}}, src::CudaArray{Quaternion{T}})
    @assert(length(dst) == (4*length(src)))
    dev = device(dst)
    w = size(src,1)
    h = size(src,2)
    block = (16,16)
    grid = (ceil(Int,w/16),ceil(Int,h/16))
    func = ptxdict[(dev, "equiv", Complex{T}, Quaternion{T})]
    CUDArt.launch(func, grid, block, (dst, src, w, h))
end

function equiv!{T}(dst::CudaArray{Quaternion{T}}, src::CudaArray{Complex{T}})
    @assert((4*length(dst)) == length(src))
    dev = device(dst)
    w = size(dst,1)
    h = size(dst,2)
    block = (16,16)
    grid = (ceil(Int,w/16),ceil(Int,h/16))
    func = ptxdict[(dev, "equiv", Quaternion{T}, Complex{T})]
    CUDArt.launch(func, grid, block, (dst, src, w, h))
end

function equiv{T}(src::CudaArray{Quaternion{T}})
    w = size(src,1)
    h = size(src,2)
    dst = CudaArray(Complex{T}, w<<1, h<<1)
    equiv!(dst, src)
    return dst
end

function equiv{T}(src::CudaArray{Complex{T}})
    w = size(src,1)
    h = size(src,2)
    dst = CudaArray(Quaternion{T}, w>>1, h>>1)
    equiv!(dst, src)
    return dst
end

function actfunc!{T}(data::CudaArray{T}; block=block_default)
    dev = device(data)
    N = length(data)
    grid = ceil(Int,N/block)
    func = ptxdict[(dev, "actfunc", T)]
    CUDArt.launch(func, grid, block, (data,N))
end

function actfunc_multi!{T<:Complex}(data::CudaArray{T}, K::Integer; block=block_default)
    dev = device(data)
    N = length(data)
    grid = ceil(Int,N/block)
    func = ptxdict[(dev, "actfunc_multi", T)]
    CUDArt.launch(func, grid, block, (data,K,N))
end

function actfunc_multi!{T<:Quaternion}(data::CudaArray{T}, A::Integer, B::Integer, C::Integer; block=block_default)
    dev = device(data)
    N = length(data)
    grid = ceil(Int,N/block)
    func = ptxdict[(dev, "actfunc_multi", T)]
    CUDArt.launch(func, grid, block, (data,A,B,C,N))
end

function fill!{T}(data::CudaArray{T}, val::T; block=block_default)
    dev = device(data)
    N = length(data)
    grid = ceil(Int,N/block)
    func = ptxdict[(dev, "fill", T)]
    CUDArt.launch(func, grid, block, (data,val,N))
end

function fill!{T}(data::CudaArray{T}, val::T; block=block_default)
    dev = device(data)
    N = length(data)
    grid = ceil(Int,N/block)
    func = ptxdict[(dev, "fill", T)]
    CUDArt.launch(func, grid, block, (data,val,N))
end

function transpose_pre!{T}(dst::CudaArray{T}; block=block_default)
    dev = device(dst)
    w = size(dst,1)
    h = size(dst,2)
    block = (16,16)
    grid = (ceil(Int,w/16),ceil(Int,h/16))
    func = ptxdict[(dev, "transpose_pre", T)]
    CUDArt.launch(func, grid, block, (dst, w, h))
end

function transpose!{T<:Quaternion}(dst::CudaArray{T}, src::CudaArray{T}; block=block_default)
    dev = device(dst)
    w = size(dst,1)
    h = size(dst,2)
    block = (16,16)
    grid = (ceil(Int,w/16),ceil(Int,h/16))
    func = ptxdict[(dev, "transpose", T)]
    CUDArt.launch(func, grid, block, (dst, src, w, h))
end

function transpose{T<:Quaternion}(src::CudaArray{T}; block=block_default)
    w = size(src,1)
    h = size(src,2)
    dst = CudaArray(T, h,w)
    transpose!(dst, src)
    return dst
end

function ctranspose!{T<:Quaternion}(dst::CudaArray{T}, src::CudaArray{T}; block=block_default)
    dev = device(dst)
    w = size(dst,1)
    h = size(dst,2)
    block = (16,16)
    grid = (ceil(Int,w/16),ceil(Int,h/16))
    func = ptxdict[(dev, "ctranspose", T)]
    CUDArt.launch(func, grid, block, (dst, src, w, h))
end

function ctranspose{T<:Quaternion}(src::CudaArray{T}; block=block_default)
    w = size(src,1)
    h = size(src,2)
    dst = CudaArray(T, h,w)
    ctranspose!(dst, src)
    return dst
end
end # module


import Base: hash,
    string, show, tryparse,
    Float16, Float32, Float64, BigFloat,
    Bool, size, axes

import Base: precision, exponent_mask, significand_mask, eps, exponent, significand,
    zero, one, floatmax, floatmin, nextfloat, prevfloat, frexp, ldexp, modf, decompose,
    ==, <, <=, >=, >, !=, isless, isequal, cmp,
    iszero, isone, isfinite, isinteger, isinf, isnan, issubnormal,
    signbit, sign, flipsign, copysign,
    +, -, *, /, \, abs, inv, sqrt, cbrt,
    mod, mod1, rem, fld, fld1, div, cld, fldmod, fldmod1, divrem,
    ceil, floor, trunc, round,
    muladd, fma, hypot, clamp,
    sum, prod, cumsum

import Base.Math: log, log1p, log10, log2, exp, expm1, exp10, exp2,
    sin, cos, tan, csc, sec, cot, sinpi, cospi,
    asin, acos, atan, acsc, asec, acot,
    sinh, cosh, tanh, csch, sech, coth,
    asinh, acosh, atanh, acsch, asech, acoth

import LinearAlgebra: norm, normalize, dot,
    *, det, tr, inv, lu, qr, factorize,
    svdvals, eigvals, eigvecs

#=
import SpecialFunctions: export
    airyai,
    airyaiprime,
    airybi,
    airybiprime,
    airyaix,
    airyaiprimex,
    airybix,
    airybiprimex,
    besselh,
    besselhx,
    besseli,
    besselix,
    besselj,
    besselj0,
    besselj1,
    besseljx,
    besselk,
    besselkx,
    bessely,
    bessely0,
    bessely1,
    besselyx,
    dawson,
    ellipk,
    ellipe,
    erf,
    erfc,
    erfcinv,
    erfcx,
    erfi,
    erfinv,
    eta,
    digamma,
    invdigamma,
    polygamma,
    trigamma,
    gamma_inc,
    beta_inc,
    beta_inc_inv,
    gamma_inc_inv,
    hankelh1,
    hankelh1x,
    hankelh2,
    hankelh2x,
    zeta,
    sinint,
    cosint,
    lbinomial
=#


###################################
## Simulation models ##
###################################

function count_active_agents(agents::Vector{Agent})
    r = 0
    @simd for i = eachindex(agents)
        @inbounds r += agents[i].active
    end
    r
end

"""
`simulation_run` run traffic simulation in two modes.

**Input parameters**
* `mode` : simulation model switch
    * `:base` : simulation with regular agents only
    * `:smart` : simulation with smart agents enabled
* `OSMmap` : OpenStreetMapX MapData object with road network data
* `inAgents` : set of agents created with generate_agents function
* `density_factor` : road length reserved for one vehicle
* `k_routes_dict` : dictionary with multiple shortest paths (values) between vertices (keys)
* `U` : period of weights updates
* `T` : distribution parameter in k-shortest path rerouting
* `k` : number of fastest routes generated in rerouting function
* `debug` : debug messages switch
* `track_avg_speeds` : return average speeds on edges if true
"""
function simulation_run(mode::Symbol,
                        OSMmap::MapData,
                        inAgents::Vector{Agent},
                        density_factor::Float64 = 5.0,
                        k_routes_dict::Dict{Tuple{Int,Int},Array{Vector{Int}}}=
                        Dict{Tuple{Int,Int},Array{Vector{Int}}}(),
                        U::Int = 100,
                        T::Float64 = 1.0,
                        k::Int = 3,
                        debug::Bool = false;
                        track_avg_speeds::Bool = false)
    if !in(mode, [:base,:smart]) error("Wrong mode specified.") end
    Agents = deepcopy(inAgents) #Creating working copy of agents
    Agents[1].start_time = 0.0
    Agents[1].active = true
    max_densities, max_speeds = traffic_constants(OSMmap, density_factor) #Traffic characteristic constants
    densities, speeds = init_traffic_variables(OSMmap, Agents) #Traffic characteristic variables
    update_weights!(speeds, densities, max_densities, max_speeds) #Initial speeds update
    curr_edges = [a.edge for a in Agents] #Initialize edges list
    if track_avg_speeds
        avg_speeds = deepcopy(speeds)
        tick = 1
    end
    #Initialize simulation variables
    simtime = 0.0
    steps = 0
    active = length(Agents)
    #Calculate initial time to nearest junction for all agents
    times_to_edge = next_edge(Agents, speeds, OSMmap.w)
    times_to_start = [a.start_time for a in Agents] #Get agents starting times
    times_to_start[1] = Inf
    event_update = Inf
    #Loop until all agents are deactivated
    while active != 0 || !all(times_to_start.==Inf)
        steps += 1
        #Calculate next event time and update sim clock
        event_edge, IDe = findmin(times_to_edge)
        event_start, IDs = findmin(times_to_start)
        if mode == :smart
            event_update = next_update = (simtime ÷ U + 1) * U - simtime
        end
        min_event, event_type = findmin([event_edge, #1
                                        event_start, #2
                                        event_update]) #3
        simtime += min_event #Increase total simulation time
        times_to_edge .-= min_event #Move agents forward to event time
        times_to_start .-= min_event
        if event_type == 1 #Switching edges
            vAgent = Agents[IDe] #Pick agent for which event is occuring
            #Process agent for which event is occuring
            changed_edges = update_event_agent!(vAgent, simtime, densities, OSMmap.v)
            length(changed_edges) == 1 ? curr_edges[IDe] = (0,0) : curr_edges[IDe] = changed_edges[2]
            #Update speeds and correct events time
            update_weights_and_events!(Agents, curr_edges, times_to_edge, speeds,
                changed_edges, densities, max_densities, max_speeds)
            vAgent.active ? times_to_edge[IDe] = next_edge(vAgent, speeds, OSMmap.w) : times_to_edge[IDe] = Inf
        elseif event_type == 2 #New agent joins traffic
            Agents[IDs].active = true
            times_to_start[IDs] = Inf
            times_to_edge[IDs] = next_edge(Agents[IDs], speeds, OSMmap.w)
            c_edge = Agents[IDs].edge
            haskey(densities, c_edge) ? densities[c_edge] += 1 : densities[c_edge] = 1
        elseif event_type == 3 #Velocities update
            if debug
                update = Int(simtime ÷ U)
                smartAgents = sum([a.smart*a.active for a in Agents])
                print("Update: $(update) | Smart agents: $(smartAgents) ")
            end
            #Process rerouting for smart agents
            for a in Agents
                a.smart && a.active &&
                k_shortest_path_rerouting!(OSMmap, k_routes_dict, a, speeds, max_speeds, k, T, U)
            end
            debug && println("Finished")
        end
        #Update average speeds every 30 secs
        if track_avg_speeds && simtime > tick*30
            #Iterative mean
            tick +=1
            avg_speeds = ((tick-1)/tick)*avg_speeds+(1/tick)*speeds
        end
        active = count_active_agents(Agents)
    end
    times = getfield.(Agents,:travel_time)
    if track_avg_speeds
        output_tuple = (Steps = steps,
                    Simtime = simtime,
                    TravelTimes = times,
                    AvgSpeeds = avg_speeds)
    else
        output_tuple = (Steps = steps,
                    Simtime = simtime,
                    TravelTimes = times)
    end
    return output_tuple
end

"""
`gather_statistics` return various measures describing TMS model quality

**Input parameters**
* `smart_ind` : vector with smart agent indicators
* `times_base` : agents travelling time in base scenario
* `times_smart` : agents travelling time in smart scenario
"""
function gather_statistics(smart_ind::BitArray{1},
                            times_base::Vector{Float64},
                            times_smart::Vector{Float64})
    overall_time = (sum(times_base) - sum(times_smart))/sum(times_base)
    smart_time = (sum(times_base[smart_ind]) - sum(times_smart[smart_ind]))/sum(times_base[smart_ind])
    other_time = (sum(times_base[.!smart_ind]) - sum(times_smart[.!smart_ind]))/sum(times_base[.!smart_ind])
    statistics_tuple = (
        overall_time = round(overall_time, digits=3),
        smart_time = round(smart_time, digits=3),
        other_time = round(other_time, digits=3),
        avg_base = round(StatsBase.mean(times_base), digits=2),
        avg_overall_V2I = round(StatsBase.mean(times_smart), digits=2),
        avg_smart_V2I = round(StatsBase.mean(times_smart[smart_ind]), digits=2),
        avg_regular_V2I = round(StatsBase.mean(times_smart[.!smart_ind]), digits=2)
                        )
    return statistics_tuple
end

"""
`run_parameter_analysis` run base and V2I simulation scenario to compare agents performance, returns comparison statistics

**Input parameters**
* `GridElement` : element from parameters grid to be used in simulation
* `ParamGrid` : array with parameters values
* `Start` : vector of areas from which agents randomly pick starting point
* `End` : vector of areas from which agents randomly pick ending point
* `density_factor` : road length reserved for one vehicle
* `K_Paths_Dict` : dictionary with multiple shortest paths (values) between vertices (keys)
* `mapdata` : OpenStreetMapX MapData object with road network data
"""
function run_parameter_analysis(GridElement::Int,
                                ParamGrid::Vector{Tuple{Int,Float64,Int,Int,Float64,Int,Float64}},
                                Start::Vector{Rect}, End::Vector{Rect}, density_factor::Float64,
                                K_Paths_Dict::Dict{Tuple{Int,Int},Array{Vector{Int}}},
                                mapdata::OpenStreetMapX.MapData)
  startTime = time()
  p = ParamGrid[GridElement]
  #Generating agents
  Agents = generate_agents(mapdata, p[3], Start, End, p[2], p[6], p[5], p[7], K_Paths_Dict)
  #Running base simulation - no V2I system
  BaseOutput = simulation_run(:base, mapdata, Agents, density_factor)
  #Running simulation with smart agents - V2I system enabled
  SmartOutput = simulation_run(:smart, mapdata, Agents, density_factor,
                              K_Paths_Dict, p[4], p[5], p[6], false)
  step_stat = gather_statistics(getfield.(Agents,:smart),
                                      BaseOutput.TravelTimes,
                                      SmartOutput.TravelTimes)
  runtime = time()-startTime
  print("$(GridElement),$(p[1]),$(p[2]),$(p[3]),$(p[4]),$(p[5]),$(p[6]),$(p[7]),")
  print("$(round(runtime,digits = 2)),$(step_stat[1]),$(step_stat[2]),$(step_stat[3]),")
  println("$(step_stat[4]),$(step_stat[5]),$(step_stat[6]),$(step_stat[7])")
end

"""
`run_parameter_analysis` run base and V2I simulation scenario to compare agents performance, returns comparison statistics

**Input parameters**
* `GridElement` : element from parameters grid to be used in simulation
* `ParamGrid` : array with parameters values
* `AgentsVec` : vector with agents used in simulation
* `density_factor` : road length reserved for one vehicle
* `K_Paths_Dict` : dictionary with multiple shortest paths (values) between vertices (keys)
* `mapdata` : OpenStreetMapX MapData object with road network data
"""
function run_parameter_analysis(GridElement::Int,
                                ParamGrid::Vector{Tuple{Int,Float64,Int,Int,Float64,Int,Float64}},
                                AgentsVec::Vector{Agent}, density_factor::Float64,
                                K_Paths_Dict::Dict{Tuple{Int,Int},Array{Vector{Int}}},
                                mapdata::OpenStreetMapX.MapData)
  startTime = time()
  p = ParamGrid[GridElement]
  #Sample N agents from supplied vector and adjust smart agents number
  Agents = deepcopy(sample(AgentsVec, p[3], replace = false))
  N_int= Int(ceil(p[3]*p[2]))
  smart_ind = [trues(N_int); falses(p[3]-N_int)]
  for i in 1:p[3] Agents[i].smart = smart_ind[i] end
  #Running base simulation - no V2I system
  BaseOutput = simulation_run(:base, mapdata, Agents, density_factor)
  #Running simulation with smart agents - V2I system enabled
  SmartOutput = simulation_run(:smart, mapdata, Agents, density_factor,
                              K_Paths_Dict, p[4], p[5], p[6], false)
  step_stat = gather_statistics(getfield.(Agents,:smart),
                                      BaseOutput.TravelTimes,
                                      SmartOutput.TravelTimes)
  runtime = time()-startTime
  print("$(GridElement),$(p[1]),$(p[2]),$(p[3]),$(p[4]),$(p[5]),$(p[6]),$(p[7]),")
  print("$(round(runtime,digits = 2)),$(step_stat[1]),$(step_stat[2]),$(step_stat[3]),")
  println("$(step_stat[4]),$(step_stat[5]),$(step_stat[6]),$(step_stat[7])")
end


import RDates: WeekendCalendar, @rd_str, setcachedcalendar!, calendar
import Dates: Date

setcachedcalendar!(CALENDARS, "UK/SETTLEMENT", WeekendCalendar() + RuleBasedCalendar([
    # New Year's Day
    PeriodicCalendarRule(rd"1y", rd"1JAN@WEEKEND[NBD]"; cal_mgr=CALENDARS),
    # Good Friday
    PeriodicCalendarRule(rd"1y", rd"0E-1FRI"),
    # Easter Monday
    PeriodicCalendarRule(rd"1y", rd"0E+1MON"),
    # Early May Bank Holiday
    PeriodicCalendarRule(rd"1y", rd"1MAY+1st MON"),
    # Spring Bank Holiday
    PeriodicCalendarRule(rd"1y", rd"1MAY+Last MON"),
    # Late Summer Bank Holiday
    PeriodicCalendarRule(rd"1y", rd"1AUG+Last MON"),
    # Christmas Day
    PeriodicCalendarRule(rd"1y", rd"25DEC+0b@WEEKEND"; cal_mgr=CALENDARS),
    # Boxing Day
    PeriodicCalendarRule(rd"1y", rd"25DEC+1b@WEEKEND"; cal_mgr=CALENDARS),
    # 50th VE Day Anniversary
    ExplicitDateRule(inclusions=[Date(1995,5,8)], exclusions=[Date(1995,5,1)]),
    # Golden Jubilee
    ExplicitDateRule(inclusions=[Date(2002,6,3), Date(2002,6,4)], exclusions=[Date(2002,5,27)]),
    # 2011 Royal Wedding
    ExplicitDateRule(inclusions=[Date(2011,4,29)]),
    # Diamond Jubilee
    ExplicitDateRule(inclusions=[Date(2012,6,4), Date(2012,6,5)], exclusions=[Date(2002,5,28)]),
    # 75th VE Day Anniversary
    ExplicitDateRule(inclusions=[Date(2020,5,8)], exclusions=[Date(2020,5,4)]),
    # Platinum Jubilee
    ExplicitDateRule(inclusions=[Date(2022,6,2), Date(2022,6,3)], exclusions=[Date(2022,5,30)])
]))

setcachedcalendar!(CALENDARS, "LONDON", calendar(CALENDARS, "UK/SETTLEMENT"))

setcachedcalendar!(CALENDARS, "UK/STOCK EXCHANGE", WeekendCalendar() + RuleBasedCalendar([
    # New Year's Day
    PeriodicCalendarRule(rd"1y", rd"1JAN@WEEKEND[NBD]"; cal_mgr=CALENDARS),
    # Good Friday
    PeriodicCalendarRule(rd"1y", rd"0E-1FRI"),
    # Easter Monday
    PeriodicCalendarRule(rd"1y", rd"0E+1MON"),
    # Early May Bank Holiday
    PeriodicCalendarRule(rd"1y", rd"1MAY+1st MON"),
    # Spring Bank Holiday
    PeriodicCalendarRule(rd"1y", rd"1MAY+Last MON"),
    # Late Summer Bank Holiday
    PeriodicCalendarRule(rd"1y", rd"1AUG+Last MON"),
    # Christmas Day
    PeriodicCalendarRule(rd"1y", rd"25DEC+0b@WEEKEND"; cal_mgr=CALENDARS),
    # Boxing Day
    PeriodicCalendarRule(rd"1y", rd"25DEC+1b@WEEKEND"; cal_mgr=CALENDARS),
    # 50th VE Day Anniversary
    ExplicitDateRule(inclusions=[Date(1995,5,8)], exclusions=[Date(1995,5,1)]),
    # Golden Jubilee
    ExplicitDateRule(inclusions=[Date(2002,6,3), Date(2002,6,4)], exclusions=[Date(2002,5,27)]),
    # 2011 Royal Wedding
    ExplicitDateRule(inclusions=[Date(2011,4,29)]),
    # Diamond Jubilee
    ExplicitDateRule(inclusions=[Date(2012,6,4), Date(2012,6,5)], exclusions=[Date(2002,5,28)]),
    # 75th VE Day Anniversary
    ExplicitDateRule(inclusions=[Date(2020,5,8)], exclusions=[Date(2020,5,4)]),
    # Platinum Jubilee
    ExplicitDateRule(inclusions=[Date(2022,6,2), Date(2022,6,3)], exclusions=[Date(2022,5,30)]),
    # 31Dec1999
    ExplicitDateRule(inclusions=[Date(1999,12,31)])
]))

setcachedcalendar!(CALENDARS, "UK/LSE", calendar(CALENDARS, "UK/STOCK EXCHANGE"))

setcachedcalendar!(CALENDARS, "UK/METAL EXCHANGE", WeekendCalendar() + RuleBasedCalendar([
    # New Year's Day
    PeriodicCalendarRule(rd"1y", rd"1JAN@WEEKEND[NBD]"; cal_mgr=CALENDARS),
    # Good Friday
    PeriodicCalendarRule(rd"1y", rd"0E-1FRI"),
    # Easter Monday
    PeriodicCalendarRule(rd"1y", rd"0E+1MON"),
    # Early May Bank Holiday
    PeriodicCalendarRule(rd"1y", rd"1MAY+1st MON"),
    # Spring Bank Holiday
    PeriodicCalendarRule(rd"1y", rd"1MAY+Last MON"),
    # Late Summer Bank Holiday
    PeriodicCalendarRule(rd"1y", rd"1AUG+Last MON"),
    # Christmas Day
    PeriodicCalendarRule(rd"1y", rd"25DEC+0b@WEEKEND"; cal_mgr=CALENDARS),
    # Boxing Day
    PeriodicCalendarRule(rd"1y", rd"25DEC+1b@WEEKEND"; cal_mgr=CALENDARS),
    # 50th VE Day Anniversary
    ExplicitDateRule(inclusions=[Date(1995,5,8)], exclusions=[Date(1995,5,1)]),
    # Golden Jubilee
    ExplicitDateRule(inclusions=[Date(2002,6,3), Date(2002,6,4)], exclusions=[Date(2002,5,27)]),
    # 2011 Royal Wedding
    ExplicitDateRule(inclusions=[Date(2011,4,29)]),
    # Diamond Jubilee
    ExplicitDateRule(inclusions=[Date(2012,6,4), Date(2012,6,5)], exclusions=[Date(2002,5,28)]),
    # 75th VE Day Anniversary
    ExplicitDateRule(inclusions=[Date(2020,5,8)], exclusions=[Date(2020,5,4)]),
    # Platinum Jubilee
    ExplicitDateRule(inclusions=[Date(2022,6,2), Date(2022,6,3)], exclusions=[Date(2022,5,30)]),
    # 31Dec1999
    ExplicitDateRule(inclusions=[Date(1999,12,31)])
]))

setcachedcalendar!(CALENDARS, "UK/LME", calendar(CALENDARS, "UK/METAL EXCHANGE"))