using .tensorboard: NodeDef, AttrValue_ListValue, AttrValue, NameAttrList

"""
	log_graph
"""
function log_graph(logger::TBLogger, g::AbstractGraph; step = nothing, nodelabel::Vector{String} = map(string, vertices(g)), nodeop::Vector{String} = map(string, vertices(g)), nodedevice::Vector{String} = fill("cpu", nv(g)), nodevalue::Vector{Any} = fill(nothing, nv(g)))
	nv(g) == length(nodelabel)  || throw(ArgumentError("length of nodelable must be same as number of vertices"))
	nv(g) == length(nodeop)  || throw(ArgumentError("length of nodeop must be same as number of vertices"))
	nv(g) == length(nodedevice)  || throw(ArgumentError("length of nodedevice must be same as number of vertices"))
	nv(g) == length(nodevalue)  || throw(ArgumentError("length of nodevalue must be same as number of vertices"))
	summ = SummaryCollection(graph_summary(g, nodelabel, nodeop, nodedevice, nodevalue))
    write_event(logger.file, make_event(logger, summ, step=step))
end

function graph_summary(g, nodelabel, nodeop, nodedevice, nodevalue)
	nodes = Vector{NodeDef}()
	for v in vertices(g)
		name = nodelabel[v]
		op = nodeop[v]
		input = [nodelabel[x] for x in inneighbors(g, v)]
		device = nodedevice[v]
		attr = Dict{String, AttrValue}()
		x = nodevalue[v]
		if isa(x, AbstractString)
			attr["value"] = AttrValue(s = Vector{UInt8}(x))
			attr["dtype"] = AttrValue(_type = jltype2tf(typeof(x)))
		elseif isa(x, Integer)
			attr["value"] = AttrValue(i = Int64(x))
			attr["dtype"] = AttrValue(_type = jltype2tf(typeof(x)))
		elseif isa(x, Real)
			attr["value"] = AttrValue(f = Float32(x))
			attr["dtype"] = AttrValue(_type = jltype2tf(typeof(x)))
		elseif isa(x, Bool)
			attr["value"] = AttrValue(b = x)
			attr["dtype"] = AttrValue(_type = jltype2tf(typeof(x)))
		elseif isa(x, AbstractArray)
			shape = TensorShapeProto(dim = [TensorShapeProto_Dim(size = d) for d in (collect(size(x)))])
			t = TensorProto(dtype = jltype2tf(eltype(x)), tensor_shape = shape, tensor_content = serialize_proto(string(x)))
			attr["value"] = AttrValue(tensor = t)
			attr["_output_shapes"] = AttrValue(list = AttrValue_ListValue(shape = [shape]))
		elseif isa(x, Tuple)
			attr["value"] = AttrValue(list = AttrValue_ListValue(s = [Vector{UInt8}(repr(y)) for y in x]))
			shape = TensorShapeProto(dim = [TensorShapeProto_Dim(size = length(x))])
			attr["_output_shapes"] = AttrValue(list = AttrValue_ListValue(shape = [shape]))
		elseif isa(x, Function)
			attr["value"] = AttrValue(func = NameAttrList(name = repr(x)))
		else
			#donothing
		end
		node = NodeDef(name = name, op = op, input = input, device = device, attr = attr)
		push!(nodes, node)
	end
	GraphDef(node = nodes)
end

function jltype2tf(dtype::DataType)
	nodetype =
	dtype == UInt8				? _DataType.DT_UINT8 :
	dtype == UInt16				? _DataType.DT_UINT16 :
	dtype == UInt32				? _DataType.DT_UINT32 :
	dtype == UInt64				? _DataType.DT_UINT64 :
	dtype == Int8				? _DataType.DT_INT8 :
	dtype == Int16				? _DataType.DT_INT16 :
	dtype == Int32				? _DataType.DT_INT32 :
	dtype == Int64				? _DataType.DT_INT64 :
	dtype == Float16			? _DataType.DT_BFLOAT16 :
	dtype == Float32			? _DataType.DT_FLOAT :
	dtype == Float64			? _DataType.DT_DOUBLE :
	dtype <: AbstractString		? _DataType.DT_STRING :
	dtype == Bool				? _DataType.DT_BOOL :
	dtype ∈ [Complex{Float32},
			Complex{Float16},
			Complex{UInt8},
			Complex{UInt16},
			Complex{UInt32},
			Complex{Int8},
			Complex{Int16},
			Complex{Int32}]		? _DataType.DT_COMPLEX64 :
	dtype ∈ [Complex{Float64},
			Complex{UInt64},
			Complex{Int64}]		? _DataType.DT_COMPLEX128 :
	_DataType.DT_INVALID
end


using Distributed

# Thea: set this path to point to the correct module file
include("SeisNoiseAWS.jl")
@everywhere include("SeisNoiseAWS.jl")

# Thea: if you want to run tests, you can hard code a path here to a data folder that contains params.csv and stations.csv
default_path = "/Users/dmikesell/Documents/GitHub/esip_julia_aws/dev/20200615_102819/41808/"

# Example of the CLI call of this Julia file.
#
# $ julia run_SeisNoiseAWS.jl /Users/dmikesell/Documents/GitHub/esip_julia_aws/dev/20200615_102819/41808/
#
# or using multiple threads (4 in this example)
#
# $ julia -p 4 run_SeisNoiseAWS.jl /Users/dmikesell/Documents/GitHub/esip_julia_aws/dev/20200615_102819/41808/

# ==============================================================================
# For testing in Atom without calling from Linux CLI.
# clearconsole()
if isempty(ARGS)
    basepath = default_path
else
    basepath = ARGS[1]
end
# ==============================================================================
using Logging
# Setup the log file
log_file = joinpath(basepath, "xcor_log.txt")
io = open(log_file, "w")
logger = SimpleLogger(io, Logging.Info)
global_logger(logger) # Set the global logger to logger
# ==============================================================================
# ncores = length(Sys.cpu_info())
# ntheads = Threads.nthreads()
# addprocs(ntheads) # start DEFAULT parallel threads
# addprocs(4)
num_procs = nprocs()
if num_procs == 1
    @info("Number of workers: $num_procs")
else
    @info("Number of workers: $(num_procs-1)", )
end
# ==============================================================================
using Dates # even though loaded in SeisNoiseAWS module, needs to be loaded here too.
using SeisIO
using Combinatorics
# ==============================================================================
# Load the input files from the web application
#
# Build the COR output directory
COR_DIR = SeisNoiseAWS.build_cor_directory(basepath)
# Get Station information
df_st = SeisNoiseAWS.load_station_file( joinpath(basepath, "stations.csv") )
# Get XCOR parameters
df_params = SeisNoiseAWS.load_parameter_file( joinpath(basepath, "params.csv") )
# ==============================================================================
# Build the data availability table
df_date = SeisNoiseAWS.build_availability_df(df_st)
# Make boolean vectors from each column
columns = collect( eachcol(df_date) )
# Set the file length for downloading data
t_duration = Dates.Second(Dates.Day(1)).value # [s] one day in seconds
# ==============================================================================
# Loop over days
t0a = now() # start timer
# for col = 7:7#20 # end-1 # skip the first column, which is channel tag
col = 10
# col=2 set up some if statements about num_pairs
# col=8 has a trace that is shorter than the cc_len (so need try/catch)
# col=7 has only a single station (the try/catch from above works)

# Get the date information
the_date    = String(names(df_date)[col  ])  # start date
the_date_p1 = String(names(df_date)[col+1])  # end date
@info("Processing: $the_date to $the_date_p1")

# Create the output folder for this date
folder_date = Dates.format(DateTime(String(names(df_date)[col])),"yyyy-mm-dd")
output_dir = joinpath(COR_DIR, folder_date)
@debug("Writing daily correlations to $output_dir")

# Download the station data for this date
stations = columns[1][columns[col]] # array of channel_tags on this day
@debug("Downloading data for $stations")
S = SeisData() # create empty seisdata object
# Download data if more than 1 channel
if length(stations) > 1
    # This does the instrument deconvolution on the fly
    get_data!(S, "FDSN", stations, src="IRIS", s=the_date, t=t_duration, rr=true)
end

# Build the cross-correlation pairs
xcor_pairs = collect(combinations(S, 2))

# We need a flag to build the autocorrelation pairs
# Then we need to create those and append to xcor_pairs

# Do the pair-wise cross correlation
if isempty(xcor_pairs)
    @info("Number of channels downloaded: $(length(S))")
    @info("No channel pairs to correlate.")
elseif any(x->x=="XX.FAIL..001", S.id)
    @info("Bad station request: nothing to correlate.")
else
    numpairs = length(xcor_pairs)
    @debug("Number of channel pair correlations: $numpairs")
    t1 = now()  # start timer for the parallel computation for this day
    # Do the channel pair correlations in parallel for a single day
    pmap(SeisNoiseAWS.xcor,
        xcor_pairs,
        fill(df_params, numpairs),
        fill(output_dir, numpairs))
    # SeisNoiseAWS.xcor(xcor_pairs, df_params, output_dir)

    cor_time = now() - t1 # compute time for this day
    @debug("Compute time for this correlation day: $cor_time")
end # if statement over xcor_pairs

# end # the FOR loop over days

run_time = now() - t0a # total run time timer
@info("Compute time for all correlations: $run_time")

close(io) # stop logger

exit()


include("typographyaux.jl")

export text, textFont, textSize
export textWidth, textHeight, textAscent, textDescent

## Loading & Displaying

#loadFont

function text(str::AbstractString, x, y)
    xo = x
    yo = y
	x = ((x+1)/2)*state.width
	y = ((y+1)/2)*state.height

	shader("fontDrawing")
	glActiveTexture(GL_TEXTURE1)
    glEnable(GL_CULL_FACE)

    posData = zeros(GLfloat, 2*6*length(str))
    texData = zeros(GLfloat, 2*6*length(str))

    n = 0
    totyadv = 0
    nforline = 0
	@inbounds for c in str
        # if we hit a newline character or we reach the max line
        # length, then move to next line, where the next line is
        # determined by the average y advance from the characters
        # of the previous line.
        if c == '\n' || nforline == fontState.maxLineLength
            cht = fontState.characters[str[1]]
            x = xo
            y -= totyadv/nforline
            totyadv = 0
            nforline = 0
            continue
        end

		ch = fontState.characters[c]

		xpos = x + ch.bearing[1] * state.textSize
        ypos = y - (ch.size[2] - ch.bearing[2]) * state.textSize

        w = ch.size[1] * state.textSize
        h = ch.size[2] * state.textSize

        totyadv += ch.advance[2] * state.textSize

        if w == 0 || h == 0
            continue
        end

        posData[n+1] = xpos
        posData[n+2] = ypos + h

        posData[n+3] = xpos
        posData[n+4] = ypos

        posData[n+5] = xpos + w
        posData[n+6] = ypos

        posData[n+7] = xpos
        posData[n+8] = ypos + h

        posData[n+9] = xpos + w
        posData[n+10] = ypos

        posData[n+11] = xpos + w
        posData[n+12] = ypos + h

        texData[n+1] = ch.atlasOffset
        texData[n+2] = 0.0

        texData[n+3] = ch.atlasOffset
        texData[n+4] = ch.size[2] / fontState.atlasHeight

        texData[n+5] = ch.atlasOffset + ch.size[1] / fontState.atlasWidth
        texData[n+6] = ch.size[2] / fontState.atlasHeight

        texData[n+7] = ch.atlasOffset
        texData[n+8] = 0.0

        texData[n+9] = ch.atlasOffset + ch.size[1] / fontState.atlasWidth
        texData[n+10] = ch.size[2] / fontState.atlasHeight

        texData[n+11] = ch.atlasOffset + ch.size[1] / fontState.atlasWidth
        texData[n+12] = 0.0

		x += ch.advance[1] * state.textSize
        n += 2*6
        nforline += 1
	end
    glBindBuffer(GL_ARRAY_BUFFER, globjs.posvbos[globjs.posind])
    glBufferData(GL_ARRAY_BUFFER, sizeof(posData), posData, GL_STATIC_DRAW)
    glBindBuffer(GL_ARRAY_BUFFER, globjs.texvbos[globjs.texind])
    glBufferData(GL_ARRAY_BUFFER, sizeof(texData), texData, GL_STATIC_DRAW)
    glDrawArrays(GL_TRIANGLES, 0, 6*length(str))

    glDisable(GL_CULL_FACE)
	shader("basicShapes")

    sfx = state.fbSize[1]/state.width; sfy = state.fbSize[2]/state.height
    x = (x./state.width - 0.5)*sfx
    y = ((y - totyadv/nforline)./state.height - 0.5)*sfy
    return [xo, yo, x, y]
end

function textFont(fontname::AbstractString)
    # for the time being, we only allow system fonts or those that have been
    # installed directly into the main system storage
    @windows_only state.fontFace = "C:/Windows/Fonts/"*fontname
    @linux_only state.fontFace = "/usr/share/fonts/"*fontname
    @osx_only state.fontFace = "/System/Library/Fonts/"*fontname
    fontState = fontStruct(newface(state.fontFace), Dict(' ' => blankChar), 0, 12)
	setpixelsize(fontState.face, fontState.fontWidth, fontState.fontHeight)
	setupFontCharacters()
end

## Attributes

#function textAlign()
#
#end

#textLeading
#textMode

function textSize(size)
	state.textSize = size
end

function textWidth(str::AbstractString)
    # extents = Cairo.text_extents(cr, str)
    # return extents[1]
end

function textHeight(str::AbstractString)
    # extents = Cairo.text_extents(cr, str)
    # return extents[2]
end

## Metrics

function textAscent(str::AbstractString)
   # extents = Cairo.scaled_font_extents(cr, str)
   # return extents[1]
end

function textDescent(str::AbstractString)
   # extents = Cairo.scaled_font_extents(cr, str)
   # return extents[2]
end

function textLineLength(l)
    fontState.maxLineLength = l
end


export Warrior
export Archer, Knight, Pikeman

export battle!, attack!, resupply!, shoot!, mount!, dismount!

abstract type Warrior end

mutable struct Archer <: Warrior
    name::String
    health::Int
    arrows::Int
end

mutable struct Pikeman <: Warrior
    name::String
    health::Int
end

mutable struct Knight <: Warrior
    name::String
    health::Int
    mounted::Bool
end

function Knight(name::AbstractString, health::Integer)
   Knight(name, health, true)
end

function Archer(name::AbstractString, health::Integer)
    archer = Archer(name, health, 0)
    resupply!(archer)
    archer
end

"""
    shoot(a::Archer)
Shoot an arrow. Deplete nubmer of arrows in quiver.
"""
function shoot!(archer::Archer)
    if archer.arrows > 0
        archer.arrows -= 1
    end
    archer
end

"""
    resupply!(a::Archer)
Ressuply archer `a` with arrows.
"""
function resupply!(archer::Archer)
    archer.arrows = 24
    archer
end

function mount!(knight::Knight)
    knight.mounted = true
    knight
end

function dismount!(knight::Knight)
    knight.mounted = false
    knight
end

"""
    battle!(a::Warrior, b::Warrior)
Display the outcome of warrior `a` attacking warrior `b`
"""
function battle!(a::Warrior, b::Warrior)
    attack!(a, b)
    if a.health == 0 && b.health == 0
        println(a.name, " and ", b.name, " destroyed each other")
    elseif a.health == 0
        println(b.name, " defeated ", a.name)
    elseif b.health == 0
        println(a.name, " defeated ", b.name)
    else
        println(b.name, " survived attack from ", a.name)
    end
end

"""
    attack!(a::Archer, b::Archer)
Archers can do a fair amount of damage on other archers as
the archers don't have good armor to protect them.
"""
function attack!(a::Archer, b::Archer)
    if a.arrows > 0
        shoot!(a)
        damage = 6 + rand(1:6)
        b.health = max(b.health - damage, 0)
    end
    a.health, b.health
end

"""
    attack!(a::Archer, b::Knight)
Archers can do some damage to a knight when when he is mounted
because the horse is vulnurable. Hurting dismounted knights is harder
as most arrows cannot pierce plate armor.
"""
function attack!(a::Archer, b::Knight)
    if a.arrows > 0
        shoot!(a)
        damage = rand(1:6)
        if b.mounted
            damage += 3
        end
        b.health = max(b.health - damage, 0)
    end
    a.health, b.health
end

"""
    attack!(a::Knight, b::Knight)
Knights vs knights is evenly matched and both have a fairly good chance of
disengaging so casualties are not that high.
"""
function attack!(a::Knight, b::Knight)
    a.health = max(a.health - rand(1:6), 0)
    b.health = max(b.health - rand(1:6), 0)
    a.health, b.health
end

"""
    attack!(a::Archer, b::Pikeman)
Because pikemen move slowly they cannot easily get away from the rain of
arrows and thus take significant damage.
"""
function attack!(a::Archer, b::Pikeman)
    if a.arrows > 0
        shoot!(a)
        damage = 4 + rand(1:6)
        b.health = max(b.health - damage, 0)
    end
    a.health, b.health
end

"""
    attack!(a::Pikeman, b::Archer)
It is hard for pikemen to attack archers are they move slowly. Thus
only when lucky do they succeed in causing some injury.
"""
function attack!(a::Pikeman, b::Archer)
   if rand(1:6) >= 5
       b.health = max(b.health - 2, 0)
   end
   a.health, b.health
end

"""
    attack!(a::Pikeman, b::Pikeman)
Pikemen vs pikemen attacks were generally avoided as
they tended to get  brutal and with heavy causualties.
It is hard to escape such a close encounter once it begins.
"""
function attack!(a::Pikeman, b::Pikeman)
    for _ in 1:3
        a.health = max(a.health - rand(1:6), 0)
        b.health = max(b.health - rand(1:6), 0)

        if a.health > 0 && b.health > 0
            break
        end
    end
    a.health, b.health
end

"""
    attack!(a::Pikeman, b::Knight)
Due to the superior mobility of knights, a pikeman cannot easily engage a
knight which does not choose to attack, unless the knight is dismounted.
"""
function attack!(a::Pikeman, b::Knight)
    if rand(1:6) >= 6
        damage = 1
        if b.mounted
            damage += rand(1:6)
        end
        b.health = max(b.health - damage, 0)
    end
    a.health, b.health
end

"""
    attack!(a::Knight, b::Archer)
Archers attacked by knights would suffer heavy losses as they have poor
armor.
"""
function attack!(a::Knight, b::Archer)
    damage = 8 + 2*rand(1:6)
    b.health = max(b.health - damage, 0)
    a.health, b.health
end

"""
    attack!(a::Knight, b::Pikeman)
Mounted knights attacking pikemen will usually suffer heavy losses
unless attacking on a flank or the rear. Unmounted knights are more
effective as pikemen cannot attack their vulnurable horses.
"""
function attack!(a::Knight, b::Pikeman)
    damage = rand(1:6)
    b.health = max(b.health - damage, 0)

    damage = rand(1:6)
    if a.mounted
        damage += 2*rand(1:6)
    end
    a.health = max(b.health - damage, 0)

    a.health, b.health
end


function PSI.construct_device!(
    optimization_container::PSI.OptimizationContainer,
    sys::PSY.System,
    model::PSI.DeviceModel{H, HydroDispatchRunOfRiverOutage},
    ::Type{S},
) where {H <: PSY.HydroGen, S <: PM.AbstractPowerModel}
    devices = PSI.get_available_components(H, sys)

    if !PSI.validate_available_devices(H, devices)
        return
    end

    # Variables
    PSI.add_variables!(
        optimization_container,
        PSI.ActivePowerVariable,
        devices,
        HydroDispatchRunOfRiverOutage(),
    )
    PSI.add_variables!(
        optimization_container,
        PSI.ReactivePowerVariable,
        devices,
        HydroDispatchRunOfRiverOutage(),
    )

    # Constraints
    PSI.add_constraints!(
        optimization_container,
        PSI.RangeConstraint,
        PSI.ActivePowerVariable,
        devices,
        model,
        S,
        PSI.get_feedforward(model),
    )
    PSI.add_constraints!(
        optimization_container,
        PSI.RangeConstraint,
        PSI.ReactivePowerVariable,
        devices,
        model,
        S,
        PSI.get_feedforward(model),
    )
    outage_constraints!(
        optimization_container,
        devices,
        model,
        S,
        PSI.get_feedforward(model),
    )
    PSI.feedforward!(optimization_container, devices, model, PSI.get_feedforward(model))

    # Cost Function
    PSI.cost_function!(optimization_container, devices, model, S, nothing)

    return
end

"""
Construct model for HydroGen with RunOfRiver Dispatch Formulation
with only Active Power.
"""
function PSI.construct_device!(
    optimization_container::PSI.OptimizationContainer,
    sys::PSY.System,
    model::PSI.DeviceModel{H, HydroDispatchRunOfRiverOutage},
    ::Type{S},
) where {H <: PSY.HydroGen, S <: PM.AbstractActivePowerModel}
    devices = PSI.get_available_components(H, sys)

    if !PSI.validate_available_devices(H, devices)
        return
    end

    # Variables
    PSI.add_variables!(
        optimization_container,
        PSI.ActivePowerVariable,
        devices,
        HydroDispatchRunOfRiverOutage(),
    )

    # Constraints
    PSI.add_constraints!(
        optimization_container,
        PSI.RangeConstraint,
        PSI.ActivePowerVariable,
        devices,
        model,
        S,
        PSI.get_feedforward(model),
    )
    outage_constraints!(
        optimization_container,
        devices,
        model,
        S,
        PSI.get_feedforward(model),
    )
    PSI.feedforward!(optimization_container, devices, model, PSI.get_feedforward(model))

    # Cost Function
    PSI.cost_function!(optimization_container, devices, model, S, nothing)

    return
end

function PSI.construct_device!(
    optimization_container::PSI.OptimizationContainer,
    sys::PSY.System,
    model::PSI.DeviceModel{H, HydroDispatchReservoirStorageOutage},
    ::Type{S},
) where {H <: PSY.HydroEnergyReservoir, S <: PM.AbstractPowerModel}
    devices = PSI.get_available_components(H, sys)

    if !PSI.validate_available_devices(H, devices)
        return
    end

    # Variables
    PSI.add_variables!(
        optimization_container,
        PSI.ActivePowerVariable,
        devices,
        HydroDispatchReservoirStorageOutage(),
    )
    PSI.add_variables!(
        optimization_container,
        PSI.ReactivePowerVariable,
        devices,
        HydroDispatchReservoirStorageOutage(),
    )
    PSI.add_variables!(
        optimization_container,
        PSI.EnergyVariable,
        devices,
        HydroDispatchReservoirStorageOutage(),
    )
    PSI.add_variables!(
        optimization_container,
        PSI.SpillageVariable,
        devices,
        HydroDispatchReservoirStorageOutage(),
    )
    PSI.add_variables!(
        optimization_container,
        PSI.EnergyShortageVariable,
        devices,
        HydroDispatchReservoirStorageOutage(),
    )
    PSI.add_variables!(
        optimization_container,
        PSI.EnergySurplusVariable,
        devices,
        HydroDispatchReservoirStorageOutage(),
    )

    # Initial Conditions
    PSI.storage_energy_initial_condition!(
        optimization_container,
        devices,
        HydroDispatchReservoirStorageOutage(),
    )
    # Energy Balance Constraint
    PSI.add_constraints!(
        optimization_container,
        PSI.EnergyBalanceConstraint,
        PSI.EnergyVariable,
        devices,
        model,
        S,
        PSI.get_feedforward(model),
    )
    PSI.energy_target_constraint!(
        optimization_container,
        devices,
        model,
        S,
        PSI.get_feedforward(model),
    )
    outage_constraints!(
        optimization_container,
        devices,
        model,
        S,
        PSI.get_feedforward(model),
    )
    PSI.feedforward!(optimization_container, devices, model, PSI.get_feedforward(model))

    # Cost Function
    PSI.cost_function!(optimization_container, devices, model, S, nothing)

    return
end

"""
Construct model for HydroGen with ReservoirStorage Dispatch Formulation
with only Active Power
"""
function PSI.construct_device!(
    optimization_container::PSI.OptimizationContainer,
    sys::PSY.System,
    model::PSI.DeviceModel{H, HydroDispatchReservoirStorageOutage},
    ::Type{S},
) where {H <: PSY.HydroEnergyReservoir, S <: PM.AbstractActivePowerModel}
    devices = PSI.get_available_components(H, sys)

    if !PSI.validate_available_devices(H, devices)
        return
    end

    # Variables

    PSI.add_variables!(
        optimization_container,
        PSI.ActivePowerVariable,
        devices,
        HydroDispatchReservoirStorageOutage(),
    )
    PSI.add_variables!(
        optimization_container,
        PSI.EnergyVariable,
        devices,
        HydroDispatchReservoirStorageOutage(),
    )
    PSI.add_variables!(
        optimization_container,
        PSI.SpillageVariable,
        devices,
        HydroDispatchReservoirStorageOutage(),
    )
    PSI.add_variables!(
        optimization_container,
        PSI.EnergyShortageVariable,
        devices,
        HydroDispatchReservoirStorageOutage(),
    )
    PSI.add_variables!(
        optimization_container,
        PSI.EnergySurplusVariable,
        devices,
        HydroDispatchReservoirStorageOutage(),
    )

    # Initial Conditions
    PSI.storage_energy_initial_condition!(
        optimization_container,
        devices,
        HydroDispatchReservoirStorageOutage(),
    )
    # Energy Balance Constraint
    PSI.add_constraints!(
        optimization_container,
        PSI.EnergyBalanceConstraint,
        PSI.EnergyVariable,
        devices,
        model,
        S,
        PSI.get_feedforward(model),
    )
    PSI.energy_target_constraint!(
        optimization_container,
        devices,
        model,
        S,
        PSI.get_feedforward(model),
    )
    outage_constraints!(
        optimization_container,
        devices,
        model,
        S,
        PSI.get_feedforward(model),
    )
    PSI.feedforward!(optimization_container, devices, model, PSI.get_feedforward(model))

    # Cost Function
    PSI.cost_function!(optimization_container, devices, model, S, nothing)

    return
end

function PSI.construct_device!(
    optimization_container::PSI.OptimizationContainer,
    sys::PSY.System,
    model::PSI.DeviceModel{H, HydroDispatchPumpedStoragewReservationOutage},
    ::Type{S},
) where {H <: PSY.HydroPumpedStorage, S <: PM.AbstractActivePowerModel}
    devices = PSI.get_available_components(H, sys)

    if !PSI.validate_available_devices(H, devices)
        return
    end

    # Variables
    PSI.add_variables!(
        optimization_container,
        PSI.ActivePowerInVariable,
        devices,
        HydroDispatchPumpedStoragewReservationOutage(),
    )
    PSI.add_variables!(
        optimization_container,
        PSI.ActivePowerOutVariable,
        devices,
        HydroDispatchPumpedStoragewReservationOutage(),
    )
    PSI.add_variables!(
        optimization_container,
        PSI.EnergyVariableUp,
        devices,
        HydroDispatchPumpedStoragewReservationOutage(),
    )
    PSI.add_variables!(
        optimization_container,
        PSI.EnergyVariableDown,
        devices,
        HydroDispatchPumpedStoragewReservationOutage(),
    )
    PSI.add_variables!(
        optimization_container,
        PSI.SpillageVariable,
        devices,
        HydroDispatchPumpedStoragewReservationOutage(),
    )
    PSI.add_variables!(
        optimization_container,
        PSI.ReserveVariable,
        devices,
        HydroDispatchPumpedStoragewReservationOutage(),
    )

    # Constraints
    PSI.add_constraints!(
        optimization_container,
        PSI.RangeConstraint,
        PSI.ActivePowerOutVariable,
        devices,
        model,
        S,
        PSI.get_feedforward(model),
    )
    PSI.add_constraints!(
        optimization_container,
        PSI.RangeConstraint,
        PSI.ActivePowerInVariable,
        devices,
        model,
        S,
        PSI.get_feedforward(model),
    )

    # Initial Conditions
    PSI.storage_energy_initial_condition!(
        optimization_container,
        devices,
        HydroDispatchPumpedStoragewReservationOutage(),
    )

    # Energy Balanace limits
    PSI.add_constraints!(
        optimization_container,
        PSI.EnergyBalanceConstraint,
        PSI.EnergyVariableUp,
        devices,
        model,
        S,
        PSI.get_feedforward(model),
    )
    PSI.add_constraints!(
        optimization_container,
        PSI.EnergyBalanceConstraint,
        PSI.EnergyVariableDown,
        devices,
        model,
        S,
        PSI.get_feedforward(model),
    )
    outage_constraints!(
        optimization_container,
        devices,
        model,
        S,
        PSI.get_feedforward(model),
    )
    PSI.feedforward!(optimization_container, devices, model, PSI.get_feedforward(model))

    # Cost Function
    PSI.cost_function!(optimization_container, devices, model, S, nothing)

    return
end


const pyjlnumbertype = pynew()
const pyjlcomplextype = pynew()
const pyjlrealtype = pynew()
const pyjlrationaltype = pynew()
const pyjlintegertype = pynew()

struct pyjlnumber_op{OP}
    op :: OP
end
(op::pyjlnumber_op)(self) = Py(op.op(self))
function (op::pyjlnumber_op)(self, other_::Py)
    if pyisjl(other_)
        other = pyjlvalue(other_)
    else
        other = @pyconvert(Number, other_, return Py(pybuiltins.NotImplemented))
    end
    Py(op.op(self, other))
end
function (op::pyjlnumber_op)(self, other_::Py, other2_::Py)
    if pyisjl(other_)
        other = pyjlvalue(other_)
    else
        other = @pyconvert(Number, other_, return Py(pybuiltins.NotImplemented))
    end
    if pyisjl(other2_)
        other2 = pyjlvalue(other2_)
    else
        other2 = @pyconvert(Number, other2_, return Py(pybuiltins.NotImplemented))
    end
    Py(op.op(self, other, other2))
end
pyjl_handle_error_type(op::pyjlnumber_op, self, exc) = exc isa MethodError && exc.f === op.op ? pybuiltins.TypeError : PyNULL

struct pyjlnumber_rev_op{OP}
    op :: OP
end
function (op::pyjlnumber_rev_op)(self, other_::Py)
    if pyisjl(other_)
        other = pyjlvalue(other_)
    else
        other = @pyconvert(Number, other_, return Py(pybuiltins.NotImplemented))
    end
    Py(op.op(other, self))
end
function (op::pyjlnumber_rev_op)(self, other_::Py, other2_::Py)
    if pyisjl(other_)
        other = pyjlvalue(other_)
    else
        other = @pyconvert(Number, other_, return Py(pybuiltins.NotImplemented))
    end
    if pyisjl(other2_)
        other2 = pyjlvalue(other2_)
    else
        other2 = @pyconvert(Number, other2_, return Py(pybuiltins.NotImplemented))
    end
    Py(op.op(other, self, other2))
end
pyjl_handle_error_type(op::pyjlnumber_rev_op, self, exc) = exc isa MethodError && exc.f === op.op ? pybuiltins.TypeError : PyNULL

pyjlreal_trunc(self::Real) = Py(trunc(Integer, self))
pyjl_handle_error_type(::typeof(pyjlreal_trunc), self, exc::MethodError) = exc.f === trunc ? pybuiltins.TypeError : PyNULL

pyjlreal_floor(self::Real) = Py(floor(Integer, self))
pyjl_handle_error_type(::typeof(pyjlreal_floor), self, exc::MethodError) = exc.f === floor ? pybuiltins.TypeError : PyNULL

pyjlreal_ceil(self::Real) = Py(ceil(Integer, self))
pyjl_handle_error_type(::typeof(pyjlreal_ceil), self, exc::MethodError) = exc.f === ceil ? pybuiltins.TypeError : PyNULL

function pyjlreal_round(self::Real, ndigits_::Py)
    ndigits = pyconvertarg(Union{Int,Nothing}, ndigits_, "ndigits")
    if ndigits === nothing
        Py(round(Integer, self))
    else
        Py(round(self; digits = ndigits))
    end
end
pyjl_handle_error_type(::typeof(pyjlreal_round), self, exc::MethodError) = exc.f === round ? pybuiltins.TypeError : PyNULL

function init_jlwrap_number()
    jl = pyjuliacallmodule
    filename = "$(@__FILE__):$(1+@__LINE__)"
    pybuiltins.exec(pybuiltins.compile("""
    class NumberValue(AnyValue):
        __slots__ = ()
        __module__ = "juliacall"
        def __bool__(self):
            return not self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(iszero))))
        def __add__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(+))), other)
        def __sub__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(-))), other)
        def __mul__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(*))), other)
        def __truediv__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(/))), other)
        def __floordiv__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(÷))), other)
        def __mod__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(%))), other)
        def __pow__(self, other, modulo=None):
            if modulo is None:
                return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(^))), other)
            else:
                return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(powermod))), other, modulo)
        def __lshift__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(<<))), other)
        def __rshift__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(>>))), other)
        def __and__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(&))), other)
        def __xor__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(⊻))), other)
        def __or__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(|))), other)
        def __radd__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(+))), other)
        def __rsub__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(-))), other)
        def __rmul__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(*))), other)
        def __rtruediv__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(/))), other)
        def __rfloordiv__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(÷))), other)
        def __rmod__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(%))), other)
        def __rpow__(self, other, modulo=None):
            if modulo is None:
                return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(^))), other)
            else:
                return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(powermod))), other, modulo)
        def __rlshift__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(<<))), other)
        def __rrshift__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(>>))), other)
        def __rand__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(&))), other)
        def __rxor__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(⊻))), other)
        def __ror__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_rev_op(|))), other)
        def __eq__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(==))), other)
        def __ne__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(!=))), other)
        def __le__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(≤))), other)
        def __lt__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(<))), other)
        def __ge__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(≥))), other)
        def __gt__(self, other):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(>))), other)
    class ComplexValue(NumberValue):
        __slots__ = ()
        __module__ = "juliacall"
        def __complex__(self):
            return self._jl_callmethod($(pyjl_methodnum(pycomplex)))
        @property
        def real(self):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(real))))
        @property
        def imag(self):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(imag))))
        def conjugate(self):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(conj))))
    class RealValue(ComplexValue):
        __slots__ = ()
        __module__ = "juliacall"
        def __float__(self):
            return self._jl_callmethod($(pyjl_methodnum(pyfloat)))
        @property
        def real(self):
            return self
        @property
        def imag(self):
            return 0
        def conjugate(self):
            return self
        def __complex__(self):
            return complex(float(self))
        def __trunc__(self):
            return self._jl_callmethod($(pyjl_methodnum(pyjlreal_trunc)))
        def __floor__(self):
            return self._jl_callmethod($(pyjl_methodnum(pyjlreal_floor)))
        def __ceil__(self):
            return self._jl_callmethod($(pyjl_methodnum(pyjlreal_ceil)))
        def __round__(self, ndigits=None):
            return self._jl_callmethod($(pyjl_methodnum(pyjlreal_round)), ndigits)
    class RationalValue(RealValue):
        __slots__ = ()
        __module__ = "juliacall"
        @property
        def numerator(self):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(numerator))))
        @property
        def denominator(self):
            return self._jl_callmethod($(pyjl_methodnum(pyjlnumber_op(denominator))))
    class IntegerValue(RationalValue):
        __slots__ = ()
        __module__ = "juliacall"
        def __int__(self):
            return self._jl_callmethod($(pyjl_methodnum(pyint)))
        def __index__(self):
            return self.__int__()
        @property
        def numerator(self):
            return self
        @property
        def denominator(self):
            return 1
    import numbers
    numbers.Number.register(NumberValue)
    numbers.Complex.register(ComplexValue)
    numbers.Real.register(RealValue)
    numbers.Rational.register(RationalValue)
    numbers.Integral.register(IntegerValue)
    del numbers
    """, filename, "exec"), jl.__dict__)
    pycopy!(pyjlnumbertype, jl.NumberValue)
    pycopy!(pyjlcomplextype, jl.ComplexValue)
    pycopy!(pyjlrealtype, jl.RealValue)
    pycopy!(pyjlrationaltype, jl.RationalValue)
    pycopy!(pyjlintegertype, jl.IntegerValue)
end

pyjl(v::Number) = pyjl(pyjlnumbertype, v)
pyjl(v::Complex) = pyjl(pyjlcomplextype, v)
pyjl(v::Real) = pyjl(pyjlrealtype, v)
pyjl(v::Rational) = pyjl(pyjlrationaltype, v)
pyjl(v::Integer) = pyjl(pyjlintegertype, v)


using TextAnalysis

model = TextAnalysis.SentimentAnalyzer()

model(StringDocument("hello world"))

model(StringDocument("an incredibly boring film"))

model(StringDocument("a highly enjoyable ride"))

model(StringDocument("a great film"))


# combination of prep_save.jl and align_prep.jl.
# Loads from NMR experiment instead of intermediate BSON file.

import NMRDataSetup
import NMRSpectraSimulator
import NMRSpecifyRegions

include("../src/NMRCalibrate.jl")
import .NMRCalibrate
#import NMRCalibrate

using LinearAlgebra
using FFTW

import PyPlot
#import PlotlyJS
#using Plots; plotly()

import BSON
import JSON

import Statistics
import Random
Random.seed!(25)

##### global constants.
#cs_config_path = "/home/roy/Documents/repo/NMRData/src/input/reduced_cs_config.txt"
SH_config_path = "/home/roy/Documents/repo/NMRData/input/SH_configs/select_compounds_SH_configs_reduce.json"
surrogate_config_path = "/home/roy/Documents/repo/NMRData/input/surrogate_configs/select_compounds_SH_configs.json"
fit_config_path = "/home/roy/Documents/repo/NMRData/input/fit_configs/align_700MHz_type1_select_compounds.json"

# get mapping from molecule names to their spin system info json files.
H_params_path = "/home/roy/Documents/repo/NMRData/input/coupling_info"
dict_compound_to_filename = JSON.parsefile("/home/roy/Documents/repo/NMRData/input/compound_mapping/select_compounds.json")

# specify the NMR experiment folder
#experiment_full_path = "/home/roy/Documents/repo/NMRData/experiments_1D1H/misc/bmse000297_ethanol/"
experiment_full_path = "/home/roy/Documents/repo/NMRData/experiments_1D1H/NRC/misc/glucose/Sep-25-2018"
#experiment_full_path = "/home/roy/Documents/repo/NMRData/experiments_1D1H/BMRB/similar_settings/BMRB-700-20mM/L-Serine"
#experiment_full_path = "/home/roy/Documents/repo/NMRData/experiments_1D1H/BMRB/similar_settings/BMRB-500-0.5mM/L-Serine"
#experiment_full_path = "/home/roy/Documents/repo/NMRData/experiments_1D1H/NRC/NRC_4_amino_acid_mixture_Jan_2022/1"
#experiment_full_path = "/home/roy/Documents/repo/NMRData/experiments_1D1H/BMRB/similar_settings/BMRB-700-20mM/L-Isoleucine"
#experiment_full_path = "/home/roy/Documents/repo/NMRData/experiments_1D1H/BMRB/similar_settings/BMRB-700-20mM/L-Glutamine"
#experiment_full_path = "/home/roy/Documents/repo/NMRData/experiments_1D1H/BMRB/similar_settings/BMRB-500-0.5mM/L-Leucine"
#experiment_full_path = "/home/roy/Documents/repo/NMRData/experiments_1D1H/misc/bmse000795_2_DSS"
#experiment_full_path = "/home/roy/Documents/repo/NMRData/experiments_1D1H/misc/gissmo_DSS"
#experiment_full_path = "/home/roy/MEGAsync/outputs/NMR/experiments/misc/bmse000900_L-Phenylalanine"

#experiment_full_path = "/home/roy/MEGAsync/outputs/NMR/experiments/misc/bmse000900_L-Phenylalanine"
#experiment_full_path = "/home/roy/Documents/repo/NMRData/experiments_1D1H/BMRB/glucose-600-bmse000855_1"

# specify where the calibration results should be saved for this experiment.
#project_name = "ethanol"
project_name = "NRC-glucose-2018"
#project_name = "Serine-BMRB-700-20mM-force-mag-eq"
#project_name = "Serine-BMRB-700-20mM-2000entry"
#project_name = "Serine-BMRB-700-20mM-mod"
#project_name = "Serine-BMRB-500-0.5mM-mod" #project_name = "Serine-BMRB-500-0.5mM"

 #project_name = "Serine-glucose-NRC-Jan2022"
#project_name = "NRC-Jan2022-serine-glucose-dss"

#project_name = "Isoleucine-BMRB-700-20mM"
#project_name = "Glutamine-BMRB-700-20mM"
#project_name = "Leucine-BMRB-500-0.5mM" # skipped.
#project_name = "bmse000795_2_DSS" # skipped.
#project_name = "gissmo_DSS"
#project_name = "Phenylalanine-GISSMO-600-bmse000900"

#project_name = "Phenylalanine-GISSMO-600-bmse000900-poster"
#project_name = "glucose-600-bmse000855_1-poster"

#molecule_names = ["Ethanol";]
molecule_names = ["alpha-D-Glucose"; "beta-D-Glucose";]
#molecule_names = ["D-Glucose - 2000 study";] #molecule_names = ["D-(+)-Glucose";]
#molecule_names = ["D-(+)-Glucose";]
#molecule_names = ["L-Serine";]
#molecule_names = ["L-Serine - 2000 study";]
#molecule_names = ["L-Serine - mod";]
#molecule_names = ["L-Serine - 2000 study";] # molecule_names = ["L-Serine";]

#molecule_names = ["L-Serine - 2000 study"; "D-Glucose - 2000 study";]
#molecule_names = ["L-Serine"; "D-(+)-Glucose"]
#molecule_names = ["L-Serine"; "D-(+)-Glucose"; "L-Isoleucine"]
#molecule_names = ["L-Serine"; "D-(+)-Glucose"; "DSS"] # for debugging code with singlets.
#molecule_names = ["L-Serine"; "D-(+)-Glucose"; "L-Glutamine"; "L-Isoleucine"; "L-Leucine"; "DSS"]

#molecule_names = ["L-Serine"; "D-(+)-Glucose"; "L-Leucine"; ]
#molecule_names = ["L-Serine"; "D-(+)-Glucose"; "DSS"; ] # really need plots that highlights regions and compounds.

#molecule_names = ["L-Isoleucine";]
#molecule_names = ["L-Glutamine";]
#molecule_names = ["L-Leucine";]
#molecule_names = ["DSS";]
#molecule_names = ["DSS";]
#molecule_names = ["L-Phenylalanine";]

#molecule_names = ["L-Phenylalanine";]
#molecule_names = ["D-(+)-Glucose";]

#molecule_names = ["L-Phenylalanine, 500 MHz"; "L-Phenylalanine"]

project_base_folder = "/home/roy/MEGAsync/outputs/NMR/align"
#project_base_folder = "/home/roy/MEGAsync/outputs/NMR/calibrate" # old.
project_folder = joinpath(project_base_folder, project_name)
isdir(project_folder) || mkpath(project_folder)

##### user input for loading data.

## for loading the experiment.
solvent_ppm_guess = 4.7
solvent_window_ppm = 0.1

# start the first entry at the frequency corresponding to offset_ppm.
# This is a method to deal with wrap-around frequency.
offset_ppm = 0.3

## for surrogate construction.

#w = [1.0;] # relative concentration.

Δcs_max_scalar_default = 0.2 # In units of ppm. interpolation border that is added to the lowest and highest resonance frequency component of the mixture being simulated.

dummy_SSFID = NMRSpectraSimulator.SpinSysParamsType1(0.0) # level 2 model.

unique_cs_atol = 1e-6 # for assessing chemical equivalence.

## for determining cost function positions
region_min_dist = 0.1 # in ppm.

#####

##### load experiment, normalize data.
s_t, S, hz2ppmfunc, ppm2hzfunc, ν_0ppm, fs, SW, α_0ppm, β_0ppm, λ_0ppm, Ω_0ppm,
    results_0ppm, dic, α_solvent, β_solvent, λ_solvent, Ω_solvent,
    results_solvent = NMRDataSetup.loadspectrum(experiment_full_path;
    solvent_ppm = solvent_ppm_guess,
    solvent_window_ppm = solvent_window_ppm)
#

# start the first entry at the frequency corresponding to offset_Hz, which we set to 0.3 ppm.

offset_Hz = ν_0ppm - (ppm2hzfunc(offset_ppm)-ppm2hzfunc(0.0))
DFT_s = fft(s_t)
U_DFT, U_y, U_inds = NMRDataSetup.getwraparoundDFTfreqs(length(s_t), fs, offset_Hz)

S_U = DFT_s[U_inds]
P_y = hz2ppmfunc.(U_y)

val, ind = findmin( abs.(U_y .- ν_0ppm) )
Z = abs(S_U[ind])

y = S_U ./ Z
#####

####### compute surrogate.
λ0 = λ_0ppm

# get a surrogate where K_{n,i} is encouraged to be no larger than `early_exit_part_size`.
println("Timing: mag equivalence")
@time MEs = NMRSpectraSimulator.getmageqinfomixture(molecule_names,
    H_params_path,
    dict_compound_to_filename;
    unique_cs_atol = unique_cs_atol)

println("Timing: setupmixtureproxies()")
@time mixture_params = NMRSpectraSimulator.setupmixtureSH(molecule_names,
    H_params_path, dict_compound_to_filename, fs, SW,
    ν_0ppm;
    MEs = MEs,
    config_path = SH_config_path)
As = mixture_params

ΩS_ppm = NMRSpectraSimulator.getPsnospininfo(As, hz2ppmfunc)
ΩS_ppm_sorted = sort(NMRSpectraSimulator.combinevectors(ΩS_ppm))

u_offset = 0.5 # in ppm.
u_min = ppm2hzfunc(ΩS_ppm_sorted[1] - u_offset)
u_max = ppm2hzfunc(ΩS_ppm_sorted[end] + u_offset)

println("Timing: fitproxies!()")
dummy_SSFID = NMRSpectraSimulator.SpinSysParamsType1(0.0)
Bs = NMRSpectraSimulator.fitproxies(As, dummy_SSFID, λ0;
    names = molecule_names,
    config_path = surrogate_config_path,
    Δcs_max_scalar_default = Δcs_max_scalar_default,
    u_min = u_min,
    u_max = u_max)

####### end mixture proxy.

##### prepare fit positions.
### prepare positions.
hz2ppmfunc = uu->(uu - ν_0ppm)*SW/fs
ppm2hzfunc = pp->(ν_0ppm + pp*fs/SW)
P_y = hz2ppmfunc.(U_y)

ΩS0 = NMRSpectraSimulator.getΩS(As)
ΩS0_ppm = NMRSpectraSimulator.getPs(ΩS0, hz2ppmfunc)

Δsys_cs, y_cost_all, U_cost_all, P_cost_all, exp_info, cost_inds, cost_inds_set,
    λ_lbs, λ_ubs, κs_β_orderings,
    κs_β_DOFs = NMRCalibrate.prepareoptim(fit_config_path, molecule_names,
    hz2ppmfunc, U_y, y, As; region_min_dist = region_min_dist)
#
a_setp, b_setp, minxs,
    rets = NMRCalibrate.setupitpab(0.1, 10, 0.7; optim_algorithm = :LN_BOBYQA)

# new normalization.
Z = maximum( maximum(abs.(y[cost_inds_set[r]])) for r = 1:length(cost_inds_set) )
y = y ./ Z

# include("inner_kappa.jl")
#@assert 1==3

# for poster.
#include("pkg_align_single_region.jl")
#include("phenylalanine.jl")

#include("pkg_calibrate.jl")
include("pkg_quantify.jl")

using Plots; plotly()

# save data plot.
file_name = "data.html"
plots_save_path = joinpath(project_folder, file_name)

canvas_size = (1000, 400)
plot_obj = Plots.plot( P_y,
    real.(y),
    title = "data spectrum",
    label = "data",
    seriestype = :line,
    ticks = :native,
    xlims = (P_y[1],P_y[end]),
    hover = P_y,
    linewidth = 4,
    xlabel = "ppm",
    ylabel = "real part of spectrum",
    xflip = true,
    size = canvas_size)
Plots.savefig(plot_obj, plots_save_path)

# julia> minxs
# 3-element Vector{Vector{Float64}}:
#  [-0.3182551682670491, 0.049183131629439616, 0.024203062792075736]
#  [0.9948724914072531, 0.06932430981123272, 0.0839330953494708]
#  [0.060496655154809946, 0.10515911754875247, 0.11996321361619477]
#
# julia> w_BLS
# ERROR: UndefVarError: w_BLS not defined
#
# julia> ws
# 3-element Vector{Vector{Float64}}:
#  [75.8591754032393, 0.21482769137077787]
#  [47.263485918404186, 0.33989540682369757]
#  [0.26376149754144756, 0.34040957238909]


import Dates
using Compat

@compat abstract type RDate end
# Calendars
@compat abstract type CalendarManager end
@compat abstract type Calendar end
# Conventions
@compat abstract type HolidayRoundingConvention end
@compat abstract type InvalidDayConvention end
@compat abstract type MonthIncrementConvention end

"""
    NullCalendarManager() <: CalendarManager

The most primitive calendar manager, that will return an error for any request to
get a calendar. The default calendar manager that is available when applying rdates
using the + operator, without an explicit calendar manager.
"""
struct NullCalendarManager <: CalendarManager end

"""
    is_holiday(calendar::Calendar, date::Dates.Date)::Bool

Determine whether the date requested is a holiday for this calendar or not.
"""
is_holiday(x::Calendar, ::Dates.Date) = error("$(typeof(x)) does not support is_holiday")

"""
    calendar(calendarmgr::CalendarManager, names::Vector{String})::Calendar

Given a set of calendar names, request the calendar manager to retrieve the associated
calendar that supports the union of them.
"""
calendar(x::CalendarManager, names)::Calendar = error("$(typeof(x)) does not support calendar")

"""
    apply(rdate::RDate, date::Dates.Date, calendarmgr::CalendarManager)::Dates.Date

The application of an rdate to a specific date, given an explicit calendar manager.
"""
apply(rd::RDate, ::Dates.Date, ::CalendarManager)::Dates.Date = error("$(typeof(rd)) does not support date apply")

"""
    multiply_no_roll(rdate::RDate, count::Integer)::RDate

An internalised multiplication of an rdate which is generated without reapplication of a relative date multiple
times. This differs from multiply_roll when either calendars or month adjustments are involved.

For example "6m" + "6m" != "12m" for all dates, due to the fact that there are different days in each month
and invalid day conventions will kick in.
"""
multiply_no_roll(rd::RDate, count::Integer) = error("$(typeof(rd)) does not support no roll multiplication")

calendar_mgr = NullCalendarManager()
function with_cal_mgr(f::Function, cal_mgr::CalendarManager)
    global calendar_mgr
    current_cal_mgr = calendar_mgr
    calendar_mgr = cal_mgr
    try
        f()
    finally
        calendar_mgr = current_cal_mgr
    end
end

"""
    apply(rdate::RDate, date::Dates.Date)::Dates.Date

The application of an rdate to a specific date, without an explicit calendar manager.
This will use the globally specified calendar manager defined at the point of request.
"""
apply(rd::RDate, date::Dates.Date) = apply(rd, date, calendar_mgr)
Base.:+(rd::RDate, date::Dates.Date) = apply(rd, date)
Base.:+(date::Dates.Date, rd::RDate) = apply(rd, date)
Base.:-(date::Dates.Date, rd::RDate) = apply(-rd, date)
Base.:-(x::RDate)::RDate = error("$(typeof(x)) does not support negation")

"""
    apply(rounding::HolidayRoundingConvention, date::Dates.Date, calendar::Calendar)::Dates.Date

Apply the appropriate adjustment to the date if it falls on a holiday for the given calendar. Be aware
that there is no strict requirement that the resolved date will not be a holiday for the given calendar.
"""
apply(x::HolidayRoundingConvention, date::Dates.Date, calendar::Calendar) = error("$(typeof(x)) does not support apply")

"""
    apply(rounding::InvalidDayConvention, day::Integer, month::Integer, year::Integer)::Dates.Date

Given a day, month and year which do not generate a valid date, adjust them in some form to a valid date.
"""
adjust(x::InvalidDayConvention, day, month, year) = error("$(typeof(x)) does not support adjust")

"""
    apply(rounding::MonthIncrementConvention, from::Dates.Date, new_month::Integer, new_year::Integer, calendar_mgr::CalendarManager)::Tuple{Integer, Integer, Integer}

Given the initial day, month and year that we're moving from and a generated new month and new year, determine the new
day, month and year that should be used. Generally used to handle specific features around preservation of month ends
or days of week, if required.

Note that the generated (day, month, year) do not need to be able to produce a valid date. The invalid day convention
should be applied, if required, after this calculation.

May use a calendar manager if required as well
"""
adjust(x::MonthIncrementConvention, from::Dates.Date, new_month, new_year, calendar_mgr::CalendarManager) = error("$(typeof(x)) does not support adjust")


module SplitExplicit

using StaticArrays

using ..Ocean
using ..HydrostaticBoussinesq
using ..ShallowWater

using ...VariableTemplates
using ...MPIStateArrays
using ...Mesh.Geometry
using ...DGMethods
using ...BalanceLaws

import ...BalanceLaws:
    initialize_states!,
    tendency_from_slow_to_fast!,
    cummulate_fast_solution!,
    reconcile_from_fast_to_slow!

HBModel = HydrostaticBoussinesqModel
SWModel = ShallowWaterModel

function initialize_states!(
    ::HBModel{C},
    ::SWModel{C},
    _...,
) where {C <: Uncoupled}
    return nothing
end
function tendency_from_slow_to_fast!(
    ::HBModel{C},
    ::SWModel{C},
    _...,
) where {C <: Uncoupled}
    return nothing
end
function cummulate_fast_solution!(
    ::HBModel{C},
    ::SWModel{C},
    _...,
) where {C <: Uncoupled}
    return nothing
end
function reconcile_from_fast_to_slow!(
    ::HBModel{C},
    ::SWModel{C},
    _...,
) where {C <: Uncoupled}
    return nothing
end

include("VerticalIntegralModel.jl")
include("Communication.jl")
include("ShallowWaterCoupling.jl")
include("HydrostaticBoussinesqCoupling.jl")

end


export MADDPGManager

"""
    MADDPGManager(; agents::Dict{<:Any, <:Agent}, args...)
Multi-agent Deep Deterministic Policy Gradient(MADDPG) implemented in Julia. By default, `MADDPGManager` uses for simultaneous environments with [continuous action space](https://spinningup.openai.com/en/latest/spinningup/rl_intro.html#stochastic-policies).
See the paper https://arxiv.org/abs/1706.02275 for more details.

# Keyword arguments
- `agents::Dict{<:Any, <:Agent}`, here each agent collects its own information. While updating the policy, each **critic** will assemble all agents'
  trajectory to update its own network. **Note that** here the policy of the `Agent` should be `DDPGPolicy` wrapped by `NamedPolicy`, see the relative
  experiment([`MADDPG_KuhnPoker`](https://juliareinforcementlearning.org/docs/experiments/experiments/Policy%20Gradient/JuliaRL_MADDPG_KuhnPoker/#JuliaRL\\_MADDPG\\_KuhnPoker) or [`MADDPG_SpeakerListener`](https://juliareinforcementlearning.org/docs/experiments/experiments/Policy%20Gradient/JuliaRL_MADDPG_SpeakerListener/#JuliaRL\\_MADDPG\\_SpeakerListener)) for references.
- `traces`, set to `SARTS` if you are apply to an environment of `MINIMAL_ACTION_SET`, or `SLARTSL` if you are to apply to an environment of `FULL_ACTION_SET`.
- `batch_size::Int`
- `update_freq::Int`
- `update_step::Int`, count the step.
- `rng::AbstractRNG`.
"""
mutable struct MADDPGManager <: AbstractPolicy
    agents::Dict{<:Any, <:Agent}
    traces
    batch_size::Int
    update_freq::Int
    update_step::Int
    rng::AbstractRNG
end

# used for simultaneous environments.
function (π::MADDPGManager)(env::AbstractEnv)
    while current_player(env) == chance_player(env)
        env |> legal_action_space |> rand |> env
    end
    Dict(
        player => agent.policy(env)
    for (player, agent) in π.agents)
end

function (π::MADDPGManager)(stage::Union{PreEpisodeStage, PostActStage}, env::AbstractEnv)
    # only need to update trajectory.
    for (_, agent) in π.agents
        update!(agent.trajectory, agent.policy, env, stage)
    end
end

function (π::MADDPGManager)(stage::PreActStage, env::AbstractEnv, actions)
    # update each agent's trajectory.
    for (player, agent) in π.agents
        update!(agent.trajectory, agent.policy, env, stage, actions[player])
    end

    # update policy
    update!(π, env)
end

function (π::MADDPGManager)(stage::PostEpisodeStage, env::AbstractEnv)
    # collect state and a dummy action to each agent's trajectory here.
    for (_, agent) in π.agents
        update!(agent.trajectory, agent.policy, env, stage)
    end

    # update policy
    update!(π, env)
end

# update policy
function RLBase.update!(π::MADDPGManager, env::AbstractEnv)
    π.update_step += 1
    π.update_step % π.update_freq == 0 || return

    for (_, agent) in π.agents
        length(agent.trajectory) > agent.policy.policy.update_after || return
        length(agent.trajectory) > π.batch_size || return
    end

    # get training data
    temp_player = collect(keys(π.agents))[1]
    t = π.agents[temp_player].trajectory
    inds = rand(π.rng, 1:length(t), π.batch_size)
    batches = Dict((player, RLCore.fetch!(BatchSampler{π.traces}(π.batch_size), agent.trajectory, inds))
                for (player, agent) in π.agents)

    # get s, a, s′ for critic
    s = vcat((batches[player][:state] for (player, _) in π.agents)...)
    a = vcat((batches[player][:action] for (player, _) in π.agents)...)
    s′ = vcat((batches[player][:next_state] for (player, _) in π.agents)...)

    for (player, agent) in π.agents
        p = agent.policy.policy # get agent's concrete DDPGPolicy.

        A = p.behavior_actor
        C = p.behavior_critic
        Aₜ = p.target_actor
        Cₜ = p.target_critic

        γ = p.γ
        ρ = p.ρ

        # by default A, C, Aₜ, Cₜ on the same device.
        batches, s, a, s′ = send_to_device(device(A), (batches, s, a, s′))
        r = batches[player][:reward]
        t = batches[player][:terminal]
        # for training behavior_actor.
        mu_actions = vcat(
            ((
                batches[p][:next_state] |>
                a.policy.policy.behavior_actor
            ) for (p, a) in π.agents)...
        )
        # for training behavior_critic.
        new_actions = vcat(
            ((
                batches[p][:next_state] |>
                a.policy.policy.target_actor
            ) for (p, a) in π.agents)...
        )

        if π.traces == SLARTSL
            # Note that by default **MADDPG** is used for the environments with continuous action space, and `legal_action_space_mask` is
            # defined in the environments with discrete action space. So we need `env.action_mapping` to transform the actions
            # getting from the trajectory.
            @assert env isa ActionTransformedEnv

            mask = batches[player][:next_legal_actions_mask]
            mu_l′ = Flux.batch(
                (begin
                    actions = env.action_mapping(mu_actions[:, i])
                    mask[actions[player]]
                end for i = 1:π.batch_size)
            )
            new_l′ = Flux.batch(
                (begin
                    actions = env.action_mapping(new_actions[:, i])
                    mask[actions[player]]
                end for i = 1:π.batch_size)
            )
        end

        qₜ = Cₜ(vcat(s′, new_actions)) |> vec
        if π.traces == SLARTSL
            qₜ .+= ifelse.(new_l′, 0.0f0, typemin(Float32))
        end
        y = r .+ γ .* (1 .- t) .* qₜ

        gs1 = gradient(Flux.params(C)) do
            q = C(vcat(s, a)) |> vec
            loss = mean((y .- q) .^ 2)
            ignore() do
                p.critic_loss = loss
            end
            loss
        end

        update!(C, gs1)

        gs2 = gradient(Flux.params(A)) do
            v = C(vcat(s, mu_actions)) |> vec
            if π.traces == SLARTSL
                v .+= ifelse.(mu_l′, 0.0f0, typemin(Float32))
            end
            reg = mean(A(batches[player][:state]) .^ 2)
            loss = -mean(v) +  reg * 1e-3
            ignore() do
                p.actor_loss = loss
            end
            loss
        end

        update!(A, gs2)

        # polyak averaging
        for (dest, src) in zip(Flux.params([Aₜ, Cₜ]), Flux.params([A, C]))
            dest .= ρ .* dest .+ (1 - ρ) .* src
        end
    end
end


using Pkg
using Base.Iterators
using BSON
using CSV
# using CuArrays
using DataFrames
using DataStructures: OrderedDict
using Dates
using FileIO
using Flux
# using JLD2
# using JLSO
using JSON
using LinearAlgebra
import MLDataUtils: shuffleobs, getobs, splitobs, stratifiedobs, RandomBatches, batchview
import MLDataPattern
using Printf
# using PyPlot
using Random
using Statistics

using LogGoodies
using MLGoodies
using FluxGoodies

# CuArrays.allowscalar(false)

function import_dataset(fname, split::AbstractFloat = 0.8)
    datafname = "$(fname).ser.gz"
    if isfile(datafname)
        @info "importing raw binary dataset: $(datafname)"
        # rawdata = JLSO.load(datafname)["data"]
        rawdata = deserialize_gzip(datafname)
    else
        df = CSV.read("$(fname).csv")
        n = size(df, 1)
        npassengers = (size(df, 2) - 4) ÷ 6
        @info "samples: $(n), #passengers: $npassengers"
        df = df[:, Symbol.(vcat(["onboard_count", "lat", "lon"], [["pickup_$(i)_lat",
                "pickup_$(i)_lon", "pickup_$(i)_maxtime",
                "dropoff_$(i)_lat",
                "dropoff_$(i)_lon", "dropoff_$(i)_maxtime"] for i in 1:npassengers]..., ["feasible"]))]
        desc = "VGA data for $npassengers passengers"
        Random.seed!(1234)
        @info "splitting"
        itrn, itst = first.(parentindices.(splitobs(shuffleobs(df), split))) #TODO there is method for this in  Dataset!
        @info "building raw dataset"
        rawdata = Dataset(df, desc, [:train => itrn, :test => itst])
        @info "saving raw binary dataset"
        # JLSO.save(datafname, "data" => rawdata; format=:julia_serialize)
        serialize_gzip(datafname, rawdata)

    end
    @info "#true: $(sum(rawdata[][!, :feasible])), #false: $(nobs(rawdata[]) - sum(rawdata[][!, :feasible]))"
    dtrans = tonumerical(rawdata, :, :T => Dict(:standardize => false))
    @info "transforming"
    data = MLGoodies.transform(rawdata, dtrans, Matrix{Float32})
    @info "storing the transform to JSON"
    open(f->JSON.print(f, dtrans, 3), "$(fname).transform.json", "w")
    data, dtrans
end

function train(Xtrn, Ttrn, Xtst, Ttst;
    model_dir = ".",
    epochs = 500,
    early_stop = 20,
    hidden = [1024, 512, 256],
    logepochs=1
    )
    mkpath(model_dir)

    Random.seed!()
    model = MLP(size(Xtrn, 1), hidden..., 1; outactivation=identity) |> gpu

    ps = params(model)
    lossforoutput(Y, T) = batchlogitbinarycrossentropy(Y, T)
    loss(X, T) = lossforoutput(model(getobs(X)), getobs(T))

    best_loss = Inf
    best_epoch = 0

    Random.seed!(1)
    total_start_time = time()

    function evaluatebatch(X, T, batchsize=4096; gpuloss=true)
        Y = similar(T)
        for idxs in partition(1:size(X, 2), batchsize)
            Y[:,idxs] .= model(X[:,idxs])
        end
        loss_ = lossforoutput(Y, T)
        Ycls = prob2class(round.(σ.(Y)))
        Tcls = prob2class(T)

        # serialize_gzip(joinpath(model_dir, "eval.ser.gz"), (Ycls, Tcls))
        acc_ = 100 * accuracy(Ycls, Tcls)
        cm_ = cmatrix(Ycls, Tcls)
        loss_, acc_, cm_
    end

    opt = ADAM()
    # bsize = 4096
    bsize = 1024

    nbatches = nobs(Xtrn) ÷ bsize
    if nbatches == 0
        nbatches = 1
        bsize = nobs(Xtrn)
    end
    @info "#batches: $nbatches, bsize: $bsize, #epochs: $epochs"
    ps = params(model)
    @info "#params: $(sum(length.(ps)))"

    test_loss, test_acc, test_cm = evaluatebatch(Xtst, Ttst)
    @info @sprintf "initial: test: %0.7f (%0.2f%%)\n" test_loss test_acc

    # dataset = flatten(repeated(RandomBatches((Xtrn, Ttrn), bsize, nbatches), epochs))
    # dataset = RandomBatches((Xtrn, Ttrn), bsize, epochs*nbatches)
    dataset = RandomBatches((Xtrn, Ttrn), bsize, nbatches)
    # @info "dataset ready: type = $(typeof(dataset))"

    for epoch in 1:epochs
        epoch_start_time = time()

        Flux.train!(loss, ps, dataset, opt; cb = () -> ())

        ste = time()
        train_loss, train_acc, train_cm = evaluatebatch(Xtrn, Ttrn)
        test_loss, test_acc, test_cm = evaluatebatch(Xtst, Ttst)
        este = time() - ste

        if test_loss < best_loss
            best_loss, best_epoch = test_loss, epoch
            cpumodel = model |> cpu
            BSON.@save joinpath(model_dir, "model.bson") cpumodel
            improvestr = "*"
        else
            improvestr = "-$(epoch-best_epoch)"
        end
        etime = time() - epoch_start_time
        ttime = time() - total_start_time

        if epoch % logepochs == 0
            @info @sprintf "E%d/%d: train: %0.7f (%0.2f%%) test: %0.7f (%0.2f%%) in %0.3fs (eval %0.3fs, total %0.3fs) %s\n" epoch epochs train_loss train_acc test_loss test_acc etime este ttime improvestr
        end
        if epoch >= best_epoch + early_stop
            @info "early stopping in epoch $epoch\n"
            break
        end
    end

    BSON.@load joinpath(model_dir, "model.bson") cpumodel
    model = cpumodel |> gpu

    train_loss, train_acc = NaN32, NaN32
    train_loss, train_acc, train_cm = evaluatebatch(Xtrn, Ttrn)
    test_loss, test_acc, test_cm = evaluatebatch(Xtst, Ttst)

    @info @sprintf "final: train: %0.5f/%0.2f%%, test: %0.5f/%0.2f%%\n" train_loss train_acc test_loss test_acc
    @info "train confusion matrix:\n$(train_cm)"
    @info "test confusion matrix:\n$(test_cm)"

    model
end

function main()
    hidden = [1024, 512, 256] # default
    # hidden = [512, 512, 256]
    # hidden = [512, 512, 256, 128]
    # hidden = [1024, 512, 256, 128]
    # hidden = [64, 64, 64, 64]
    # hidden = [1024, 1024, 1024]
    # hidden = [2048, 2048, 2048]

    # data_dir = "full/1/group_data_size"
    # data_dir = "full/2/group_data_size"
    # data_dir = "full/3/group_data_size"
    # data_dir = "full/4/group_data_size"
    # data_dir = "full/5/group_data_size"
    # data_dir = "full/6/group_data_size"
    # data_dir = "full/7/group_data_size"
    data_dir = "full/8/group_data_size"
    # data_dir = "full/9/group_data_size"
    # data_dir = "full/10/group_data_size"

    model_dir = "exp/$(data_dir)/$(join(hidden, "_"))"
    ispath(model_dir) && error("path $model_dir exists!")
    mkpath(model_dir)

    append_default_flushed_logger(joinpath(model_dir, "log.txt"))

    @info "importing dataset: $(data_dir)"
    data, dtrans = import_dataset("data/$(data_dir)", 0.98)
    @info "dataset imported"
    Xtrn = data[:train, :X] |> getobs |> gpu
    Ttrn = data[:train, :T] |> getobs |> gpu
    Xtst = data[:test, :X] |> getobs |> gpu
    Ttst = data[:test, :T] |> getobs |> gpu

    @info "copied to GPU"
    @info "Xtrn: $(summary(Xtrn)), Ttrn: $(summary(Ttrn)), #true: $(Int(sum(Ttrn))), #false: $(size(Ttrn, 2) - Int(sum(Ttrn)))"
    @info "Xtst: $(summary(Xtst)), Ttst: $(summary(Ttst)), #true: $(Int(sum(Ttst))), #false: $(size(Ttst, 2) - Int(sum(Ttst)))"

    train(Xtrn, Ttrn, Xtst, Ttst; model_dir=model_dir, hidden=hidden)
end

main()


function triangulate(polygon::Vector{Vector{Point{2, Float64}}})
    lengths = map(x-> UInt32(length(x)), polygon)
    len = UInt32(length(lengths))
    array = ccall(
        (:u32_triangulate_f64, libearcut),
        Tuple{Ptr{GLTriangleFace}, Cint},
        (Ptr{Ptr{Float64}}, Ptr{UInt32}, UInt32),
        polygon, lengths, len
    )
    return unsafe_wrap(Vector{GLTriangleFace}, array[1], array[2])
end

function triangulate(polygon::Vector{Vector{Point{2, Float32}}})
    lengths = map(x-> UInt32(length(x)), polygon)
    len = UInt32(length(lengths))
    array = ccall(
        (:u32_triangulate_f32, libearcut),
        Tuple{Ptr{GLTriangleFace}, Cint},
        (Ptr{Ptr{Float32}}, Ptr{UInt32}, UInt32),
        polygon, lengths, len
    )
    return unsafe_wrap(Vector{GLTriangleFace}, array[1], array[2])
end

function triangulate(polygon::Vector{Vector{Point{2, Int64}}})
    lengths = map(x-> UInt32(length(x)), polygon)
    len = UInt32(length(lengths))
    array = ccall(
        (:u32_triangulate_i64, libearcut),
        Tuple{Ptr{GLTriangleFace}, Cint},
        (Ptr{Ptr{Int64}}, Ptr{UInt32}, UInt32),
        polygon, lengths, len
    )
    return unsafe_wrap(Vector{GLTriangleFace}, array[1], array[2])
end

function triangulate(polygon::Vector{Vector{Point{2, Int32}}})
    lengths = map(x-> UInt32(length(x)), polygon)
    len = UInt32(length(lengths))
    array = ccall(
        (:u32_triangulate_i32, libearcut),
        Tuple{Ptr{GLTriangleFace}, Cint},
        (Ptr{Ptr{Int32}}, Ptr{UInt32}, UInt32),
        polygon, lengths, len
    )
    return unsafe_wrap(Vector{GLTriangleFace}, array[1], array[2])
end


#### DEFINE A PEDESTRIAN MODEL AND A PEDESTRIAN FLOW #####

const PED_WIDTH = 1.0
const PED_LENGTH = 1.0
const PEDESTRIAN_DEF = VehicleDef(AgentClass.PEDESTRIAN, PED_LENGTH, PED_WIDTH)

"""
Action type for the pedestrian model
"""
struct ConstantSpeedDawdling
    v::Float64 # instantaneous change in velocity
    lat::Float64 # lateral displacement
end

"""
    Simple model, pedestrian walking at constant speed with noise
    and dawdling in the y direction
"""
@with_kw mutable struct ConstantPedestrian <: DriverModel{ConstantSpeedDawdling}
    update_freq::Int64 = 1
    tick::Int64 = 0.
    a::ConstantSpeedDawdling = ConstantSpeedDawdling(0., 0.)
    dawdling_amp::Float64 = 0.
    v_noise::Float64 = 0.5
    v_desired::Float64 = 1.0
    dt::Float64 = 0.1
end

AutomotiveDrivingModels.get_name(model::ConstantPedestrian) = "ConstantPedestrian"
Base.rand(rng::AbstractRNG, model::ConstantPedestrian) = model.a
Distributions.pdf(model::ConstantPedestrian, a::ConstantSpeedDawdling) = 1/3*pdf(Normal(0., model.dawdling_amp))
AutomotiveDrivingModels.reset_hidden_state!(model::ConstantPedestrian) = model

function AutomotiveDrivingModels.observe!(model::ConstantPedestrian,
                                          scene::EntityFrame{VehicleState, VehicleDef, Int64},
                                          roadway::Roadway,
                                          egoid::Int)
    if model.tick % model.update_freq == 0
        model.a = ConstantSpeedDawdling(model.v_desired + rand([-model.v_noise, 0, model.v_noise]),
                                        model.dawdling_amp*randn())
    end
    model.a = ConstantSpeedDawdling(model.a.v, model.dawdling_amp*randn())
    model.tick += 1
end

function AutomotiveDrivingModels.propagate(veh::VehicleState, action::ConstantSpeedDawdling, roadway::Roadway, ΔT::Float64)
    lane_tag_orig = veh.posF.roadind.tag

    # update value in Frenet coordinates
    v = veh.v
    lat = action.lat
    ϕ = veh.posF.ϕ
    s = veh.posF.s
    Δs = v*cos(ϕ)*ΔT + lat*sin(ϕ)
    t = veh.posF.t
    Δt = v*sin(ϕ)*ΔT + lat*cos(ϕ)
    lane = get_lane(roadway, veh)
    # cannot go offroad
    if t + Δt > lane.width/2
        Δt = 0.
    elseif t - Δt < -lane.width/2
        Δt = 0.
    end

    speed = action.v
    # println(s + Δs)
    s_ = clamp(s + Δs, 0., get_end(lane))
    # println(s_)

    #XXX what exactly is happening here, ask Tim ?
    # roadind = move_along(veh.posF.roadind, roadway, Δs)
    # footpoint = roadway[roadind]
    # posG = convert(VecE2, footpoint.pos) + polar(t + Δt, footpoint.pos.θ + π/2)

    # posG = VecSE2(posG.x, posG.y, footpoint.pos.θ + ϕ)

    # state = VehicleState(posG, roadway, action.v)

    # roadproj = proj(state.posG, roadway[lane_tag_orig], roadway, move_along_curves=false)
    # retval = VehicleState(Frenet(roadproj, roadway), roadway, state.v)
    # return retval
    return VehicleState(Frenet(lane, s_, t + Δt, ϕ), roadway, speed)
end

function AutomotiveDrivingModels.propagate(veh::Vehicle, action::ConstantSpeedDawdling, roadway::Roadway, ΔT::Float64)
    return propagate(veh.state, action, roadway, ΔT)
end

## Discrete distribution
# P(a | s)

function action_space(model::ConstantPedestrian)
    return (ConstantSpeedDawdling(0., 0.), ConstantSpeedDawdling(1., 0.), ConstantSpeedDawdling(2., 0.))
end

function get_distribution(model::ConstantPedestrian)
    actions = action_space(model)
    probs = (1/3, 1/3, 1/3)
    return actions, probs
end


using Fastback
using Test
using Dates

inst = Instrument("TICKER");

# generate data
data = Dict{Instrument, Vector{BidAsk}}();
dt = DateTime(2018, 1, 2, 9, 30, 0);
data[inst] = [
    BidAsk(dt + Second(1), 100.0, 101.0),
    BidAsk(dt + Second(2), 100.5, 102.0),
    BidAsk(dt + Second(3), 102.5, 103.0),
    BidAsk(dt + Second(4), 101.0, 102.0),
    BidAsk(dt + Second(5), 99.5, 100.0),
    BidAsk(dt + Second(6), 101.5, 102.0),
    BidAsk(dt + Second(7), 103.5, 104.0),
    BidAsk(dt + Second(8), 104.5, 105.0)
];

@testset "Dummy trading" begin

    # create trading account
    acc = Account(10_000.0)
    collect_balance, balance_curve = periodic_collector(Float64, Second(1));
    collect_equity, equity_curve = periodic_collector(Float64, Second(1));

    # dummy backtest
    for (i, ba) in enumerate(data[inst])
        if i == 1
            execute_order!(acc, OpenOrder(inst, 500.0, Long::TradeDir), ba)
        end

        update_account!(acc, inst, ba)

        if i == 7
            execute_order!(acc, CloseAllOrder(), ba)
        end

        # collect data for analysis
        collect_balance(ba.dt, acc.balance)
        collect_equity(ba.dt, acc.equity)
    end

    # display(acc)

end

@testset "Dummy trading #2" begin

    # create trading account
    acc = Account(100_000.0)
    execute_order!(acc, OpenOrder(inst, 100.0, Long::TradeDir), data[inst][1])  # 100.0 101.0
    update_account!(acc, inst, data[inst][2]) # 100.5, 102.0
    execute_order!(acc, CloseOrder(acc.open_positions[1]), data[inst][3]) # 102.5, 103.0
    update_account!(acc, inst, data[inst][4]) # 101.0, 102.0

    # display(acc)

    p::Position = acc.closed_positions[1]
    @test p.open_price == 101.0
    @test p.last_price == 102.5

    @test pnl_gross(p) ≈ p.size * (midprice(p.last_quote) - midprice(p.open_quote))
    @test pnl_net(p) ≈ p.size * (p.last_price - p.open_price)

    @test return_gross(p) ≈ (midprice(p.last_quote) - midprice(p.open_quote)) / midprice(p.open_quote)
    @test return_net(p) ≈ (p.last_price - p.open_price) / p.open_price

    @test length(acc.open_positions) == 0
    @test length(acc.closed_positions) == 1
    @test acc.initial_balance == 100_000.0
    @test acc.equity ≈ acc.initial_balance + pnl_net(p)
    @test acc.balance ≈ acc.equity

end


"""
    Rel

Ability to define frames in a relative fashion.

# Example
```
 Action(1:100, ground; in_global_layer=true),
 Action(1:90, :red_ball, (args...)->circ(p1, "red"), Rotation(from_rot, to_rot)),
 Action(Rel(10), :blue_ball, (args...)->circ(p2, "blue"), Rotation(2π, from_rot, :red_ball)),
 Action((video, args...)->path!(path_of_red, pos(:red_ball), "red"))
```
is the same as
```
Action(1:100, ground; in_global_layer=true),
Action(1:90, :red_ball, (args...)->circ(p1, "red"), Rotation(from_rot, to_rot)),
Action(91:100, :blue_ball, (args...)->circ(p2, "blue"), Rotation(2π, from_rot, :red_ball)),
Action(91:100, (video, args...)->path!(path_of_red, pos(:red_ball), "red"))
```

# Fields
- rel::UnitRange defines the frames in a relative fashion.
"""
struct Rel
    rel::UnitRange
end

"""
    Rel(i::Int)

Shorthand for Rel(1:i)
"""
Rel(i::Int) = Rel(1:i)


"""
The abstract base for types that manage the objective function evaluation.
`P` is the optimization problem it is used for.
"""
abstract type Evaluator{P <: OptimizationProblem} end

fitness_scheme(e::Evaluator) = fitness_scheme(problem(e))
fitness_type(e::Evaluator) = fitness_type(fitness_scheme(e))
worst_fitness(e::Evaluator) = worst_fitness(fitness_scheme(e))
numdims(e::Evaluator) = numdims(problem(e))
search_space(e::Evaluator) = search_space(problem(e))
describe(e::Evaluator) = "Problem: $(name(e.problem)) (dimensions = $(numdims(e)))"
problem_summary(e::Evaluator) = "$(name(e.problem))_$(numdims(e))d"

shutdown!(e::Evaluator) = e # do nothing

"""
Default implementation of the `Evaluator`.

`FP` is the original problem's fitness type
`FA` is the fitness type actually stored by the archive.
"""
mutable struct ProblemEvaluator{FP, FA, A<:Archive, P<:OptimizationProblem} <: Evaluator{P}
    # FIXME F is the fitness type of the problem, but with current Julia it's
    # not possible to get and use it at declaration type
    problem::P
    archive::A
    num_evals::Int
    last_fitness::FP

    ProblemEvaluator(problem::P, archive::A) where {P<:OptimizationProblem, A<:Archive} =
        new{fitness_type(fitness_scheme(problem)),fitness_type(archive),A,P}(
                problem, archive,
                0, nafitness(fitness_scheme(problem)))
end

ProblemEvaluator(problem::OptimizationProblem; archiveCapacity::Int = 10) =
    ProblemEvaluator(problem, TopListArchive(fitness_scheme(problem), numdims(problem), archiveCapacity))

problem(e::Evaluator) = e.problem
num_evals(e::ProblemEvaluator) = e.num_evals

"""
    fitness(params::Individual, e::ProblemEvaluator, tag::Int=0)

Evaluate the fitness and implicitly update the archive with the provided
parameters and calculated fitness.

Returns the fitness in the archived format.
"""
function fitness(params::Individual, e::ProblemEvaluator, tag::Int=0)
    e.last_fitness = fit = fitness(params, e.problem)
    e.num_evals += 1
    fita = archived_fitness(fit, e.archive)
    candi = add_candidate!(e.archive, fita, params, tag, e.num_evals)
    return fita
end

"""
    last_fitness(e::Evaluator)

Get the fitness of the last evaluated candidate.

Leads to nicer code if we can first test if it is better or worse than existing candidates
and only want the fitness itself if the criteria fulfilled.
"""
last_fitness(e::Evaluator) = e.last_fitness

is_better(f1::F, f2::F, e::Evaluator) where {F} = is_better(f1, f2, fitness_scheme(e))

is_better(candidate, f, e::Evaluator) = is_better(fitness(candidate, e), f, fitness_scheme(e))

function best_of(candidate1::Individual, candidate2::Individual, e::Evaluator)
    f1 = fitness(candidate1, e)
    f2 = fitness(candidate2, e)
    if is_better(f1, f2, e)
        return candidate1, f1
    else
        return candidate2, f2
    end
end

function update_fitness!(e::ProblemEvaluator{FP,FA}, candidate::Candidate{FA}) where {FP,FA}
    # evaluate fitness if not known yet
    if isnafitness(candidate.fitness, fitness_scheme(e.archive))
        candidate.fitness = fitness(candidate.params, e, candidate.tag)
    end
    return candidate
end

function update_fitness!(e::ProblemEvaluator{FP,FA}, candidates::AbstractVector{Candidate{FA}}) where {FP,FA}
    @inbounds for candidate in candidates
        update_fitness!(e, candidate)
    end
    return candidates
end

function rank_by_fitness!(e::Evaluator, candidates::AbstractVector{<:Candidate})
    fs = fitness_scheme(e)
    sort!(update_fitness!(e, candidates);
          # FIXME use lt=fitness_scheme(a) when v0.5 #14919 would be fixed
          by=fitness, lt=(x, y) -> is_better(x, y, fs))
end

# called by check_stop_condition(OptRunController)
check_stop_condition(e::Evaluator, ctrl) = check_stop_condition(e.archive, e.problem, ctrl)


abstract type AdversarialMDP{S,A} <: MDP{S,A} end
abstract type AdversarialPOMDP{S,A,O} <: POMDP{S,A,O} end

struct AdditiveAdversarialMDP{S,A}  <: AdversarialMDP{S,A}
    mdp::MDP{S,A}
    x_distribution
end

POMDPs.initialstate(mdp::AdditiveAdversarialMDP) = initialstate(mdp.mdp)

POMDPs.actions(mdp::AdditiveAdversarialMDP) = actions(mdp.mdp)
POMDPs.actionindex(mdp::AdditiveAdversarialMDP, a) = actionindex(mdp.mdp, a)

disturbances(mdp::AdditiveAdversarialMDP) = mdp.x_distribution.vs
disturbanceindex(mdp::AdditiveAdversarialMDP, x) = findfirst(mdp.x_distribution.vs .== x)

POMDPs.isterminal(mdp::AdditiveAdversarialMDP, s) = isterminal(mdp.mdp, s)
POMDPs.discount(mdp::AdditiveAdversarialMDP) = discount(mdp.mdp)

render(mdp::AdditiveAdversarialMDP, s, a = nothing; kwargs...) = render(mdp.mdp, s, a; kwargs...)
render(mdp::AdditiveAdversarialMDP; kwargs...) = render(mdp.mdp; kwargs...)

POMDPs.gen(mdp::AdditiveAdversarialMDP, s, a, x, rng::AbstractRNG = Random.GLOBAL_RNG; kwargs...) = gen(mdp.mdp, s, a .+ x, rng; kwargs...)

POMDPs.gen(mdp::AdditiveAdversarialMDP, s, a, rng::AbstractRNG = Random.GLOBAL_RNG; kwargs...) = gen(mdp, s, a, typeof(a)(rand(mdp.x_distribution)), rng; kwargs...)

isfailure(mdp::AdditiveAdversarialMDP, s) = isfailure(mdp.mdp, s)

struct AdditiveAdversarialPOMDP{S,A,O}  <: AdversarialPOMDP{S,A, O}
    pomdp::POMDP{S,A,O}
    x_distribution
end

POMDPs.initialstate(pomdp::AdditiveAdversarialPOMDP) = initialstate(pomdp.pomdp)
POMDPs.initialobs(pomdp::AdditiveAdversarialPOMDP, s) = initialobs(pomdp.pomdp, s)

POMDPs.isterminal(pomdp::AdditiveAdversarialPOMDP, s) = isterminal(pomdp.pomdp, s)
POMDPs.discount(pomdp::AdditiveAdversarialPOMDP) = discount(pomdp.pomdp)

render(pomdp::AdditiveAdversarialPOMDP, s, a = nothing; kwargs...) = render(pomdp.pomdp, s, a; kwargs...)
render(pomdp::AdditiveAdversarialPOMDP; kwargs...) = render(pomdp.pomdp; kwargs...)

POMDPs.gen(pomdp::AdditiveAdversarialPOMDP, s, a, x, rng::AbstractRNG = Random.GLOBAL_RNG; kwargs...) = gen(pomdp.pomdp, s, a .+ x, rng; kwargs...)

POMDPs.gen(pomdp::AdditiveAdversarialPOMDP, s, a, rng::AbstractRNG = Random.GLOBAL_RNG; kwargs...) = gen(pomdp, s, a, typeof(a)(rand(pomdp.x_distribution)), rng; kwargs...)

isfailure(pomdp::AdditiveAdversarialPOMDP, s) = isfailure(pomdp.pomdp, s)


using .ReverseDiff: compile, GradientTape
using .ReverseDiff.DiffResults: GradientResult

struct ReverseDiffAD{cache} <: ADBackend end
const RDCache = Ref(false)
setrdcache(b::Bool) = setrdcache(Val(b))
setrdcache(::Val{false}) = RDCache[] = false
setrdcache(::Val) = throw("Memoization.jl is not loaded. Please load it before setting the cache to true.")
function emptyrdcache end

getrdcache() = RDCache[]
ADBackend(::Val{:reversediff}) = ReverseDiffAD{getrdcache()}
function _setadbackend(::Val{:reversediff})
    ADBACKEND[] = :reversediff
end

function gradient_logp(
    backend::ReverseDiffAD{false},
    θ::AbstractVector{<:Real},
    vi::VarInfo,
    model::Model,
    sampler::AbstractSampler = SampleFromPrior(),
    context::DynamicPPL.AbstractContext = DynamicPPL.DefaultContext()
)
    T = typeof(getlogp(vi))

    # Specify objective function.
    function f(θ)
        new_vi = VarInfo(vi, sampler, θ)
        model(new_vi, sampler, context)
        return getlogp(new_vi)
    end
    tp, result = taperesult(f, θ)
    ReverseDiff.gradient!(result, tp, θ)
    l = DiffResults.value(result)
    ∂l∂θ::typeof(θ) = DiffResults.gradient(result)

    return l, ∂l∂θ
end

tape(f, x) = GradientTape(f, x)
function taperesult(f, x)
    return tape(f, x), GradientResult(x)
end

# @require Memoization = "6fafb56a-5788-4b4e-91ca-c0cea6611c73" @eval begin
setrdcache(::Val{true}) = RDCache[] = true
function emptyrdcache()
    Memoization.empty_cache!(memoized_taperesult)
    return
end

function gradient_logp(
    backend::ReverseDiffAD{true},
    θ::AbstractVector{<:Real},
    vi::VarInfo,
    model::Model,
    sampler::AbstractSampler = SampleFromPrior(),
    context::DynamicPPL.AbstractContext = DynamicPPL.DefaultContext()
)
    T = typeof(getlogp(vi))

    # Specify objective function.
    function f(θ)
        new_vi = VarInfo(vi, sampler, θ)
        model(new_vi, sampler, context)
        return getlogp(new_vi)
    end
    ctp, result = memoized_taperesult(f, θ)
    ReverseDiff.gradient!(result, ctp, θ)
    l = DiffResults.value(result)
    ∂l∂θ = DiffResults.gradient(result)

    return l, ∂l∂θ
end

# This makes sure we generate a single tape per Turing model and sampler
struct RDTapeKey{F, Tx}
    f::F
    x::Tx
end
function Memoization._get!(f, d::Dict, keys::Tuple{Tuple{RDTapeKey}, Any})
    key = keys[1][1]
    return Memoization._get!(f, d, (key.f, typeof(key.x), size(key.x), Threads.threadid()))
end
memoized_taperesult(f, x) = memoized_taperesult(RDTapeKey(f, x))
Memoization.@memoize Dict function memoized_taperesult(k::RDTapeKey)
    return compiledtape(k.f, k.x), GradientResult(k.x)
end
compiledtape(f, x) = compile(GradientTape(f, x))
# end


module AtomModule
export Atom, Element
import Base.==

using ..BaseModule

atomicNumbers = Dict{String,Int}(
  "H" => 	1,
  "He" => 	2,
  "Li" => 	3,
  "Be" =>	4,
  "B" =>	5,
  "C" =>	6,
  "N" =>	7,
  "O" =>	8,
  "F" =>	9,
  "Ne" =>	10,
  "Na" =>	11,
  "Mg" =>	12,
  "Al" =>	13,
  "Si" =>	14,
  "P" =>	15,
  "S" =>	16,
  "Cl" =>	17,
  "Ar" =>	18,
  "K" =>	19,
  "Ca" =>	20,
  "Sc" =>	21,
  "Ti" =>	22,
  "V" =>	23,
  "Cr" =>	24,
  "Mn" =>	25,
  "Fe" =>	26,
  "Co" =>	27,
  "Ni" =>	28,
  "Cu" =>	29,
  "Zn" =>	30,
  "Ga" =>	31,
  "Ge" =>	32,
  "As" =>	33,
  "Se" =>	34,
  "Br" =>	35,
  "Kr" =>	36,
  "Rb" =>	37,
  "Sr" =>	38,
  "Y" =>	39,
  "Zr" =>	40,
  "Nb" =>	41,
  "Mo" =>	42,
  "Tc" =>	43,
  "Ru" =>	44,
  "Rh" =>	45,
  "Pd" =>	46,
  "Ag" =>	47,
  "Cd" =>	48,
  "In" =>	49,
  "Sn" =>	50,
  "Sb" =>	51,
  "Te" =>	52,
  "I" =>	53,
  "Xe" =>	54,
  "Cs" =>	55,
  "Ba" =>	56,
  "La" =>	57,
  "Ce" =>	58,
  "Pr" =>	59,
  "Nd" =>	60,
  "Pm" =>	61,
  "Sm" =>	62,
  "Eu" =>	63,
  "Gd" =>	64,
  "Tb" =>	65,
  "Dy" =>	66,
  "Ho" =>	67,
  "Er" =>	68,
  "Tm" =>	69,
  "Yb" =>	70,
  "Lu" =>	71,
  "Hf" =>	72,
  "Ta" =>	73,
  "W" =>	74,
  "Re" =>	75,
  "Os" =>	76,
  "Ir" =>	77,
  "Pt" =>	78,
  "Au" =>	79,
  "Hg" =>	80,
  "Tl" =>	81,
  "Pb" =>	82,
  "Bi" =>	83,
  "Po" =>	84,
  "At" =>	85,
  "Rn" =>	86,
  "Fr" =>	87,
  "Ra" =>	88,
  "Ac" =>	89,
  "Th" =>	90,
  "Pa" =>	91,
  "U" =>	92,
  "Np" =>	93,
  "Pu" =>	94,
  "Am" =>	95,
  "Cm" =>	96,
  "Bk" =>	97,
  "Cf" =>	98,
  "Es" =>	99,
  "Fm" =>	100,
  "Md" =>	101,
  "No" =>	102,
  "Lr" =>	103,
  "Rf" =>	104,
  "Db" =>	105,
  "Sg" =>	106,
  "Bh" =>	107,
  "Hs" =>	108,
  "Mt" =>	109,
  "Ds" =>	110,
  "Rg" =>	111,
  "Cp" =>	112,
  "Uut" =>	113,
  "Uuq" =>	114,
  "Uup" =>	115,
  "Uuh" =>	116,
  "Uus" =>	117,
  "Uuo" =>	118
)

atomicWeight = Dict{Int64,Float64}(# according to IUPAC Pure Appl. Chem. 2016, 88(12), 1203-1224
  1  => 1.008,
  2  => 4.002,
  3  => 6.94,
  4  => 9.012,
  5  => 10.81,
  6  => 12.011,
  7  => 14.007,
  8  => 15.999,
  9  => 18.998,
  10 => 20.1797,
  11 => 22.989,
  12 => 24.305,
  13 => 26.981,
  14 => 28.085,
  15 => 30.973,
  16 => 32.06,
  17 => 35.45,
  18 => 39.948,
  19 => 39.0983,
  20 => 40.078,
  21 => 44.955,
  22 => 47.867,
  23 => 50.9415,
  24 => 51.9961,
  25 => 54.938,
  26 => 55.845,
  27 => 58.933,
  28 => 58.6934,
  29 => 63.546,
  30 => 65.38,
  31 => 69.723,
  32 => 72.630,
  33 => 74.921,
  34 => 78.971,
  35 => 79.904,
  36 => 83.798,
  37 => 85.4678,
  38 => 87.62,
  39 => 88.905,
  40 => 91.224,
  41 => 92.906,
  42 => 95.95,
  44 => 101.07,
  45 => 102.905,
  46 => 106.42,
  47 => 107.8682,
  48 => 112.414,
  49 => 114.818,
  50 => 118.710,
  51 => 121.760,
  52 => 127.60,
  53 => 126.904,
  54 => 131.293,
  55 => 132.905,
  56 => 137.327,
  57 => 138.905,
  58 => 140.116,
  59 => 140.907,
  60 => 144.242,
  62 => 150.36,
  63 => 151.964,
  64 => 157.25,
  65 => 158.925,
  66 => 162.500,
  67 => 164.930,
  68 => 167.259,
  69 => 168.934,
  70 => 173.045,
  71 => 174.9668,
  72 => 178.49,
  73 => 180.947,
  74 => 183.84,
  75 => 186.207,
  76 => 190.23,
  77 => 192.217,
  78 => 195.084,
  79 => 196.966,
  80 => 200.592,
  81 => 204.38,
  82 => 207.2,
  83 => 208.980,
  90 => 232.0377,
  91 => 231.035,
  92 => 238.028
)

struct Element
	symbol::String
	atomicNumber::Int

	Element(sym::AbstractString) = new(sym,atomicNumbers[sym])
end

==(el1::Element,el2::Element) = el1.symbol==el2.symbol && el1.atomicNumber==el2.atomicNumber
#isequal(el1::Element,el2::Element) = isequal(el1.symbol,el2.symbol) && isequal(el1.atomicNumber,el2.atomicNumber)

import Base.hash
hash(el::Element) = hash(el.symbol,hash(el.atomicNumber))

mutable struct Atom
	element::Element
	position::Position
end

import ..BaseModule.distance
distance(at1::Atom,at2::Atom) = distance(at1.position,at2.position)

end # module


function position(decal::Maple.Decal)
    return (
        floor(Int, decal.x),
        floor(Int, decal.y)
    )
end

function decalSelection(decal::Maple.Decal)
    texture = "decals/$(decal.texture)"
    sprite = getTextureSprite(texture)

    x, y = round(Int, decal.x), round(Int, decal.y)
    sx, sy = round(Int, decal.scaleX), round(Int, decal.scaleY)

    width, height = sprite.width, sprite.height
    realWidth, realHeight = sprite.realWidth, sprite.realHeight

    drawX, drawY = floor(Int, x - (realWidth / 2 + sprite.offsetX) * sx), floor(Int, y - (realHeight / 2 + sprite.offsetY) * sy)
    drawX += sx < 0 ? width * sx : 0
    drawY += sy < 0 ? height * sy : 0

    return Rectangle(drawX, drawY, width * abs(sx), height * abs(sy))
end

function drawDecal(ctx::Cairo.CairoContext, decal::Maple.Decal; alpha::Number=getGlobalAlpha())
    texture = "decals/$(decal.texture)"
    sprite = getTextureSprite(texture, "Gameplay")

    realWidth, realHeight = sprite.realWidth, sprite.realHeight
    width, height = sprite.width, sprite.height

    x, y = round(Int, decal.x), round(Int, decal.y)
    sx, sy = round(Int, decal.scaleX), round(Int, decal.scaleY)

    if width > 0 && height > 0
        Cairo.save(ctx)

        translate(ctx, x, y)
        scale(ctx, sx, sy)
        translate(ctx, -sprite.offsetX, -sprite.offsetY)
        translate(ctx, -realWidth / 2, -realHeight / 2)

        drawImage(ctx, sprite, 0, 0, alpha=alpha)

        restore(ctx)

    else
        debug.log("Couldn't render decal with texture '$texture' at ($x, $y)", "DRAWING_DECAL_MISSING")
    end
end

function spritesToDecalTextures(sprites::Dict{String, SpriteHolder})
    res = String[]

    for (path, spriteHolder) in sprites
        if startswith(path, "decals/")
            if spriteHolder.sprite.width > 0 && spriteHolder.sprite.height > 0
                push!(res, path[8:end])
            end
        end
    end

    return res
end

animationRegex = r"\D+0*?$"
filterAnimations(s::String) = occursin(animationRegex, s)

function decalTextures(animationFrames::Bool=false)
    Ahorn.loadExternalSprites!()
    textures = Ahorn.spritesToDecalTextures(Ahorn.getAtlas("Gameplay"))

    if !animationFrames
        filter!(filterAnimations, textures)
    end

    sort!(textures)

    return textures
end

function decalConfigOptions(decal::Maple.Decal, ignores::Array{String, 1}=String[])
    res = Form.Option[]

    names = get(langdata, ["placements", "decals", "names"])
    tooltips = get(langdata, ["placements", "decals", "tooltips"])

    ignores = vcat("__name", ignores)
    data = Dict(decal)

    for (attr, value) in data
        if attr in ignores
            continue
        end

        name = expandTooltipText(get(names, Symbol(attr), ""))
        displayName = isempty(name) ? humanizeVariableName(attr) : name
        tooltip = expandTooltipText(get(tooltips, Symbol(attr), ""))
        textures = attr == "texture" ? decalTextures() : nothing

        push!(res, Form.suggestOption(displayName, value, dataName=attr, tooltip=tooltip, choices=textures, editable=true))
    end

    return res
end