"""
    getimports(imports::Dict)

Get a .wat string with any used functions that are not builtins or userdefined.

# Details
- a few basic are builtin to wasm. these can be translated.
- other basic functions are bultin to JavaScripts global Math object, these can be imported
- many more are builtin to julia... so they need to be implemented either in .jl, .wat or .js
- give warning if the function cant be imported from js Math.
"""
function getimports(imports::Dict)
    watimports = []
    jsimports = []
    for func in keys(imports)
        push!(watimports, imports[func][1])
        push!(jsimports, imports[func][2])
    end

    if length(jsimports) > 0
        println("INFO: Suggested import object when instantiating WebAssembly from JavaScript:")
        println(jsimportsstring(jsimports))
        println()
    end
    return join(watimports, "\n")
end

function jsimportsstring(jsimports)
    str = ["imports = {\n"]
    for func in jsimports
        push!(str, "  ")
        push!(str, func)
        push!(str, ",\n")
    end
    push!(str, "}")
    return join(str)
end

"""
    jsimportentry(func, argtypes)

Get a string of possible javascript Math module import, assuming it exists in the Math module.

# Details
from a func such as sin, return a string like
\"sin: (x) => Math.sin(x)\"
"""
function jsimportentry(func, argtypes)
    Nparams = length(argtypes)

    # the global Math object in js have the same function names as julia has, except for pow and random
    # ofcourse it may not exist in js, but this allows the ones that do to be imported with correct name
    if func == :(^)
        jsfunc = "pow"
    elseif func == :(rand)
        jsfunc = "random"
    else
        jsfunc = func
    end

    jskey = "\"$(func)\": "
    jsval = " => Math.$(jsfunc)"
    paramnames = []
    for i = 1:Nparams
        push!(paramnames, index2abc(i))
    end
    pn = join(["("; join(paramnames, ", "); ")"])

    return join([jskey,pn,jsval,pn])
end

index2abc(i) = '`' + i # unicode char, '`'+1 means a, '`'+2 means b

isgenericfunction(func) = length(methods(Base.eval(Evalscope, func)).ms) > 0
imports!(imports::Dict, ci::CodeInfo, funcs::Dict, builtinfuncs::Dict, item) = nothing
function imports!(imports::Dict, ci::CodeInfo, funcs::Dict, builtinfuncs::Dict, items::Array)
    jsglobalMath = ["^","rand","acos","acosh","asin","asinh","atan","atanh","atan2","cbrt","cos","cosh","exp","expm1","hypot","imul","log","log1p","log10","log2","sign","sin","sinh","tan","tanh","trunc"]

    if isimport(funcs, builtinfuncs, items[1])
        func = items[1]
        if isgenericfunction(func)
            argtypes = itemtype.((ci,), items[2:end])
            ct = code_typed(Base.eval(Evalscope, func), Tuple{argtypes...}; optimize=false, debuginfo=:none)[1]
            cinfo = ct[1]
            Rtype = ct[2]
            imports[func] = [importdeclaration(func.name, argtypes, Rtype), jsimportentry(func.name, argtypes)]

            isavailableinjavascript = string(func.name) in jsglobalMath
            if !isavailableinjavascript
                println("INFO: JavaScript does not have Math.$(func.name). Import your own function instead or write it in julia.")
            end
        end
    end
    imports!.((imports,), (ci,), (funcs,), (builtinfuncs,), items)
end

isimport(funcs, builtinfuncs, item) = false
isimport(funcs, builtinfuncs, item::GlobalRef) = !haskey(builtinfuncs, item.name) && !haskey(funcs, item.name) && !haskey(floatops, item.name) && !haskey(intops, item.name)

function importdeclaration(func, argtypes, Rtype)
    Nparams = length(argtypes)
    paramtypes = [paramtype <: AbstractFloat ? "f32" : "i32" for paramtype in argtypes]

    decl = ["func \$$func (import \"imports\" \"$func\")"]
    for i = 1:Nparams
        push!(decl, "(param \$$('`'+i) $(paramtypes[i]))")
    end
    if !(Rtype <: Nothing) && Rtype <: Number
        rt = Rtype <: AbstractFloat ? "f32" : "i32"
        push!(decl, "(result $rt)")
    end
    decl = join(decl," ")
    return "($decl)"
end


#  ======================================================
#  Constructors
#  ======================================================

mutable struct SimpleValueGraph{V <: Integer,
                                E_VAL,
                                E_VAL_C <: Union{Nothing, EdgeValContainer{E_VAL}}
                               } <: AbstractSimpleValueGraph{V, E_VAL}
    ne::Int
    fadjlist::Adjlist{V}
    edgevals::E_VAL_C
end

const ConstValueGraph{V, E_VAL} = SimpleValueGraph{V, E_VAL, Nothing}
const InOutValueGraph{V, E_VAL} = SimpleValueGraph{V, E_VAL, <: EdgeValContainer{E_VAL}}

function SimpleValueGraph(nv::V,
                          E_VAL::Type=default_edgeval_type;
                          const_edgevals::Bool = false
                         ) where {V<:Integer}
    fadjlist = Adjlist{V}(nv)
    if const_edgevals
        return ConstValueGraph{V, E_VAL, Nothing}(nv, fadjlist, nothing)
    end
    edgevals = create_edgeval_list(nv, E_VAL)
    return InOutValueGraph{V, E_VAL, typeof(edgevals)}(V(0), fadjlist, edgevals)
end

SimpleValueGraph{V}(nv::Integer, E_VAL::Type=default_edgeval_type; args...) where {V} = SimpleValueGraph(V(nv), E_VAL, args...)
SimpleValueGraph{V, E_VAL}(nv::Integer, args...) where {V, E_VAL} = SimpleValueGraph(V(nv), E_VAL, args...)

# TODO weights are not symmetric
function SimpleValueGraph(g::SimpleGraph{V},
                          E_VAL::Type=default_edgeval_type;
                          const_edgevals::Bool=false,
                          edgeval_initializer = (s, d) -> default_edgeval(E_VAL)) where {V}
    n = nv(g)
    fadjlist = deepcopy_adjlist(g.fadjlist)
    if const_edgevals
        return ConstValueGraph{V, E_VAL}(nv, fadjlist, nothing)
    end
    edgevals = Vector{Vector{E_VAL}}(undef, n)
    for s in Base.OneTo(n)
        edgevals[s] = [edgeval_initializer(V(s), d) for d in fadjlist[s]]
    end
    InOutValueGraph{V, E_VAL, typeof(edgevals)}(ne(g), fadjlist, edgevals)
end

# TODO this function has some issues with typesafety
# TODO weights are not symmetric
# TODO const edgevals
function SimpleValueGraph(g::SimpleGraph{V},
                          E_VAL::Type{<:TupleOrNamedTuple};
                          const_edgevals::Bool=false,
                          edgeval_initializer = (s, d) -> default_edgeval(E_VAL)) where {V}
    n = nv(g)
    fadjlist = deepcopy_adjlist(g.fadjlist)
    if const_edgevals
        return ConstValueGraph{V, E_VAL}(nv, fadjlist, nothing)
    end

    E_VAL_C = edgevals_container_type(Val(E_VAL))
    edgevals = E_VAL_C( T(undef, n) for T in E_VAL_C.types )
    for s in Base.OneTo(n)
        for (i, T) in enumerate(E_VAL.types)
            edgevals[i][s] = Vector{T}(undef, length(fadjlist[s]))
        end
    end
    for s in Base.OneTo(n)
        for (j, d) in enumerate(fadjlist[s])
            w = edgeval_initializer(V(s), d)
            for (i, T) in enumerate(E_VAL.types)
                edgevals[i][s][j] = w[i]
            end
        end
    end
    InOutValueGraph{V, E_VAL, E_VAL_C}(ne(g), fadjlist, edgevals)
end

# =========================================================
# Interface
# =========================================================

function add_edge!(g::InOutValueGraph{V, E_VAL},
                   s::Integer,
                   d::Integer,
                   value=default_edgeval(E_VAL)) where {V, E_VAL}
    verts = vertices(g)
    (s in verts && d in verts) || return false # edge out of bounds
    edgevals = g.edgevals
    @inbounds list = g.fadjlist[s]
    index = searchsortedfirst(list, d)
    @inbounds if index <= length(list) && list[index] == d
        # edge already there, replace value, but return false
        set_value_for_index!(edgevals, s, index, value)
        # TODO maybe shortcircuit if it is a self-loop
        index = searchsortedfirst(g.fadjlist[d], s)
        set_value_for_index!(edgevals, d, index, value)
        return false
    end

    insert!(list, index, d)
    insert_value_for_index!(edgevals, s, index, value)
    g.ne += 1

    s == d && return true # selfloop

    @inbounds list = g.fadjlist[d]
    index = searchsortedfirst(list, s)
    insert!(list, index, s)
    insert_value_for_index!(edgevals, d, index, value)
    return true # edge successfully added
end

function add_edge!(g::ConstValueGraph,
                   s::Integer,
                   d::Integer)
    verts = vertices(g)
    (s in verts && d in verts) || return false # edge out of bounds
    @inbounds list = g.fadjlist[s]
    index = searchsortedfirst(list, d)
    @inbounds if index <= length(list) && list[index] == d
        # edge already there, return false
        return false
    end

    insert!(list, index, d)
    g.ne += 1

    s == d && return true # selfloop

    @inbounds list = g.fadjlist[d]
    index = searchsortedfirst(list, s)
    insert!(list, index, s)
    return true # edge successfully added
end

add_edge!(g::SimpleValueGraph, e::SimpleEdge)      = add_edge!(g, src(e), dst(e))
add_edge!(g::InOutValueGraph, e::SimpleEdge, u)   = add_edge!(g, src(e), dst(e), u)
add_edge!(g::InOutValueGraph, e::SimpleValueEdge) = add_edge!(g, src(e), dst(e), val(e))

function rem_edge!(g::InOutValueGraph, s::Integer, d::Integer)
    verts = vertices(g)
    (s in verts && d in verts) || return false # edge out of bounds
    edgevals = g.edgevals
    @inbounds list = g.fadjlist[s]
    index = searchsortedfirst(list, d)
    @inbounds (index <= length(list) && list[index] == d) || return false
    deleteat!(list, index)
    delete_value_for_index!(edgevals, s, index)

    g.ne -= 1
    s == d && return true # self-loop

    @inbounds list = g.fadjlist[d]
    index = searchsortedfirst(list, s)
    deleteat!(list, index)
    delete_value_for_index!(edgevals, d, index)

    return true
end

function rem_edge!(g::ConstValueGraph, s::Integer, d::Integer)
    verts = vertices(g)
    (s in verts && d in verts) || return false # edge out of bounds
    edgevals = g.edgevals
    @inbounds list = g.fadjlist[s]
    index = searchsortedfirst(list, d)
    @inbounds (index <= length(list) && list[index] == d) || return false
    deleteat!(list, index)

    g.ne -= 1
    s == d && return true # self-loop

    @inbounds list = g.fadjlist[d]
    index = searchsortedfirst(list, s)
    deleteat!(list, index)

    return true
end

rem_edge!(g::SimpleValueGraph, e::SimpleEdge) = rem_edge!(g, src(e), dst(e))
rem_edge!(g::InOutValueGraph, e::SimpleValueEdge) = rem_edge!(g, src(e), dst(e))

# TODO rem_vertex!, rem_vertices!

function has_edge(g::SimpleValueGraph, s::Integer, d::Integer)
    verts = vertices(g)
    (s in verts && d in verts) || return false # edge out of bounds
    @inbounds list_s = g.fadjlist[s]
    @inbounds list_d = g.fadjlist[d]
    if length(list_s) > length(list_d)
        d = s
        list_s = list_d
    end
    return LightGraphs.insorted(d, list_s)
end

function has_edge(g::SimpleValueGraph{V, E_VAL}, s::Integer, d::Integer, value) where {V, E_VAL}
    verts = vertices(g)
    (s in verts && d in verts) || return false # edge out of bounds
    @inbounds list_s = g.fadjlist[s]
    @inbounds list_d = g.fadjlist[d]
    if length(list_s) > length(list_d)
        s, d = d, s
        list_s = list_d
    end
    index = searchsortedfirst(list_s, d)
    @inbounds return (index <= length(list_s) && list_s[index] == d && value_for_index(g.edgevals, E_VAL, s, index) == value)
end

has_edge(g::SimpleValueGraph, e::SimpleEdge)      = has_edge(g, src(e), dst(e))
has_edge(g::SimpleValueGraph, e::SimpleEdge, u)   = has_edge(g, src(e), dst(e), u)
has_edge(g::SimpleValueGraph, e::SimpleValueEdge) = has_edge(g, src(e), dst(e), val(e))

# TODO rest methods for get_value
# TODO lots of duplicated code
function get_edgeval(g::InOutValueGraph{V, E_VAL}, s::Integer, d::Integer) where {V, E_VAL}
    verts = vertices(g)
    (s in verts && d in verts) || return nothing # TODO may raise bounds error?
    @inbounds list_s = g.fadjlist[s]
    @inbounds list_d = g.fadjlist[d]
    if length(list_s) > length(list_d)
        s, d = d, s
        list_s = list_d
    end
    index = searchsortedfirst(list_s, d)
    @inbounds if index <= length(list_s) && list_s[index] == d
        return value_for_index(g.edgevals, E_VAL, s, index)
    end
    return nothing
end

function get_edgeval(g::InOutValueGraph{V, E_VAL}, s::Integer, d::Integer, key) where {V <: Integer, E_VAL <: TupleOrNamedTuple}
    verts = vertices(g)
    (s in verts && d in verts) || return nothing # TODO may raise bounds error?
    @inbounds list_s = g.fadjlist[s]
    @inbounds list_d = g.fadjlist[d]
    if length(list_s) > length(list_d)
        s, d = d, s
        list_s = list_d
    end
    index = searchsortedfirst(list_s, d)
    @inbounds if index <= length(list_s) && list_s[index] == d
        return value_for_index(g.edgevals, E_VAL, s, index, key)
    end
    return nothing
end

function get_edgeval(g::ConstValueGraph{V, E_VAL}, s::Integer, d::Integer) where {V, E_VAL}
    if has_edge(g, s, d)
        return default_edgeval(E_VAL)
    end
    return nothing
end

function get_edgeval(g::ConstValueGraph{V, E_VAL}, s::Integer, d::Integer, key) where {V, E_VAL}
    if has_edge(g, s, d)
        return default_edgeval(E_VAL)[key]
    end
    return nothing
end

get_edgeval!(g::SimpleValueGraph, e::SimpleEdge)   = get_edgeval(g, src(e), dst(e))
get_edgeval!(g::SimpleValueGraph, e::SimpleEdge, key)   = get_edgeval(g, src(e), dst(e), key)

function set_edgeval!(g::InOutValueGraph, s::Integer, d::Integer, value)
    verts = vertices(g)
    (s in verts && d in verts) || return false
    @inbounds list = g.fadjlist[s]
    index = searchsortedfirst(list, d)
    @inbounds index <= length(list) && list[index] == d || return false
    edgevals = g.edgevals
    set_value_for_index!(edgevals, s, index, value)

    @inbounds list = g.fadjlist[d]
    index = searchsortedfirst(list, s)
    set_value_for_index!(edgevals, d, index, value)

    return true
end

# TODO create generated methods for NamedTuple (and also Tuple?)
function set_edgeval!(g::InOutValueGraph{V, <: TupleOrNamedTuple}, s::Integer, d::Integer, key, value) where {V}
    verts = vertices(g)
    (s in verts && d in verts) || return false
    @inbounds list = g.fadjlist[s]
    index = searchsortedfirst(list, d)
    @inbounds index <= length(list) && list[index] == d || return false
    edgevals = g.edgevals
    set_value_for_index!(edgevals, s, index, key, value)

    @inbounds list = g.fadjlist[d]
    index = searchsortedfirst(list, s)
    set_value_for_index!(edgevals, d, index, key, value)

    return true
end

set_edgeval!(g::InOutValueGraph, e::SimpleEdge, u)   = set_value!(g, src(e), dst(e), u)
set_edgeval!(g::InOutValueGraph, e::SimpleValueEdge) = set_value!(g, src(e), dst(e), edgeval(e))

is_directed(::Type{<:SimpleValueGraph}) = false
is_directed(g::SimpleValueGraph)        = false

outneighbors(g::SimpleValueGraph, v::Integer) = g.fadjlist[v]
inneighbors(g::SimpleValueGraph,  v::Integer) = outneighbors(g, v)

outedgevals(g::InOutValueGraph{V, E_VAL, <: Adjlist}, v::Integer) where {V, E_VAL} =
    g.edgevals[v]
outedegevals(g::ConstValueGraph{V, E_VAL}, v::Integer) where {V, E_VAL} = Base.Iterators.repeated(default_edgeval(E_VAL), length(g.fadjlist[v]))

inedgevals(g::SimpleValueGraph, v::Integer) = outedgevals(g, v)
all_edgevals(g::SimpleValueGraph, v::Integer) = outedgevals(g, v)

# TODO implement these with iterators instead of generators
outedgevals(g::InOutValueGraph{V, E_VAL, <: Tuple}, v::Integer) where {V, E_VAL} =
    ( Tuple( adjlist[v][i] for adjlist in g.edgevals ) for i in OneTo(length(g.fadjlist[v])) )
outedgevals(g::InOutValueGraph{V, E_VAL, T}, v::Integer) where {V, E_VAL, T <: NamedTuple} =
    ( NamedTuple{Tuple(T.names)}( adjlist[v][i] for adjlist in g.edgevals ) for i in OneTo(length(g.fadjlist[v])) )

outedgevals(g::InOutValueGraph{V, E_VAL, <: TupleOrNamedTuple}, v::Integer, key) where {V, E_VAL} =
    g.edgevals[key][v]
outedegevals(g::ConstValueGraph{V, E_VAL}, v::Integer, key) where {V, E_VAL} = Base.Iterators.repeated(default_edgeval(E_VAL)[key], length(g.fadjlist[v]))

inedgevals(g::SimpleValueGraph{V, E_VAL}, v::Integer, key) where {V, E_VAL} =
    outedgevals(g, v, key)
all_edgevals(g::SimpleValueGraph{V, E_VAL}, v::Integer, key) where {V, E_VAL} =
    outedgevals(g, v, key)

# TODO maybe add a sizehint kwarg
function add_vertex!(g::InOutValueGraph{V, E_VAL, E_VAL_C}) where {V, E_VAL, E_VAL_C}
    # TODO There are overflow checks in Julia Base, use these
    (nv(g) + one(V) <= nv(g)) && return false # overflow
    push!(g.fadjlist, V[])
    if E_VAL_C <: TupleOrNamedTuple
        for (i, T) in enumerate(E_VAL.types)
            push!(g.edgevals[i], T[])
        end
    else
        push!(g.edgevals, E_VAL[])
    end
    return true
end

function add_vertex!(g::ConstValueGraph{V}) where {V}
    # TODO There are overflow checks in Julia Base, use these
    (nv(g) + one(V) <= nv(g)) && return false # overflow
    push!(g.fadjlist, V[])
    return true
end

# ====================================================================
# Iterators
# ====================================================================

function iterate(iter::SimpleValueEdgeIter{<:InOutValueGraph}, state=(one(eltype(iter.g)), 1) )
    g = iter.g
    fadjlist = g.fadjlist
    V = eltype(g)
    E_VAL = edgeval_type(g)
    n::V = nv(g)
    u::V, i = state

    @inbounds while u < n
        if i > length(fadjlist[u])
            u += V(1)
            i = searchsortedfirst(fadjlist[u], u)
            continue
        end
        e = SimpleValueEdge(u, fadjlist[u][i], value_for_index(g.edgevals, E_VAL, u, i))
        return e, (u, i + 1)
    end

    (n == 0 || i > length(fadjlist[n])) && return nothing

    e = SimpleValueEdge(n, n, value_for_index(g.edgevals, E_VAL, n, 1))
    return e, (u, i + 1)
end

function iterate(iter::SimpleValueEdgeIter{<:ConstValueGraph}, state=(one(eltype(iter.g)), 1) )
    g = iter.g
    fadjlist = g.fadjlist
    V = eltype(g)
    n::V = nv(g)
    u::V, i = state

    @inbounds while u < n
        if i > length(fadjlist[u])
            u += V(1)
            i = searchsortedfirst(fadjlist[u], u)
            continue
        end
        e = SimpleEdge(u, fadjlist[u][i])
        return e, (u, i + 1)
    end

    (n == 0 || i > length(fadjlist[n])) && return nothing

    e = SimpleEdge(n, n, default_edgeval(E_VAL))
    return e, (u, i + 1)
end


using NamedArrays, StatsPlots, JLD2, Measures

sumdimdrop(x::AbstractArray; dims) = dropdims(sum(x, dims=dims), dims=dims)

# Convert a JuMPArray to an AxisArray. Uses current JuMPArray internals, will need a rewrite in the next JuMP version.
AxisArrays.AxisArray(ja::JuMP.JuMPArray) = AxisArray(ja.innerArray, ja.indexsets...)

# Convert a JuMPDict to an AxisArray. Uses current JuMPDict internals, will need a rewrite in the next JuMP version.
# function AxisArrays.AxisArray(jd::JuMP.JuMPDict)
#   d = jd.tupledict
#   k = keys(d)
#   ndims = length(iterate(k)[1])
#   sets = [unique(getindex.(k,dim)) for dim=1:ndims]
#   a = AxisArray(zeros(length.(sets)...), sets...)
#   for (key, value) in d
#       a[key...] = value
#   end
#   return a
# end

# Convert a JuMPDict to a Dict. Uses current JuMPDict internals, will need a rewrite in the next JuMP version.
getdict(jd::JuMP.JuMPDict) = jd.tupledict

function readresults(model::ModelInfo, status::Symbol)
    @unpack REGION, TECH, CLASS, HOUR, techtype, STORAGECLASS = model.sets
    @unpack Systemcost, CO2emissions, FuelUse, Electricity, Charging, StorageLevel, Transmission, TransmissionCapacity, Capacity = model.vars
    @unpack demand, classlimits, hydrocapacity = model.params
    storagetechs = [k for k in TECH if techtype[k] == :storage]

    params = Dict(:demand => demand, :classlimits => classlimits, :hydrocapacity => hydrocapacity)

    cost = AxisArray(getvalue(Systemcost))
    emis = AxisArray(getvalue(CO2emissions))
    fuel = AxisArray(getvalue(FuelUse))
    # getting Electricity for all set combos is slowest, so let's optimize storage format and use faster internal function call
    elec = Dict((k,c) => [JuMP._getValue(Electricity[r,k,c,h]) for h in HOUR, r in REGION] for k in TECH for c in CLASS[k]);
    charge = AxisArray(getvalue(Charging))
    # oops, StorageLevel is also slow
    storage = Dict((k,c) => [JuMP._getValue(StorageLevel[r,k,c,h]) for h in HOUR, r in REGION] for k in storagetechs for c in STORAGECLASS[k]);
    transmission = AxisArray(getvalue(Transmission))
    transmissioncapac = AxisArray(getvalue(TransmissionCapacity))
    capac = getdict(getvalue(Capacity))

    return Results(status, model.options, model.hourinfo, model.sets, params, cost, emis, fuel, elec, charge, storage, transmission, transmissioncapac, capac)
end

function saveresults(results::Results, runname; resultsfile="", group="", compress=true)
    isempty(resultsfile) && return nothing
    if !isempty(group) && group[end] != '/'
        group *= "/"
    end
    runname = "$group$runname"
    JLD2.jldopen(resultsfile, "a+", compress=compress) do file
        if haskey(file, runname)
            @warn "The run $runname already exists in $resultsfile (new run not saved to disk). "
        else
            file[runname] = results
        end
    end
    return nothing
end

function listresults(; resultsfile="results.jld2", group="")
    if !isempty(group) && group[end] != '/'
        group *= "/"
    end
    JLD2.jldopen(resultsfile, "r") do file
        return keys(file)
    end
end

function loadresults(; resultsfile="results.jld2", group="", loadoptions...)
    options = merge(defaultoptions(), loadoptions)
    loadresults(autorunname(options); resultsfile=resultsfile, group=group)
end

function loadresults(runname::String; resultsfile="results.jld2", group="")
    if !isempty(group) && group[end] != '/'
        group *= "/"
    end
    runname = "$group$runname"
    results = nothing
    JLD2.jldopen(resultsfile, "r") do file
        if haskey(file, runname)
            results = file[runname]
        else
            println("\nThe run $runname does not exist in $resultsfile.")
        end
    end
    return results
end

const CHARTTECHS = Dict(
    :palette => [
        #RGB([68,131,208]/255...),  #hydroRoR
        RGB([216,137,255]/255...),  #nuclear            or RGBA{Float64}(0.76444,0.444112,0.824298,1.0)
        RGB([119,112,71]/255...),   #coal
        #RGB([164,155,104]/255...), #coal CCS
        RGB([199,218,241]/255...),  #wind
        RGB([149,179,215]/255...),  #wind offshore
        RGB([214,64,64]/255...),    #solarCSP
        RGB([255,255,64]/255...),   #solarPV
        RGB([240,224,0]/255...),    #PVrooftop
        RGB([255,192,0]/255...),    #CCGT
        RGB([99,172,70]/255...),    #bioCCGT
        RGB([100,136,209]/255...),  #hydro
        RGB([144,213,93]/255...),   #bioGT
        RGB([148,138,84]/255...),   #gasGT
        RGB([157,87,205]/255...)    #battery
    ],

    :labellookup => Dict(:nuclear => "nuclear", :coal => "coal", :wind => "onshore wind", :offwind => "offshore wind",
        :csp => "CSP", :pv => "PV plant", :pvroof => "PV rooftop", :gasCCGT => "gas CCGT", :bioCCGT => "bio CCGT",
        :hydro => "hydro", :bioGT => "bio GT", :gasGT => "gas GT", :battery => "battery"),

    :displaytechs => [:nuclear, :coal, :wind, :offwind, :csp, :pv, :pvroof, :gasCCGT, :bioCCGT, :hydro, :bioGT, :gasGT, :battery]
)

function analyzeresults(results::Results)
    @unpack REGION, FUEL, TECH, CLASS, HOUR, techtype, STORAGECLASS = results.sets
    @unpack demand, classlimits, hydrocapacity = results.params
    @unpack CO2emissions, FuelUse, Electricity, Transmission, Capacity, TransmissionCapacity, Charging, StorageLevel, Systemcost = results

    hoursperperiod = results.hourinfo.hoursperperiod

    capacmat = [sum(Capacity[r,k,c] for c in CLASS[k]) for k in TECH, r in REGION]
    capac = NamedArray([capacmat sum(capacmat, dims=2)], (TECH, [REGION; :TOTAL]))
    elec = zeros(length(HOUR), length(TECH), length(REGION))
    for (i,k) in enumerate(TECH)
        elec[:,i,:] = sum(Electricity[k,c] for c in CLASS[k])
    end
    annualelec = NamedArray([sumdimdrop(elec,dims=1) sumdimdrop(elec, dims=(1,3))], (TECH, [REGION; :TOTAL]), (:TECH, :REGION))
    charge = [Charging[r,:battery,h] for h in HOUR, r in REGION]
    storagetechs = [k for k in TECH if techtype[k] == :storage]
    storage = zeros(length(HOUR), length(storagetechs), length(REGION))
    for (i,k) in enumerate(storagetechs)
        storage[:,i,:] = sum(StorageLevel[k,c] for c in STORAGECLASS[k])
    end
    existingstoragelevel = NamedArray(storage, (collect(HOUR), storagetechs, REGION), (:HOUR, :TECH, :REGION))
    tcapac = NamedArray([TransmissionCapacity[r1,r2] for r1 in REGION, r2 in REGION], (REGION,REGION))

    # @unpack ElecDemand = model.constraints
    # prices = [getdual(ElecDemand[r,h]) for r in REGION, h in HOUR]
    # prices = NamedArray([getdual(ElecDemand[r,h]) for r in REGION, h in HOUR], (REGION, collect(HOUR)))       # €/kWh

    plotly()

    @unpack palette, labellookup, displaytechs = CHARTTECHS
    techlabels = [labellookup[k] for r=1:1, k in displaytechs]
    displayorder = [i for (i,k) in enumerate(TECH), d in displaytechs if d == k]

    function chart(country::Symbol; plotstoragetech=:none, plotbatterycharge=false, plotbatterylevel=false, optionlist...)
        if country == :BARS
            existinghydro = vec(sum(Electricity[:hydro,:x0], dims=1))
            regcost = Systemcost ./ (vec(sum(annualelec, dims=1)[1:end-1]) - existinghydro) * 1000
            totcost = sum(Systemcost) / (sum(annualelec[:,:TOTAL]) - sum(existinghydro)) * 1000
            lcoe = NamedArray(collect([regcost; totcost]'), (["system cost (€/MWh)"], [REGION; :TOTAL]))
            println("Regional system cost per MWh generated (€/MWh):")
            display(round.(lcoe, digits=2))

            lr = length(REGION)
            stackedbar(String.(REGION), collect(annualelec[displayorder,1:end-1]'/1000); labels=techlabels, size=(340+70*lr,550), left_margin=25px,
                legend=:outertopright, line=0, tickfont=10, legendfont=14, guidefont=14, color_palette=palette, ylabel="TWh/year", yformatter=:plain, optionlist...)

            xpos = (1:lr)' .- 0.5
            demandtext = ["demand" permutedims(repeat([""],lr-1))]
            display(plot!([xpos; xpos], [zeros(lr)'; sum(demand,dims=2)'*hoursperperiod/1000], line=3, color=:black,
                            labels=demandtext))
            if lr == 21
                totelec = [sumdimdrop(annualelec[:,1:8],dims=2) sumdimdrop(annualelec[:,9:15],dims=2) sumdimdrop(annualelec[:,16:21],dims=2) annualelec[:,:TOTAL]]
                stackedbar(["EU","CAS","China","TOTAL"], collect(totelec[displayorder,:]'/1e3); labels=techlabels, left_margin=20px,
                    legend=:outertopright, size=(500,950), line=0, tickfont=14, legendfont=14, color_palette=palette, yformatter=:plain, optionlist...)
                xpos = (1:4)' .- 0.5
                totdemand = [sum(demand[1:8,:]) sum(demand[9:15,:]) sum(demand[16:21,:]) sum(demand)]
                display(plot!([xpos; xpos], [zeros(4)'; totdemand*hoursperperiod/1e3], line=3, color=:black, labels=permutedims(repeat([""],4))))
                totcost2 = [sum(Systemcost[1:8]) sum(Systemcost[9:15]) sum(Systemcost[16:21]) sum(Systemcost)]
                tothydro = [sum(existinghydro[1:8]) sum(existinghydro[9:15]) sum(existinghydro[16:21]) sum(existinghydro)]
                lcoe_tot = NamedArray(totcost2./(totdemand .- tothydro) * 1000, (["system cost (€/MWh)"], ["EU","CAS","China","TOTAL"]))
                println("\nSystem cost per MWh demand (€/MWh):")
                display(round.(lcoe_tot, digits=2))
            else
                stackedbar(["TOTAL"], collect(annualelec[displayorder,:TOTAL]')/1e3; labels=techlabels, left_margin=30px,
                    size=(350,600), line=0, tickfont=14, legendfont=14, color_palette=palette, yformatter=:plain,
                    legend = :outertopright, optionlist...)
                xpos = (1:1)' .- 0.5
                totdemand = [sum(demand)]
                display(plot!([xpos; xpos], [zeros(1)'; totdemand*hoursperperiod/1e3], line=3, color=:black,
                    xlims=(-0.2,1.2), label="demand"))
            end
            country == :BARS && return nothing
        end

        if country == :TOTAL || country == :TOT || country == :total || country == :tot
            regs = 1:length(REGION)
        elseif country == :EU || country == :eu
            regs = 1:8
        elseif country == :ASIA || country == :asia
            regs = 9:15
        elseif country == :CHINA || country == :china || country == :CH || country == :ch
            regs = 16:21
        else
            countryindex = findfirst(REGION .== country)
            countryindex == nothing && error("Region $country not in $REGION.")
            regs = [countryindex]
        end

        regelec = sumdimdrop(elec[:,:,regs], dims=3)[:,displayorder] / hoursperperiod
        regcharge = sumdimdrop(charge[:,regs], dims=2) / hoursperperiod
        regdischarge = regelec[:, findfirst(displaytechs .== :battery)]
        regdemand = sumdimdrop(demand[regs,:], dims=1)

        composite = plot(; layout=grid(2,3,widths=[.4,.2,.4,.4,.2,.4]), size=(1850,950),
                        legend=false, tickfont=16, titlefont=20, optionlist...)
        for (i,k) in enumerate([:wind, :offwind, :hydro, :pv, :pvroof, :csp])
            colors = [palette[findfirst(displaytechs .== k)]; RGB(0.9,0.9,0.9)]
            classes = (k == :offwind || k == :pvroof) ? CLASS[k][1:length(CLASS[k])÷2] : CLASS[k]
            used = [sum(Capacity[r,k,c] for r in REGION[regs]) for c in classes]
            lims = [sum(k == :hydro ? hydrocapacity[r,c] : classlimits[r,k,c] for r in REGION[regs]) for c in classes]
            stackedbar!(String.(classes), [used lims-used]; subplot=i, title=labellookup[k], tickfont=14, legendfont=14, guidefont=14,
                            ylabel="GW", line=0, color_palette=colors, bottom_margin=30px, optionlist...)
        end
        display(composite)

        if plotstoragetech != :none
            regstorage = sumdimdrop(existingstoragelevel[:,[plotstoragetech],regs], dims=3, optionlist...)
            p = plot(regstorage; size=(900,550), tickfont=16, legendfont=16, label="storage level (TWh)")
            if plotstoragetech == :battery
                plot!(regcharge/1000, label="charge (TWh/h)")
                batteryelec = sumdimdrop([Electricity[:battery,:_][r,h] for r in REGION, h in HOUR][regs,:], dims=1) / hoursperperiod
                plot!(batteryelec/1000, label="discharge (TWh/h)")
            end
            display(p)
        end

        level = StorageLevel[:battery,:_]*1000
        reglevel = sumdimdrop(level[:,regs], dims=2)
        # display(plot(HOUR,[regcharge regelec[:,12] reglevel],size=(1850,950)))

        stackedarea(HOUR, regelec; labels=techlabels, size=(900,550), line=(0.03,1,:black), tickfont=14, legendfont=14, guidefont=14,
                                    xlabel="hour of year", ylabel="GW", yformatter=:plain, color_palette=palette, optionlist...)
        plotbatterycharge && plot!(HOUR, -regcharge, color=RGB([157,87,205]/255...))
        plotbatterylevel && plot!(HOUR, reglevel, line=(:black,:dash))
        # plot!(HOUR, regdischarge, color=:green)
        display(plot!(HOUR, regdemand, c=:black, line=2, label="demand"))
        nothing
    end

    # chart(:NOR)

    # if true   # plot hydro storage & shadow prices
    #   plot(elec[:,:hydro,:NOR]/hoursperperiod/1000, size=(1850,950), tickfont=16, legendfont=16)
    #   plot!(sum(elec[:,:wind,:],2)/hoursperperiod/1000)
    #   display(plot!(vec(mean(prices,1))))
    # end

    return annualelec, capac, tcapac, chart
end

function chart_energymix_scenarios(scenarios, resultsnames, resultsfile; size=(900,550), options...)
    numscen = length(scenarios)
    scenelec, demands, hoursperperiod, displayorder, techlabels, palette = allscenarioresults(scenarios, resultsnames, resultsfile)

    println("\nShare of demand (%):")
    demshare = [techlabels "-"; round.(scenelec[displayorder,:]'./demands .* 100, digits=1)  scenarios]
    display(reverse(permutedims(demshare), dims=1))

    stackedbar(collect(scenelec[displayorder,:]')/1e3, label=techlabels, size=size, left_margin=20px,
            xticks=(1:numscen,scenarios), line=0, tickfont=12, legendfont=12, guidefont=12,
            color_palette=palette, ylabel="TWh/year", yformatter=:plain; options...)
    xpos = (1:numscen)'
    lab = fill("",(1,numscen))
    lab[1] = "demand"
    display(plot!([xpos; xpos], [zeros(numscen)'; demands'*hoursperperiod/1e3], line=3, color=:black, label=lab))
    nothing
end

function allscenarioresults(scenarios, resultsnames, resultsfile)
    numscen = length(scenarios)
    scenelec = zeros(13,numscen)
    demands = zeros(numscen)
    hoursperperiod, displayorder, techlabels, palette = nothing, nothing, nothing, nothing

    for (i,s) in enumerate(scenarios)
        println("\nLoading results: $s...")
        totalelec, totaldemand, hoursperperiod, displayorder, techlabels, palette =
                readscenariodata(resultsnames[i], resultsfile)
        scenelec[:,i] = totalelec
        demands[i] = totaldemand
    end
    return scenelec, demands, hoursperperiod, displayorder, techlabels, palette
end

function readscenariodata(resultname, resultsfile)
    println(resultname, ": ", resultsfile)
    results = loadresults(resultname, resultsfile=resultsfile)

    annualelec, capac, tcapac, chart = analyzeresults(results);
    chart(:BARS)
    println()
    display(round.(Int, tcapac))

    @unpack TECH, REGION, CLASS, HOUR = results.sets
    hoursperperiod = results.hourinfo.hoursperperiod
    totaldemand = sum(results.params[:demand])
    totalelec = [sum(sum(results.Electricity[k,c]) for c in CLASS[k]) for k in TECH]

    @unpack palette, labellookup, displaytechs = CHARTTECHS
    techlabels = [labellookup[k] for r=1:1, k in displaytechs]
    displayorder = [i for (i,k) in enumerate(TECH), d in displaytechs if d == k]

    return totalelec, totaldemand, hoursperperiod, displayorder, techlabels, palette
end

function getlines(r, tcapac)
    reg = r.sets.REGION
    ii = findall(tcapac.>0)
    tc = [round(Int, tcapac[i]*1000) for i in ii if i[1]>i[2]]
    conn = [(reg[i[1]], reg[i[2]]) for i in ii if i[1]>i[2]]
    return sortslices([tc conn], dims=1, rev=true)
end

function exportwindresults(r, name)
    classes = r.sets.CLASS[:wind][1:5]
    electricity = Dict(c => sum(r.Electricity[:wind, c], dims=1) for c in classes)
    off_electricity = Dict(c => sum(r.Electricity[:offwind, c], dims=1) for c in classes)
    inputdata = getdatafolder(Dict(:datafolder => ""))
    open(joinpath(inputdata, "windresults_$name.csv"), "w") do io
        write(io, "region,cap1,cap2,cap3,cap4,cap5,")
        write(io, "offcap1,offcap2,offcap3,offcap4,offcap5,")
        write(io, "elec1,elec2,elec3,elec4,elec5,")
        write(io, "offelec1,offelec2,offelec3,offelec4,offelec5\n")
        for (i, reg) in enumerate(r.sets.REGION)
            cap = [round(1000 * r.Capacity[reg, :wind, c], digits=3) for c in classes]
            offcap = [round(1000 * r.Capacity[reg, :offwind, c], digits=3) for c in classes]
            elec = [round(electricity[c][i], digits=3) for c in classes]
            offelec = [round(off_electricity[c][i], digits=3) for c in classes]
            write(io, string(reg), ", ")
            write(io, join(cap, ", "), ", ")
            write(io, join(offcap, ", "), ", ")
            write(io, join(elec, ", "), ", ")
            write(io, join(offelec, ", "))
            write(io, "\n")
        end
    end
end

#=
default colors
get_color_palette(:auto, default(:bgcolor), 13)

 RGBA{Float64}(0.0,0.605603,0.97868,1.0)
 RGBA{Float64}(0.888874,0.435649,0.278123,1.0)
 RGBA{Float64}(0.242224,0.643275,0.304449,1.0)
 RGBA{Float64}(0.76444,0.444112,0.824298,1.0)
 RGBA{Float64}(0.675544,0.555662,0.0942343,1.0)
 RGBA{Float64}(4.82118e-7,0.665759,0.680997,1.0)
 RGBA{Float64}(0.930767,0.367477,0.57577,1.0)
 RGBA{Float64}(0.776982,0.509743,0.146425,1.0)
 RGBA{Float64}(3.80773e-7,0.664268,0.552951,1.0)
 RGBA{Float64}(0.558465,0.593485,0.117481,1.0)
 RGBA{Float64}(5.94762e-7,0.660879,0.798179,1.0)
 RGBA{Float64}(0.609671,0.499185,0.911781,1.0)
 RGBA{Float64}(0.380002,0.551053,0.966506,1.0)

function test()
_Systemcost = AxisArray(getvalue(m.vars.Systemcost))
_CO2emissions = AxisArray(getvalue(m.vars.CO2emissions))
_FuelUse = AxisArray(getvalue(m.vars.FuelUse))
_Electricity = Supergrid.getdict(getvalue(m.vars.Electricity))
_Charging = Supergrid.getdict(getvalue(m.vars.Charging))
_StorageLevel = Supergrid.getdict(getvalue(m.vars.StorageLevel))
_Transmission = AxisArray(getvalue(m.vars.Transmission))
_TransmissionCapacity = AxisArray(getvalue(m.vars.TransmissionCapacity))
_Capacity = Supergrid.getdict(getvalue(m.vars.Capacity))
end

@time elec = [sum(getvalue(m.vars.Electricity[r,k,c,h]) for c in CLASS[k]) for h in HOUR, k in TECH, r in REGION];

@time elec = Dict(getvalue(m.vars.Electricity[r,k,c,h]) for r in REGION, k in TECH, c in CLASS[k], h in HOUR);
@time elec = Dict((k,c) => [JuMP._getValue(m.vars.Electricity[r,k,c,h]) for r in REGION, h in HOUR] for k in TECH for c in CLASS[k]);
=#


module VisualCircuits

# Implement math for two-tuples
import Base: +, -, *, /, getindex, connect
using Compose, Colors, Measures

include("TupleMath.jl")
include("core.jl")
export Block, IC

include("LogicGates.jl")
export Gate

include("BlockUtils.jl")
export connect, dots, render, autosize

end # module


using Test
using LinearAlgebra
using DataFlowTasks
using DataFlowTasks: DAG, num_nodes, num_edges, addnode!, addedge!,
                    has_edge, outneighbors, inneighbors, isconnected,
                    adjacency_matrix, addedge_transitive!, remove_node!, R,W,RW, TaskGraph

@testset "DAG{Int}" begin
    dag = DAG{Int}()
    @test num_nodes(dag) == 0
    addnode!(dag,1)
    @test num_nodes(dag) == 1
    @test num_edges(dag) == 0
    addnode!(dag, 2)
    @test num_nodes(dag) == 2
    addedge!(dag, 1, 2)
    @test num_edges(dag) == 1
    @test has_edge(dag, 1, 2) == true
    @test has_edge(dag, 2, 1) == false
    @test outneighbors(dag, 1) == Set(2)
    @test inneighbors(dag, 1) == Set()
    @test outneighbors(dag, 2) == Set()
    @test inneighbors(dag, 2) == Set(1)
    addnode!(dag, 3)
    addnode!(dag, 4)
    addedge!(dag,2,4)
    @test isconnected(dag, 1, 3) == false
    @test isconnected(dag, 2, 4) == true
    @test isconnected(dag, 1, 4) == true # 1 --> 2 --> 4
    @test adjacency_matrix(dag) == [0 1 0 0;
                                    0 0 0 1;
                                    0 0 0 0;
                                    0 0 0 0]
    @test num_edges(addedge_transitive!(dag, 1, 4)) == 2 # vtx 1 is already connected to vtx through vtx2
end

@testset "TaskGraph" begin
    dag = TaskGraph()
    C = rand(3,3)
    cl1 = @dtask rmul!(C,2) (C,) (RW,)
    @test num_nodes(dag) == 0
    addnode!(dag,cl1)
    @test num_nodes(dag) == 1
    @test num_edges(dag) == 0
    cl2 = @dtask rmul!(C,2) (C,) (RW,)
    addnode!(dag, cl2)
    @test num_nodes(dag) == 2
    addedge!(dag, cl1, cl2)
    @test num_edges(dag) == 1
    @test has_edge(dag, cl1, cl2) == true
    @test has_edge(dag, cl2, cl1) == false
    @test outneighbors(dag, cl1) == Set((cl2,))
    @test inneighbors(dag, cl1) == Set()
    @test outneighbors(dag, cl2) == Set()
    @test inneighbors(dag, cl2) == Set([cl1])
    cl3 = @dtask rmul!(C,2) (C,) (RW,)
    cl4 = @dtask rmul!(C,2) (C,) (RW,)
    addnode!(dag, cl3)
    addnode!(dag, cl4)
    addedge!(dag,cl2,cl4)
    @test isconnected(dag, cl1, cl3) == false
    @test isconnected(dag, cl2, cl4) == true
    @test isconnected(dag, cl1, cl4) == true # 1 --> 2 --> 4
    @test adjacency_matrix(dag) == [0 1 0 0;
                                    0 0 0 1;
                                    0 0 0 0;
                                    0 0 0 0]
    @test num_edges(addedge_transitive!(dag, cl1, cl4)) == 2 # vtx 1 is already connected to vtx through vtx2
end

@testset "Buffering" begin
    sz = 10
    dag = DAG{Int}(sz)
    for i in 1:sz
        addnode!(dag, i)
    end
    @test num_nodes(dag) == sz
    t = @async addnode!(dag, 11)
    sleep(0.1)
    @test num_nodes(dag) == sz
    remove_node!(dag,1)
    sleep(0.1)
    @test num_nodes(dag) == sz
end


using DataFrames
using PooledArrays
using BenchmarkTools
using Random

Random.seed!(1)

grouping_benchmarks = BenchmarkGroup()

for n in (1000, 10_000_000),
    k in (10, 10_000),
    x in (rand(1:k, n),
          rand([1:k; missing], n),
          rand(string.(1:k), n),
          rand([string.(1:k); missing], n),
          PooledArray(rand(1:k, n)),
          PooledArray(rand([missing; 1:k], n)))
        df = DataFrame(x=x)

        grouping_benchmarks[n, k, typeof(x), "skipmissing=false"] =
            @benchmarkable groupby($df, :x)

        # Skipping missing values
        grouping_benchmarks[n, k, typeof(x), "skipmissing=true"] =
            @benchmarkable groupby($df, :x, skipmissing=true)

        for l in (10, 10_000),
            y in (rand(1:l, n),
                  rand([1:l; missing], n),
                  rand(string.(1:l), n),
                  rand([string.(1:l); missing], n),
                  PooledArray(rand(1:l, n)),
                  PooledArray(rand([missing; 1:l], n)))
            df.y = y

            grouping_benchmarks[n, (k, l), (typeof(x), typeof(y)),
                                "skipmissing=false"] =
                @benchmarkable groupby($df, [:x, :y])

            # Skipping missing values
            grouping_benchmarks[n, (k, l), (typeof(x), typeof(y)),
                                "skipmissing=true"] =
                @benchmarkable groupby($df, [:x, :y], skipmissing=true)
        end

        if df.x isa PooledArray
            # Empty group which requires adjusting group indices
            replace!(df.x, levels(df.x)[1] => levels(df.x)[2])
            grouping_benchmarks[n, k, typeof(x), "empty group"] =
                @benchmarkable groupby($df, :x)
        end
end

# If a cache of tuned parameters already exists, use it, otherwise, tune and cache
# the benchmark parameters. Reusing cached parameters is faster and more reliable
# than re-tuning `suite` every time the file is included.
paramspath = joinpath(dirname(@__FILE__), "params.json")

if isfile(paramspath)
    loadparams!(grouping_benchmarks, BenchmarkTools.load(paramspath)[1], :evals);
else
    tune!(grouping_benchmarks)
    BenchmarkTools.save(paramspath, params(grouping_benchmarks));
end

grouping_results = run(grouping_benchmarks, verbose=true)
# using Serialization
# serialize("grouping_results.jls", grouping_results)
# leaves(judge(median(grouping_results1), median(grouping_results2)))
# leaves(regressions(judge(median(grouping_results1), median(grouping_results2))))
# leaves(improvements(judge(median(grouping_results1), median(grouping_results2))))


export MtlDim

const MTLDim = MtSize
"""
    CuDim3(x)

    CuDim3((x,))
    CuDim3((x, y))
    CuDim3((x, y, x))

A type used to specify dimensions, consisting of 3 integers for respectively the `x`, `y`
and `z` dimension. Unspecified dimensions default to `1`.

Often accepted as argument through the `CuDim` type alias, eg. in the case of
[`cudacall`](@ref) or [`launch`](@ref), allowing to pass dimensions as a plain integer or a
tuple without having to construct an explicit `CuDim3` object.
"""
#=struct MtlDim3
    x::NsUInteger
    y::NsUInteger
    z::NsUInteger
end
=#
const MtlDim3 = MTLDim

MtlDim3(dims::Integer)             = MtlDim3(dims,    NsUInteger(1), NsUInteger(1))
MtlDim3(dims::NTuple{1,<:Integer}) = MtlDim3(dims[1], NsUInteger(1), NsUInteger(1))
MtlDim3(dims::NTuple{2,<:Integer}) = MtlDim3(dims[1], dims[2],       NsUInteger(1))
MtlDim3(dims::NTuple{3,<:Integer}) = MtlDim3(dims[1], dims[2],       dims[3])

# Type alias for conveniently specifying the dimensions
# (e.g. `(len, 2)` instead of `CuDim3((len, 2))`)
const MtlDim = Union{Integer,
                     Tuple{Integer},
                     Tuple{Integer, Integer},
                     Tuple{Integer, Integer, Integer}}


using Abaco
using Test

interval=10

function onresult(ts, ne, name, value, inputs)
    @debug "timestamp [$ts]: ne: [$ne] $name = $value"
end

function onresult(timestamp, ne, name, value::Abaco.PValue, inputs)
    @debug "progressive timestamp [$ts]: ne: [$ne] $name = $value"
end

abaco = abaco_init(onresult, interval=interval)

@debug "starting demo with time span: $interval, values timestamp: $ts"

setup_settings(abaco, "state", oncomplete=onresult)

formula(abaco, "total", "sum(region.city.footprint)", "state")

italy = node(abaco, "italy", "state")
veneto = node(abaco, italy, "veneto", "region")
trentino = node(abaco, italy, "trentino", "region")

node(abaco, veneto, "belluno", "city")
node(abaco, veneto, "padova", "city")
node(abaco, trentino, "rovereto", "city")
node(abaco, trentino, "pergine", "city")
node(abaco, trentino, "mori", "city")
node(abaco, trentino, "bondone", "mountain")

nodes = split("region.city.footprint", ".")[1:end-1]
count = Abaco.deep_count(abaco, italy.ne, nodes)
@test count == 5


using PyCall
pyimport("d4rl")

function h5toEB(filename)
    # Load the data from the file
    actions = h5read(filename, "actions")
    obs = h5read(filename, "observations")
    rewards = h5read(filename, "rewards")
    dones = Bool.(h5read(filename, "terminals"))
    timeouts = Bool.(h5read(filename, "timeouts"))

    # we are going to remove all of the observations that included a terminal or done signal
    keep = .!(dones .| timeouts)
    keep[end] = false # remove the last one regardless

    next_obs = obs[:, 2:end][:, keep[1:end-1]] # shift the observations and then keep the ones that we want
    # Keep the rest of the elements consistent
    obs = obs[:,keep]
    actions = actions[:,keep]
    rewards = reshape(rewards[keep], 1, :)
    dones = reshape(dones[keep], 1, :)

    ExperienceBuffer(Dict(:s => obs, :sp=>next_obs, :a=>actions, :r=>rewards, :done=>dones))
end


using Mass2
using ReferenceMicrocalFiles

# type GetPulsesStep{T} <: AbstractStep
# 	pulse_source::T
# 	outputs::Vector{Symbol}
# 	previous_pulse_index::Int # used to keep track of location in an ljh file for example
# 	max_pulses_per_step::Int
# end
# inputs(s::GetPulsesStep) = Symbol[]
# function dostep(s::GetPulsesStep{LJHGroup},c::Channel)
# 	r = s.previous_pulse_index+1:min(s.previous_pulse_index+s.max_pulses_per_step, length(s.pulse_source))
# 	length(r)==0 && (return r)
# 	pulses, rowstamps = collect(s.pulse_source[r])
# 	pulses_out, rowstamps_out = perpulse_outputs(s,c)
# 	append!(pulses_out, pulses)
# 	assert(length(pulses_out)==last(r))
# 	append!(rowstamps_out, rowstamps)
# 	assert(length(rowstamps_out)==last(r))
# 	s.previous_pulse_index=last(r)
# 	r
# end
# graphlabel(s::GetPulsesStep) = repr(typeof(s))

# type LJHPulseEmitter
# 	ljh_group::LJHGroup
# 	max_pulses_per_step::Int
# 	donethru::Int
# end
# type LJHPulseEmitterSlice
# 	ljhpulsemitter::LJHPulseEmitter
# 	r::UnitRange{Int64}
# end
# LJHPulseEmitter(filename,max_pulses_per_step=1000) = LJHPulseEmitter(LJHGroup(filename), max_pulses_per_step, max_pulses_per_step)
# donethru(x::LJHPulseEmitter) = x.donethru
# Base.getindex(x::LJHPulseEmitter, r::UnitRange{Int64}) = LJHPulseEmitterSlice(x, r)
# function getpulses!(ljhpulsemitterslight::LJHPulseEmitterSlice, max_pulses_per_step::Int)
# 	pulses = Array(Vector{UInt16},length(ljh_group_slice))
# 	rowstamps = Array(UInt64, length(ljh_group_slice))
# 	ljh_group_slice = ljhpulsemitterslice.ljhpulsemitter.ljh_group[r]
# 	ljhpulsemitterslice.ljhpulsemitter.donethru = last(ljhpulsemitter.r)
# 	for (i,(pulse, rowstamp)) in enumerate(ljh_group_slice)
# 		pulses[i]=pulse
# 		rowstamps[i]=rowstamp
# 	end
# 	pulses, rowstamps
# end

ljhpaths = ReferenceMicrocalFiles.getljhpaths()
ljh = LJHGroup(ljhpaths[1])

steps = AbstractStep[]
# generate a spectra with multiple peaks for calibration testing
dist_energies = [4000,4700,5500,6000,6800,7500,8200,9600,10000]
dist_filt_values = 5*dist_energies.^0.8
dist_use_inds = [1,3,6,7,8,9]
dist_use_filt_values = dist_filt_values[dist_use_inds]
dist_use_energies = dist_energies[dist_use_inds]
filt_value_distribution = MixtureModel([Normal(mu,1) for mu in dist_filt_values])
push!(steps, GetPulsesStep(ljh, [:pulse, :rowstamp], 0,500))
s=steps[1]

push!(steps, PerPulseStep(compute_summary, [:pulse, :pre_samples, :frame_time],
	[:pretrig_mean, :pretrig_rms, :pulse_average, :pulse_rms, :rise_time, :postpeak_deriv, :peak_index, :peak_value, :min_value]))
push!(steps, PerPulseStep(selectfromcriteria, [:pretrig_rms, :pretrig_rms_criteria, :peak_index, :peak_index_criteria, :postpeak_deriv, :postpeak_deriv_criteria], [:selection_good]))
push!(steps, PerPulseStep(filter1lag, [:pulse, :whitenoise_filter], [:filt_value]))
push!(steps, HistogramStep(update_histogram!, [:filt_value_hist, :selection_good, :filt_value]))
push!(steps, ThresholdStep(calibrate_nofit, [:filt_value_dc_hist,:known_energies],[:calibration],:filt_value_dc_hist, counted, 5000, true))
push!(steps, ThresholdStep(compute_whitenoise_filter, [:pulse, :selection_good], [:whitenoise_filter], :selection_good, sum, 100, true))
push!(steps, ThresholdStep(calc_dc, [:pretrig_mean, :filt_value, :selection_good], [:ptm_dc],:filt_value_hist, counted, 3000, true))
push!(steps, PerPulseStep(applycorrection, [:ptm_dc, :pretrig_mean, :filt_value], [:filt_value_dc]))
push!(steps, HistogramStep(update_histogram!, [:filt_value_dc_hist, :selection_good, :filt_value_dc]))
push!(steps, PerPulseStep(apply_calibration, [:calibration, :filt_value_dc], [:energy]) )
push!(steps, HistogramStep(update_histogram!, [:energy_hist, :selection_good, :energy]))

# push!(steps, ToJLDStep([:filt_value,:pretrig_rms, :energy]))
# push!(steps, FreeMemoryStep())

push!(perpulse_symbols, :filt_value, :selection_good, :energy, :pulse, :rowstamp,
	:pretrig_mean, :pretrig_rms, :pulse_average, :pulse_rms, :rise_time, :postpeak_deriv,
	:peak_index, :peak_value, :min_value, :selection_good, :filt_value_dc,:ljh_group)

c=MassChannel()
# c[:ljh_group] = ljh
c[:pretrig_mean] = RunningVector(Float64)
c[:pretrig_rms] = RunningVector(Float64)
c[:pulse_average] = RunningVector(Float64)
c[:pulse_rms] = RunningVector(Float64)
c[:rise_time] = RunningVector(Float64)
c[:postpeak_deriv] = RunningVector(Float64)
c[:peak_index] = RunningVector(UInt16)
c[:peak_value] = RunningVector(UInt16)
c[:min_value] = RunningVector(UInt16)
c[:filt_value] = RunningVector(Float64)
c[:filt_value_dc] = RunningVector(Float64)
c[:pretrig_rms] = RunningVector(Float64)
c[:selection_good] = RunningSumBitVector()
c[:energy] = RunningVector(Float64)
c[:filt_value_hist] = Histogram(0:1:20000)
c[:filt_value_dc_hist] = Histogram(0:1:20000)
c[:energy_hist] = Histogram(0:1:20000)
c[:pulse] = RunningVector(Vector{UInt16})
c[:rowstamp] = RunningVector(Int)
c[:pre_samples] = 100
c[:frame_time] = 1/100000
c[:peak_index_criteria] = (2280,2360)
c[:pretrig_rms_criteria] = (0.0,10.)
c[:postpeak_deriv_criteria] = (0.0,100.0)
c[:known_energies] = [4952, 5898, 6929, 8040]

workstat(n, s::MockPulsesStep, t) = "MockPulsesStep "*@sprintf("%0.2f pulses/s",n/t)
workstat(n, s::PerPulseStep, t) = "PerPulse:$(graphlabel(s)) "*@sprintf("%0.2f pulses/s",n/t)
workstat(n, s::GetPulsesStep, t) = "GetPulsesStep:$(graphlabel(s)) "*@sprintf("%0.2f pulses/s",n/t)
workstat(n, s::ToJLDStep, t) = "ToJLDStep $n executions at "*@sprintf("%0.2f executions/s",n/t)
workstat(n, s::HistogramStep, t) = "HistogramStep:$(inputs(s)[1]) "*@sprintf("%0.2f pulses/s",n/t)
workstat(n, s::ThresholdStep, t) = "ThresholdStep:$(graphlabel(s)) $n executions at "*@sprintf("%0.2f executions/s",n/t)
workstat(n, s::FreeMemoryStep, t) = "FreeMemoryStep $n executions at "*@sprintf("%0.2f executions/s",n/t)

workdone_cumulative = zeros(Int, length(steps))
stepelapsed_cumulative = Array(Float64, length(steps))
errors = Any[]
close(jldopen(filename(c),"w")) # wipe the test file
for i = 1:30
	println("** loop iteration $i")
	#do steps
	for (j,s) in enumerate(steps)
		tstart = time()
		try
			wu = workunits(dostep(s,c))
			stepelapsed = time()-tstart
			workdone_cumulative[j] += wu
			stepelapsed_cumulative[j] += stepelapsed
		catch ex
			println((i,j))
			println(sprint(io->showerror(io, ex)))
			println(sprint(io->Base.show_backtrace(io, catch_backtrace())))
			debug(s,c)
			break
		end

	end

end
[println("step $i: "*workstat(workdone_cumulative[i], steps[i], stepelapsed_cumulative[i])) for i = 1:length(steps)];
[println("$n donethru $(donethru(c[n]))") for n in keys(c)];
jld = jldopen(filename(c), "r+")


__precompile__()

module DiffEqJump

using DiffEqBase, Compat, RandomNumbers, TreeViews, LinearAlgebra
using DataStructures, PoissonRandom, Random
using FunctionWrappers, Parameters

import DiffEqBase: DiscreteCallback, init, solve, solve!, plot_indices
import Base: size, getindex, setindex!, length, similar, show
import DataStructures: length, update!

import RecursiveArrayTools: recursivecopy!

abstract type AbstractJump end
abstract type AbstractAggregatorAlgorithm end
abstract type AbstractJumpAggregator end

import Base.Threads
@static if VERSION < v"1.3"
  seed_multiplier() = Threads.threadid()
else
  seed_multiplier() = 1
end

include("jumps.jl")
include("massaction_rates.jl")
include("aggregators/aggregators.jl")
include("aggregators/ssajump.jl")
include("aggregators/direct.jl")
include("aggregators/frm.jl")
include("aggregators/sortingdirect.jl")
include("aggregators/nrm.jl")
include("aggregators/bracketing.jl")
include("aggregators/rssa.jl")
include("aggregators/prioritytable.jl")
include("aggregators/directcr.jl")
include("problem.jl")
include("callbacks.jl")
include("solve.jl")
include("extended_jump_array.jl")
include("coupled_array.jl")
include("coupling.jl")
include("SSA_stepper.jl")
include("simple_regular_solve.jl")

export ConstantRateJump, VariableRateJump, RegularJump, MassActionJump,
       JumpSet

export JumpProblem

export SplitCoupledJumpProblem

export Direct, DirectFW, SortingDirect, DirectCR
export BracketData, RSSA
export FRM, FRMFW, NRM

export get_num_majumps, needs_depgraph, needs_vartojumps_map

export init, solve, solve!

export ExtendedJumpArray

end # module


# MIT License
#
# Copyright (c) 2018 Martin Biel
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

# Type shorthands #
# ========================== #
const _Variable = VariableRef
const _Decision = DecisionRef
const _Scalar = Union{_Variable, _Decision, _Constant}
const _DecisionAffOrQuadExpr{C} = Union{DecisionAffExpr{C}, DecisionQuadExpr{C}}

# Basic operations #
# ========================== #
MA.mutability(::Type{<:_DecisionAffOrQuadExpr}) = MA.IsMutable()
function MA.mutable_copy(expr::_DecisionAffOrQuadExpr)
    return map_coefficients(MA.copy_if_mutable, expr)
end

# Promote operation #
# ========================== #
# _Constant--_Decision
function MA.promote_operation(::Union{typeof(+), typeof(-), typeof(*)},
                               ::Type{<:_Constant}, S::Type{<:_Decision})
    return DAE
end
function MA.promote_operation(::Union{typeof(+), typeof(-), typeof(*)},
                               S::Type{<:_Decision}, ::Type{<:_Constant})
    return DAE
end
# _Variable--Decision
function MA.promote_operation(::Union{typeof(+), typeof(-)},
                               ::Type{<:_Variable}, S::Type{<:_Decision})
    return DAE
end
function MA.promote_operation(::Union{typeof(+), typeof(-)},
                               S::Type{<:_Decision}, ::Type{<:_Variable})
    return DAE
end
# _Constant--_DecisionAffOrQuadExpr
function MA.promote_operation(::Union{typeof(+), typeof(-), typeof(*)},
                               ::Type{<:_Constant}, S::Type{<:_DecisionAffOrQuadExpr})
    return S
end
function MA.promote_operation(::Union{typeof(+), typeof(-), typeof(*)},
                               S::Type{<:_DecisionAffOrQuadExpr}, ::Type{<:_Constant})
    return S
end
# _Variable/_Decision--_DecisionAffOrQuadExpr
function MA.promote_operation(
    ::Union{typeof(+), typeof(-)}, ::Type{<:Union{_Variable, _Decision}},
    S::Type{<:_DecisionAffOrQuadExpr})
    return S
end
function MA.promote_operation(
    ::Union{typeof(+), typeof(-)}, S::Type{<:_DecisionAffOrQuadExpr},
    ::Type{<:Union{_Variable, _Decision}})
    return S
end
# (_Variable/_Decision)AffExpr--(_Variable/_Decision)AffExpr
function MA.promote_operation(
    op::Union{typeof(+), typeof(-)}, ::Type{GenericAffExpr{C1, _Variable}},
    ::Type{GenericAffExpr{C2, _Decision}}) where {C1, C2}
    C = MA.promote_operation(op, C1, C2)
    return DecisionAffExpr{C}
end
function MA.promote_operation(
    op::Union{typeof(+), typeof(-)}, ::Type{GenericAffExpr{C1, _Decision}},
    ::Type{GenericAffExpr{C2, _Variable}}) where {C1, C2}
    C = MA.promote_operation(op, C1, C2)
    return DecisionAffExpr{C}
end
# (_Variable/_Decision)AffExpr--_DecisionAffOrQuadExpr
function MA.promote_operation(
    ::Union{typeof(+), typeof(-)}, ::Type{GenericAffExpr{C,V}},
    S::Type{_DecisionAffOrQuadExpr{C}}) where {C, V <: Union{_Variable, _Decision}}
    return S
end
function MA.promote_operation(
    ::Union{typeof(+), typeof(-)}, S::Type{_DecisionAffOrQuadExpr{C}},
    ::Type{GenericAffExpr{C,V}}) where {C, V <: Union{_Variable, _Decision}}
    return S
end
# _DecisionAffOrQuadExpr--_DecisionAffOrQuadExpr
function MA.promote_operation(::Union{typeof(+), typeof(-)}, ::Type{A},
                               ::Type{A}) where {A <: _DecisionAffOrQuadExpr}
    return A
end
function MA.promote_operation(::Union{typeof(+), typeof(-)},
                               ::Type{<:DecisionAffExpr},
                               S::Type{<:DecisionQuadExpr})
    return S
end
function MA.promote_operation(::Union{typeof(+), typeof(-)},
                               S::Type{<:DecisionQuadExpr},
                               ::Type{<:DecisionAffExpr})
    return S
end
# _Variable--DecisionRef
function MA.promote_operation(::typeof(*), ::Type{<:_Variable}, ::Type{<:_Decision})
    return DQE
end
function MA.promote_operation(::typeof(*), ::Type{<:_Decision}, ::Type{<:_Variable})
    return DQE
end
# _Decision--_Decision
function MA.promote_operation(::typeof(*), ::Type{<:_Decision}, ::Type{<:_Decision})
    return DQE
end
# _Variable/_Decision--(_Variable/_Decision)AffExpr
function MA.promote_operation(::typeof(*), ::Type{<:_Variable}, ::Type{GenericAffExpr{C, _Decision}}) where {C, D <: _Decision}
    return DecisionQuadExpr{C}
end
function MA.promote_operation(::typeof(*), ::Type{GenericAffExpr{C, D}}, ::Type{<:_Variable}) where {C, D <: _Decision}
    return DecisionQuadExpr{C}
end
function MA.promote_operation(::typeof(*), ::Type{<:_Decision}, ::Type{GenericAffExpr{C, V}}) where {C, V <: _Variable}
    return DecisionQuadExpr{C}
end
function MA.promote_operation(::typeof(*), ::Type{GenericAffExpr{C, V}}, ::Type{<:_Decision}) where {C, V <: _Variable}
    return DecisionQuadExpr{C}
end
function MA.promote_operation(::typeof(*), ::Type{D}, ::Type{GenericAffExpr{C, D}}) where {C, D <: _Decision}
    return DecisionQuadExpr{C}
end
function MA.promote_operation(::typeof(*), ::Type{GenericAffExpr{C, D}}, ::Type{D}) where {C, D <: _Decision}
    return DecisionQuadExpr{C}
end
# _Variable/_Decision--DecisionAffExpr
function MA.promote_operation(::typeof(*), ::Type{V}, ::Type{DecisionAffExpr{C}}) where {C, V <: Union{_Variable, _Decision}}
    return DecisionQuadExpr{C}
end
function MA.promote_operation(::typeof(*), ::Type{DecisionAffExpr{C}}, ::Type{V}) where {C, V <: Union{_Variable, _Decision}}
    return DecisionQuadExpr{C}
end

function MA.scaling(aff::DecisionAffExpr{C}) where C
    if !isempty(aff.variables.terms) || !isempty(aff.decisions.terms)
        throw(InexactError("Cannot convert `$aff` to `$C`."))
    end
    return MA.scaling(aff.variables.constant)
end
function MA.scaling(quad::DecisionQuadExpr{C}) where C
    if !isempty(quad.variables.terms) || !isempty(quad.decisions.terms) || !isempty(quad.decisions.quad.terms) || !isempty(quad.cross_terms)
        throw(InexactError("Cannot convert `$quad` to `$C`."))
    end
    return MA.scaling(quad.variables.aff)
end

# Mutable operate #
# ========================== #
# zero/one #
function MA.mutable_operate!(op::Union{typeof(zero), typeof(one)}, aff::DecisionAffExpr)
    MA.mutable_operate!(op, aff.variables)
    MA.mutable_operate!(op, aff.decisions)
    return aff
end
function MA.mutable_operate!(op::Union{typeof(zero), typeof(one)}, quad::DecisionQuadExpr)
    MA.mutable_operate!(op, quad.variables)
    MA.mutable_operate!(op, quad.decisions)
    empty!(quad.cross_terms)
    return quad
end
# * #
function MA.mutable_operate!(::typeof(*), expr::_DecisionAffOrQuadExpr, α::_Constant)
    if iszero(α)
        return MA.mutable_operate!(zero, expr)
    else
        return map_coefficients_inplace!(x -> MA.mul!(x, α), expr)
    end
end
# +/- #
function MA.mutable_operate!(::typeof(+), expr::_DecisionAffOrQuadExpr, x)
    return JuMP.add_to_expression!(expr, x)
end
function MA.mutable_operate!(::typeof(-), expr::_DecisionAffOrQuadExpr, x)
    return JuMP.add_to_expression!(expr, -1.0, x)
end
# add/sub_mul #
function MA.mutable_operate!(::typeof(MA.add_mul), expr::_DecisionAffOrQuadExpr, x::_Scalar)
    return JuMP.add_to_expression!(expr, x)
end
function MA.mutable_operate!(::typeof(MA.add_mul), expr::_DecisionAffOrQuadExpr, x::_Scalar, y::_Scalar)
    return JuMP.add_to_expression!(expr, x, y)
end
function MA.mutable_operate!(::typeof(MA.sub_mul), expr::_DecisionAffOrQuadExpr, x::_Scalar)
    return JuMP.add_to_expression!(expr, -1.0, x)
end
function MA.mutable_operate!(::typeof(MA.sub_mul), expr::_DecisionAffOrQuadExpr, x::_Scalar, y::_Scalar)
    return JuMP.add_to_expression!(expr, -x, y)
end
function MA.mutable_operate!(::typeof(MA.sub_mul), expr::_DecisionAffOrQuadExpr, x::Union{_Variable, _Decision}, y::_Constant)
    return JuMP.add_to_expression!(expr, x, -y)
end
function MA.mutable_operate!(op::MA.AddSubMul, expr::_DecisionAffOrQuadExpr, x, y)
    return MA.mutable_operate!(op, expr, x * y)
end
@generated function _add_sub_mul_reorder!(op::MA.AddSubMul, expr::_DecisionAffOrQuadExpr, args::Vararg{Any, N}) where N
    n = length(args)
    @assert n ≥ 3
    varidx = findall(t -> (t <: Union{_Variable, _Decision}), collect(args))
    allscalar = all(t -> (t <: _Constant), args[setdiff(1:n, varidx)])
    idx = (allscalar && length(varidx) == 1) ? varidx[1] : n
    coef = Expr(:call, :*, [:(args[$i]) for i in setdiff(1:n, idx)]...)
    return :(MA.mutable_operate!(op, expr, $coef, args[$idx]))
end
function MA.mutable_operate!(op::MA.AddSubMul, expr::_DecisionAffOrQuadExpr, x, y, z, other_args::Vararg{Any, N}) where N
    return _add_sub_mul_reorder!(op, expr, x, y, z, other_args...)
end
function JuMP.add_to_expression!(expr::_DecisionAffOrQuadExpr, α::_Constant, β::_Constant)
    return JuMP.add_to_expression!(expr, *(α, β))
end


"""
    FullFeaturedCPNN(;
        graphChain::Flux.Chain
        nodeChain::Flux.Chain
        globalChain::Flux.Chain
        outputChain::Flux.Dense
    )

This structure is here to provide a flexible way to create a nn model which respect this approach:
Making modification on the graph, then extract one node feature and modify it.
"""
Base.@kwdef struct FullFeaturedCPNN <: NNStructure
    graphChain::Flux.Chain = Flux.Chain()
    nodeChain::Flux.Chain = Flux.Chain()
    globalChain::Flux.Chain = Flux.Chain()
    outputChain::Union{Flux.Dense, Flux.Chain} = Flux.Chain()
end

# Enable the `|> gpu` syntax from Flux
Flux.@functor FullFeaturedCPNN

# TODO make it possible to use global features
function (nn::FullFeaturedCPNN)(states::BatchedDefaultTrajectoryState)
    @assert !isnothing(states.allValuesIdx)

    variableIdx = states.variableIdx
    batchSize = length(variableIdx)
    allValuesIdx = states.allValuesIdx
    actionSpaceSize = size(allValuesIdx, 1)

    # chain working on the graph(s) with the GNNs
    featuredGraph = nn.graphChain(states.fg)
    nodeFeatures = featuredGraph.nf
    globalFeatures = featuredGraph.gf

    # Extract the features corresponding to the varibales
    variableIndices = nothing
    Zygote.ignore() do
        variableIndices = Flux.unsqueeze(CartesianIndex.(variableIdx, 1:batchSize), 1)
    end
    variableFeatures = nodeFeatures[:, variableIndices]
    variableFeatures = reshape(nn.nodeChain(RL.flatten_batch(variableFeatures)), :, 1, batchSize)

    # Extract the features corresponding to the values
    valueIndices = nothing
    Zygote.ignore() do
        valueIndices = CartesianIndex.(allValuesIdx, repeat(transpose(1:batchSize); outer=(actionSpaceSize, 1)))
    end
    valueFeatures = nodeFeatures[:, valueIndices]
    valueFeatures = reshape(nn.nodeChain(RL.flatten_batch(valueFeatures)), :, actionSpaceSize, batchSize)

    finalFeatures = nothing
    if sizeof(globalFeatures) != 0

        # Extract the global features
        globalFeatures = reshape(nn.globalChain(globalFeatures), :, 1, batchSize)

        # Prepare the input of the outputChain
        finalFeatures = vcat(
            repeat(variableFeatures; outer=(1, actionSpaceSize, 1)),
            repeat(globalFeatures; outer=(1, actionSpaceSize, 1)),
            valueFeatures,
        )
        finalFeatures = RL.flatten_batch(finalFeatures)
    else
        # Prepare the input of the outputChain
        finalFeatures = vcat(
            repeat(variableFeatures; outer=(1, actionSpaceSize, 1)),
            valueFeatures,
        )
        finalFeatures = RL.flatten_batch(finalFeatures)
    end

    # output layer
    predictions = nn.outputChain(finalFeatures)
    output = reshape(predictions, actionSpaceSize, batchSize)

    return output
end


#=
This file is auto-generated. Do not edit.
=#
"""
    mutable struct StorageManagementCost <: OperationalCost
        variable::VariableCost
        fixed::Float64
        start_up::Float64
        shut_down::Float64
        energy_shortage_cost::Float64
        energy_surplus_cost::Float64
    end

Data Structure for Operational Cost Data like variable cost and start - stop costs and energy storage cost.

# Arguments
- `variable::VariableCost`: variable cost
- `fixed::Float64`: fixed cost
- `start_up::Float64`: start-up cost, validation range: `(0, nothing)`, action if invalid: `warn`
- `shut_down::Float64`: shut-down cost, validation range: `(0, nothing)`, action if invalid: `warn`
- `energy_shortage_cost::Float64`: Cost incurred by the model for being short of the energy target., validation range: `(0, nothing)`, action if invalid: `warn`
- `energy_surplus_cost::Float64`: Cost incurred by the model for surplus energy stored., validation range: `(0, nothing)`, action if invalid: `warn`
"""
mutable struct StorageManagementCost <: OperationalCost
    "variable cost"
    variable::VariableCost
    "fixed cost"
    fixed::Float64
    "start-up cost"
    start_up::Float64
    "shut-down cost"
    shut_down::Float64
    "Cost incurred by the model for being short of the energy target."
    energy_shortage_cost::Float64
    "Cost incurred by the model for surplus energy stored."
    energy_surplus_cost::Float64
end

function StorageManagementCost(; variable, fixed, start_up, shut_down, energy_shortage_cost, energy_surplus_cost, )
    StorageManagementCost(variable, fixed, start_up, shut_down, energy_shortage_cost, energy_surplus_cost, )
end

# Constructor for demo purposes; non-functional.
function StorageManagementCost(::Nothing)
    StorageManagementCost(;
        variable=VariableCost((0.0, 0.0)),
        fixed=0.0,
        start_up=0.0,
        shut_down=0.0,
        energy_shortage_cost=0.0,
        energy_surplus_cost=0.0,
    )
end

"""Get [`StorageManagementCost`](@ref) `variable`."""
get_variable(value::StorageManagementCost) = value.variable
"""Get [`StorageManagementCost`](@ref) `fixed`."""
get_fixed(value::StorageManagementCost) = value.fixed
"""Get [`StorageManagementCost`](@ref) `start_up`."""
get_start_up(value::StorageManagementCost) = value.start_up
"""Get [`StorageManagementCost`](@ref) `shut_down`."""
get_shut_down(value::StorageManagementCost) = value.shut_down
"""Get [`StorageManagementCost`](@ref) `energy_shortage_cost`."""
get_energy_shortage_cost(value::StorageManagementCost) = value.energy_shortage_cost
"""Get [`StorageManagementCost`](@ref) `energy_surplus_cost`."""
get_energy_surplus_cost(value::StorageManagementCost) = value.energy_surplus_cost

"""Set [`StorageManagementCost`](@ref) `variable`."""
set_variable!(value::StorageManagementCost, val) = value.variable = val
"""Set [`StorageManagementCost`](@ref) `fixed`."""
set_fixed!(value::StorageManagementCost, val) = value.fixed = val
"""Set [`StorageManagementCost`](@ref) `start_up`."""
set_start_up!(value::StorageManagementCost, val) = value.start_up = val
"""Set [`StorageManagementCost`](@ref) `shut_down`."""
set_shut_down!(value::StorageManagementCost, val) = value.shut_down = val
"""Set [`StorageManagementCost`](@ref) `energy_shortage_cost`."""
set_energy_shortage_cost!(value::StorageManagementCost, val) = value.energy_shortage_cost = val
"""Set [`StorageManagementCost`](@ref) `energy_surplus_cost`."""
set_energy_surplus_cost!(value::StorageManagementCost, val) = value.energy_surplus_cost = val


gr_poly_line(points::Vector{Vec2f0}) = GR.polyline(first.(points), last.(points))
gr_poly_line(points::Vector{Vec3f0}) = GR.polyline(first.(points), getindex.(points, 2), last.(points))

"""
    `lines(x, y, z)` / `lines(x, y)` / `lines(positions)`

Creates a connected line plot for each element in `(x, y, z)`, `(x, y)` or `positions`.
"""
function draw(scene::Scene, plot::Lines)
    # TODO:
    # - Continuous color (`color = 1:10`)
    positions = map(plot[1][]) do pos
        # take camera from scene + model transformation matrix and apply it to pos
        project_position(scene, pos, plot[:model][])
    end

    c = if plot.color[] isa Symbol
            parse(Colorant, plot.color[])
        elseif plot.color[] isa Tuple{Symbol, Float64}
            RGBA(parse(Colorant, plot.color[][1]), plot.color[][2])
        else
            plot.color[]
        end
    # TODO change this to an array

    GR.settransparency(alpha(c))
    GR.setlinecolorind(gr_colorind(c))
    GR.setlinewidth(plot.linewidth[])
    gr_poly_line(positions)
    # broadcast_foreach(1:length(positions), plot.color[], plot.linewidth[], plot.linestyle[]) do i, c, lw, ls
        # GR.settransparency(alpha(c))
        # GR.setlinecolorind(gr_color(c))
    #     GR.polyline(x, y, )
    # end
end

"""
    linesegments(positions)

Plots a line for each pair of points in `positions`.

**Attributes**:
The same as for [`lines`](@ref)
"""
function draw(scene::Scene, plot::LineSegments)
    positions = map(plot[1][]) do pos
        # take camera from scene + model transformation matrix and apply it to pos
        project_position(scene, pos, plot[:model][])
    end
    @get_attribute(plot, (color, linewidth, linestyle))
    if color isa Union{RGBA, Symbol}
        color = fill(RGBA(color), length(positions))
    end
    if linewidth isa Number
        linewidth = fill(linewidth, length(positions))
    end
    #@show positions
    #@show color
    #@show linewidth
    #@show linestyle
    for i in 1:2:length(positions)
        GR.settransparency(alpha(color[i]))
        GR.setlinecolorind(gr_colorind(color[i]))
        GR.setlinewidth(linewidth[i])
        gr_poly_line(positions[i:(i+1)])
    end
end

"""
    text(string)

Plots a text.
"""
function draw(scene::Scene, plot::AbstractPlotting.Text)
    @get_attribute(plot, (textsize, color, font, align, rotation, model))
    txt = to_value(plot[1])
    position = plot.attributes[:position][]
    N = length(txt)
    broadcast_foreach(1:N, position, textsize, color, font, rotation) do i, p, ts, cc, f, r
        pos = project_position(scene, p, model)
        chup = r * Vec2f0(0, 1)
        GR.setcharup(chup[1], chup[2])
        GR.settextfontprec(233, 3)
        GR.setcharheight(0.018) # ts ???
        GR.settextcolorind(Int(GR.inqcolorfromrgb(cc.r, cc.g, cc.b)))
        GR.settransparency(cc.alpha)
        GR.settextalign(1, 4)
        GR.text(pos[1], pos[2], string(txt[i]))
    end
end

"""
    `scatter(x, y, z)` / `scatter(x, y)` / `scatter(positions)`

Plots a marker for each element in `(x, y, z)`, `(x, y)`, or `positions`.
"""
function draw(scene::Scene, plot::Scatter)
    fields = @get_attribute(plot, (color, colormap, colorrange, markersize, strokecolor, strokewidth, marker))
    model = plot[:model][]
    color isa AbstractVector && (color = to_image(float.(collect(color)), plot))
    broadcast_foreach(plot[1][], color, markersize, strokecolor, strokewidth, marker) do point, c, markersize, strokecolor, strokewidth, marker
        scale = project_scale(scene, markersize)[1] * 2500 / 6
        pos = project_position(scene, point, model)
        GR.setmarkertype(get(marker_translation_table, marker, GR.MARKERTYPE_SOLID_CIRCLE))
        GR.setmarkersize(scale)
        GR.setbordercolorind(gr_colorind(strokecolor))
        GR.setborderwidth(strokewidth)
        GR.setmarkercolorind(gr_colorind(c))
        GR.settransparency(alpha(c))
        GR.polymarker([pos[1]], [pos[2]])
    end
end

"""
    `image(x, y, image)` / `image(image)`

Plots an image on range `x, y` (defaults to dimensions).
"""
function draw(scene::Scene, plot::Image)
    @get_attribute plot (colormap, colorrange)
end

to_image(values::VecOrMat{<: Colorant}, attributes) = RGBA.(values)
function to_image(values::VecOrMat{<: Real}, attributes)
    AbstractPlotting.@get_attribute attributes (colormap, colorrange)
    return AbstractPlotting.interpolated_getindex.(Ref(colormap), values, (colorrange,))
end
# could be implemented via image, but might be optimized specifically by the backend
"""
    `heatmap(x, y, values)` or `heatmap(values)`

Plots a heatmap as an image on `x, y` (defaults to interpretation as dimensions).
"""
function draw(scene::Scene, plot::Heatmap)
    # TODO:
    # - get interpolation working consistently
    # - get levels to work, if they even do
    @get_attribute(plot, (colormap,colorrange,linewidth,levels,fxaa,interpolate))

    image = plot[3][]
    x, y = plot[1][], plot[2][]
    model = plot.model[]

    interp = plot.interpolate[]

    imsize = (AbstractPlotting.extrema_nan(x), AbstractPlotting.extrema_nan(y))

    _xy_min = project_position(scene, Point2f0(first.(imsize)), model)
    _xy_max = project_position(scene, Point2f0(last.(imsize)), model)

    if is_uniformly_spaced(x) && is_uniformly_spaced(y)

        xy_min = min.(_xy_min, _xy_max)
        xy_max = max.(_xy_min, _xy_max)

        colors = gr_color.(to_image(image, plot))

        # Main.@infiltrate

        GR.drawimage(
            xy_min[1],
            xy_max[1],
            xy_min[2],
            xy_max[2],
            size(image)[1],
            size(image)[2],
            colors
        )
    end
end

"""
    `volume(volume_data)`

Plots a volume. Available algorithms are:
* `:iso` => IsoValue
* `:absorption` => Absorption
* `:mip` => MaximumIntensityProjection
* `:absorptionrgba` => AbsorptionRGBA
* `:indexedabsorption` => IndexedAbsorptionRGBA
"""
function draw(scene::Scene, plot::Volume)
    @get_attribute(plot, (algorithm,absorption,isovalue,isorange,colormap,colorrange))

    x, y, z = plot[1][], plot[2][], plot[3][]
    if algorithm == AbstractPlotting.MaximumIntensityProjection

    end
end

"""
    `surface(x, y, z)`

Plots a surface, where `(x, y, z)` are supposed to lie on a grid.
"""
function draw(scene::Scene, plot::Surface)
    # TODO:
    # - transform x and y points into the screen's space
    @get_attribute(plot, (colormap,colorrange,shading))
    cmap = AbstractPlotting.to_colormap(colormap)

    x, y, z = plot[1][], plot[2][], plot[3][]

    GR.setcolormapfromrgb(red.(cmap), green.(cmap), blue.(cmap))

    if is_uniformly_spaced(x) && is_uniformly_spaced(y)
        x = ensure_vector(x; size = size(z)[1])
        y = ensure_vector(y; size = size(z)[2])
        # points = Point2f0.(x, y)
        # positions = map(points) do pos
        #     # take camera from scene + model transformation matrix and apply it to pos
        #     project_position(scene, pos, plot[:model][])
        # end
        # x, y = first.(positions), last.(positions)
        GR.surface(x, y, z, 4) # 4 => COLORED_MESH
    end
end

"""
    `mesh(x, y, z)`, `mesh(mesh_object)`, `mesh(x, y, z, faces)`, or `mesh(xyz, faces)`

Plots a 3D mesh.
"""
function draw(scene::Scene, plot::Mesh)
    @get_attribute(plot, (interpolate,shading,colormap,colorrange))
end

"""
    `meshscatter(x, y, z)` / `meshscatter(x, y)` / `meshscatter(positions)`

Plots a mesh for each element in `(x, y, z)`, `(x, y)`, or `positions` (similar to `scatter`).
`markersize` is a scaling applied to the primitive passed as `marker`
"""
function draw(scene::Scene, plot::MeshScatter)
    @get_attribute(plot, (marker,markersize,rotations,colormap,colorrange))
end

function setup_plot(scene)
    mwidth, mheight, width, height = GR.inqdspsize()
    size_factor = 0.25  # 25% of the display width
    w, h = scene.camera.resolution[]
    if w > h
        GR.setwsviewport(0, size_factor * mwidth, 0, size_factor * mwidth * h / w)
        GR.setwswindow(0, 1, 0, h / w)
    else
        GR.setwsviewport(0, size_factor * mheight * w / h, 0, size_factor * mheight)
        GR.setwswindow(0, w / h, 0, 1)
    end
    GR.selntran(0)
end

# The toplevel method - this sets up GR attributes, et cetera
function gr_draw(scene::Scene)
    setup_plot(scene)
    draw(scene)
end
# The main method - this draws all plots and child scenes
function draw(scene::Scene)
    foreach(plot-> draw(scene, plot), scene.plots)
    foreach(child-> draw(child), scene.children)
end
# The lower level method.  This dispatches on primitives, meaning that
# we have some guarantees about their attributes.
function draw(scene::Scene, primitive::AbstractPlotting.Combined)
    foreach(x-> draw(scene, x), filter(x -> x.visible[], primitive.plots))
end


# ======================
# Dependencies
# ======================

using LinearAlgebra, SparseArrays, # modules from the Julia standard library
      Reexport,                    # see @reexport macro below
      RecipesBase,                 # plotting
      Parameters,                  # structs with kwargs
      StaticArrays,                # statically sized arrays
      RecursiveArrayTools          # vector of arrays type

# manipulate function definition expressions
using ExprTools: splitdef, combinedef

# the reexport macro ensures that the names exported by the following libraries
# are made available after loading ReachabilityAnalysis
@reexport using HybridSystems,
                IntervalMatrices,
                LazySets,
                MathematicalSystems,
                TaylorIntegration

# required to avoid conflicts with MathematicalSystems
using LazySets: LinearMap, AffineMap, ResetMap

# required to avoid conflicts with IntervalMatrices
using LazySets: Interval, radius, sample, ∅, dim, scale, scale!

# in-place set operations
using LazySets: linear_map!

# LazySets internal functions frequently used
using LazySets.Arrays: projection_matrix, SingleEntryVector
using LazySets.Approximations: AbstractDirections
using LazySets: @commutative

# aliases for intervals
const IM = IntervalMatrices
import IntervalArithmetic
const IA = IntervalArithmetic
const TimeInterval = IA.Interval{Float64}
import TaylorModels
const TM = TaylorModels

# method extensions for Taylor model reach-sets
import TaylorModels: domain, remainder, polynomial, get_order, evaluate

# aliases for set types
const CPA = CartesianProductArray

# convenience union for dispatch on structs that are admissible as initial sets or inputs
const AdmissibleSet = Union{LazySet, UnionSet, UnionSetArray, IA.Interval, IA.IntervalBox}

# method extensions
import LazySets: dim, overapproximate, box_approximation, project, Projection,
                 intersection, directions, linear_map, LinearMap, _split, split!,
                 set, array, _isapprox

import MathematicalSystems: discretize
import Base: ∈, ∩, convert, isdisjoint
import LinearAlgebra: normalize

import CommonSolve: solve # common solve name

# ======================
# Useful constants
# ======================

@inline zeroBox(m) = IntervalBox(zeroI, m)
@inline unitBox(m) = IntervalBox(IA.Interval(0.0, 1.0), m)
@inline symBox(n::Integer) = IntervalBox(symI, n)
const zeroI = IA.Interval(0.0) # TODO use number type
const oneI = IA.Interval(1.0)
const symI = IA.Interval(-1.0, 1.0)

# Interval constructor given a float
LazySets.Interval(x::Float64) = Interval(interval(x))

# common aliases for system's names
const LCS = LinearContinuousSystem
const LDS = LinearDiscreteSystem
const CLCS = ConstrainedLinearContinuousSystem
const CLDS = ConstrainedLinearDiscreteSystem
const CLCCS = ConstrainedLinearControlContinuousSystem
const CLCDS = ConstrainedLinearControlDiscreteSystem
const ACS = AffineContinuousSystem
const ADS = AffineDiscreteSystem
const CACCS = ConstrainedAffineControlContinuousSystem
const CACDS = ConstrainedAffineControlDiscreteSystem
const CACS = ConstrainedAffineContinuousSystem
const CADS = ConstrainedAffineDiscreteSystem
const BBCS = BlackBoxContinuousSystem
const CBBCS = ConstrainedBlackBoxContinuousSystem
const CBBCCS = ConstrainedBlackBoxControlContinuousSystem
const SOLCS = SecondOrderLinearContinuousSystem
const SOACS = SecondOrderAffineContinuousSystem
const SOCLCCS = SecondOrderConstrainedLinearControlContinuousSystem
const SOCACCS = SecondOrderConstrainedAffineControlContinuousSystem
const SecondOrderSystem = Union{SOLCS, SOACS, SOCLCCS, SOCACCS}
const NonlinearSystem = Union{BBCS, CBBCS, CBBCCS}

@inline function _isapprox(Δt::TimeInterval, Δs::TimeInterval)
    return (inf(Δt) ≈ inf(Δs)) && (sup(Δt) ≈ sup(Δs))
end

const VecOrTuple = Union{<:AbstractVector{Int}, NTuple{D, Int}} where {D}
const VecOrTupleOrInt = Union{<:AbstractVector{Int}, NTuple{D, Int}, Int} where {D}

# ======================
# Optional dependencies
# ======================

using Requires

function __init__()
    # numerical differential equations suite
    @require DifferentialEquations = "0c46a032-eb83-5123-abaf-570d42b7fbaa" include("init_DifferentialEquations.jl")

    # exponentiation methods using Krylov subspace approximations
    @require ExponentialUtilities = "d4d017d3-3776-5f7e-afef-a10c40355c18" include("init_ExponentialUtilities.jl")

    # methods that require optimization modeling and solvers
    @require JuMP = "4076af6c-e467-56ae-b986-b466b2749572" include("init_JuMP.jl")

    # tools for symbolic computation
    #@require ModelingToolkit = "961ee093-0014-501f-94e3-6117800e7a78" include("init_ModelingToolkit.jl")
    @require Symbolics = "0c5d862f-8b57-4792-8d23-62f2024744c7" include("init_Symbolics.jl")

    # tools for symbolic algebra
    @require MultivariatePolynomials = "102ac46a-7ee4-5c85-9060-abc95bfdeaa3" include("init_MultivariatePolynomials.jl")

    # sparse dynamic representation of multivariate polynomials
    @require DynamicPolynomials = "7c1d4256-1411-5781-91ec-d7bc3513ac07" include("init_DynamicPolynomials.jl")
end

# ===========================
# Utility macros
# ===========================

"""
    @requires(module_name)

Convenience macro to annotate that a package is required to use a certain function.

### Input

- `module_name` -- name of the required package

### Output

The macro expands to an assertion that checks whether the module `module_name` is
known in the calling scope.

### Notes

Usage:

```julia
function foo(...)
    @require MyPackage
    ... # functionality that requires MyPackage to be loaded
end
```
"""
macro requires(module_name)
    m = Meta.quot(Symbol(module_name))
    return esc(:(@assert isdefined(@__MODULE__, $m) "package `$($m)` is required " *
                    "for this function; do `using $($m)` and try again"))
end


using Glutils
using ModernGL
using GLFW
using Images

window = GLFW.CreateWindow(800, 600, "Textures")
GLFW.MakeContextCurrent(window)

vao = vertexarray()
vbo = arraybuffer(Cfloat[
#    Positions    Texture Coords
     0.5,  0.5,     1.0, 1.0,
     0.5, -0.5,     1.0, 0.0,
    -0.5, -0.5,     0.0, 0.0,
    -0.5,  0.5,     0.0, 1.0,
], g"static draw")

ibo = indexbuffer(Cuint[
    0, 1, 3,
    1, 2, 3
], g"static draw")

bindbuffer(:Array, vbo)
vertexattrib(Cfloat, 4, @map {
    0 := (size = 2, offset = 0),
    1 := (size = 2, offset = 2),
})

glActiveTexture(g"texture0")

image   = Images.load(assets("lenna.png")) |> rotr90
texture = texture2D(image, size(image)..., @map {
    min_filter = g"linear",
    mag_filter = g"linear",
    wrap_s = g"clamp",
    wrap_t = g"clamp"
})

shader = sprogram(assets("texture.glsl"), file = true)
@uniforms u_Texture = shader
glClearColor(0.0, 0.0, 0.0, 1.0)

while true
    # Break the loop when the window closes
    GLFW.WindowShouldClose(window) && break
    # Wait for events
    GLFW.PollEvents()
    # Clear the screen
    glClear(g"color buffer bit")
    glUseProgram(shader)
    uniform!(u_Texture, 0)
    glDrawElements(g"triangles", 6, g"unsigned int", C_NULL)
    # Swap the buffers
    GLFW.SwapBuffers(window)
end

GLFW.DestroyWindow(window)


using ColorVectorSpace: AbstractGray, TransparentGray, TransparentRGB

const dotops = VERSION < v"0.6.0-dev.1839"

if isdefined(Base.Broadcast, :containertype)
    # Specialize the low-level broadcasting machinery for ImageMeta.
    # This make it work for all operators, rather than specializing
    # each operator individually.
    using Base.Broadcast: _broadcast_eltype, broadcast_indices
    Base.Broadcast._containertype(::Type{<:ImageMeta}) = ImageMeta
    Base.Broadcast.promote_containertype(::Type{ImageMeta}, ::Type{ImageMeta}) = ImageMeta
    Base.Broadcast.promote_containertype(::Type{Array}, ::Type{ImageMeta}) = ImageMeta
    Base.Broadcast.promote_containertype(::Type{ImageMeta}, ::Type{Array}) = ImageMeta
    Base.Broadcast.promote_containertype(::Type{ImageMeta}, ct) = ImageMeta
    Base.Broadcast.promote_containertype(ct, ::Type{ImageMeta}) = ImageMeta
    Base.Broadcast.broadcast_indices(::Type{ImageMeta}, A) = indices(A)
    function Base.Broadcast.broadcast_c(f, ::Type{ImageMeta}, As...)
        T = _broadcast_eltype(f, As...)
        shape = broadcast_indices(As...)
        Mt = imagemeta(As...)
        M = Mt[1]
        if length(Mt) > 1
            for i = 2:length(Mt)
                Mt[i] == M || return ambigop(Symbol(f))
            end
        end
        broadcast!(f, shareproperties(M, similar(data(M), T, shape)), As...)
    end
    # Select all the ImageMeta arrays
    @inline imagemeta(As...) = _imagemeta((), As...)
    _imagemeta(out) = out
    @inline _imagemeta(out, A::ImageMeta, As...) = _imagemeta((out..., A), As...)
    @inline _imagemeta(out, A, As...) = _imagemeta(out, As...)
end

if dotops
    import Base: .+, .-, .*, ./, .^, .<, .>, .==
    batch1 = (:+, :.+, :-, :.-, :*, :.*)
    batch2 = (:+, :.+, :-, :.-)
else
    batch1 = (:+, :-, :*)
    batch2 = (:+, :-)
end

(-)(img::ImageMeta) = shareproperties(img, -data(img))

for op in batch1
    @eval begin
        ($op)(img::ImageMeta{Bool}, n::Bool) = shareproperties(img, ($op)(data(img), n))
        ($op)(n::Bool, img::ImageMeta{Bool}) = shareproperties(img, ($op)(n, data(img)))
        ($op)(img::ImageMeta, n::Number) = shareproperties(img, ($op)(data(img), n))
        ($op)(n::Number, img::ImageMeta) = shareproperties(img, ($op)(n, data(img)))
    end
    if op !== :*
        @eval begin
            ($op)(img::ImageMeta, B::BitArray) = shareproperties(img, ($op)(data(img), B))
            ($op)(B::BitArray, img::ImageMeta) = shareproperties(img, ($op)(B, data(img)))
            ($op)(img::ImageMeta, B::ImageMeta) = ambigop(Symbol($op))
            ($op)(img::ImageMeta, B::AbstractArray) = shareproperties(img, ($op)(data(img), B))
            ($op)(B::AbstractArray, img::ImageMeta) = shareproperties(img, ($op)(B, data(img)))
        end
    end
end

for op in batch2
    for CV in (:AbstractGray, :TransparentGray, :AbstractRGB, :TransparentRGB)
        @eval begin
            ($op)(img::ImageMeta{CV}, n::$CV) where {CV<:$CV} =
                shareproperties(img, ($op)(data(img), n))
            ($op)(n::$CV, img::ImageMeta{CV}) where {CV<:$CV} =
                shareproperties(img, ($op)(n, data(img)))
        end
    end
end

(/)(img::ImageMeta, n::Number) = shareproperties(img, data(img)/n)

if dotops
    # evaling in a string avoids a parser depwarn
    include_string("""
    (./)(img::ImageMeta, A::BitArray) = shareproperties(img, data(img)./A)  # needed to avoid ambiguity warning
    (./)(img1::ImageMeta, img2::ImageMeta) = ambigop(:./)
    (./)(img::ImageMeta, A::AbstractArray) = shareproperties(img, data(img)./A)
    (.^)(img::ImageMeta, p::Number) = shareproperties(img, data(img).^p)

    # Logical operations
    (.<)(img::ImageMeta, n::Number) = data(img) .< n
    (.>)(img::ImageMeta, n::Number) = data(img) .> n
    (.<)(img::ImageMeta{Bool}, A::AbstractArray{Bool}) = data(img) .< A
    (.<)(img::ImageMeta, A::AbstractArray) = data(img) .< A
    (.>)(img::ImageMeta, A::AbstractArray) = data(img) .> A
    (.==)(img::ImageMeta, n::Number) = data(img) .== n
    (.==)(img::ImageMeta{Bool}, A::AbstractArray{Bool}) = data(img) .== A
    (.==)(img::ImageMeta, A::AbstractArray) = data(img) .== A
    """)
end

ambigop(s::Symbol) = error("$s with two ImageMeta arrays: dictionary choice is ambiguous")


module ShaderToy
using GeometryTypes, GLAbstraction, GLVisualize, ColorTypes, Reactive
using GLWindow, FileIO, ModernGL

function shadertoy(shaderpath, additional_uniforms::Dict{Symbol, Any}=Dict{Symbol, Any}(); templates=Dict{ASCIIString, ASCIIString}())
    @gen_defaults! additional_uniforms begin
        iResolution         = map(Vec2f0, screen.inputs[:framebuffer_size])
        iMouse              = filterwhen(map(isempty, screen.inputs[:mouse_buttons_pressed]), Vec2f0(0), map(Vec2f0, screen.inputs[:mouseposition]))
        iGlobalTime         = bounce(0f0:0.016f0:2000f0)
        preferred_camera    = :nothing
        primitive::GLMesh2D = SimpleRectangle(-1f0,-1f0,2f0,2f0)
    end
    templates = merge(templates, Dict("SHADERTOY_INPUTS" => """
    uniform vec2 iResolution;
    uniform vec2 iMouse;
    uniform float iGlobalTime;
    uniform sampler2D iChannel0;
    uniform sampler2D iChannel1;
    uniform sampler2D iChannel2;
    uniform sampler2D iChannel3;
    """
    ))
    program = LazyShader(
        load(Pkg.dir("ShaderToy", "src", "fullscreen.vert")),
        load(Pkg.dir("ShaderToy", "src", "shadertoy.frag")),
        load(shaderpath),
        view=templates
    )
    robj = std_renderobject(additional_uniforms, program)
    view(robj)
    isinteractive() ? @async(renderloop(screen)) : renderloop(screen)
    robj
end
export shadertoy

function __init__()
	global screen
	screen = glscreen()
end

end # module


"""
    SynchronousExecution

Functor object for using synchronous execution in a progressive-hedging algorithm (assuming multiple Julia cores are available). Create by supplying a [`Synchronous`](@ref) object through `execution` in the `ProgressiveHedgingSolver` factory function and then pass to a `StochasticPrograms.jl` model.

"""
struct SynchronousExecution{T <: AbstractFloat,
                            A <: AbstractVector,
                            S <: LQSolver,
                            PT <: PenaltyTerm} <: AbstractExecution
    subworkers::Vector{SubWorker{T,A,S,PT}}

    function SynchronousExecution(::Type{T}, ::Type{A}, ::Type{S}, ::Type{PT}) where {T <: AbstractFloat, A <: AbstractVector, S <: LQSolver, PT <: PenaltyTerm}
        return new{T,A,S,PT}(Vector{SubWorker{T,A,S,PT}}(undef, nworkers()))
    end
end

function initialize_subproblems!(ph::AbstractProgressiveHedgingSolver, subsolver::QPSolver, penaltyterm::PenaltyTerm, execution::SynchronousExecution)
    return initialize_subproblems!(ph, subsolver, penaltyterm, execution.subworkers)
end

function resolve_subproblems!(ph::AbstractProgressiveHedgingSolver, execution::SynchronousExecution{T}) where T <: AbstractFloat
    partial_objectives = Vector{T}(undef, nworkers())
    @sync begin
        for (i,w) in enumerate(workers())
            @async partial_objectives[i] = remotecall_fetch(resolve_subproblems!, w, execution.subworkers[w-1], ph.ξ, penalty(ph))
        end
    end
    return sum(partial_objectives)
end

function update_iterate!(ph::AbstractProgressiveHedgingSolver, execution::SynchronousExecution{T,A}) where {T <: AbstractFloat, A <: AbstractVector}
    partial_primals = Vector{A}(undef, nworkers())
    @sync begin
        for (i,w) in enumerate(workers())
            @async partial_primals[i] = remotecall_fetch(collect_primals, w, execution.subworkers[w-1], length(ph.ξ))
        end
    end
    ξ_prev = copy(ph.ξ)
    ph.ξ .= sum(partial_primals)
    # Update δ₁
    ph.data.δ₁ = norm(ph.ξ-ξ_prev, 2)^2
    return nothing
end

function update_subproblems!(ph::AbstractProgressiveHedgingSolver, execution::SynchronousExecution)
    # Update dual prices
    @sync begin
        for w in workers()
            @async remotecall_fetch((sw,ξ,r)->begin
                subproblems = fetch(sw)
                if length(subproblems) > 0
                    update_subproblems!(subproblems, ξ, r)
                end
                end,
                w,
                execution.subworkers[w-1],
                ph.ξ,
                penalty(ph))
        end
    end
    return nothing
end

function update_dual_gap!(ph::AbstractProgressiveHedgingSolver, execution::SynchronousExecution)
    return update_dual_gap!(ph, execution.subworkers)
end

function calculate_objective_value(ph::AbstractProgressiveHedgingSolver, execution::SynchronousExecution)
    return calculate_objective_value(ph, execution.subworkers)
end

function fill_first_stage!(ph::AbstractProgressiveHedgingSolver, stochasticprogram::StochasticProgram, nrows::Integer, ncols::Integer, execution::SynchronousExecution)
    return fill_first_stage!(ph, stochasticprogram, execution.subworkers, nrows, ncols)
end

function fill_submodels!(ph::AbstractProgressiveHedgingSolver, scenarioproblems, nrows::Integer, ncols::Integer, execution::SynchronousExecution)
    return fill_submodels!(ph, scenarioproblems, execution.subworkers, nrows, ncols)
end
# API
# ------------------------------------------------------------
function (execution::Synchronous)(::Type{T}, ::Type{A}, ::Type{S}, ::Type{PT}) where {T <: AbstractFloat, A <: AbstractVector, S <: LQSolver, PT <: PenaltyTerm}
    return SynchronousExecution(T,A,S,PT)
end

function str(::Synchronous)
    return "Synchronous "
end