using SimilaritySearch
using TextModel
using SnowballStemmer
using JSON

function set_oracle(index, fastlinks::Dict{UInt64,KnnResult})
    function oracle(q::VBOW)::Vector{Int32}
        L = Int32[]
        for term in q.tokens
            if haskey(fastlinks, term.id)
                for p in fastlinks[term.id]
                    push!(L, p.objID)
                end
            end
        end

        if length(L) == 0
            # just link randomly for orthogonal vectors
            n = length(index.db)
            return rand(1:n, floor(Int, log2(n+1)))
        end

        # @show L, fastlinks
        L
    end

    index.options.oracle = oracle
    fastlinks
end

function update_oracle(index, fastlinks, bow::VBOW, num_fast_links)
    for term in bow.tokens
        if !haskey(fastlinks, term.id)
            fastlinks[term.id] = KnnResult(num_fast_links)
        end
        push!(fastlinks[term.id], length(index.db), term.weight)
    end
end

function create_index()
    index = LocalSearchIndex(VBOW, angle_distance, recall=0.90, neighborhood=LogSatNeighborhood(1.5))
    fastlinks = set_oracle(index, Dict{UInt64,KnnResult}())
    # index.options.verbose = false
    return index, fastlinks
end

function create_normalizer(stemming, tabufiles, config)
    tabu = Set{String}()
    if length(tabufiles) > 0
        for filename in split(tabufiles, ':')
            words = split(TextModel.normalize_text(readstring(filename), config) |> join)
            union!(tabu, words)
        end
    end

    emptystring = ""
    if length(stemming) > 0
        S = Stemmer(stemming)
        if length(tabu) > 0
            info("CREATING stemming && tabu $(length(tabu))")
            return (w) -> if w ∈ tabu
                emptystring
            else
                stem(S, w)
            end
        else
            return (w) -> stem(S, w)
        end
    else
        if length(tabu) > 0
            return (w) -> if w ∈ tabu
                emptystring
            else
                w
            end
        else
            return identity
        end
    end
end

function main(filename, dup_threshold, key, num_fast_links; stemming="", verbose=false, tabu="")
    config = TextConfig()
    config.nlist = [1]
    config.qlist = []
    config.skiplist = []
    config.normalize = create_normalizer(stemming, tabu, config)
    index, fastlinks = create_index()
    index.options.verbose = verbose

    lineno = 0

    DB = []

    iterlines(filename) do line
        tweet = TextModel.parsetweet(line)
        lineno += 1
        rawbow = compute_bow(tweet[key], config)
        # info(rawbow)
        bow = VBOW(rawbow)
        knn, N = find_neighborhood(index, bow)
        if length(knn) > 0 && first(knn).dist < dup_threshold
            if verbose
                info("- DISCARDED:", line)
                info("-  ORIGINAL:", DB[first(knn).objID])
                info("-  DISTANCE:", first(knn).dist, " < ", dup_threshold)
            end
        else
            push_neighborhood!(index, bow, N, length(index.db))
            update_oracle(index, fastlinks, bow, num_fast_links)  # adding fast links
            push!(DB, line)
            println(line)
        end

        if lineno % 1000 == 0
            info("selected $(length(DB)) of $(lineno), drop-ratio: $(length(DB) / lineno)")
        end

    end
end

if length(ARGS) == 0
    info("""
    neardup-simsearch.jl removes the number of near duplicated items in text databases. It keeps only the original version of the document.

    Usage: [environment-variables] julia neardup-simsearch.jl files...

    Each input file must contain a json-dictionary per line, the indexes are independent per file

    For simplicity, the arguments are passed as environment-variables

    - dup_threshold: items under this value are considered as duplicated
       valid range: 0 < dup_threshold < pi/2
       default: 1.0

    - key: the keyword containing the text for each json
       default: text

    - num_fast_links: the number of documents stored per token to help localsearch indexes to perform fast lookups
       default: 7

    - stemming: specifies stemming rules to apply.
      e.g. stemming=spanish
       valid values: "", "danish", "dutch", "english", "finnish", "french", "german", "hungarian", "italian", "norwegian", "porter", "portuguese", "romanian", "russian", "spanish", "swedish", "turkish"
    - verbose: if enabled (verbose=true) the output becomes too noisy :S
    """)
else
    for filename in ARGS
        main(
            filename,
            parse(Float64, get(ENV, "dup_threshold", "1.0")),
            get(ENV, "key", "text"),
            parse(Int, get(ENV, "num_fast_links", "7")),
            verbose=parse(Bool, get(ENV, "verbose", "false")),
            tabu=get(ENV, "tabu", ""),
            stemming=get(ENV, "stemming", ""),
        )
    end
end


abstract type AbstractMechanism{T,Nn,Nb,Ne,Ni} end

"""
$(TYPEDEF)

A `Mechanism` contains the [`Origin`](@ref), [`Body`](@ref)s, and [`EqualityConstraint`](@ref)s of a system and can be used for simulation.
# Important attributes
* `origin`:        The origin of a mechanism.
* `bodies`:        The bodies of a mechanism (Dict).
* `eqconstraints`: The equality constraints (joints) of a mechanism (Dict).
* `Δt`:            The time step of the mechanism.
* `g`:             The gravitational acceleration in z-direction.

# Constuctors
    Mechanism(origin, bodies; Δt, g)
    Mechanism(origin, bodies, eqcs; Δt, g)
    Mechanism(urdf_filename; floating, Δt, g)
"""
mutable struct Mechanism{T,Nn,Nb,Ne,Ni} <: AbstractMechanism{T,Nn,Nb,Ne,Ni}
    origin::Origin{T}
    bodies::UnitDict{Base.OneTo{Int64},Body{T}}
    eqconstraints::UnitDict{UnitRange{Int64},<:EqualityConstraint{T}}
    ineqconstraints::UnitDict{UnitRange{Int64},<:InequalityConstraint{T}}

    graph::Graph{Nn}
    ldu::SparseLDU{T}

    # TODO remove once EqualityConstraint is homogenous
    normf::T
    normΔs::T

    Δt::T
    g::T

    α::T
    μ::T

    function Mechanism(origin::Origin{T},bodies::Vector{Body{T}},
            eqcs::Vector{<:EqualityConstraint{T}}, ineqcs::Vector{<:InequalityConstraint{T}};
            Δt::Real = .01, g::Real = -9.81
        ) where T

        resetGlobalID()
        order = getGlobalOrder()

        for body in bodies
            if norm(body.m) == 0 || norm(body.J) == 0
                @info "Potentially bad inertial properties detected"
            end
        end

        Nb = length(bodies)
        Ne = length(eqcs)
        Ni = length(ineqcs)
        Nn = Nb + Ne

        if Nb < Ne
            @info "More constraints than bodies. Potentially bad behavior."
        end

        currentid = 1

        bdict = Dict{Int64,Int64}()
        for (ind, body) in enumerate(bodies)
            initknotpoints!(body.state, order)

            for eqc in eqcs
                eqc.parentid == body.id && (eqc.parentid = currentid)
                for (ind, bodyid) in enumerate(eqc.childids)
                    if bodyid == body.id
                        eqc.childids = setindex(eqc.childids, currentid, ind)
                        eqc.constraints[ind].childid = currentid
                    end
                end
            end

            for ineqc in ineqcs
                ineqc.parentid == body.id && (ineqc.parentid = currentid)
            end

            body.id = currentid
            currentid += 1

            bdict[body.id] = ind
        end

        eqdict = Dict{Int64,Int64}()
        for (ind, eqc) in enumerate(eqcs)
            eqc.id = currentid
            currentid += 1

            eqdict[eqc.id] = ind
        end

        ineqdict = Dict{Int64,Int64}()
        for (ind, ineqc) in enumerate(ineqcs)
            ineqc.id = currentid
            currentid += 1

            ineqdict[ineqc.id] = ind
        end

        normf = 0
        normΔs = 0

        graph = Graph(origin, bodies, eqcs, ineqcs)
        ldu = SparseLDU(graph, bodies, eqcs, ineqcs, bdict, eqdict, ineqdict)

        # storage = Storage{T}(steps, Nb, Ne)

        bodies = UnitDict(bodies)
        eqcs = UnitDict((eqcs[1].id):(eqcs[Ne].id), eqcs)
        if Ni > 0
            ineqcs = UnitDict((ineqcs[1].id):(ineqcs[Ni].id), ineqcs)
        else
            ineqcs = UnitDict(0:0, ineqcs)
        end

        α = 1
        μ = 1

        new{T,Nn,Nb,Ne,Ni}(origin, bodies, eqcs, ineqcs, graph, ldu, normf, normΔs, Δt, g, α, μ)
    end

    function Mechanism(origin::Origin{T},bodies::Vector{Body{T}},eqcs::Vector{<:EqualityConstraint{T}};
            Δt::Real = .01, g::Real = -9.81
        ) where T

        ineqcs = InequalityConstraint{T}[]
        return Mechanism(origin, bodies, eqcs, ineqcs, Δt = Δt, g = g)
    end

    function Mechanism(origin::Origin{T},bodies::Vector{Body{T}},ineqcs::Vector{<:InequalityConstraint{T}};
            Δt::Real = .01, g::Real = -9.81
        ) where T

        eqc = EqualityConstraint{T}[]
        for body in bodies
            push!(eqc, EqualityConstraint(Floating(origin, body)))
        end
        return Mechanism(origin, bodies, eqc, ineqcs, Δt = Δt, g = g)
    end

    function Mechanism(origin::Origin{T},bodies::Vector{Body{T}};
            Δt::Real = .01, g::Real = -9.81
        ) where T

        eqc = EqualityConstraint{T}[]
        for body in bodies
            push!(eqc, EqualityConstraint(Floating(origin, body)))
        end
        return Mechanism(origin, bodies, eqc, Δt = Δt, g = g)
    end

    function Mechanism(filename::AbstractString; floating::Bool=false, type::Type{T} = Float64, Δt::Real = .01, g::Real = -9.81) where T
        origin, links, joints, loopjoints = parse_urdf(filename, floating, T)
        mechanism = Mechanism(origin, links, [joints;loopjoints], Δt = Δt, g = g)
        set_parsed_values!(mechanism, loopjoints)

        return mechanism
    end
end

mutable struct LinearMechanism{T,Nn,Nb,Ne,Ni} <: AbstractMechanism{T,Nn,Nb,Ne,Ni}
    ## Mechanism attributes
    origin::Origin{T}
    bodies::UnitDict{Base.OneTo{Int64},Body{T}}
    eqconstraints::UnitDict{UnitRange{Int64},<:EqualityConstraint{T}}
    ineqconstraints::UnitDict{UnitRange{Int64},<:InequalityConstraint{T}}

    graph::Graph{Nn}
    ldu::SparseLDU{T}

    # TODO remove once EqualityConstraint is homogenous
    normf::T
    normΔs::T

    Δt::T
    g::T

    α::T
    μ::T

    ## LinearMechanism attributes
    A::AbstractMatrix{T}
    Bu::AbstractMatrix{T}
    Bλ::AbstractMatrix{T}
    G::AbstractMatrix{T}

    xd::Vector{SVector{3,T}}
    vd::Vector{SVector{3,T}}
    qd::Vector{UnitQuaternion{T}}
    ωd::Vector{SVector{3,T}}
    # Fτd::Vector{SVector{3,T}}

    z::Vector{T}
    zsol::Vector{Vector{T}}
    Δz::Vector{T}
    λ::Vector{T}
    λsol::Vector{Vector{T}}
    Δλ::Vector{T}
    u::Vector{T}

    function LinearMechanism(mechanism::Mechanism{T,Nn,Nb,Ne,Ni}, xd, vd, qd, ωd, Fτd, eqcids) where {T,Nn,Nb,Ne,Ni}

        A, Bu, Bλ, G = linearsystem(mechanism, xd, vd, qd, ωd, Fτd, getid.(mechanism.bodies), eqcids)

        z = zeros(T,Nb*12)
        zsol = [zeros(T,Nb*12) for i=1:2]
        Δz = zeros(T,Nb*12)

        nc = 0
        for eqc in mechanism.eqconstraints
            nc += length(eqc)
        end
        λ = zeros(T,nc)
        λsol = [zeros(T,nc) for i=1:2]
        Δλ = zeros(T,nc)

        u = zeros(T,size(Bu)[2])

        new{T,Nn,Nb,Ne,Ni}([getfield(mechanism,i) for i=1:getfieldnumber(mechanism)]..., A, Bu, Bλ, G, xd, vd, qd, ωd, z, zsol, Δz, λ, λsol, Δλ, u)
    end

    function LinearMechanism(mechanism::Mechanism{T,Nn,Nb,Ne,Ni};
        xd = [mechanism.bodies[i].state.xc for i=1:Nb],
        vd = [mechanism.bodies[i].state.vc for i=1:Nb],
        qd = [mechanism.bodies[i].state.qc for i=1:Nb],
        ωd = [mechanism.bodies[i].state.ωc for i=1:Nb],
        eqcids = [],
        Fτd = []
    ) where {T,Nn,Nb,Ne,Ni}

        return LinearMechanism(mechanism, xd, vd, qd, ωd, Fτd, eqcids)
    end
end

function Base.show(io::IO, mime::MIME{Symbol("text/plain")}, mechanism::AbstractMechanism{T,Nn,Nb,Ne,0}) where {T,Nn,Nb,Ne}
    summary(io, mechanism)
    println(io, " with ", Nb, " bodies and ", Ne, " constraints")
    println(io, " Δt: "*string(mechanism.Δt))
    println(io, " g:  "*string(mechanism.g))
end

function Base.show(io::IO, mime::MIME{Symbol("text/plain")}, mechanism::AbstractMechanism{T,Nn,Nb,Ne,Ni}) where {T,Nn,Nb,Ne,Ni}
    summary(io, mechanism)
    println(io, " with ", Nb, " bodies, ", Ne, " equality constraints, and ", Ni, " inequality constraints")
    println(io, " Δt: "*string(mechanism.Δt))
    println(io, " g:  "*string(mechanism.g))
end


## Feacalc.jl Full implementation of feature calculation for speech files.
## (c) 2013-2016 David A. van Leeuwen

## Feacalc.  Feature calculation as used for speaker and language recognition.

nrow(x) = size(x,1)
ncol(x) = size(x,2)

## compute features according to standard settingsm directly from a wav file.
## this does channel at the time.
function feacalc(wavfile::AbstractString; method=:wav, kwargs...)
    if method == :wav
        (x, sr) = wavread(wavfile)
    elseif method == :sox
        (x, sr) = soxread(wavfile)
    elseif method == :sphere
        (x, sr) = sphread(wavfile)
    end
    sr = Float64(sr)       # more reasonable sr
    feacalc(x; sr=sr, source=wavfile, kwargs...)
end

## assume we have an array already
function feacalc(x::Array; augtype=:ddelta, normtype=:warp, sadtype=:energy, dynrange::Real=30., nwarp::Int=399, chan=:mono, sr::AbstractFloat=8000.0, source=":array", defaults=:nbspeaker, mfccargs...)
    if ndims(x) > 1
        nsamples, nchan = size(x)
        if chan == :mono
            x = vec(mean(x, 2))            # averave multiple channels for now
        elseif in(chan, [:a, :b])
            channum = findin([:a, :b], [chan])
            x = vec(x[:,channum])
        elseif isa(chan, Integer)
            if !(chan in 1:nchan)
                error("Bad channel specification: ", chan)
            end
            x = vec(x[:,chan])
            chan=[:a, :b][chan]
        else
            error("Unknown channel specification: ", chan)
        end
    else
        nsamples, nchan = length(x), 1
    end
    ## save some metadata
    meta = Dict("nsamples" => nsamples, "sr" => sr, "source" => source, "nchan" => nchan,
            "chan" => chan)
    preemph = 0.97
    preemph ^= 16000. / sr

    ## basic features
    (m, pspec, params) = mfcc(x, sr, defaults; preemph=preemph, mfccargs...)
    meta["totnframes"] = nrow(m)

    ## augment features
    if augtype==:delta || augtype==:ddelta
        d = deltas(m)
        if augtype==:ddelta
            dd = deltas(d)
            m = hcat(m, d, dd)
        else
            m = hcat(m, d)
        end
    elseif augtype==:sdc
        m = sdc(m)
    end
    meta["augtype"] = augtype

    if nrow(m)>0
        if sadtype==:energy
            ## integrate power
            deltaf = size(pspec,2) / (sr/2)
            minfreqi = round(Int, 300deltaf)
            maxfreqi = round(Int, 4000deltaf)
            power = 10log10(sum(pspec[:,minfreqi:maxfreqi], 2))

            maxpow = maximum(power)
            speech = find(power .> maxpow - dynrange)
            params["dynrange"] = dynrange
        elseif sadtype==:none
            speech = collect(1:nrow(m))
        end
    else
        speech=Int[]
    end
    meta["sadtype"] = sadtype
    ## perform SAD
    m = m[speech,:]
    meta["speech"] = map(UInt32, speech)
    meta["nframes"] , meta["nfea"] = size(m)

    ## normalization
    if nrow(m)>0
        if normtype==:warp
            m = warp(m, nwarp)
            params["warp"] = nwarp          # the default
        elseif normtype==:mvn
            if nrow(m)>1
                znorm!(m,1)
            else
                fill!(m, 0)
            end
        end
        meta["normtype"] = normtype
    end
    return(map(Float32, m), meta, params)
end

## When called with a specific application in mind, call with two arguments
function feacalc(wavfile::AbstractString, application::Symbol; kwargs...)
    if (application in [:speaker, :nbspeaker])
        feacalc(wavfile; defaults=:nbspeaker, kwargs...)
    elseif application==:wbspeaker
        feacalc(wavfile; defaults=:wbspeaker, kwargs...)
    elseif (application==:language)
        feacalc(wavfile; defaults=:rasta, nwarp=299, augtype=:sdc, kwargs...)
    elseif (application==:diarization)
        feacalc(wavfile; defaults=:rasta, sadtype=:none, normtype=:mvn, augtype=:none, kwargs...)
    else
        error("Unknown application ", application)
    end
end

function sad(pspec::Matrix{Float64}, sr::Float64, method=:energy; dynrange::Float64=30.)
    deltaf = size(pspec,2) / (sr/2)
    minfreqi = round(Int, 300deltaf)
    maxfreqi = round(Int, 4000deltaf)
    power = 10log10(sum(pspec[:,minfreqi:maxfreqi], 2))
    maxpow = maximum(power)
    speech = find(power .> maxpow - dynrange)
end

## listen to SAD
function sad(wavfile::AbstractString, speechout::AbstractString, silout::AbstractString; dynrange::Float64=30.)
    (x, sr, nbits) = wavread(wavfile)
    sr = float64(sr)               # more reasonable sr
    x = vec(mean(x, 2))            # averave multiple channels for now
    (m, pspec, meta) = mfcc(x, sr; preemph=0)
    sp = sad(pspec, sr, dynrange=dynrange)
    sl = round(Int, meta["steptime"] * sr)
    xi = falses(size(x))
    for i in sp
        xi[(i-1)*sl+(1:sl)] = true
    end
    y = x[find(xi)]
    wavwrite(y, speechout, Fs=sr, nbits=nbits, compression=WAVE_FORMAT_PCM)
    y = x[find(.!xi)]
    wavwrite(y, silout, Fs=sr, nbits=nbits, compression=WAVE_FORMAT_PCM)
end

## this should probably be called soxread...
function soxread(file)
    nch = parse(Int, readall(`soxi -c $file`))
    sr = parse(i=Int, readall(`soxi -r $file`))
    sox = `sox $file -t raw -e signed -r $sr -b 16 -`
    fd, proc = open(sox, "r")
    x = Int16[]
    while !eof(fd)
        push!(x, read(fd, Int16))
    end
    ns = length(x) ÷ nch
    reshape(x, nch, ns)' / (1<<15), sr
end

## similarly using sphere tools
function sphread(file)
    nch = parse(Int, readstring(`h_read -n -F channel_count $file`))
    sr = parse(Int, readstring(`h_read -n -F sample_rate $file`))
    open(pipeline(`w_decode -o pcm $file -`, `h_strip - - `), "r") do fd
        x = Int16[]
        while !eof(fd)
            push!(x, read(fd, Int16))
        end
        ns = length(x) ÷ nch
        return reshape(x, nch, ns)' / (1<<15), sr
    end
end


struct EarthType <: abstractPlanet end
Earth = Planet(EarthType)

#Data, as previously used in AstroDynBase
mu(::Type{EarthType}) = 3.986004418e5 #km^3/s^2
j2(::Type{EarthType}) = 1.08262668e-3
j3(::Type{EarthType}) = -2.532e-6
j4(::Type{EarthType}) = -1.61e-6
mean_radius(::Type{EarthType}) = 6371.0084
equatorial_radius(::Type{EarthType}) = 6378.1366
polar_radius(::Type{EarthType}) = 6356.7519

naif_id(::Type{EarthType}) = 399
parent(::Type{EarthType}) = EarthMoonBarycenter
frame(::Type{EarthType}) = "IAU_EARTH"


# This proposal samples from prior and updates exogenous variables
# Using inverse transform

function stdproposal(qω, x::Member{<:PrimDist}, ω)
  if x in keys(ω.data)
    nothing
  else
    x_ = rand(qω, x.class)
    @show x => x_
  end
end

stdproposal(qω, x, ω) = nothing

function stdproposal(qω, x::Member{<:Distribution}, ω)
  @show x
  @show ω.data[x]
  if x in keys(ω)
    @show "here"
    @assert false
  else
    @show "not here"
    @assert false
  end
end


export defΩ, SimpleΩ, LazyΩ

using ..IDS, ..Util, ..Tagging
using Distributions: Distribution
using Random: AbstractRNG
import ..Basis: replacetags

"Simplest, immutable Omega"
struct SimpleΩ{TAGS <: Tags, T} <: AbstractΩ
  data::T
  tags::TAGS
end

idtype(::SimpleΩ{TAGS, Dict{T, V}}) where {TAGS, T, V} = T

SimpleΩ(data) = SimpleΩ(data, Tags())
replacetags(ω::SimpleΩ, tags) = SimpleΩ(ω.data, tags)
(T::Type{<:Distribution})(π::SimpleΩ, args...) = ω.data[scope(ω)]

recurse(d::D, ω::SimpleΩ) where {D<:Distribution} = getindex(ω.data, scope(ω))::eltype(D)

traithastag(t::Type{SimpleΩ{TAGS, T}}, tag) where {TAGS, T} = traithastag(TAGS, tag)
traits(::Type{SimpleΩ{TAGS, T}}) where {TAGS, T} = traits(TAGS)

function resolve(dist, id, ω::SimpleΩ)
  # @assert false
  @show ω
  @show id
  @show ω.data[id]
  d, val = ω.data[id]
  val::eltype(dist)
end


#batched CuArray gemm by BatchedRoutines.jl
function batched_gemm!(transA::AbstractChar,
                       transB::AbstractChar,
                       alpha::Float32,
                       A::CuArrays.CuArray{Float32, 3},
                       B::CuArrays.CuArray{Float32, 3},
                       beta::Float32,
                       C::CuArrays.CuArray{Float32, 3})
    CuArrays.CUBLAS.gemm_strided_batched!(transA, transB, alpha, A, B, beta, C)
end

function batched_gemm!(transA::AbstractChar,
                       transB::AbstractChar,
                       alpha::Float64,
                       A::CuArrays.CuArray{Float64, 3},
                       B::CuArrays.CuArray{Float64, 3},
                       beta::Float64,
                       C::CuArrays.CuArray{Float64, 3})
    CuArrays.CUBLAS.gemm_strided_batched!(transA, transB, alpha, A, B, beta, C)
end


""
function get_solution_tf(pm::GenericPowerModel, sol::Dict{String,Any})
    PowerModels.add_bus_voltage_setpoint(sol, pm)
    PowerModels.add_generator_power_setpoint(sol, pm)
    PowerModels.add_branch_flow_setpoint(sol, pm)
    PowerModels.add_dcline_flow_setpoint(sol, pm)
    add_risk(sol, pm)
    add_branch_shift_setpoint(sol, pm)
    add_branch_tap_setpoint(sol, pm)
    add_load_power_setpoint(sol, pm)
    add_nodal_power_setpoint(sol, pm)
    add_pdelta(sol, pm)
    add_qdelta(sol, pm)
    return sol
end

""
function add_branch_shift_setpoint(sol, pm::GenericPowerModel)
  PowerModels.add_setpoint(sol, pm, "branch", "shiftf", :va_shift; extract_var = (var,idx,item) -> var[(idx, item["f_bus"], item["t_bus"])], default_value = (item) -> 0)
  PowerModels.add_setpoint(sol, pm, "branch", "shiftt", :va_shift; extract_var = (var,idx,item) -> var[(idx, item["t_bus"], item["f_bus"])], default_value = (item) -> 0)
end

""
function add_branch_tap_setpoint(sol, pm::GenericPowerModel)
    dict_name = "branch"
    index_name = "index"
    if pm.data["multinetwork"] == false
        sol_dict = get(sol, dict_name, Dict{String,Any}())
        if length(pm.data[dict_name]) > 0
            sol[dict_name] = sol_dict
        end
        for (i,item) in pm.data[dict_name]
            idx = Int(item[index_name])
            fbus = Int(item["f_bus"])
            tbus = Int(item["t_bus"])
            sol_item = sol_dict[i] = get(sol_dict, i, Dict{String,Any}())
            sol_item["tapf"] = 1
            sol_item["tapt"] = 1
            try
                extract_vtap_fr = (var,idx,item) -> var[(idx, item["f_bus"], item["t_bus"])]
                extract_vtap_to = (var,idx,item) -> var[(idx, item["t_bus"], item["f_bus"])]
                vtap_fr = getvalue(extract_vtap_fr(PowerModels.var(pm, :vm_tap), idx, item))
                vtap_to = getvalue(extract_vtap_to(PowerModels.var(pm, :vm_tap), idx, item))
                vf = getvalue(PowerModels.var(pm,:vm))[fbus]
                vt = getvalue(PowerModels.var(pm,:vm))[tbus]
                sol_item["tapf"] = vf / vtap_fr
                sol_item["tapt"] = vt / vtap_to
            catch
            end
        end
    else
        for (n,nw_data) in pm.data["nw"]
            sol_dict = get(sol, dict_name, Dict{String,Any}())
            if length(pm.data["nw"][n][dict_name]) > 0
                sol[dict_name] = sol_dict
            end
            for (i,item) in pm.data["nw"][n][dict_name]
                idx = Int(item[index_name])
                fbus = Int(item["f_bus"])
                tbus = Int(item["t_bus"])
                sol_item = sol_dict[i] = get(sol_dict, i, Dict{String,Any}())
                sol_item["tapf"] = 1
                sol_item["tapt"] = 1
                try
                    extract_vtap_fr = (var,idx,item) -> var[(idx, item["f_bus"], item["t_bus"])]
                    extract_vtap_to = (var,idx,item) -> var[(idx, item["t_bus"], item["f_bus"])]
                    vtap_fr = getvalue(extract_vtap_fr(PowerModels.var(pm, :vm_tap), idx, item))
                    vtap_to = getvalue(extract_vtap_to(PowerModels.var(pm, :vm_tap), idx, item))
                    vf = getvalue(PowerModels.var(pm,:vm))[fbus]
                    vt = getvalue(PowerModels.var(pm,:vm))[tbus]
                    sol_item["tapf"] = vf / vtap_fr
                    sol_item["tapt"] = vt / vtap_to
                catch
                end
            end
        end
    end
end
""
function add_load_power_setpoint(sol, pm::GenericPowerModel)
    mva_base = pm.data["baseMVA"]
    PowerModels.add_setpoint(sol, pm, "load", "pl", :pl)
    PowerModels.add_setpoint(sol, pm, "load", "ql", :ql)
end
""
function add_nodal_power_setpoint(sol, pm::GenericPowerModel)
    mva_base = pm.data["baseMVA"]
    PowerModels.add_setpoint(sol, pm, "bus", "pnode", :pnode)
    PowerModels.add_setpoint(sol, pm, "bus", "qnode", :qnode)
end
""
function add_pdelta(sol, pm::GenericPowerModel)
    mva_base = pm.data["baseMVA"]
    PowerModels.add_setpoint(sol, pm, "load", "pdelta", :pl_delta)
    PowerModels.add_setpoint(sol, pm, "gen", "pdelta", :pg_delta)
end
""
function add_qdelta(sol, pm::GenericPowerModel)
    mva_base = pm.data["baseMVA"]
    PowerModels.add_setpoint(sol, pm, "load", "qdelta", :ql_delta)
    PowerModels.add_setpoint(sol, pm, "gen", "qdelta", :qg_delta)
end
""
function add_risk(sol, pm::GenericPowerModel)
  add_case_level_setpoint(sol, pm, "first_stage_cost", :first_stage_cost)
  add_case_level_setpoint(sol, pm, "second_stage_risk", :second_stage_risk)
  PowerModels.add_setpoint(sol, pm, "contingencies", "dispatch_cost", :dispatch_cost)
  PowerModels.add_setpoint(sol, pm, "contingencies", "redispatch_cost", :redispatch_cost)
  PowerModels.add_setpoint(sol, pm, "contingencies", "loadshedding_cost", :loadshedding_cost)
  # add_cont_level_setpoint(sol, pm, "contingencies", "dispatch_cost", :dispatch_cost)
  # add_cont_level_setpoint(sol, pm, "contingencies", "redispatch_cost", :redispatch_cost)
  # add_cont_level_setpoint(sol, pm, "contingencies", "loadshedding_cost", :loadshedding_cost)
end
""
function add_case_level_setpoint(sol, pm::GenericPowerModel, param_name, variable_symbol; scale = (x) -> x, extract_var = var -> var)
        if !haskey(sol, param_name)
            sol[param_name] = Dict()
        end
        # print(pm.var[:nw][1][:cnd][pm.ccnd][variable_symbol])
        # print(sol)
        variable = extract_var(pm.var[:nw][1][:cnd][pm.ccnd][variable_symbol])
        sol[param_name] = scale(getvalue(variable))
        # catch
        # end
end
""
function add_cont_level_setpoint(sol, pm::GenericPowerModel, dict_name, param_name, variable_symbol; scale = (x) -> x, extract_var = var -> var)
    if !haskey(sol, dict_name)
        sol[dict_name] = Dict()
    end
    sol[dict_name][param_name] = Dict()
    sol[dict_name][param_name] = getvalue(extract_var(pm.var[:nw][1][:cnd][pm.ccnd][variable_symbol]))
end


################################################################################
#
#   METHODS FOR CONSTRUCTION OF BANDSTRUCTURES (ALONG PATHS)
#
#   STRUCTURE OF THE FILE
#
#   1) TYPE BANDSTRUCTURE
#       - type definition
#       - printInfo function
#
#   2) CALCULATION OF BAND STRUCTURES OF UNTICELL OBJECTS
#
#   3) PLOTTING OF BAND STRUCTURES
#       - plotting of Bandstructure objects
#       - plotting of bandstructures of unitcells along paths
#
################################################################################

################################################################################
#
#   TYPE BANDSTRUCTURE
#       - type definition
#       - printInfo function
#
################################################################################
"""
    struct Bandstructure

The type that contains information on a band structure (energy values in momentum space). Fields are

    path   :: Path
    bands  :: Array{Array{Array{Float64, 1}, 1}, 1}

Note that the notation of `bands` is the following:
- `bands[i]`       gives all bands of path segment `i`
- `bands[i][j]`    gives all energy values for band `j` of path segment `i`
- `bands[i][j][k]` gives the energy value at kpoint index `k` of band `j` in path segment `i`

New `Bandstructure` objects can be created only by the default constructor or one of
the several functions to calculate band structures.

# Examples

```julia-repl
julia> bandstructure = Bandstructure(path, bands)
LatticePhysics.Bandstructure(...)
```
"""
struct Bandstructure

    # the path along which the band structure is calcualted
    path::Path

    # bands for each segment
    # bands[i] gives all bands of segment i
    # bands[i][j] gives all energy values for band j of segment i
    # bands[i][j][k] gives the energy value at kpoint index k of band j in segment i
    bands::Array{Array{Array{Float64, 1}, 1}, 1}

    # ONLY DEFAULT CONSTRUCTOR

end

# export the type
export Bandstructure

# INFORMATION FUNCTION
"""
    printInfo(bandstructure::Bandstructure)

Prints information about a `Bandstructure` in terms of
how many bands are in the given object and along which path the structure is calculated.

# Examples

```julia-repl
julia> printInfo(bandstructure)
...

```
"""
function printInfo(bandstructure::Bandstructure)
    # print the header
    println("Bandstructure information:")
    print("$(bandstructure.path.point_names[1])")
    for p in bandstructure.path.point_names[2:end]
        print("\t->-\t$(p)")
    end
    println("")
    # first the point
    print("|")
    # then all segments with closing points
    for s in 1:length(bandstructure.bands)
        # segment s starts
        print("\t")
        # calculate the number of bands
        print("$(length(bandstructure.bands[s])) bands")
        # print the next point
        print("\t|")
    end
    # print a new line
    println("")
end

################################################################################
#
#   BAND STRUCTURE CALCULATION
#
################################################################################
"""
    getBandstructure(
                unitcell::Unitcell,
                path::Path
             [; resolution::Int64=-1,
                enforce_hermitian::Bool=false ]
            )

    getBandstructure(
                matrixFunction::Function,
                path::Path
             [; resolution::Int64=-1 ]
            )

Calculates the band struture of a `Unitcell` object (or a matrix given by a function `matrixFunction`)
along some path given by a `Path` object and returns a `Bandstructure` object.

# Examples

```julia-repl
julia> bandstructure = getBandstructure(unitcell, path)
LatticePhysics.Bandstructure(...)

julia> bandstructure = getBandstructure(unitcell, path, resolution=1000)
LatticePhysics.Bandstructure(...)
```
"""
function getBandstructure(
                unitcell::Unitcell,
                path::Path;
                resolution::Int64=-1,
                enforce_hermitian::Bool=false
            )

    # maybe modify the path resolution
    if resolution > 0
        setTotalResolution!(path, resolution)
    end

    # build up the list of all bands of all segments
    segments_total = Array{Array{Float64,1},1}[]
    # iterate over all path segments and push empty lists into the segments list
    for i in 1:length(path.segment_resolution)
        # build an empty band structure for this segment
        segment = Array{Float64, 1}[]
        for b in 1:length(unitcell.basis)
            push!(segment, zeros(Float64, path.segment_resolution[i]))
        end
        # push the segment band structure into the complete segment list
        push!(segments_total, segment)
    end

    # iterate over all segments
    for s in 1:length(path.segment_resolution)
        # get the grid in between two points
        segment_resolution = path.segment_resolution[s]
        # get all multipliers of k vectors (i.e. all alpha in (1-alpha)*k_1 + alpha*k_2)
        multipliers = linspace(0, 1, segment_resolution)
        # get the local start and end point of the segment
        k1 = path.points[s]
        k2 = path.points[s+1]
        # calculate all energies
        for i in 1:segment_resolution
            # get the current k
            k = (k2 .* multipliers[i]) .+ (k1 .* (1-multipliers[i]))
            # get the interaction matrix for this k
            matrix = getInteractionMatrixKSpace(unitcell, k, enforce_hermitian=enforce_hermitian)
            # diagonalize the matrix
            eigenvalues = eigvals(matrix)
            # save all the eigenvalues to their lists
            for b in 1:length(unitcell.basis)
                segments_total[s][b][i] = eigenvalues[b]
            end
        end
    end

    # generate a new band structure object
    bandstructure = Bandstructure(path, segments_total)

    # return the band structure
    return bandstructure
end
function getBandstructure(
                matrixFunction::Function,
                path::Path;
                resolution::Int64=-1
            )

    # maybe modify the path resolution
    if resolution > 0
        setTotalResolution!(path, resolution)
    end

    # build up the list of all bands of all segments
    segments_total = Array{Array{Float64,1},1}[]
    # get the number of bands
    N_bands = size(matrixFunction(path.points[1]),1)
    # iterate over all path segments and push empty lists into the segments list
    for i in 1:length(path.segment_resolution)
        # build an empty band structure for this segment
        segment = Array{Float64, 1}[]
        for b in 1:N_bands
            push!(segment, zeros(Float64, path.segment_resolution[i]))
        end
        # push the segment band structure into the complete segment list
        push!(segments_total, segment)
    end

    # iterate over all segments
    for s in 1:length(path.segment_resolution)
        # get the grid in between two points
        segment_resolution = path.segment_resolution[s]
        # get all multipliers of k vectors (i.e. all alpha in (1-alpha)*k_1 + alpha*k_2)
        multipliers = linspace(0, 1, segment_resolution)
        # get the local start and end point of the segment
        k1 = path.points[s]
        k2 = path.points[s+1]
        # calculate all energies
        for i in 1:segment_resolution
            # get the current k
            k = (k2 .* multipliers[i]) .+ (k1 .* (1-multipliers[i]))
            # get the interaction matrix for this k
            matrix = matrixFunction(k)
            # diagonalize the matrix
            eigenvalues = eigvals(matrix)
            # save all the eigenvalues to their lists
            for b in 1:length(eigenvalues)
                segments_total[s][b][i] = eigenvalues[b]
            end
        end
    end

    # generate a new band structure object
    bandstructure = Bandstructure(path, segments_total)

    # return the band structure
    return bandstructure
end
export getBandstructure

################################################################################
#
#   BAND STRUCTURE PLOTTING
#
################################################################################

"""
    plotBandstructure(
            bandstructure::Bandstructure
         [; limits_energy="AUTO",
            plot_title::String="",
            plot_color="b",
            figsize::Tuple=(6,4),
            showPlot::Bool=true,
            save_filename::String="NONE" ]
            )

    plotBandstructure(
            unitcell::Unitcell,
            path::Path
         [; resolution::Int64=-1,
            enforce_hermitian::Bool=false,
            ... ]
        )

    plotBandstructure(
            matrixFunction::Function,
            path::Path
         [; resolution::Int64=-1,
            ... ]
        )

Plots the band struture of a passed `Bandstructure` object along some its path and returns the plot as a `PyPlot.Figure` object.
Alternatively, one can pass a `Unitcell` (or a matrix function) and `Path` to calculate the band structure which is plotted.

Additional options include setting plotting related options of `PyPlot` as well as determining if the plot is saved or shown.

# Examples

```julia-repl
julia> plotBandstructure(unitcell, path)
PyPlot.Figure(...)

julia> plotBandstructure(unitcell, path, showPlot=false)
PyPlot.Figure(...)

julia> plotBandstructure(unitcell, save_filename="myplot.pdf")
PyPlot.Figure(...)

julia> plotBandstructure(bandstructure)
PyPlot.Figure(...)
```
"""
function plotBandstructure(
            bandstructure::Bandstructure;
            limits_energy="AUTO",
            plot_title::String="",
            plot_color="b",
            figsize::Tuple=(6,4),
            showPlot::Bool=true,
            save_filename::String="NONE"
        )

    ###########################
    #   INITIAL SETTINGS
    ###########################

    # get the path from the bandstructure
    path = bandstructure.path

    # configure plot environment
    rc("font", family="serif")

    # create a new figure
    fig = figure(figsize=figsize)

    ###########################
    #   PLOT BANDS
    ###########################

    # plot the band structure
    for s in 1:length(bandstructure.bands)
        # plot the segment
        for b in bandstructure.bands[s]
            # plot the band
            plot(
                collect(1:path.segment_resolution[s]) .+ sum(path.segment_resolution[1:s-1]), # xvalues
                b,
                "-$(plot_color)"
            )
        end
    end

    ###########################
    #   SET ALL TICKS (POINTS)
    ###########################

    # get the current axis
    ax = gca()
    axx = ax[:get_xaxis]()
    # compile tick positions and labels
    point_pos = Int64[]
    push!(point_pos, 1)
    for l in 1:length(path.segment_resolution)
        push!(point_pos, sum(path.segment_resolution[1:l]))
    end
    point_labels = String[path.point_names[i] for i in 1:length(path.points)]
    # configure tick labels
    xticks(point_pos, point_labels)
    # configure ticks
    axx[:set_tick_params](which="both", direction="out")
    axx[:set_tick_params](which="top", color="none")
    axy = ax[:get_yaxis]()
    axy[:set_tick_params](which="both", direction="out")

    # plot vertical lines for each point
    for p in point_pos
        axvline(p,color=[0.6, 0.6, 0.6], linestyle="--")
    end

    ###########################
    #   CONFIGURE AXIS & TITLE
    ###########################

    # label the axis
    xlabel("momentum")
    ylabel("energy")

    # energy limits
    # check if specific boundaries are desired
    if !(limits_energy == "AUTO")
        ylim(limits_energy[1], limits_energy[2])
    end

    # momentum limits (x axis)
    xlim(0, maximum(point_pos)+1)

    # set the title
    if plot_title == "AUTO"
        # set the title to an automatically generated title
        title("energy spectrum along path $(getPathString(path))")
    elseif plot_title == ""
        # do nothing title related
    else
        # set the title to the given title
        title(plot_title)
    end

    ###########################
    #   FINISH THE PLOT
    ###########################

    # tighten the layout
    tight_layout()

    # save the plot
    if save_filename != "NONE"
        # make sure the directory exists
        if contains(save_filename, "/")
    		# get the containing folder
    		folder = save_filename[1:findlast(save_filename, '/')]
    		# build the path to that folder
    		mkpath(folder)
    	end
        # save the plot
        savefig(save_filename)
    end

    # maybe show the plot
    if showPlot
        show()
    end

    # return the figure object
    return fig
end
function plotBandstructure(
            unitcell::Unitcell,
            path::Path;
            resolution::Int64=-1,
            enforce_hermitian::Bool=false,
            limits_energy="AUTO",
            plot_title::String="",
            plot_color="b",
            figsize::Tuple=(6,4),
            showPlot::Bool=true,
            save_filename::String="NONE"
        )
    # calculate the bandstructure
    bandstructure = getBandstructure(unitcell, path, resolution=resolution, enforce_hermitian=enforce_hermitian)
    # call the respective function
    return plotBandstructure(
                bandstructure;
                limits_energy=limits_energy,
                plot_title=plot_title,
                plot_color=plot_color,
                figsize=figsize,
                showPlot=showPlot,
                save_filename=save_filename
            )
end
function plotBandstructure(
            matrixFunction::Function,
            path::Path;
            resolution::Int64=-1,
            limits_energy="AUTO",
            plot_title::String="",
            plot_color="b",
            figsize::Tuple=(6,4),
            showPlot::Bool=true,
            save_filename::String="NONE"
        )
    # calculate the bandstructure
    bandstructure = getBandstructure(matrixFunction, path, resolution=resolution)
    # call the respective function
    return plotBandstructure(
                bandstructure;
                limits_energy=limits_energy,
                plot_title=plot_title,
                plot_color=plot_color,
                figsize=figsize,
                showPlot=showPlot,
                save_filename=save_filename
            )
end
export plotBandstructure


function Base.show(io::IO, cms::Array{T, 1}) where T <: ConfusionMatrix
    for i in eachindex(cms)
        cm = cms[i]
        println(io, "$i --> TP: $(cm.tp) FP: $(cm.fp) TN: $(cm.tn) FN $(cm.fn) [P = :$(cm.pos_class), N = :$(cm.neg_class)]")
    end
end

function listfiles(path)
    files = readdir(path)
    res = []
    for f in joinpath.(path, files)
        if isfile(f)
            endswith(f, ".json") && push!(res, f)
        else
            res = vcat(res, listfiles(f))
        end
    end
    return res
end

findResultFile(path, hash) = filter(x -> occursin(hash, x), listfiles(path))[1]

getResultFromFile(path, hash) = Unmarshal.unmarshal(OneClassActiveLearning.Result, JSON.parsefile(findResultFile(path, hash)))

function load(res::Result, iteration::Int; solver = with_optimizer(Gurobi.Optimizer, OutputFlag = 0))
    data_file = res.experiment[:data_file]
    data, labels = load_data(data_file)

    queries = res.al_history[:query_history].values[1:iteration]

    pools = Symbol.(res.experiment[:param][:initial_pools])
    pools[queries] = ifelse.(labels[queries] .== :inlier, :Lin, :Lout)

    kernels = eval.(Meta.parse.(res.experiment[:model][:fitted][:kernel]))
    gamma_strategy = FixedGammaStrategy(kernels)
    C_strategy = FixedCStrategy(res.experiment[:model][:fitted][:model_params][:C])
    init_strategy = SVDD.SimpleSubspaceStrategy(gamma_strategy, C_strategy, gamma_scope=Val(:Subspace))

    subspaces = convert.(Vector{Int}, res.experiment[:model][:param][:subspaces])

    model = SubSVDD(data, subspaces, pools)
    model.adjust_K = res.experiment[:param][:adjust_K]

    v_Lin = res.experiment[:model][:fitted][:model_params][:weight_update_strategy][:v_Lin]
    v_Lout = res.experiment[:model][:fitted][:model_params][:weight_update_strategy][:v_Lout]

    model.v[queries] = ifelse.(labels[queries] .== :inlier, v_Lin, v_Lout)

    SVDD.initialize!(model, init_strategy)
    return data, labels, model
end

#### Subspace Analysis

function n_outlying_subspaces(predictions; sorted=true)
    tmp = map(x -> ifelse.(SVDD.classify.(x) .== :outlier, 1, 0), predictions)
    subspace_outliers = foldl((x,y) -> x .+ y, tmp)
    sort(collect(zip(eachindex(subspace_outliers), subspace_outliers)), by = x -> x[2], rev=true)
end

function outlying_subspaces(observation_id, predictions, subspaces)
    classifications = SVDD.classify(predictions, Val(:Subspace))
    subspaces[map(x-> x[observation_id], classifications).==:outlier]
end

### Publication plotting
COLORS = (inlier = :lightblue, outlier = :red)
SHAPES = (inlier = (inlier = :circle, outlier = :pentagon), outlier = (inlier = :cross, outlier = :+))

@recipe function plot_svdd(m::SVDD.SVDDClassifier, labels::Vector{Symbol}; grid_resolution = 100, axis_overhang = 0.2, db_color=:black)
    grid_range, grid_data = OCALPlots.get_grid(extrema(m.data)..., grid_resolution, axis_overhang)
    grid_scores = reshape(SVDD.predict(m, grid_data), grid_resolution, grid_resolution)
    data_class = SVDD.classify.(SVDD.predict(m, m.data))
    title := "Decision Boundary"
    @series begin
        seriestype := :contourf
        seriescolor --> :greens
        levels := range(0, maximum(grid_scores), length=10)
        grid_range, grid_range, grid_scores
    end

    markeralpha --> 0.8
    markersize --> 5

    for l in [:inlier, :outlier]
        markercolor := COLORS[l]
        for dc in [:inlier, :outlier]
            markershape := SHAPES[dc][l]
            @series begin
                seriestype := :scatter
                label := OCALPlots.get_legend_text(l, dc)
                OCALPlots.split_2d_array(m.data, (labels.==l) .& (data_class .== dc))
            end
       end
    end

    @series begin
        seriestype := :contour
        levels := [0]
        linewidth := 3
        seriescolor := [db_color]
        cbar:= false
        grid_range, grid_range, grid_scores
    end
end

@recipe function plot_svdd(m::SVDD.SubSVDD, labels::Vector{Symbol}, subspace_idx; grid_resolution = 100, axis_overhang = 0.2, db_color=:black)
    data = m.data[m.subspaces[subspace_idx], :]
    grid_range, grid_data = OCALPlots.get_grid(extrema(data)..., grid_resolution, axis_overhang)
    grid_scores = reshape(SVDD.predict(m, grid_data, subspace_idx), grid_resolution, grid_resolution)
    data_class = SVDD.classify.(SVDD.predict(m, data, subspace_idx))

    title --> "Subspace $(m.subspaces[subspace_idx])"
    @series begin
        seriestype := :contourf
        seriescolor --> :greens
        levels := range(0, maximum(grid_scores), length=10)
        grid_range, grid_range, grid_scores
    end

    markeralpha --> 0.8
    markersize --> 5

    for l in [:inlier, :outlier]
        markercolor := COLORS[l]
        for dc in [:inlier, :outlier]
            markershape := SHAPES[dc][l]
            @series begin
                seriestype := :scatter
                label := OCALPlots.get_legend_text(l, dc)
                OCALPlots.split_2d_array(data, (labels.==l) .& (data_class .== dc))
            end
       end
    end

    @series begin
        seriestype := :contour
        levels := [0]
        linewidth := 3
        seriescolor := [db_color]
        cbar:= false
        grid_range, grid_range, grid_scores
    end
end


# -------------------------------------------------------------------------------------------------------------------------------------------------------------
# Functions
# -------------------------------------------------------------------------------------------------------------------------------------------------------------

function import_data(dir::String, source::String, revision::String, year::Integer)

    if source == "OECD"

        if revision == "2021"

            file = source * "/" * revision * "/"
            path = ifelse(contains(dir[end-5:end], '.'), dir, dir * file)

            time = 1995:2018
            index = findfirst(time .== year) # get index of specified year

            Z = RData.load(path * "ICIO2021econZ" * ".RData")["ICIO2021econZ"][index,:,:] # intermediate demand
            Y = RData.load(path * "ICIO2021econFD" * ".RData")["ICIO2021econFD"][index,:,:] # final demand
            W = RData.load(path * "ICIO2021econVA" * ".RData")["ICIO2021econVA"][index,:,:] # value added
            X = RData.load(path * "ICIO2021econX" * ".RData")["ICIO2021econX"][index,:,:] # gross output

            println(" ✓ Raw data from $source (rev. $revision) for $year was successfully imported!")

        else
            println(" × The revision - $revision - of $source IO tables is not available!")
        end

    end

    return Z, Y, W, X

end

function remove_reorder(Z::Matrix{Float64}, Y::Matrix{Float64}, W::Matrix{Float64}, X::Matrix{Float64}, N::Integer, S::Integer)

    # merge MX1 and MX2 and substitute for MEX
    # rows
    Z[(N-43)*S+1:(N-42)*S, :] .= Z[(N-43)*S+1:(N-42)*S, :] .+ Z[N*S+1:(N+1)*S, :] .+ Z[(N+1)*S+1:(N+2)*S, :]
    Y[(N-43)*S+1:(N-42)*S, :] .= Y[(N-43)*S+1:(N-42)*S, :] .+ Y[N*S+1:(N+1)*S, :] .+ Y[(N+1)*S+1:(N+2)*S, :]
    W[(N-43)*S+1:(N-42)*S] .= W[(N-43)*S+1:(N-42)*S] .+ W[N*S+1:(N+1)*S] .+ W[(N+1)*S+1:(N+2)*S]
    X[(N-43)*S+1:(N-42)*S] .= X[(N-43)*S+1:(N-42)*S] .+ X[N*S+1:(N+1)*S] .+ X[(N+1)*S+1:(N+2)*S]
    # columns
    Z[:, (N-43)*S+1:(N-42)*S] .= Z[:, (N-43)*S+1:(N-42)*S] .+ Z[:, N*S+1:(N+1)*S] .+ Z[:, (N+1)*S+1:(N+2)*S]
    Y[:, (N-42)] .= Y[:, (N-42)] .+ Y[:, (N+1)] .+ Y[:, (N+2)]
    # added domest trade twice! (actually no double counting because columns/rows are designed to avoid)
    # Z[(N-43)*S+1:(N-42)*S, (N-43)*S+1:(N-42)*S] .= Z[(N-43)*S+1:(N-42)*S, (N-43)*S+1:(N-42)*S] ./ 2
    # Y[(N-43)*S+1:(N-42)*S, (N-42)] .= Y[(N-43)*S+1:(N-42)*S, (N-42)] ./ 2

    # merge CH1 and CH2 and substitute for CHN
    # rows
    Z[(N-24)*S+1:(N-23)*S, :] .= Z[(N-24)*S+1:(N-23)*S, :] .+ Z[(N+2)*S+1:(N+3)*S, :] .+ Z[(N+3)*S+1:(N+4)*S, :]
    Y[(N-24)*S+1:(N-23)*S, :] .= Y[(N-24)*S+1:(N-23)*S, :] .+ Y[(N+2)*S+1:(N+3)*S, :] .+ Y[(N+3)*S+1:(N+4)*S, :]
    W[(N-24)*S+1:(N-23)*S] .= W[(N-24)*S+1:(N-23)*S] .+ W[(N+2)*S+1:(N+3)*S] .+ W[(N+3)*S+1:(N+4)*S]
    X[(N-24)*S+1:(N-23)*S] .= X[(N-24)*S+1:(N-23)*S] .+ X[(N+2)*S+1:(N+3)*S] .+ X[(N+3)*S+1:(N+4)*S]
    # columns
    Z[:, (N-24)*S+1:(N-23)*S] .= Z[:, (N-24)*S+1:(N-23)*S] .+ Z[:, (N+2)*S+1:(N+3)*S] .+ Z[:, (N+3)*S+1:(N+4)*S]
    Y[:, (N-23)] .= Y[:, (N-23)] .+ Y[:, (N+3)] .+ Y[:, (N+4)]
    # added domestic trade twice! (actually no double counting because columns/rows are designed to avoid)
    # Z[(N-24)*S+1:(N-23)*S, (N-24)*S+1:(N-23)*S] .= Z[(N-24)*S+1:(N-23)*S, (N-24)*S+1:(N-23)*S] ./ 2
    # Y[(N-24)*S+1:(N-23)*S, (N-23)] .= Y[(N-24)*S+1:(N-23)*S, (N-23)] ./ 2

    # delete MX1/CH1 and MX2/CH2
    Z = Z[1:N*S, 1:N*S]
    Y = Y[1:N*S, 1:N]
    W = W[1:N*S]
    X = X[1:N*S]

    return Z, Y, W, X

end

function trade(Z::Matrix{Float64}, Y::Matrix{Float64}, N::Integer, S::Integer, dimension::String, flow::String)

    if dimension == "industry"

        # bilateral trade by exporting industry/country and importing country
        #   - remove domestic trade
        GRTR_INT = [sum(Z[i, j:j+S-1]) for i in 1:N*S, j in 1:S:N*S] # NS×N
        GRTR_FNL = copy(Y) # NS×N

        for i in 1:N*S
            GRTR_INT[i, ceil(Int, i/S)] = 0.0
            GRTR_FNL[i, ceil(Int, i/S)] = 0.0
        end

    else

        # bilateral trade by exporting country and importing country
        #   - remove domestic trade
        GRTR_INT = [sum(Z[i:i+S-1, j:j+S-1]) for i in 1:S:N*S, j in 1:S:N*S] # N×N
        GRTR_FNL = [sum(Y[i:i+S-1, j]) for i in 1:S:N*S, j in 1:N] # N×N

        for i in 1:N
            GRTR_INT[i, i] = 0.0
            GRTR_FNL[i, i] = 0.0
        end

    end

    GRTR = GRTR_INT .+ GRTR_FNL

    if flow == "exports"
        return GRTR, GRTR_INT, GRTR_FNL
    else
        return GRTR', GRTR_INT', GRTR_FNL'
    end

end

function exports(Z::Matrix{Float64}, Y::Matrix{Float64}, N::Integer, S::Integer, dimension::String)

    if dimension == "industry"

        # bilateral trade by exporting industry/country and importing country
        #   - remove domestic trade
        EXGR_INT = [sum(Z[i, j:j+S-1]) for i in 1:N*S, j in 1:S:N*S] # NS×N
        EXGR_FNL = copy(Y) # NS×N

        for i in 1:N*S
            EXGR_INT[i, ceil(Int, i/S)] = 0.0
            EXGR_FNL[i, ceil(Int, i/S)] = 0.0
        end

    else

        # bilateral trade by exporting country and importing country
        #   - remove domestic trade
        EXGR_INT = [sum(Z[i:i+S-1, j:j+S-1]) for i in 1:S:N*S, j in 1:S:N*S] # N×N
        EXGR_FNL = [sum(Y[i:i+S-1, j]) for i in 1:S:N*S, j in 1:N] # N×N

        for i in 1:N
            EXGR_INT[i, i] = 0.0
            EXGR_FNL[i, i] = 0.0
        end

    end

    EXGR = EXGR_INT .+ EXGR_FNL

    return EXGR, EXGR_INT, EXGR_FNL

end

function imports(Z::Matrix{Float64}, Y::Matrix{Float64}, N::Integer, S::Integer, dimension::String)

    if dimension == "industry"

        # bilateral trade by exporting industry/country and importing country
        #   - remove domestic trade
        IMGR_INT = [sum(Z[i:i+S-1, j]) for i in 1:S:N*S, j in 1:N*S] # N×NS
        IMGR_FNL = [sum(Y[i:i+S-1, j]) for i in 1:S:N*S, j in 1:N] # N×N

        for i in 1:N*S
            IMGR_INT[ceil(Int, i/S), i] = 0.0
            IMGR_FNL[ceil(Int, i/S), ceil(Int, i/S)] = 0.0
        end

        # cannot add up since final demand is not differentiated along industries

        return IMGR_INT, IMGR_FNL

    else

        # bilateral trade by exporting country and importing country
        #   - remove domestic trade
        IMGR_INT = [sum(Z[i:i+S-1, j:j+S-1]) for i in 1:S:N*S, j in 1:S:N*S] # N×N
        IMGR_FNL = [sum(Y[i:i+S-1, j]) for i in 1:S:N*S, j in 1:N] # N×N

        for i in 1:N
            IMGR_INT[i, i] = 0.0
            IMGR_FNL[i, i] = 0.0
        end

        IMGR = IMGR_INT .+ IMGR_FNL # N×N

        return IMGR, IMGR_INT, IMGR_FNL

    end

end


using Revise; using BenchmarkTools; using FunctionWrappers:FunctionWrapper;

RheoFloat = Float64

struct TT
   G::FunctionWrapper{RheoFloat, Tuple{RheoFloat, Vector{RheoFloat}}}
   J::FunctionWrapper{RheoFloat, Tuple{RheoFloat, Vector{RheoFloat}}}

   Ga::FunctionWrapper{Vector{RheoFloat}, Tuple{Vector{RheoFloat}, Vector{RheoFloat}}}
   Ja::FunctionWrapper{Vector{RheoFloat}, Tuple{Vector{RheoFloat}, Vector{RheoFloat}}}
end

function ftest()
   # test base case
   p=(:a1, :a2)
   unpack_expr = Meta.parse(string(join(string.(p), ","), ", = a"))
   G = :(a1*t + a2)
   J = :(a2*t + a1)
   @eval return(TT(
      ((t, a) -> ($unpack_expr; $G)) |> FunctionWrapper{RheoFloat, Tuple{RheoFloat, Vector{RheoFloat}}},
      ((t, a) -> ($unpack_expr; $J)) |> FunctionWrapper{RheoFloat, Tuple{RheoFloat, Vector{RheoFloat}}},

      ((ta, a) -> ($unpack_expr; [$G for t in ta])) |> FunctionWrapper{Vector{RheoFloat}, Tuple{Vector{RheoFloat}, Vector{RheoFloat}}},
      ((ta, a) -> ($unpack_expr; [$J for t in ta])) |> FunctionWrapper{Vector{RheoFloat}, Tuple{Vector{RheoFloat}, Vector{RheoFloat}}},
      ))
end

macro mtest()
   # test base case
   p=(:a1, :a2)
   unpack_expr = Meta.parse(string(join(string.(p), ","), ", = a"))
   G = :(a1*t + a2)
   J = :(a2*t + a1)
   return quote TT(
      ((t, a) -> ($unpack_expr; $G)) |> FunctionWrapper{RheoFloat, Tuple{RheoFloat, Vector{RheoFloat}}},
      ((t, a) -> ($unpack_expr; $J)) |> FunctionWrapper{RheoFloat, Tuple{RheoFloat, Vector{RheoFloat}}},

      ((ta, a) -> ($unpack_expr; [$G for t in ta])) |> FunctionWrapper{Vector{RheoFloat}, Tuple{Vector{RheoFloat}, Vector{RheoFloat}}},
      ((ta, a) -> ($unpack_expr; [$J for t in ta])) |> FunctionWrapper{Vector{RheoFloat}, Tuple{Vector{RheoFloat}, Vector{RheoFloat}}},
     ) end
end

params = [1.0, 1.0];
testvector = Vector(0.0:0.01:100.0)

t1 = ftest()
t2 = @mtest

#println("Function")
@btime t1.Ga($testvector, $params) # 8.739 μs (3 allocations: 78.30 KiB)
@btime t2.Ga($testvector, $params) # 8.437 μs (3 allocations: 78.30 KiB)


module Stat

import Gadfly
import StatsBase
import Contour
using Color
using Compose
using DataArrays
using DataStructures
using Hexagons
using Loess

import Gadfly: Scale, Coord, element_aesthetics, default_scales, isconcrete,
               nonzero_length, setfield!
import KernelDensity
import Distributions: Uniform, Distribution, qqbuild
import Iterators: chain, cycle, product, partition, distinct

include("bincount.jl")

# Apply a series of statistics.
#
# Args:
#   stats: Statistics to apply in order.
#   scales: Scales used by the plot.
#   aes: A Aesthetics instance.
#
# Returns:
#   Nothing, modifies aes.
#
function apply_statistics(stats::Vector{Gadfly.StatisticElement},
                          scales::Dict{Symbol, Gadfly.ScaleElement},
                          coord::Gadfly.CoordinateElement,
                          aes::Gadfly.Aesthetics)
    for stat in stats
        apply_statistic(stat, scales, coord, aes)
    end
    nothing
end

immutable Nil <: Gadfly.StatisticElement
end

const nil = Nil

immutable Identity <: Gadfly.StatisticElement
end

function apply_statistic(stat::Identity,
                         scales::Dict{Symbol, Gadfly.ScaleElement},
                         coord::Gadfly.CoordinateElement,
                         aes::Gadfly.Aesthetics)
    nothing
end

const identity = Identity

immutable HistogramStatistic <: Gadfly.StatisticElement
    minbincount::Int
    maxbincount::Int
    orientation::Symbol

    function HistogramStatistic(; bincount=nothing,
                                  minbincount=3,
                                  maxbincount=150,
                                  orientation::Symbol=:vertical)
        if bincount != nothing
            new(bincount, bincount, orientation)
        else
            new(minbincount, maxbincount, orientation)
        end
    end
end

element_aesthetics(::HistogramStatistic) = [:x]

function default_scales(stat::HistogramStatistic)
    if stat.orientation == :vertical
        return [Gadfly.Scale.y_continuous()]
    else
        return [Gadfly.Scale.x_continuous()]
    end
end

const histogram = HistogramStatistic

function apply_statistic(stat::HistogramStatistic,
                         scales::Dict{Symbol, Gadfly.ScaleElement},
                         coord::Gadfly.CoordinateElement,
                         aes::Gadfly.Aesthetics)
    if stat.orientation == :horizontal
        var = :y
        othervar = :x
        minvar = :ymin
        maxvar = :ymax
        drawmaxvar = :xdrawmax
        labelvar = :x_label
    else
        var = :x
        othervar = :y
        minvar = :xmin
        maxvar = :xmax
        drawmaxvar = :ydrawmax
        labelvar = :y_label
    end

    Gadfly.assert_aesthetics_defined("HistogramStatistic", aes, var)

    values = getfield(aes, var)
    if stat.minbincount > stat.maxbincount
        error("Histogram minbincount > maxbincount")
    end

    if isempty(getfield(aes, var))
        setfield!(aes, minvar, Float64[1.0])
        setfield!(aes, maxvar, Float64[1.0])
        setfield!(aes, var, Float64[1.0])
        setfield!(aes, othervar, Float64[0.0])
        return
    end

    if haskey(scales, var) && isa(scales[var], Scale.DiscreteScale)
        isdiscrete = true
        x_min = minimum(values)
        x_max = maximum(values)
        d = x_max - x_min + 1
        bincounts = zeros(Int, d)
        for x in values
            bincounts[x - x_min + 1] += 1
        end
    else
        isdiscrete = false
        value_set = collect(Set(values[Bool[Gadfly.isconcrete(v) for v in values]]))
        sort!(value_set)

        if  length(value_set) / length(values) < 0.9
            d, bincounts, x_max = choose_bin_count_1d_discrete(
                        values, value_set, stat.minbincount, stat.maxbincount)
        else
            d, bincounts, x_max = choose_bin_count_1d(
                        values, stat.minbincount, stat.maxbincount)
        end
    end

    x_min = Gadfly.concrete_minimum(values)
    binwidth = isdiscrete ? 1 : (x_max - x_min) / d

    if aes.color === nothing
        setfield!(aes, othervar, Array(Float64, d))
        if isdiscrete
            setfield!(aes, var, collect(Int, 1:d))
            setfield!(aes, othervar, bincounts)
        else
            setfield!(aes, minvar, Array(isdiscrete ? Int : Float64, d))
            setfield!(aes, maxvar, Array(isdiscrete ? Int : Float64, d))
            setfield!(aes, var, Array(isdiscrete ? Int : Float64, d))
            for j in 1:d
                getfield(aes, minvar)[j] = x_min + (j - 1) * binwidth
                getfield(aes, maxvar)[j] = x_min + j * binwidth
                getfield(aes, var)[j] = x_min + (j - 0.5) * binwidth
                getfield(aes, othervar)[j] = bincounts[j]
            end
        end
    else
        groups = Dict()
        for (x, c) in zip(values, cycle(aes.color))
            if !Gadfly.isconcrete(x)
                continue
            end

            if !haskey(groups, c)
                groups[c] = Float64[x]
            else
                push!(groups[c], x)
            end
        end

        if isdiscrete
            setfield!(aes, var, Array(Int, d * length(groups)))
        else
            setfield!(aes, minvar, Array(isdiscrete ? Int : Float64, d * length(groups)))
            setfield!(aes, maxvar, Array(isdiscrete ? Int : Float64, d * length(groups)))
            setfield!(aes, var, Array(isdiscrete ? Int : Float64, d * length(groups)))
        end

        setfield!(aes, othervar, Array(Float64, d * length(groups)))
        colors = Array(RGB{Float32}, d * length(groups))

        x_min = Gadfly.concrete_minimum(values)
        stack_height = zeros(Int, d)
        for (i, (c, xs)) in enumerate(groups)
            fill!(bincounts, 0)
            for x in xs
                if !Gadfly.isconcrete(x)
                    continue
                end
                if isdiscrete
                    bincounts[int(x)] += 1
                else
                    bin = max(1, min(d, int(ceil((x - x_min) / binwidth))))
                    bincounts[bin] += 1
                end
            end
            stack_height += bincounts[1:d]

            if isdiscrete
                for j in 1:d
                    idx = (i-1)*d + j
                    getfield(aes, var)[idx] = j
                    getfield(aes, othervar)[idx] = bincounts[j]
                    colors[idx] = c
                end
            else
                for j in 1:d
                    idx = (i-1)*d + j
                    getfield(aes, minvar)[idx] = x_min + (j - 1) * binwidth
                    getfield(aes, maxvar)[idx] = x_min + j * binwidth
                    getfield(aes, var)[idx] = x_min + (j - 0.5) * binwidth
                    getfield(aes, othervar)[idx] = bincounts[j]
                    colors[idx] = c
                end
            end

        end

        drawmax = float64(maximum(stack_height))
        aes_drawmax = getfield(aes, drawmaxvar)
        if aes_drawmax === nothing || aes_drawmax < drawmax
            setfield!(aes, drawmaxvar, drawmax)
        end

        aes.color = PooledDataArray(colors)
    end

    if haskey(scales, othervar)
        data = Gadfly.Data()
        setfield!(data, othervar, getfield(aes, othervar))
        Scale.apply_scale(scales[othervar], [aes], data)
    else
        setfield!(aes, labelvar, Scale.identity_formatter)
    end
end

immutable DensityStatistic <: Gadfly.StatisticElement
    # Number of points sampled
    n::Int

    function DensityStatistic(n=300)
        new(n)
    end
end

const density = DensityStatistic

element_aesthetics(::DensityStatistic) = [:x, :y]

default_scales(::DensityStatistic) = [Gadfly.Scale.y_continuous()]

function apply_statistic(stat::DensityStatistic,
                         scales::Dict{Symbol, Gadfly.ScaleElement},
                         coord::Gadfly.CoordinateElement,
                         aes::Gadfly.Aesthetics)
    Gadfly.assert_aesthetics_defined("DensityStatistic", aes, :x)

    if aes.color === nothing
        if !isa(aes.x[1], Real)
            error("Kernel density estimation only works on Real types.")
        end

        x_f64 = collect(Float64, aes.x)
        # When will stat.n ever be <= 1? Seems pointless
        # certainly its length will always be 1
        window = stat.n > 1 ? KernelDensity.default_bandwidth(x_f64) : 0.1
        f = KernelDensity.kde(x_f64, bandwidth=window, npoints=stat.n)
        aes.x = collect(Float64, f.x)
        aes.y = f.density
    else
        groups = Dict()
        for (x, c) in zip(aes.x, cycle(aes.color))
            if !haskey(groups, c)
                groups[c] = Float64[x]
            else
                push!(groups[c], x)
            end
        end

        colors = Array(RGB{Float32}, 0)
        aes.x = Array(Float64, 0)
        aes.y = Array(Float64, 0)
        for (c, xs) in groups
            window = stat.n > 1 ? KernelDensity.default_bandwidth(xs) : 0.1
            f = KernelDensity.kde(xs, bandwidth=window, npoints=stat.n)
            append!(aes.x, f.x)
            append!(aes.y, f.density)
            for _ in 1:length(f.x)
                push!(colors, c)
            end
        end
        aes.color = PooledDataArray(colors)
    end
    aes.y_label = Gadfly.Scale.identity_formatter
end

immutable Histogram2DStatistic <: Gadfly.StatisticElement
    xminbincount::Int
    xmaxbincount::Int
    yminbincount::Int
    ymaxbincount::Int

    function Histogram2DStatistic(; xbincount=nothing,
                                    xminbincount=3,
                                    xmaxbincount=150,
                                    ybincount=nothing,
                                    yminbincount=3,
                                    ymaxbincount=150)
        if xbincount != nothing
            xminbincount = xbincount
            xmaxbincount = xbincount
        end

        if ybincount != nothing
            yminbincount = ybincount
            ymaxbincount = ybincount
        end

        new(xminbincount, xmaxbincount, yminbincount, ymaxbincount)
    end
end

element_aesthetics(::Histogram2DStatistic) = [:x, :y, :color]

default_scales(::Histogram2DStatistic) = [Gadfly.Scale.continuous_color()]

const histogram2d = Histogram2DStatistic

function apply_statistic(stat::Histogram2DStatistic,
                         scales::Dict{Symbol, Gadfly.ScaleElement},
                         coord::Gadfly.CoordinateElement,
                         aes::Gadfly.Aesthetics)

    Gadfly.assert_aesthetics_defined("Histogram2DStatistic", aes, :x, :y)

    x_min, x_max = Gadfly.concrete_minimum(aes.x), Gadfly.concrete_maximum(aes.x)
    y_min, y_max = Gadfly.concrete_minimum(aes.y), Gadfly.concrete_maximum(aes.y)

    if haskey(scales, :x) && isa(scales[:x], Scale.DiscreteScale)
        x_categorial = true
        xminbincount = x_max - x_min + 1
        xmaxbincount = xminbincount
    else
        x_categorial = false
        xminbincount = stat.xminbincount
        xmaxbincount = stat.xmaxbincount
    end

    if haskey(scales, :y) && isa(scales[:y], Scale.DiscreteScale)
        y_categorial = true
        yminbincount = y_max - y_min + 1
        ymaxbincount = yminbincount
    else
        y_categorial = false
        yminbincount = stat.yminbincount
        ymaxbincount = stat.ymaxbincount
    end

    dy, dx, bincounts = choose_bin_count_2d(aes.x, aes.y,
                                            xminbincount, xmaxbincount,
                                            yminbincount, ymaxbincount)

    wx = x_categorial ? 1 : (x_max - x_min) / dx
    wy = y_categorial ? 1 : (y_max - y_min) / dx

    n = 0
    for cnt in bincounts
        if cnt > 0
            n += 1
        end
    end

    if x_categorial
        aes.x = Array(Int64, n)
    else
        aes.xmin = Array(Float64, n)
        aes.xmax = Array(Float64, n)
    end

    if y_categorial
        aes.y = Array(Int64, n)
    else
        aes.ymin = Array(Float64, n)
        aes.ymax = Array(Float64, n)
    end

    k = 1
    for i in 1:dy, j in 1:dx
        cnt = bincounts[i, j]
        if cnt > 0
            if x_categorial
                aes.x[k] = x_min + (j - 1)
            else
                aes.xmin[k] = x_min + (j - 1) * wx
                aes.xmax[k] = x_min + j * wx
            end

            if y_categorial
                aes.y[k] = y_min + (i - 1)
            else
                aes.ymin[k] = y_min + (i - 1) * wy
                aes.ymax[k] = y_min + i * wy
            end
            k += 1
        end
    end
    @assert k - 1 == n

    if !haskey(scales, :color)
        error("Histogram2DStatistic requires a color scale.")
    end
    color_scale = scales[:color]
    if !(typeof(color_scale) <: Scale.ContinuousColorScale)
        error("Histogram2DStatistic requires a continuous color scale.")
    end

    aes.color_key_title = "Count"

    data = Gadfly.Data()
    data.color = Array(Int, n)
    k = 1
    for cnt in transpose(bincounts)
        if cnt > 0
            data.color[k] = cnt
            k += 1
        end
    end

    if x_categorial
        aes.x = PooledDataArray(aes.x)
    end

    if y_categorial
        aes.y = PooledDataArray(aes.y)
    end

    Scale.apply_scale(color_scale, [aes], data)
    nothing
end

# Find reasonable places to put tick marks and grid lines.
immutable TickStatistic <: Gadfly.StatisticElement
    in_vars::Vector{Symbol}
    out_var::String

    # fixed ticks, or nothing
    ticks::Union(Nothing, AbstractArray)
end

function xticks(ticks::Union(Nothing, AbstractArray)=nothing)
    TickStatistic([:x, :xmin, :xmax, :xdrawmin, :xdrawmax], "x", ticks)
end

function yticks(ticks::Union(Nothing, AbstractArray)=nothing)
    TickStatistic(
        [:y, :ymin, :ymax, :middle, :lower_hinge, :upper_hinge,
         :lower_fence, :upper_fence, :ydrawmin, :ydrawmax], "y", ticks)
end

# Apply a tick statistic.
#
# Args:
#   stat: statistic.
#   aes: aesthetics.
#
# Returns:
#   nothing
#
# Modifies:
#   aes
#
function apply_statistic(stat::TickStatistic,
                         scales::Dict{Symbol, Gadfly.ScaleElement},
                         coord::Gadfly.CoordinateElement,
                         aes::Gadfly.Aesthetics)

    if isa(coord, Coord.SubplotGrid)
        error("TickStatistic cannot be applied to subplot coordinates.")
    end

    in_group_var = symbol(string(stat.out_var, "group"))
    minval, maxval = nothing, nothing
    in_values = {}
    categorical = (:x in stat.in_vars && Scale.iscategorical(scales, :x)) ||
                  (:y in stat.in_vars && Scale.iscategorical(scales, :y))

    for var in stat.in_vars
        vals = getfield(aes, var)
        if vals != nothing && eltype(vals) != Function
            if minval == nothing
                minval = first(vals)
            end
            if maxval == nothing
                maxval = first(vals)
            end
            T = promote_type(typeof(minval), typeof(maxval))
            T = promote_type(T, eltype(vals))
            minval = convert(T, minval)
            maxval = convert(T, maxval)

            if stat.out_var == "x"
                dsize = aes.xsize === nothing ? [nothing] : aes.xsize
            elseif stat.out_var == "y"
                dsize = aes.ysize === nothing ? [nothing] : aes.ysize
            else
                dsize = [nothing]
            end

            size = aes.size === nothing ? [nothing] : aes.size

            minval, maxval = apply_statistic_typed(minval, maxval, vals, size, dsize)
            push!(in_values, vals)
        end
    end

    if isempty(in_values)
        return
    end

    in_values = chain(in_values...)

    # consider forced tick marks
    if stat.ticks != nothing
        minval = min(minval, minimum(stat.ticks))
        maxval = max(maxval, maximum(stat.ticks))
    end

    # TODO: handle the outliers aesthetic

    n = Gadfly.concrete_length(in_values)

    # take into account a forced viewport in cartesian coordinates.
    if typeof(coord) == Coord.Cartesian
        if stat.out_var == "x"
            if !is(coord.xmin, nothing)
                minval = min(minval, coord.xmin)
            end
            if !is(coord.xmax, nothing)
                maxval = max(maxval, coord.xmax)
            end
        elseif stat.out_var == "y"
            if !is(coord.ymin, nothing)
                minval = min(minval, coord.ymin)
            end
            if !is(coord.ymax, nothing)
                maxval = max(maxval, coord.ymax)
            end
        end
    end

    # check the x/yviewmin/max pesudo-aesthetics
    if stat.out_var == "x"
        if aes.xviewmin != nothing
            minval = aes.xviewmin
        end
        if aes.xviewmax != nothing
            maxval = aes.xviewmax
        end
    elseif stat.out_var == "y"
        if aes.yviewmin != nothing
            minval = aes.yviewmin
        end
        if aes.yviewmax != nothing
            maxval = aes.yviewmax
        end
    end

    # all the input values in order.
    if stat.ticks != nothing
        grids = ticks = stat.ticks
        viewmin = minval
        viewmax = maxval
        tickvisible = fill(true, length(ticks))
        tickscale = fill(1.0, length(ticks))
    elseif categorical
        ticks = Set()
        for val in in_values
            push!(ticks, val)
        end
        ticks = Float64[t for t in ticks]
        sort!(ticks)
        grids = (ticks .- 0.5)[2:end]
        viewmin = minimum(ticks)
        viewmax = maximum(ticks)
        tickvisible = fill(true, length(ticks))
        tickscale = fill(1.0, length(ticks))
    else
        minval, maxval = promote(minval, maxval)

        ticks, viewmin, viewmax =
            Gadfly.optimize_ticks(minval, maxval, extend_ticks=true)
        grids = ticks
        multiticks = Gadfly.multilevel_ticks(viewmin - (viewmax - viewmin),
                                             viewmax + (viewmax - viewmin))
        tickcount = length(ticks) + sum([length(ts) for ts in values(multiticks)])
        tickvisible = Array(Bool, tickcount)
        tickscale = Array(Float64, tickcount)
        i = 1
        for t in ticks
            tickscale[i] = 1.0
            tickvisible[i] = viewmin <= t <= viewmax
            i += 1
        end

        for (scale, ts) in multiticks
            for t in ts
                push!(ticks, t)
                tickvisible[i] = false
                tickscale[i] = scale
                i += 1
            end
        end
    end

    # We use the first label function we find for any of the aesthetics. I'm not
    # positive this is the right thing to do, or would would be.
    labeler = getfield(aes, symbol(string(stat.out_var, "_label")))

    setfield!(aes, symbol(string(stat.out_var, "tick")), ticks)
    setfield!(aes, symbol(string(stat.out_var, "grid")), grids)
    setfield!(aes, symbol(string(stat.out_var, "tick_label")), labeler)
    setfield!(aes, symbol(string(stat.out_var, "tickvisible")), tickvisible)
    setfield!(aes, symbol(string(stat.out_var, "tickscale")), tickscale)

    viewmin_var = symbol(string(stat.out_var, "viewmin"))
    if getfield(aes, viewmin_var) === nothing ||
       getfield(aes, viewmin_var) > viewmin
        setfield!(aes, viewmin_var, viewmin)
    end

    viewmax_var = symbol(string(stat.out_var, "viewmax"))
    if getfield(aes, viewmax_var) === nothing ||
       getfield(aes, viewmax_var) < viewmax
        setfield!(aes, viewmax_var, viewmax)
    end

    nothing
end

function apply_statistic_typed{T}(minval::T, maxval::T, vals, size, dsize)
#     for (val, s, ds) in zip(vals, cycle(size), cycle(dsize))
    lensize  = length(size)
    lendsize = length(dsize)
    for (i, val) in enumerate(vals)
        if !Gadfly.isconcrete(val) || !isfinite(val)
            continue
        end

        s = size[mod1(i, lensize)]
        ds = dsize[mod1(i, lendsize)]

        minval, maxval = minvalmaxval(minval, maxval, convert(T, val), s, ds)
    end
    minval, maxval
end

function apply_statistic_typed{T}(minval, maxval, vals::DataArray{T}, size, dsize)
    lensize  = length(size)
    lendsize = length(dsize)
    for i = 1:length(vals)
        if vals.na[i]
            continue
        end

        val::T = vals.data[i]
        s = size[mod1(i, lensize)]
        ds = dsize[mod1(i, lendsize)]

        minval, maxval = minvalmaxval(minval, maxval, val, s, ds)
    end
    minval, maxval
end

function minvalmaxval{T}(minval::T, maxval::T, val, s, ds)
    if val < minval || !isfinite(minval)
        minval = val
    end

    if val > maxval || !isfinite(maxval)
        maxval = val
    end

    if s != nothing
        minval = min(minval, val - s)::T
        maxval = max(maxval, val + s)::T
    end

    if ds != nothing
        minval = min(minval, val - ds)::T
        maxval = max(maxval, val + ds)::T
    end

    minval, maxval
end

immutable BoxplotStatistic <: Gadfly.StatisticElement
end

element_aesthetics(::BoxplotStatistic) = [:x, :y]

const boxplot = BoxplotStatistic

function apply_statistic(stat::BoxplotStatistic,
                         scales::Dict{Symbol, Gadfly.ScaleElement},
                         coord::Gadfly.CoordinateElement,
                         aes::Gadfly.Aesthetics)
    if aes.y === nothing
        Gadfly.assert_aesthetics_defined("BoxplotStatistic", aes,
            :x, :lower_hinge, :upper_hinge, :lower_fence, :upper_fence)

        aes_color = aes.color === nothing ? [nothing] : aes.color
        groups = {}
        for (x, c) in zip(aes.x, cycle(aes_color))
            push!(groups, (x, c))
        end

        if !is(aes.color, nothing)
            aes.color = PooledDataArray(ColorValue[c for (x, c) in groups],
                                        levels(aes.color))
        end

        return
    end

    if aes.x === nothing
        aes_x = [1]
        aes.x_label = x -> fill("", length(x))
    else
        aes_x = aes.x
    end
    aes_color = aes.color === nothing ? [nothing] : aes.color

    T = isempty(aes.y) ? eltype(aes.y) : typeof(aes.y[1] / 1)
    groups = DefaultOrderedDict(() -> T[])

    for (x, y, c) in zip(cycle(aes_x), aes.y, cycle(aes_color))
        push!(groups[(x, c)], y)
    end

    if aes.y != nothing
        m = length(groups)
        aes.x = Array(eltype(aes.x), m)
        aes.middle = Array(T, m)
        aes.lower_hinge = Array(T, m)
        aes.upper_hinge = Array(T, m)
        aes.lower_fence = Array(T, m)
        aes.upper_fence = Array(T, m)
        aes.outliers = Vector{T}[]

        for (i, ((x, c), ys)) in enumerate(groups)
            sort!(ys)

            aes.x[i] = x
            aes.lower_hinge[i], aes.middle[i], aes.upper_hinge[i] =
                    quantile!(ys, [0.25, 0.5, 0.75])
            iqr = aes.upper_hinge[i] - aes.lower_hinge[i]

            idx = searchsortedfirst(ys, aes.lower_hinge[i] - 1.5iqr)
            aes.lower_fence[i] = ys[idx]

            idx = searchsortedlast(ys, aes.upper_hinge[i] + 1.5iqr)
            aes.upper_fence[i] = ys[idx]

            push!(aes.outliers,
                 filter(y -> y < aes.lower_fence[i] || y > aes.upper_fence[i], ys))
        end
    end

    if isa(aes_x, PooledDataArray)
        aes.x = PooledDataArray(aes.x, aes_x.pool)
    end

    if !is(aes.color, nothing)
        aes.color = PooledDataArray(RGB{Float32}[c for (x, c) in keys(groups)],
                                    levels(aes.color))
    end

    nothing
end

immutable SmoothStatistic <: Gadfly.StatisticElement
    method::Symbol
    smoothing::Float64

    function SmoothStatistic(; method::Symbol=:loess, smoothing::Float64=0.75)
        new(method, smoothing)
    end
end

const smooth = SmoothStatistic

element_aesthetics(::SmoothStatistic) = [:x, :y]

function apply_statistic(stat::SmoothStatistic,
                         scales::Dict{Symbol, Gadfly.ScaleElement},
                         coord::Gadfly.CoordinateElement,
                         aes::Gadfly.Aesthetics)

    Gadfly.assert_aesthetics_defined("Stat.smooth", aes, :x, :y)
    Gadfly.assert_aesthetics_equal_length("Stat.smooth", aes, :x, :y)

    if !(stat.method in [:loess,:lm])
        error("The only Stat.smooth methods currently supported are loess and lm.")
    end

    num_steps = 750

    if aes.color === nothing
        x_min, x_max = minimum(aes.x), maximum(aes.x)

        if x_min == x_max
            error("Stat.smooth requires more than one distinct x value")
        end

        local xs, ys

        try
            xs = convert(Vector{Float64}, aes.x)
            ys = convert(Vector{Float64}, aes.y)
        catch e
            error("Stat.loess and Stat.lm require that x and y be bound to arrays of plain numbers.")
        end

        # loess can't predict points <x_min or >x_max. Make sure that doesn't
        # happen through a floating point fluke
        nudge = 1e-5 * (x_max - x_min)
        aes.x = collect((x_min + nudge):((x_max - x_min) / num_steps):(x_max - nudge))

        if stat.method == :loess
            aes.y = predict(loess(xs, ys, span=stat.smoothing), aes.x)
        elseif stat.method == :lm
            lmcoeff = linreg(xs,ys)
            aes.y = lmcoeff[2].*aes.x .+ lmcoeff[1]
        end
    else
        groups = Dict()
        aes_color = aes.color === nothing ? [nothing] : aes.color
        for (x, y, c) in zip(aes.x, aes.y, cycle(aes_color))
            if !haskey(groups, c)
                groups[c] = (Float64[], Float64[])
            end

            try
                push!(groups[c][1], x)
                push!(groups[c][2], y)
            catch
                error("Stat.loess and Stat.lm require that x and y be bound to arrays of plain numbers.")
            end
        end

        aes.x = Array(Float64, length(groups) * num_steps)
        aes.y = Array(Float64, length(groups) * num_steps)
        colors = Array(RGB{Float32}, length(groups) * num_steps)

        for (i, (c, (xs, ys))) in enumerate(groups)
            x_min, x_max = minimum(xs), maximum(xs)
            if x_min == x_max
                error("Stat.smooth requires more than one distinct x value")
            end
            nudge = 1e-5 * (x_max - x_min)
            steps = collect((x_min + nudge):((x_max - x_min) / num_steps):(x_max - nudge))

            if stat.method == :loess
                smoothys = predict(loess(xs, ys, span=stat.smoothing), steps)
            elseif stat.method == :lm
                lmcoeff = linreg(xs,ys)
                smoothys = lmcoeff[2].*steps .+ lmcoeff[1]
            end

            for (j, (x, y)) in enumerate(zip(steps, smoothys))
                aes.x[(i - 1) * num_steps + j] = x
                aes.y[(i - 1) * num_steps + j] = y
                colors[(i - 1) * num_steps + j] = c
            end
        end
        aes.color = PooledDataArray(colors)
    end
end

immutable HexBinStatistic <: Gadfly.StatisticElement
    xbincount::Int
    ybincount::Int

    function HexBinStatistic(; xbincount=50, ybincount=50)
        new(xbincount, ybincount)
    end
end

const hexbin = HexBinStatistic

function apply_statistic(stat::HexBinStatistic,
                         scales::Dict{Symbol, Gadfly.ScaleElement},
                         coord::Gadfly.CoordinateElement,
                         aes::Gadfly.Aesthetics)
    xmin, xmax = minimum(aes.x), maximum(aes.x)
    ymin, ymax = minimum(aes.y), maximum(aes.y)
    xspan, yspan = xmax - xmin, ymax - ymin

    xsize = xspan / stat.xbincount
    ysize = yspan / stat.ybincount

    counts = Dict{(Any, Any), Int}()
    for (x, y) in zip(aes.x, aes.y)
        h = convert(HexagonOffsetOddR, pointhex(x - xmin + xspan/2,
                                                y - ymin + yspan/2,
                                                xsize, ysize))
        idx = (h.q, h.r)
        if !haskey(counts, idx)
            counts[idx] = 1
        else
            counts[idx] += 1
        end
    end

    N = length(counts)
    aes.x = Array(Float64, N)
    aes.y = Array(Float64, N)
    data = Gadfly.Data()
    data.color = Array(Int, N)
    k = 1
    for (idx, cnt) in counts
        x, y = center(HexagonOffsetOddR(idx[1], idx[2]), xsize, ysize,
                      xmin - xspan/2, ymin - yspan/2)
        aes.x[k] = x
        aes.y[k] = y
        data.color[k] = cnt
        k += 1
    end
    aes.xsize = [xsize]
    aes.ysize = [ysize]

    color_scale = scales[:color]
    if !(typeof(color_scale) <: Scale.ContinuousColorScale)
        error("HexBinGeometry requires a continuous color scale.")
    end

    Scale.apply_scale(color_scale, [aes], data)
end

function default_scales(::HexBinStatistic)
    return [Gadfly.Scale.continuous_color()]
end

immutable StepStatistic <: Gadfly.StatisticElement
    direction::Symbol

    function StepStatistic(; direction::Symbol=:hv)
        return new(direction)
    end
end

const step = StepStatistic

function element_aesthetics(::StepStatistic)
    return [:x, :y]
end

function apply_statistic(stat::StepStatistic,
                         scales::Dict{Symbol, Gadfly.ScaleElement},
                         coord::Gadfly.CoordinateElement,
                         aes::Gadfly.Aesthetics)
    Gadfly.assert_aesthetics_defined("StepStatistic", aes, :x)
    Gadfly.assert_aesthetics_defined("StepStatistic", aes, :y)
    Gadfly.assert_aesthetics_equal_length("StepStatistic", aes, :x, :y)

    points = collect(zip(aes.x, aes.y))
    sort!(points, by=first)
    n = length(points)
    x_step = Array(eltype(aes.x), 2n - 1)
    y_step = Array(eltype(aes.y), 2n - 1)

    for i in 1:(2n-1)
        if isodd(i)
            x_step[i] = points[div(i-1,2)+1][1]
            y_step[i] = points[div(i-1,2)+1][2]
        elseif stat.direction == :hv
            x_step[i] = points[div(i-1,2)+2][1]
            y_step[i] = points[div(i-1,2)+1][2]
        else
            x_step[i] = points[div(i-1,2)+1][1]
            y_step[i] = points[div(i-1,2)+2][2]
        end
    end

    aes.x = x_step
    aes.y = y_step
end

immutable FunctionStatistic <: Gadfly.StatisticElement
    # Number of points to evaluate the function at
    num_samples::Int

    function FunctionStatistic(; num_samples=250)
        return new(num_samples)
    end
end

const func = FunctionStatistic

function default_scales(::FunctionStatistic)
    return [Gadfly.Scale.x_continuous(), Gadfly.Scale.y_continuous()]
end

function element_aesthetics(::FunctionStatistic)
    return [:y, :xmin, :xmax, :ymin, :ymax]
end

function apply_statistic(stat::FunctionStatistic,
                         scales::Dict{Symbol, Gadfly.ScaleElement},
                         coord::Gadfly.CoordinateElement,
                         aes::Gadfly.Aesthetics)
    Gadfly.assert_aesthetics_defined("FunctionStatistic", aes, :y)
    Gadfly.assert_aesthetics_defined("FunctionStatistic", aes, :xmin)
    Gadfly.assert_aesthetics_defined("FunctionStatistic", aes, :xmax)
    Gadfly.assert_aesthetics_equal_length("FunctionStatistic", aes, :xmin, :xmax)

    aes.x = Array(Float64, length(aes.y) * stat.num_samples)
    ys = Array(Float64, length(aes.y) * stat.num_samples)

    i = 1
    for (f, xmin, xmax) in zip(aes.y, cycle(aes.xmin), cycle(aes.xmax))
        for x in linspace(xmin, xmax, stat.num_samples)
            aes.x[i] = x
            ys[i] = f(x)
            i += 1
        end
    end

    # color was bound explicitly
    if aes.color != nothing
        func_color = aes.color
        aes.color = DataArray(eltype(aes.color), length(aes.y) * stat.num_samples)
        groups = DataArray(Int, length(aes.y) * stat.num_samples)
        for i in 1:length(aes.y)
            aes.color[1+(i-1)*stat.num_samples:i*stat.num_samples] = func_color[i]
            groups[1+(i-1)*stat.num_samples:i*stat.num_samples] = i
        end
        aes.group = PooledDataArray(groups)
    elseif length(aes.y) > 1 && haskey(scales, :color)
        data = Gadfly.Data()
        data.color = Array(String, length(aes.y) * stat.num_samples)
        for i in 1:length(aes.y)
            fname = "f<sub>$(i)</sub>"
            data.color[1+(i-1)*stat.num_samples:i*stat.num_samples] = fname
        end
        Scale.apply_scale(scales[:color], [aes], data)
    end

    data = Gadfly.Data()
    data.y = ys
    Scale.apply_scale(scales[:y], [aes], data)
end

immutable ContourStatistic <: Gadfly.StatisticElement
    levels
    samples::Int

    function ContourStatistic(; n=15, samples=150)
        new(n, samples)
    end

    function ContourStatistic(; levels=15, samples=150)
        new(levels, samples)
    end
end

element_aesthetics(::ContourStatistic) = [:z, :xmin, :xmax, :ymin, :ymax]

const contour = ContourStatistic

function default_scales(::ContourStatistic)
    return [Gadfly.Scale.z_func(), Gadfly.Scale.x_continuous(),
            Gadfly.Scale.y_continuous(),
            Gadfly.Scale.continuous_color_gradient()]
end

function apply_statistic(stat::ContourStatistic,
                         scales::Dict{Symbol, Gadfly.ScaleElement},
                         coord::Gadfly.CoordinateElement,
                         aes::Gadfly.Aesthetics)
    xs = aes.x === nothing ? nothing : convert(Vector{Float64}, aes.x)
    ys = aes.y === nothing ? nothing : convert(Vector{Float64}, aes.y)

    if typeof(aes.z) <: Function
        if xs == nothing && aes.xmin != nothing && aes.xmax != nothing
            xs = linspace(aes.xmin[1], aes.xmax[1], stat.samples)
        end

        if ys == nothing && aes.ymin != nothing && aes.ymax != nothing
            ys = linspace(aes.ymin[1], aes.ymax[1], stat.samples)
        end

        zs = Float64[aes.z(x, y) for x in xs, y in ys]

    elseif typeof(aes.z) <: Matrix
        zs = convert(Matrix{Float64}, aes.z)

        if xs == nothing
            xs = float([1:size(zs)[1]])
        end
        if ys == nothing
            ys = float([1:size(zs)[2]])
        end
        if size(zs) != (length(xs), length(ys))
            error("Stat.contour requires dimension of z to be length(x) by length(y)")
        end
    else
        error("Stat.contour requires either a matrix or a function")
    end

    levels = Float64[]
    contour_xs = eltype(xs)[]
    contour_ys = eltype(ys)[]

    groups = PooledDataArray(Int[])
    group = 0
    for contour in Contour.contours(xs, ys, zs, stat.levels)
        for curve in contour.lines
            for v in curve.vertices
                push!(contour_xs, v[1])
                push!(contour_ys, v[2])
                push!(levels, contour.level)
                push!(groups, group)
            end
            group += 1
        end
    end

    aes.group = groups
    color_scale = get(scales, :color, Gadfly.Scale.continuous_color_gradient())
    Scale.apply_scale(color_scale, [aes], Gadfly.Data(color=levels))
    Scale.apply_scale(scales[:x], [aes],  Gadfly.Data(x=contour_xs))
    Scale.apply_scale(scales[:y], [aes], Gadfly.Data(y=contour_ys))
end

immutable QQStatistic <: Gadfly.StatisticElement
end

element_aesthetics(::QQStatistic) = [:x, :y]

const qq = QQStatistic

function default_scales(::QQStatistic)
    return [Gadfly.Scale.x_continuous(), Gadfly.Scale.y_continuous]
end

function apply_statistic(stat::QQStatistic,
                         scales::Dict{Symbol, Gadfly.ScaleElement},
                         coord::Gadfly.CoordinateElement,
                         aes::Gadfly.Aesthetics)

    Gadfly.assert_aesthetics_defined("Stat.qq", aes, :x, :y)

    # NOTES:
    #
    # apply_scales happens before apply_statistics, so we need to handle in
    # apply_scales the Distributions that might be bound to x and y... By
    # analogy with Stat.func, we can add a check in apply_statistic which defers
    # application.  Stat.func though requires an ARRAY of Functions, and doesn't
    # work on naked functions bound to aes.y.  If we want to bind Distributions,
    # we'd need to extend the types that are allowed for aes.y/.x (e.g. change
    # type of Aesthetics fields x and y).  Right now these are of type
    # NumericalOrCategoricalAesthetic.  The .x and .y fields are the _only_
    # place where this type is used, but I'm not sure if there's a reason that
    # changing this typealias would be a bad idea...for now I've just used a
    # direct `Union(NumericalOrCategoricalAesthetic, Distribution)`.
    #
    # TODO:
    #
    # Grouping by color etc.?

    # a little helper function to convert either numeric or distribution
    # variables to a format suitable to input to qqbuild.
    toVecOrDist = v -> typeof(v) <: Distribution ? v : convert(Vector{Float64}, v)

    # check and convert :x and :y to proper types for input to qqbuild
    local xs, ys
    try
        (xs, ys) = map(toVecOrDist, (aes.x, aes.y))
    catch e
        error("Stat.qq requires that x and y be bound to either a Distribution or to arrays of plain numbers.")
    end

    qqq = qqbuild(xs, ys)

    aes.x = qqq.qx
    aes.y = qqq.qy

    # apply_scale to Distribution-bound aesthetics is deferred, so re-apply here
    # (but only for Distribution, numeric data is already scaled).  Only one of
    # :x or :y can be a Distribution since qqbuild will throw an error for two
    # Distributions.
    data = Gadfly.Data()
    if typeof(xs) <: Distribution
        data.x = aes.x
        Scale.apply_scale(scales[:x], [aes], data)
    elseif typeof(ys) <: Distribution
        data.y = aes.y
        Scale.apply_scale(scales[:y], [aes], data)
    end

end

end # module Stat


module WiringDiagrams

using Reexport

include("Directed.jl")
include("CPortGraphs.jl")
include("Undirected.jl")
include("MonoidalDirected.jl")
include("MonoidalUndirected.jl")
include("Algebras.jl")
include("Algorithms.jl")
include("Expressions.jl")
include("ScheduleUndirected.jl")

@reexport using .DirectedWiringDiagrams
@reexport using .CPortGraphs
@reexport using .UndirectedWiringDiagrams
@reexport using .MonoidalDirectedWiringDiagrams
@reexport using .MonoidalUndirectedWiringDiagrams
@reexport using .WiringDiagramAlgebras
@reexport using .WiringDiagramAlgorithms
@reexport using .WiringDiagramExpressions
@reexport using .ScheduleUndirectedWiringDiagrams

include("Serialization.jl")
include("GraphML.jl")
include("JSON.jl")

using .WiringDiagramSerialization
export convert_from_graph_data, convert_to_graph_data
@reexport using .GraphMLWiringDiagrams
@reexport using .JSONWiringDiagrams

end