mutable struct DoorKeyDirected{T, R} <: AbstractGridWorld
    world::GridWorldBase{Tuple{Agent, Wall, Goal, Door, Key}}
    agent_pos::CartesianIndex{2}
    agent_dir::AbstractDirection
    reward::T
    rng::R
    terminal_reward::T
    goal_pos::CartesianIndex{2}
    door_pos::CartesianIndex{2}
    key_pos::CartesianIndex{2}
    partition::CartesianIndices{2,Tuple{UnitRange{Int},UnitRange{Int}}}
    has_key::Bool
    done::Bool
end

@generate_getters(DoorKeyDirected)
@generate_setters(DoorKeyDirected)

mutable struct DoorKeyUndirected{T, R} <: AbstractGridWorld
    world::GridWorldBase{Tuple{Agent, Wall, Goal, Door, Key}}
    agent_pos::CartesianIndex{2}
    reward::T
    rng::R
    terminal_reward::T
    goal_pos::CartesianIndex{2}
    door_pos::CartesianIndex{2}
    key_pos::CartesianIndex{2}
    partition::CartesianIndices{2,Tuple{UnitRange{Int},UnitRange{Int}}}
    has_key::Bool
    done::Bool
end

@generate_getters(DoorKeyUndirected)
@generate_setters(DoorKeyUndirected)

#####
# Directed
#####

function DoorKeyDirected(; T = Float32, height = 8, width = 8, rng = Random.GLOBAL_RNG)
    objects = (AGENT, WALL, GOAL, DOOR, KEY)
    world = GridWorldBase(objects, height, width)

    room = Room(CartesianIndex(1, 1), height, width)
    place_room!(world, room)

    goal_pos = CartesianIndex(height - 1, width - 1)
    world[GOAL, goal_pos] = true

    agent_pos = CartesianIndex(2, 2)
    world[AGENT, agent_pos] = true

    door_pos = CartesianIndex(2, 3)
    world[DOOR, door_pos] = true

    partition = CartesianIndices((2:height-1, door_pos[2]:door_pos[2]))
    world[WALL, partition] .= true
    world[WALL, door_pos] = false

    key_pos = CartesianIndex(3, 2)
    world[KEY, key_pos] = true

    agent_dir = RIGHT
    reward = zero(T)
    terminal_reward = one(T)
    has_key = false
    done = false

    env = DoorKeyDirected(world, agent_pos, agent_dir, reward, rng, terminal_reward, goal_pos, door_pos, key_pos, partition, has_key, done)

    RLBase.reset!(env)

    return env
end

RLBase.state_space(env::DoorKeyDirected, ::RLBase.Observation, ::RLBase.DefaultPlayer) = nothing
const DOOR_KEY_DIRECTED_LAYERS = SA.SVector(2, 3, 4, 5)
RLBase.state(env::DoorKeyDirected, ::RLBase.Observation, ::RLBase.DefaultPlayer) = get_grid(get_world(env), get_agent_pos(env), get_agent_dir(env), get_half_size(env), DOOR_KEY_DIRECTED_LAYERS)

RLBase.state_space(env::DoorKeyDirected, ::RLBase.InternalState, ::RLBase.DefaultPlayer) = nothing
RLBase.state(env::DoorKeyDirected, ::RLBase.InternalState, ::RLBase.DefaultPlayer) = (copy(get_grid(env)), get_agent_dir(env))

RLBase.action_space(env::DoorKeyDirected, ::RLBase.DefaultPlayer) = (DIRECTED_NAVIGATION_ACTIONS..., PICK_UP)
RLBase.reward(env::DoorKeyDirected, ::RLBase.DefaultPlayer) = get_reward(env)
RLBase.is_terminated(env::DoorKeyDirected) = get_done(env)

function RLBase.reset!(env::DoorKeyDirected{T}) where {T}
    world = get_world(env)
    height = get_height(env)
    width = get_width(env)
    rng = get_rng(env)

    world[AGENT, get_agent_pos(env)] = false
    world[WALL, get_partition(env)] .= false
    world[GOAL, get_goal_pos(env)] = false
    world[DOOR, get_door_pos(env)] = false
    world[KEY, get_key_pos(env)] = false

    new_door_pos, new_partition, region1, region2 = generate_partition(rng, height, width)
    region1, region2 = Random.shuffle(SA.SVector(region1, region2))

    set_door_pos!(env, new_door_pos)
    world[DOOR, new_door_pos] = true

    set_partition!(env, new_partition)
    world[WALL, new_partition] .= true
    world[WALL, new_door_pos] = false

    new_goal_pos = rand(rng, pos -> !any(@view world[:, pos]), region2)
    set_goal_pos!(env, new_goal_pos)
    world[GOAL, new_goal_pos] = true

    new_key_pos = rand(rng, pos -> !any(@view world[:, pos]), region1)
    set_key_pos!(env, new_key_pos)
    world[KEY, new_key_pos] = true

    new_agent_pos = rand(rng, pos -> !any(@view world[:, pos]), region1)
    set_agent_pos!(env, new_agent_pos)
    world[AGENT, new_agent_pos] = true

    set_agent_dir!(env, rand(rng, DIRECTIONS))

    set_reward!(env, zero(T))
    set_has_key!(env, false)
    set_done!(env, false)

    return nothing
end

function (env::DoorKeyDirected{T})(action::AbstractTurnAction) where {T}
    new_dir = turn(action, get_agent_dir(env))
    set_agent_dir!(env, new_dir)
    world = get_world(env)

    if world[GOAL, get_agent_pos(env)]
        set_done!(env, true)
        set_reward!(env, get_terminal_reward(env))
    else
        set_done!(env, false)
        set_reward!(env, zero(T))
    end

    return nothing
end

function (env::DoorKeyDirected{T})(action::AbstractMoveAction) where {T}
    world = get_world(env)

    agent_pos = get_agent_pos(env)
    dest = move(action, get_agent_dir(env), agent_pos)
    if !(world[DOOR, dest] || world[WALL, dest]) || (world[DOOR, dest] && get_has_key(env))
        world[AGENT, agent_pos] = false
        set_agent_pos!(env, dest)
        world[AGENT, dest] = true
    end

    if world[GOAL, get_agent_pos(env)]
        set_done!(env, true)
        set_reward!(env, get_terminal_reward(env))
    else
        set_done!(env, false)
        set_reward!(env, zero(T))
    end

    return nothing
end

#####
# Undirected
#####

function DoorKeyUndirected(; T = Float32, height = 8, width = 8, rng = Random.GLOBAL_RNG)
    objects = (AGENT, WALL, GOAL, DOOR, KEY)
    world = GridWorldBase(objects, height, width)

    room = Room(CartesianIndex(1, 1), height, width)
    place_room!(world, room)

    goal_pos = CartesianIndex(height - 1, width - 1)
    world[GOAL, goal_pos] = true

    agent_pos = CartesianIndex(2, 2)
    world[AGENT, agent_pos] = true

    door_pos = CartesianIndex(2, 3)
    world[DOOR, door_pos] = true

    partition = CartesianIndices((2:height-1, door_pos[2]:door_pos[2]))
    world[WALL, partition] .= true
    world[WALL, door_pos] = false

    key_pos = CartesianIndex(3, 2)
    world[KEY, key_pos] = true

    reward = zero(T)
    terminal_reward = one(T)
    has_key = false
    done = false

    env = DoorKeyUndirected(world, agent_pos, reward, rng, terminal_reward, goal_pos, door_pos, key_pos, partition, has_key, done)

    RLBase.reset!(env)

    return env
end

RLBase.state_space(env::DoorKeyUndirected, ::RLBase.Observation, ::RLBase.DefaultPlayer) = nothing
const DOOR_KEY_UNDIRECTED_LAYERS = SA.SVector(2, 3, 4, 5)
RLBase.state(env::DoorKeyUndirected, ::RLBase.Observation, ::RLBase.DefaultPlayer) = get_grid(get_world(env), get_agent_pos(env), get_half_size(env), DOOR_KEY_UNDIRECTED_LAYERS)

RLBase.state_space(env::DoorKeyUndirected, ::RLBase.InternalState, ::RLBase.DefaultPlayer) = nothing
RLBase.state(env::DoorKeyUndirected, ::RLBase.InternalState, ::RLBase.DefaultPlayer) = copy(get_grid(env))

RLBase.action_space(env::DoorKeyUndirected, player::RLBase.DefaultPlayer) = (UNDIRECTED_NAVIGATION_ACTIONS..., PICK_UP)
RLBase.reward(env::DoorKeyUndirected, ::RLBase.DefaultPlayer) = get_reward(env)
RLBase.is_terminated(env::DoorKeyUndirected) = get_done(env)

function RLBase.reset!(env::DoorKeyUndirected{T}) where {T}
    world = get_world(env)
    height = get_height(env)
    width = get_width(env)
    rng = get_rng(env)

    world[AGENT, get_agent_pos(env)] = false
    world[WALL, get_partition(env)] .= false
    world[GOAL, get_goal_pos(env)] = false
    world[DOOR, get_door_pos(env)] = false
    world[KEY, get_key_pos(env)] = false

    new_door_pos, new_partition, region1, region2 = generate_partition(rng, height, width)
    region1, region2 = Random.shuffle(SA.SVector(region1, region2))

    set_door_pos!(env, new_door_pos)
    world[DOOR, new_door_pos] = true

    set_partition!(env, new_partition)
    world[WALL, new_partition] .= true
    world[WALL, new_door_pos] = false

    new_goal_pos = rand(rng, pos -> !any(@view world[:, pos]), region2)
    set_goal_pos!(env, new_goal_pos)
    world[GOAL, new_goal_pos] = true

    new_key_pos = rand(rng, pos -> !any(@view world[:, pos]), region1)
    set_key_pos!(env, new_key_pos)
    world[KEY, new_key_pos] = true

    new_agent_pos = rand(rng, pos -> !any(@view world[:, pos]), region1)
    set_agent_pos!(env, new_agent_pos)
    world[AGENT, new_agent_pos] = true

    set_reward!(env, zero(T))
    set_has_key!(env, false)
    set_done!(env, false)

    return nothing
end

function (env::DoorKeyUndirected{T})(action::AbstractMoveAction) where {T}
    world = get_world(env)

    agent_pos = get_agent_pos(env)
    dest = move(action, agent_pos)
    if !(world[DOOR, dest] || world[WALL, dest]) || (world[DOOR, dest] && get_has_key(env))
        world[AGENT, agent_pos] = false
        set_agent_pos!(env, dest)
        world[AGENT, dest] = true
    end

    if world[GOAL, get_agent_pos(env)]
        set_done!(env, true)
        set_reward!(env, get_terminal_reward(env))
    else
        set_done!(env, false)
        set_reward!(env, zero(T))
    end

    return nothing
end

#####
# Common
#####

function generate_partition(rng, height, width)
    if rand(rng) < 0.5
        # vertical partition
        door_pos = rand(rng, CartesianIndices((2:height-1, 3:width-2)))
        partition = CartesianIndices((2:height-1, door_pos[2]:door_pos[2]))
        region1 = CartesianIndices((2:height-1, 2:door_pos[2]-1))
        region2 = CartesianIndices((2:height-1, door_pos[2]+1:width-1))
        return door_pos, partition, region1, region2
    else
        # horizontal partition
        door_pos = rand(rng, CartesianIndices((3:height-2, 2:width-1)))
        partition = CartesianIndices((door_pos[1]:door_pos[1], 2:width-1))
        region1 = CartesianIndices((2:door_pos[1]-1, 2:width-1))
        region2 = CartesianIndices((door_pos[1]+1:height-1, 2:width-1))
        return door_pos, partition, region1, region2
    end
end

function (env::Union{DoorKeyDirected{T}, DoorKeyUndirected{T}})(::PickUp) where {T}
    world = get_world(env)
    agent_pos = get_agent_pos(env)

    if world[KEY, agent_pos]
        set_has_key!(env, true)
        world[KEY, agent_pos] = false
    end

    if world[GOAL, get_agent_pos(env)]
        set_done!(env, true)
        set_reward!(env, get_terminal_reward(env))
    else
        set_done!(env, false)
        set_reward!(env, zero(T))
    end

    return nothing
end


function _precompile_()
    ccall(:jl_generating_output, Cint, ()) == 1 || return nothing

    VecBitSites = Sites{Bit, Int, 1}
    for L = 4:10, BC in [Lattices.Periodic, Lattices.Fixed], DType in [Float32, Float64, ComplexF32, ComplexF64]
        precompile(Base.getindex, (SumSingleSiteType{DType, Matrix{DType}, Lattices.Chain{L, BC}}, VecBitSites, VecBitSites))
    end
end


using POMDPs

POMDPs.add("BasicPOMCP")
POMDPs.add("QMDP")
POMDPs.add("POMCPOW")
POMDPs.add("DESPOT")
POMDPs.add("DiscreteValueIteration")
POMDPs.add("ARDESPOT")

try Pkg.clone("https://github.com/slundberg/PmapProgressMeter.jl.git") end

try Pkg.clone("https://github.com/zsunberg/LightDarkPOMDPs.jl") end
try Pkg.clone("https://github.com/zsunberg/VDPTag.jl.git") end
try Pkg.clone("https://github.com/zsunberg/VDPTag2.jl.git") end
try Pkg.clone("https://github.com/zsunberg/LaserTag.jl.git") end
try Pkg.clone("https://github.com/zsunberg/SubHunt.jl.git") end


using KUparser, KUnet, HDF5, JLD, ArgParse, Compat, CUDArt
VERSION < v"0.4-" && eval(Expr(:using,:Dates))
require("fscore.jl")

type Fselect allfeats; bestfeats; bestscore; idxqueue; scores; Fselect()=new(); end

function main()
    args = parse_commandline()
    (data, ndeps) = initdata(args)
    pt = eval(parse(args["arctype"]))

    # We want a single copy of the state variables
    fs = Fselect()
    fs.allfeats = eval(parse("Flist."*args["allfeats"]))
    fs.idxqueue = randperm(length(fs.allfeats))
    fs.bestfeats = eval(parse("Flist."*args["feats"]))
    fs.bestscore = getcache(args["cache"], fs.bestfeats)
    if fs.bestscore < 0
        fs.bestscore = @fetchfrom workers()[1] fscore(pt, data, ndeps, fs.bestfeats, args)
        updatecache(args["cache"], fs.bestfeats, fs.bestscore)
    end

    function getnextidx()
        # See if we can improve fs.bestfeats with single feature flip steps from cache
        while true
            fs.scores = map(fs.allfeats) do f
                getcache(args["cache"], fs.bestfeats, f)
            end
            (smax,imax) = findmax(fs.scores)
            smax <= fs.bestscore && break
            fs.bestscore = smax
            fs.bestfeats = flip!(fs.bestfeats, fs.allfeats[imax])
            fs.idxqueue = randperm(length(fs.allfeats))
        end

        @show fs; flush(STDOUT)

        # Find the next neighbor to compute
        while (!isempty(fs.idxqueue) && (fs.scores[fs.idxqueue[1]] >= 0))
            shift!(fs.idxqueue)
        end

        return (minimum(fs.scores) >= 0 ? nothing : # terminate if all scores known
                isempty(fs.idxqueue) ? 0 :          # wait for remaining computations if queue empty
                shift!(fs.idxqueue))                # compute the next element in queue
    end

    # Feeder tasks based on multi.jl:pmap implementation:
    while !(isempty(fs.idxqueue) && (getnextidx()==nothing))
        @show fs; flush(STDOUT)
        @sync for wpid in workers()
            @async begin
                idx = getnextidx()
                # This got messed up because gc() is a leaky bucket
                # while idx != nothing
                if idx == nothing
                    # do nothing, time to terminate
                elseif idx == 0
                    sleep(10)
                else
                    @show ("$wpid gets fs.allfeats[$idx]=$(fs.allfeats[idx])"); flush(STDOUT)
                    @show feats = flip(fs.bestfeats, fs.allfeats[idx])
                    try
                        score = remotecall_fetch(wpid, fscore, pt, data, ndeps, feats, args)
                        @show ("$wpid gets score[$(fs.allfeats[idx])]=$score"); flush(STDOUT)
                        if isa(score, Number)
                            updatecache(args["cache"], feats, score)
                        else
                            push!(fs.idxqueue, idx)
                            @show ("Got $score from $wpid"); flush(STDOUT); sleep(10)
                        end
                    catch ex
                        push!(fs.idxqueue, idx)
                        @show ("Caught $ex from $wpid"); flush(STDOUT); sleep(10)
                    end
                end  # if idx == 0
                # idx = getnextidx()
                # end
            end  # @async begin
        end  # @sync for wpid in workers()
        @show fs; flush(STDOUT)
        @date Main.restartmachines(); flush(STDOUT)
        require("KUparser")
        require("fscore.jl")
    end  # while
    info("$(fs.bestscore)\t$(join(sort(fs.bestfeats), ' '))")
end

function parse_commandline()
    s = ArgParseSettings()
    @add_arg_table s begin
        "datafiles"
        help = "JLD files for input: first one will be used for training second for scoring"
        nargs = '+'
        required = true
        "--cache"
        help = "Cache file (plain text) for saving and reading results"
        default = ""
        "--nogpu"
        help = "Do not use gpu"
        action = :store_true
        # We use oparser on the training set to get training data.
        # We train the net for n epochs with fixed training data, no parsing.
        # Finally we evaluate by parsing the dev corpus using greedy or beam parser specified below.
        "--parser"
        help = "Parsing algorithm to use: g(reedy)parser, b(eam)parser"
        default = "gparser"
        "--arctype"
        help = "Move set to use: ArcEager{R1,13}, ArcHybrid{R1,13}"
        default = "ArcEager13"
        "--feats"
        help = "Features to start with from KUparser.Flist"
        default = "acl11eager"
        "--allfeats"
        help = "Complete feature set to use from KUparser.Flist"
        default = "acl11eager"
        "--hidden"
        help = "One or more hidden layer sizes"
        nargs = '+'
        arg_type = Int
        default = [20000]
        "--epochs"
        help = "Number of epochs to train"
        arg_type = Int
        default = 10
        # Don't need this, we only parse at the end:
        # "--pepochs"
        # help = "Number of epochs between parsing"
        # arg_type = Int
        # default = 1
        # We will just use whatever julia was started with
        # "--ncpu"
        # help = "Number of workers for multithreaded parsing"
        # arg_type = Int
        # default = 6
        "--pbatch"
        help = "Minibatch size for parsing"
        arg_type = Int
        default = 500
        "--tbatch"
        help = "Minibatch size for training"
        arg_type = Int
        default = 128
        "--nbeam"
        help = "Beam size for bparser"
        arg_type = Int
        default = 10
        "--adagrad"
        help = "If nonzero apply adagrad using arg as epsilon"
        arg_type = Float32
        nargs = '+'
        default = [1f-8]
        "--dropout"
        help = "Dropout probability"
        arg_type = Float32
        nargs = '+'
        # No dropout for 10-epoch training
        # default = [0.2f0, 0.7f0]
        "--l1reg"
        help = "L1 regularization parameter"
        arg_type = Float32
        nargs = '+'
        "--l2reg"
        help = "L2 regularization parameter"
        arg_type = Float32
        nargs = '+'
        "--learningRate"
        help = "Learning rate"
        arg_type = Float32
        nargs = '+'
        default = [2f-2]
        "--maxnorm"
        help = "If nonzero upper limit on weight matrix row norms"
        arg_type = Float32
        nargs = '+'
        "--momentum"
        help = "Momentum"
        arg_type = Float32
        nargs = '+'
        "--nesterov"
        help = "Apply nesterov's accelerated gradient"
        arg_type = Float32
        nargs = '+'
    end
    args = parse_args(s)
    for (arg,val) in args
        print("$arg:$val ")
    end
    println("")
    return args
end

function getcache(cachefile::String, feats::Fvec, f::Feature="")
    isempty(f) || (feats = flip(feats, f))
    fkey = join(sort(feats), ' ')
    fval = -1.0
    if isfile(cachefile)
        open(cachefile) do fp
            for l in eachline(fp)
                (score, fstr) = split(chomp(l), '\t')
                fstr == fkey && (fval = float(score); break)
            end
        end
    end
    return fval
end

function updatecache(cachefile::String, feats::Fvec, score::Real)
    open(cachefile, "a") do fp
        fkey = join(sort(feats), ' ')
        write(fp, "$score\t$fkey\n")
    end
end

function updatecache(cachefile::String, feats::Fvec, toflip::Fvec, scores::AbstractArray)
    @assert length(toflip) == length(scores)
    open(cachefile, "a") do fp
        for i=1:length(toflip)
            fkey = join(sort(flip(feats,toflip[i])), ' ')
            fval = scores[i]
            write(fp, "$fval\t$fkey\n")
        end
    end
end

function initdata(args::Dict)
    data = Corpus[]
    deprel = nothing
    ndeps = 0
    for f in args["datafiles"]
        @date d=load(f)
        push!(data, d["corpus"])
        if deprel == nothing
            deprel = d["deprel"]
            ndeps = length(deprel)
        else
            @assert deprel == d["deprel"]
        end
    end
    @assert length(data)==2 "fselect needs a training and a development set"
    (data, ndeps)
end

main()


"""Plate motion data requests using the [UNAVCO Plate Motion Calculator](https://www.unavco.org/software/geodetic-utilities/plate-motion-calculator/plate-motion-calculator.html).

Exported names:
$(EXPORTS)

"""
module PlateMotionRequests

export platemotion
export write_platemotion
export read_platemotion

using DelimitedFiles

using DocStringExtensions
using HTTP
using TypedTables

"""
    platemotion(lats, lons, heights; kwargs...)
    platemotion(lats, lons; kwargs...)
    platemotion(XYZ, kwargs...)

Request plate motion data from the [UNAVCO Plate Motion Calculator](https://www.unavco.org/software/geodetic-utilities/plate-motion-calculator/plate-motion-calculator.html). Headers and metadata are stripped from the output, which is parsed into a [`Table`](https://typedtables.juliadata.org/latest/man/table/). Accepts either separate vectors for latitude, longitude and optionally height. The XYZ input method is deprecated, and will be removed in the next release.

!!! note

    Site names are not supported.
    Only 'ansii', 'ansii_xyz' and 'psvelo' formats are supported.
    Not all optional argument permutations are permitted,
    see the website linked above for further details.

Optional arguments:
- `model`: The plate motion model to use for calculations, or GSRM v2.1 by default.
- `plate`: The tectonic plate of attributed motion, or automatic plate selection by default.
- `reference`: The fixed reference plate, or NNR (No Net Rotation) by default.
- `up_lat`: Custom coordinate of the Euler pole (attributed motion) in decimal degrees.
- `up_lon`: See above.
- `up_w`: Custom rotation rate (attributed motion) in degrees per million years.
- `up_x`: Custom component of the Euler pole (attributed motion) in degrees per million years.
- `up_y`: See above.
- `up_z`: See above.
- `ur_lat`: Custom coordinate of the Euler pole (reference velocity) in decimal degrees.
- `ur_lon`: See above.
- `ur_w`: Custom rotation rate of the Euler pole (reference velocity) in degrees per million years.
- `ur_x`: Custom component of the Euler pole (reference velocity) in degrees per million years.
- `ur_y`: See above.
- `ur_z`: See above.
- `format`: Output format for the data section of the response, or ASCII by default.

"""
function platemotion(lats, lons, heights = fill(0, length(lats)); kwargs...)
    request = _kwargparser(kwargs)
    push!(
        request,
        :geo =>
            join(mapslices(x -> join(x, " "), hcat(lons, lats, heights), dims = 2), ",\n"),
    )
    return _parse!(_submit(request), request[:format])
end

function platemotion(XYZ; kwargs...)
    X, Y, Z = XYZ
    request = _kwargparser(kwargs)
    push!(
        request,
        :xyz => join(mapslices(x -> join(x, " "), hcat(X, Y, Z), dims = 2), ",\n"),
    )
    return _parse!(_submit(request), request[:format])
end

function _kwargparser(kwargs::Base.Iterators.Pairs)
    kwargs = Dict(kwargs)
    return Dict(
        :model => get(kwargs, :model, "gsrm_2014"),
        :plate => get(kwargs, :plate, ""),
        :reference => get(kwargs, :reference, "NNR"),
        :up_lat => String(get(kwargs, :up_lat, "")),
        :up_lon => String(get(kwargs, :up_lon, "")),
        :up_w => String(get(kwargs, :up_w, "")),
        :up_x => String(get(kwargs, :up_x, "")),
        :up_y => String(get(kwargs, :up_y, "")),
        :up_z => String(get(kwargs, :up_z, "")),
        :ur_lat => String(get(kwargs, :ur_lat, "")),
        :ur_lon => String(get(kwargs, :ur_lon, "")),
        :ur_w => String(get(kwargs, :ur_w, "")),
        :ur_x => String(get(kwargs, :ur_x, "")),
        :ur_y => String(get(kwargs, :ur_y, "")),
        :ur_z => String(get(kwargs, :ur_z, "")),
        :format => :format in keys(kwargs) ? _validate_format(kwargs[:format]) : "ascii",
    )
end

function _validate_format(format)
    supported_formats = ("ascii", "ascii_xyz", "psvelo")
    if !(format in supported_formats)
        throw(
            ArgumentError(
                "`format` must be one of `$(supported_formats)`." *
                " You've supplied `format = $(format)`.",
            ),
        )
    end
    return String(format)
end

function _submit(request)
    return HTTP.post(
        "https://www.unavco.org/software/geodetic-utilities/plate-motion-calculator/plate-motion/model",
        [],  # Empty header.
        HTTP.Form(request),
    )
end

function _parse!(raw::HTTP.Response, format)
    body = raw.body
    # Data is inside the <pre> tag.
    bytes = body[findfirst(b"<pre>", body)[end]+1:findfirst(b"</pre>", body)[1]-1]
    # Replace trailing tab(s) with 8-bit newline char, otherwise readdlm() fails.
    replace!(bytes, codepoint('\t') => UInt8('\n'))

    if format == "ascii"
        Format = _FormatASCII
    elseif format == "ascii_xyz"
        Format = _FormatASCIIxyz
    elseif format == "psvelo"
        Format = _FormatPsvelo
    end

    table = _mktable(Format)
    for line in eachrow(readdlm(bytes, ' ', '\n'))
        line = line[line.!=""]
        floats = line[isa.(line, Float64)]
        ints = line[isa.(line, Int64)]
        strings = line[isa.(line, SubString{String})]
        plate_and_ref, model = strings[1], join(strings[2:end], ' ')
        rowdata = Format(floats..., ints..., plate_and_ref, model)
        push!(table, _mkrow(rowdata))
    end
    return table
end

struct _FormatPsvelo
    lon::Float64
    lat::Float64
    velocity_east::Float64
    velocity_north::Float64
    err_east::Float64
    err_north::Float64
    correlation_NE::Float64
    plate_and_reference::String
    model::String
end

struct _FormatASCII
    lon::Float64
    lat::Float64
    velocity_east::Float64
    velocity_north::Float64
    plate_and_reference::String
    model::String
end

struct _FormatASCIIxyz
    lon::Float64
    lat::Float64
    velocity_x::Float64
    velocity_y::Float64
    velocity_z::Float64
    plate_and_reference::String
    model::String
end

struct ParseError <: Exception
    msg::AbstractString
end
ParseError() = ParseError("")

function _mktable(t::DataType)
    return Table(NamedTuple{fieldnames(t)}(type[] for type in fieldtypes(t)))
end
function _mktable(d::Matrix, t::DataType)
    return Table(;
        (colname => coldata for (colname, coldata) in zip(fieldnames(t), eachcol(d)))...,
    )
end

function _mkrow(x::Union{_FormatASCII,_FormatASCIIxyz,_FormatPsvelo})
    fields = fieldnames(typeof(x))
    return NamedTuple{fields}((getfield(x, field) for field in fields))
end

"""
    write_platemotion(file, table)

Write plate motion table to `file` as tab-delimited text columns.
The first line written is a tab-delimited header containing the column names.
Note that the first header column is a comment marker (`#`), not a column name.

"""
function write_platemotion(file, table)
    open(file, "w") do io
        writedlm(io, [append!(["#"], String.(columnnames(table)))])
        writedlm(io, table)
    end
end

"""
    read_platemotion(file)

Read tab-delimited plate motion data from `file`.
Expects a single tab-delimited header line (starting with a comment marker),
with column names that match one of the supported formats.
See [`platemotion`](@ref) for details.

"""
function read_platemotion(file)
    data, header = readdlm(file, '\t', Any, '\n', header = true)
    # First cell of the header is the comment marker.
    isformat(format) = Set(header[2:end]) == Set(String.(fieldnames(format)))
    if isformat(_FormatASCII)
        table = _mktable(data, _FormatASCII)
    elseif isformat(_FormatASCIIxyz)
        table = _mktable(data, _FormatASCIIxyz)
    elseif isformat(_FormatPsvelo)
        table = _mktable(data, _FormatPsvelo)
    else
        throw(
            ParseError(
                "columns in `file` must match a supported format." *
                " You've supplied a corrupt or unsupported file." *
                " If possible, try to write the file again using `write_platemotion`.",
            ),
        )
    end
    return table
end

end # PlateMotionRequests


__precompile__()

module Integrators

    using ..Equations
    using ..Solvers
    using ..Utils

    import ..Solvers.function_stages!

    include("utils/macro_utils.jl")

    export Tableau, TableauRK, TableauERK, TableauDIRK, TableauFIRK, TableauSIRK,
           TableauEPRK, TableauIPRK, TableauSARK, TableauSPARK, TableauGLM,
           showTableau, writeTableauToFile, readTableauERKFromFile

    include("integrators/tableaus.jl")

    export Timeseries, compute_timeseries!

    include("integrators/timeseries.jl")

    export Solution, SolutionODE, SolutionPODE, SolutionDAE, SolutionPDAE,
           reset!, set_initial_conditions!, compute_time_steps!,
           createHDF5, writeSolutionToHDF5

    include("integrators/solutions.jl")
    include("integrators/solutions_ode.jl")
    include("integrators/solutions_pode.jl")
    include("integrators/solutions_dae.jl")
    include("integrators/solutions_pdae.jl")

    export InitialGuess, evaluate, initialize!, update!

    export Integrator, IntegratorERK, IntegratorDIRK, IntegratorFIRK, IntegratorSIRK,
           IntegratorEPRK, IntegratorIPRK, IntegratorSARK, IntegratorSPARK,
           integrate, integrate!, function_stages!

    include("integrators/integrators.jl")
    include("integrators/integrators_erk.jl")
    include("integrators/integrators_eprk.jl")

    include("integrators/initial_guess.jl")

    include("integrators/integrators_dirk.jl")
    include("integrators/integrators_firk.jl")
    include("integrators/integrators_sirk.jl")
    include("integrators/integrators_iprk.jl")
    include("integrators/integrators_sark.jl")
    include("integrators/integrators_spark.jl")

end


module InverseProblems

using OrderedCollections
using Suppressor: @suppress

using ..Observations: obs_str, AbstractObservation, SyntheticObservations, initialize_simulation!, FieldTimeSeriesCollector,
    observation_times, observation_names

using ..TurbulenceClosureParameters: free_parameters_str, new_closure_ensemble

using OffsetArrays, Statistics

using Oceananigans: short_show, run!, fields, FieldTimeSeries, CPU
using Oceananigans.OutputReaders: InMemory
using Oceananigans.Fields: interior, location
using Oceananigans.Grids: Flat, Bounded,
                          Face, Center,
                          RectilinearGrid, offset_data,
                          topology, halo_size,
                          interior_parent_indices

using Oceananigans.Models.HydrostaticFreeSurfaceModels: SingleColumnGrid, YZSliceGrid, ColumnEnsembleSize

import ..Observations: normalize!

#####
##### Output maps (maps from simulation output to observation space)
#####

abstract type AbstractOutputMap end

output_map_type(fp) = output_map_str(fp)

struct ConcatenatedOutputMap{T}
    time_indices::T
end

ConcatenatedOutputMap(; time_indices = Colon()) = ConcatenatedOutputMap(time_indices)

output_map_str(::ConcatenatedOutputMap) = "ConcatenatedOutputMap"

"""
    ConcatenatedVectorNormMap()

Forward map transformation of simulation output to a scalar by
taking a naive `norm` of the difference between concatenated vectors of the
observations and simulation output.
"""
struct ConcatenatedVectorNormMap end

output_map_str(::ConcatenatedVectorNormMap) = "ConcatenatedVectorNormMap"

#####
##### InverseProblems
#####

struct InverseProblem{F, O, S, T, P}
    observations :: O
    simulation :: S
    time_series_collector :: T
    free_parameters :: P
    output_map :: F
end

"""
    InverseProblem(observations, simulation, free_parameters; output_map=ConcatenatedOutputMap())

Return an `InverseProblem`.
"""
function InverseProblem(observations,
                        simulation,
                        free_parameters;
                        output_map = ConcatenatedOutputMap(),
                        time_series_collector = nothing)

    if isnothing(time_series_collector) # attempt to construct automagically
        simulation_fields = fields(simulation.model)
        collected_fields = NamedTuple(name => simulation_fields[name] for name in observation_names(observations))
        time_series_collector = FieldTimeSeriesCollector(collected_fields, observation_times(observations))
    end

    return InverseProblem(observations, simulation, time_series_collector, free_parameters, output_map)
end

function Base.show(io::IO, ip::InverseProblem)
    sim_str = "Simulation on $(short_show(ip.simulation.model.grid)) with Δt=$(ip.simulation.Δt)"

    out_map_type = output_map_type(ip.output_map)
    out_map_str = output_map_str(ip.output_map)

    print(io, "InverseProblem{$out_map_type}", '\n',
        "├── observations: $(obs_str(ip.observations))", '\n',
        "├── simulation: $sim_str", '\n',
        "├── free_parameters: $(free_parameters_str(ip.free_parameters))", '\n',
        "└── output map: $out_map_str")

    return nothing
end

tupify_parameters(ip, θ) = NamedTuple{ip.free_parameters.names}(Tuple(θ))
tupify_parameters(ip, θ::NamedTuple) = NamedTuple(name => θ[name] for name in ip.free_parameters.names)

"""
    expand_parameters(ip, θ)

Convert `θ` to `Vector{<:NamedTuple}`, where the elements
correspond to `ip.free_parameters`.

`θ` may represent an ensemble of parameter sets via:

* `θ::Vector{<:Vector}` (caution: parameters must be ordered correctly!)
* `θ::Matrix` (caution: parameters must be ordered correctly!)
* `θ::Vector{<:NamedTuple}`

or a single parameter set if `θ::Vector{<:Number}`.

If `length(θ)` is less the the number of ensemble members in `ip.simulation`, the
last parameter set is copied to fill the parameter set ensemble.
"""
function expand_parameters(ip, θ::Vector)
    Nfewer = n_ensemble(ip) - length(θ)
    Nfewer < 0 && throw(ArgumentError("There are $(-Nfewer) more parameter sets than ensemble members!"))

    θ = [tupify_parameters(ip, θi) for θi in θ]

    # Fill out parameter set ensemble
    Nfewer > 0 && append!(θ, [θ[end] for _ = 1:Nfewer])

    return θ
end

# Expand single parameter set
expand_parameters(ip, θ::Union{NamedTuple,Vector{<:Number}}) = expand_parameters(ip, [θ])
expand_parameters(ip, θ::NamedTuple) = expand_parameters(ip, [θ])

# Convert matrix to vector of vectors
expand_parameters(ip, θ::Matrix) = expand_parameters(ip, [θ[:, i] for i = 1:size(θ, 2)])

#####
##### Forward map evaluation given vector-of-vector (one parameter vector for each ensemble member)
#####

const OneDimensionalEnsembleGrid = RectilinearGrid{<:Any, Flat, Flat, Bounded}
const TwoDimensionalEnsembleGrid = RectilinearGrid{<:Any, Flat, Bounded, Bounded}

n_ensemble(grid::Union{OneDimensionalEnsembleGrid, TwoDimensionalEnsembleGrid}) = grid.Nx
n_observations(grid::OneDimensionalEnsembleGrid) = grid.Ny
n_observations(grid::TwoDimensionalEnsembleGrid) = 1
n_z(grid::Union{OneDimensionalEnsembleGrid, TwoDimensionalEnsembleGrid}) = grid.Nz
n_y(grid::TwoDimensionalEnsembleGrid) = grid.Ny
n_ensemble(ip::InverseProblem) = n_ensemble(ip.simulation.model.grid)

""" Transform and return `ip.observations` appropriate for `ip.output_map`. """
observation_map(ip::InverseProblem) = transform_time_series(ip.output_map, ip.observations)

"""
    forward_run!(ip, parameters)

Initialize `ip.simulation` with `parameters` and run it forward. Output is stored
in `ip.time_series_collector`.
"""
function forward_run!(ip::InverseProblem, parameters)
    observations = ip.observations
    simulation = ip.simulation
    closures = simulation.model.closure

    θ = expand_parameters(ip, parameters)

    simulation.model.closure = new_closure_ensemble(closures, θ)

    initialize_simulation!(simulation, observations, ip.time_series_collector)

    @suppress run!(simulation)

    return nothing
end

"""
    forward_map(ip, parameters)

Run `ip.simulation` forward with `parameters` and return the data,
transformed into an array format expected by `EnsembleKalmanProcesses.jl`.
"""
function forward_map(ip, parameters)

    # Run the simulation forward and populate the time series collector
    # with model data.
    forward_run!(ip, parameters)

    # Transform the model data according to `ip.output_map` into
    # the array format expected by EnsembleKalmanProcesses.jl
    # The result has `size(output) = (output_size, ensemble_capacity)`,
    # where `output_size` is determined by both the `output_map` and the
    # data collection dictated by `ip.observations`.
    output = transform_output(ip.output_map, ip.observations, ip.time_series_collector)

    # (output_size, ensemble_size)
    return output
end

(ip::InverseProblem)(θ) = forward_map(ip, θ)

"""
    transform_time_series(::ConcatenatedOutputMap, time_series::SyntheticObservations)

Concatenates flattened, normalized data for each field in the `time_series`.
"""
function transform_time_series(output_map::ConcatenatedOutputMap, time_series::SyntheticObservations)
    flattened_normalized_data = []

    for field_name in keys(time_series.field_time_serieses)
        field_time_series = time_series.field_time_serieses[field_name]
        field_time_series_data = Array(interior(field_time_series))[:, :, :, output_map.time_indices]

        Nx, Ny, Nz, Nt = size(field_time_series_data)

        field_time_series_data = reshape(field_time_series_data, Nx, Ny * Nz * Nt)

        normalize!(field_time_series_data, time_series.normalization[field_name])

        push!(flattened_normalized_data, field_time_series_data)
    end

    transformed = hcat(flattened_normalized_data...)

    return Matrix(transpose(transformed))
end

"""
    transform_time_series(map, time_serieses::Vector)

Return the `transform_time_series` of each `time_series` in `time_serieses` vector.
"""
transform_time_series(map, time_serieses::Vector) =
    vcat(Tuple(transform_time_series(map, time_series) for time_series in time_serieses)...)

"""
    observation_map_variance_across_time(map::ConcatenatedOutputMap, observation::SyntheticObservations)

Returns an (Nx, Ny*Nz*Nfields, Ny*Nz*Nfields) array storing the covariance of each element of the observation
map measured across time, for each ensemble member, where `Nx` is the ensemble size, `Ny` is the batch size,
`Nz` is the number of grid elements in the vertical, and `Nfields` is the number of fields in `observation`.
"""
function observation_map_variance_across_time(map::ConcatenatedOutputMap, observation::SyntheticObservations)

    N_fields = length(keys(observation.field_time_serieses))

    a = transform_time_series(map, observation)
    a = transpose(a) # (Nx, Ny*Nz*Nt)

    example_field_time_series = values(observation.field_time_serieses)[1]

    Nx, Ny, Nz, Nt = size(interior(example_field_time_series))

    # Assume all fields have the same size
    b = reshape(a, Nx, Ny * Nz, Nt, N_fields) # (Nx, Ny*Nz, Nt, Nfields)

    c = cat((b[:, :, :, i] for i = 1:N_fields)..., dims = 2) # (Nx, Ny*Nz*Nfields, Nt)

    ds = [reshape(var(c[:, :, 1:t], dims = 3), Nx, Ny * Nz, N_fields) for t = 1:Nt]

    e = cat(ds..., dims = 2)

    replace!(e, NaN => 0) # variance for first time step is zero

    return reshape(e, Nx, Ny * Nz * Nt * N_fields)
end

observation_map_variance_across_time(map::ConcatenatedOutputMap, observations::Vector) =
    hcat(Tuple(observation_map_variance_across_time(map, observation) for observation in observations)...)

observation_map_variance_across_time(ip::InverseProblem) = observation_map_variance_across_time(ip.output_map, ip.observations)

function transform_output(map::ConcatenatedOutputMap,
    observations::Union{SyntheticObservations,Vector{<:SyntheticObservations}},
    time_series_collector)

    # transposed_output isa Vector{SyntheticObservations} where SyntheticObservations is Nx by Nz by Nt
    transposed_output = transpose_model_output(time_series_collector, observations)

    return transform_time_series(map, transposed_output)
end

vectorize(observation) = [observation]
vectorize(observations::Vector) = observations

const YZSliceObservations = SyntheticObservations{<:Any,<:YZSliceGrid}

transpose_model_output(time_series_collector, observations::YZSliceObservations) =
    SyntheticObservations(time_series_collector.field_time_serieses,
                          time_series_collector.grid,
                          time_series_collector.times,
                          nothing,
                          nothing,
                          observations.normalization)

"""
    transpose_model_output(time_series_collector, observations)

Transpose a `NamedTuple` of 4D `FieldTimeSeries` model output collected by `time_series_collector`
into a Vector of `SyntheticObservations` for each member of the observation batch.

Return a 1-vector in the case of singleton observations.
"""
function transpose_model_output(time_series_collector, observations)
    observations = vectorize(observations)
    times = time_series_collector.times

    transposed_output = []

    n_ensemble = time_series_collector.grid.Nx
    n_batch = time_series_collector.grid.Ny
    Nz = time_series_collector.grid.Nz
    Hz = time_series_collector.grid.Hz
    Nt = length(times)

    grid = drop_y_dimension(time_series_collector.grid)

    for j = 1:n_batch
        observation = observations[j]
        time_serieses = OrderedDict{Any,Any}()

        for name in keys(observation.field_time_serieses)
            loc = LX, LY, LZ = location(observation.field_time_serieses[name])
            topo = topology(grid)

            field_time_series = time_series_collector.field_time_serieses[name]

            raw_data = parent(field_time_series.data)
            data = OffsetArray(view(raw_data, :, j:j, :, :), 0, 0, -Hz, 0)

            time_series = FieldTimeSeries{LX, LY, LZ, InMemory}(data, grid, nothing, times)
            time_serieses[name] = time_series
        end

        # Convert to NamedTuple
        time_serieses = NamedTuple(name => time_series for (name, time_series) in time_serieses)

        batch_output = SyntheticObservations(time_serieses,
                                             grid,
                                             times,
                                             nothing,
                                             nothing,
                                             observation.normalization)

        push!(transposed_output, batch_output)
    end

    return transposed_output
end

function drop_y_dimension(grid::RectilinearGrid{<:Any, <:Flat, <:Flat, <:Bounded})
    new_size = ColumnEnsembleSize(Nz=grid.Nz, ensemble=(grid.Nx, 1), Hz=grid.Hz)
    z_domain = (grid.zᵃᵃᶠ[1], grid.zᵃᵃᶠ[grid.Nz])
    new_grid = RectilinearGrid(size=new_size, z=z_domain, topology=(Flat, Flat, Bounded))
    return new_grid
end

end # module


### Julia OpenStreetMap Package ###
### MIT License                 ###
### Copyright 2014              ###

### Crop map elements without copying data ###
function cropMap!(nodes::@compat(Union{Dict{Int,LLA},Dict{Int,ENU}}), bounds::Bounds;
                  highways::@compat(Union{@compat(Void),Dict{Int,Highway}}) = nothing,
                  buildings::@compat(Union{@compat(Void),Dict{Int,Building}}) = nothing,
                  features::@compat(Union{@compat(Void),Dict{Int,Feature}}) = nothing,
                  delete_nodes::Bool=true)

    if !isa(highways, @compat(Void))
        crop!(nodes, bounds, highways)
    end
    if !isa(buildings, @compat(Void))
        crop!(nodes, bounds, buildings)
    end
    if !isa(features, @compat(Void))
        crop!(nodes, bounds, features)
    end

    if delete_nodes
        crop!(nodes, bounds)
    end

    return nothing
end

### Crop nodes ###
function crop!(nodes::Dict, bounds::Bounds)
    for (key, node) in nodes
        if !inBounds(node, bounds)
            delete!(nodes, key)
        end
    end

    return nothing
end

### Crop highways ###
function crop!(nodes::Dict, bounds::Bounds, highways::Dict{Int,Highway})
    missing_nodes = Int[]

    for (key, highway) in highways
        valid = falses(length(highway.nodes))
        n = 1
        while n <= length(highway.nodes)
            if haskey(nodes, highway.nodes[n])
                valid[n] = inBounds(nodes[highway.nodes[n]], bounds)
                n += 1
            else
                push!(missing_nodes, highway.nodes[n])
                splice!(highway.nodes, n)
                splice!(valid, n)
            end
        end

        nodes_in_bounds = sum(valid)

        if nodes_in_bounds == 0
            delete!(highways, key)   # Remove highway from list
        elseif nodes_in_bounds < length(valid)
            cropHighway!(nodes, bounds, highway, valid) # Crop highway length
        end
    end

    if length(missing_nodes) > 0
        warn("$(length(missing_nodes)) missing nodes were removed from highways.")
    end

    return missing_nodes
end

### Crop buildings ###
function crop!(nodes::Dict, bounds::Bounds, buildings::Dict{Int,Building})
    for (key, building) in buildings
        valid = falses(length(building.nodes))
        for n = 1:length(building.nodes)
            if haskey(nodes, building.nodes[n])
                valid[n] = inBounds(nodes[building.nodes[n]], bounds)
            end
        end

        nodes_in_bounds = sum(valid)
        if nodes_in_bounds == 0
            delete!(buildings, key)   # Remove building from list
        elseif nodes_in_bounds < length(valid)
            # TODO: Interpolate buildings to bounds?
            delete!(buildings, key)   # Remove building from list
        end
    end

    return nothing
end

### Crop features ###
function crop!(nodes::Dict, bounds::Bounds, features::Dict{Int,Feature})
    for key in keys(features)
        if !haskey(nodes, key) || !inBounds(nodes[key], bounds)
            delete!(features, key)
        end
    end

    return nothing
end

function cropHighway!(nodes::Dict, bounds::Bounds, highway::Highway, valids::BitVector)
    prev_id, prev_valid = highway.nodes[1], valids[1]
    ni = 1
    for valid in valids
        id = highway.nodes[ni]

        if !valid
            deleteat!(highway.nodes, ni)
            ni -= 1
        end
        if valid != prev_valid
            prev_node, node = nodes[prev_id], nodes[id]
            if !(Geodesy.onBounds(prev_node, bounds) ||
                 Geodesy.onBounds(node, bounds))
                new_node = Geodesy.boundaryPoint(prev_node, node, bounds)
                new_id = addNewNode!(nodes, new_node)
                insert!(highway.nodes, ni + !valid, new_id)
                ni += 1
            end
        end

        ni += 1

        prev_id, prev_valid = id, valid
    end

    return nothing
end


# Driver functions for the integrators.

struct MolecularDynamicsParameters
    "Centroid friction (1 / ps)."
    gamma0::Float64
    "Time step (ps)."
    dt::Float64
    "Total duration (ps)."
    duration::Float64
end

"""
    num_steps(mdp::MolecularDynamicsParameters)

Number of molecular dynamics steps for `mdp`.
"""
num_steps(mdp::MolecularDynamicsParameters) = ceil(Int, mdp.duration / mdp.dt)

function run_constrained_equilibration(
        model_T::Type{<:WaterModel}, int_T::Type{<:Integrator}, N::Int,
        T::Float64, R::Float64, P::Int, mdp_equil::MolecularDynamicsParameters;
        qs_init::Maybe{Vector{Float64}}=nothing, output_skip::Int=100)
    is_constrained(int_T) || error("Integrator is not constrained.")

    # Number of atoms in each water molecule.
    A = 3
    # Number of spatial dimensions.
    D = 3

    if !isnothing(qs_init)
        s = State(P, D, A, N; qs_init)
    else
        s = State(P, D, A, N; R)
    end

    model = model_T(N)
    fc = ForceContainer(P, D, A, N)
    int_equil = int_T(model, R, mdp_equil.dt, beta(T), mdp_equil.gamma0, fc, s)

    for i in 1:num_steps(mdp_equil)
        step!(s, int_equil)

        i % output_skip == 0 || continue

        time = i * mdp_equil.dt
        E = energy(int_equil, s)
        E_K = pop!(E, "kinetic")
        E_Vspring = pop!(E, "spring")
        E_Vmodel = pop!(E, "model")
        isempty(E) || error("Unexpected energy contribution.")
        E_total = E_K + E_Vspring + E_Vmodel
        T = get_temperature(int_equil)

        println("$(time) $(E_K) $(E_Vspring) $(E_Vmodel) $(E_total) $(T)")
    end

    s
end

function run_constrained_production(
        model_T::Type{<:WaterModel}, int_T::Type{<:Integrator}, N::Int,
        T::Float64, R::Float64, P::Int, mdp_equil::MolecularDynamicsParameters,
        mdp_prod::MolecularDynamicsParameters;
        qs_init::Maybe{Vector{Float64}}=nothing, aux_output::IO=stderr)
    is_constrained(int_T) || error("Integrator is not constrained.")

    # Number of atoms in each water molecule.
    A = 3
    # Number of spatial dimensions.
    D = 3

    # Number of molecular dynamics steps for equilibration.
    N_equil = num_steps(mdp_equil)
    # Number of molecular dynamics steps for production.
    N_prod = num_steps(mdp_prod)

    if !isnothing(qs_init)
        s = State(P, D, A, N; qs_init)
    else
        s = State(P, D, A, N; R)
    end

    model = model_T(N)
    fc = ForceContainer(P, D, A, N)
    int_equil = int_T(model, R, mdp_equil.dt, beta(T), mdp_equil.gamma0, fc, s)
    int_prod = int_T(model, R, mdp_prod.dt, beta(T), mdp_prod.gamma0, fc, s)

    meter_equil = Progress(N_equil, output=aux_output)

    for _ in ProgressWrapper(1:N_equil, meter_equil)
        step!(s, int_equil)
    end

    constraints = Array{Float64}(undef, N_prod)
    E_Ks = Array{Float64}(undef, N_prod)
    E_Vsprings = Array{Float64}(undef, N_prod)
    E_Vmodels = Array{Float64}(undef, N_prod)
    E_totals = Array{Float64}(undef, N_prod)
    temperatures = Array{Float64}(undef, N_prod)

    forces1 = Array{Float64}(undef, N_prod)
    forces2 = Array{Float64}(undef, N_prod)

    estim1 = estimator1(R, beta(T), int_prod, s)
    estim2 = estimator2(R, beta(T), int_prod, s)

    meter = Progress(N_prod, output=aux_output)

    for i in ProgressWrapper(1:N_prod, meter)
        step!(s, int_prod)

        constraints[i] = com_r(CN1, CN2, CJ, s)
        E = energy(int_prod, s)
        E_Ks[i] = pop!(E, "kinetic")
        E_Vsprings[i] = pop!(E, "spring")
        E_Vmodels[i] = pop!(E, "model")
        isempty(E) || error("Unexpected energy contribution.")
        E_totals[i] = E_Ks[i] + E_Vsprings[i] + E_Vmodels[i]
        temperatures[i] = get_temperature(int_prod)

        forces1[i] = estim1()
        forces2[i] = estim2()
    end

    data_table = []

    for (label, data) in [("xi (nm)", constraints),
                          ("K (kJ / mol)", E_Ks),
                          ("Vspring (kJ / mol)", E_Vsprings),
                          ("Vmodel (kJ / mol)", E_Vmodels),
                          ("E (kJ / mol)", E_totals),
                          ("T (K)", temperatures),
                          ("est1 (1 / nm)", forces1),
                          ("est2 (1 / nm)", forces2)]
        data_mean, data_stderr, data_actime = aggregate(mdp_prod.dt, data)
        push!(data_table, [label, data_mean, data_stderr, data_actime])
    end

    pretty_table(aux_output, permutedims(hcat(data_table...)),
                 ["", "Value", "Error", "Autocorrelation time (ps)"])

    s, aggregate(mdp_prod.dt, forces1), aggregate(mdp_prod.dt, forces2)
end

function run_restrained_equilibration(
        model_T::Type{<:WaterModel}, int_T::Type{<:Integrator}, N::Int,
        T::Float64, R::Float64, P::Int, mdp_equil::MolecularDynamicsParameters,
        bias_force_constant::Float64; qs_init::Maybe{Vector{Float64}}=nothing,
        output_skip::Int=100)
    is_constrained(int_T) && error("Integrator is constrained.")

    # Number of atoms in each water molecule.
    A = 3
    # Number of spatial dimensions.
    D = 3

    if !isnothing(qs_init)
        s = State(P, D, A, N; qs_init)
    else
        s = State(P, D, A, N; R)
    end

    bias = HarmonicRestraint(bias_force_constant, R, CN1, CN2, CJ)
    model = model_T(N)
    fc = ForceContainer(P, D, A, N)
    int_equil = int_T(model, mdp_equil.dt, beta(T), mdp_equil.gamma0, fc, s;
                      extra_forces=Dict("bias" => [bias]))

    for i in 1:num_steps(mdp_equil)
        step!(s, int_equil)

        i % output_skip == 0 || continue

        time = i * mdp_equil.dt
        xi = com_r(CN1, CN2, CJ, s)
        E = energy(int_equil, s)
        E_K = pop!(E, "kinetic")
        E_Vspring = pop!(E, "spring")
        E_Vmodel = pop!(E, "model")
        E_Vbias = pop!(E, "bias")
        isempty(E) || error("Unexpected energy contribution.")
        E_total = E_K + E_Vspring + E_Vmodel + E_Vbias
        T = get_temperature(int_equil)

        print("$(time) $(xi) $(E_K) $(E_Vspring) $(E_Vmodel) $(E_Vbias) ")
        println("$(E_total) $(T)")
    end

    s
end

function run_restrained_production(
        model_T::Type{<:WaterModel}, int_T::Type{<:Integrator}, N::Int,
        T::Float64, R::Float64, P::Int, mdp_equil::MolecularDynamicsParameters,
        mdp_prod::MolecularDynamicsParameters, bias_force_constant::Float64;
        qs_init::Maybe{Vector{Float64}}=nothing, output_skip::Int=100)
    is_constrained(int_T) && error("Integrator is constrained.")

    # Number of atoms in each water molecule.
    A = 3
    # Number of spatial dimensions.
    D = 3

    # Number of molecular dynamics steps for equilibration.
    N_equil = num_steps(mdp_equil)
    # Number of molecular dynamics steps for production.
    N_prod = num_steps(mdp_prod)

    if !isnothing(qs_init)
        s = State(P, D, A, N; qs_init)
    else
        s = State(P, D, A, N; R)
    end

    bias = HarmonicRestraint(bias_force_constant, R, CN1, CN2, CJ)
    model = model_T(N)
    fc = ForceContainer(P, D, A, N)
    int_equil = int_T(model, mdp_equil.dt, beta(T), mdp_equil.gamma0, fc, s;
                      extra_forces=Dict("bias" => [bias]))
    int_prod = int_T(model, mdp_prod.dt, beta(T), mdp_prod.gamma0, fc, s;
                     extra_forces=Dict("bias" => [bias]))

    for _ in 1:N_equil
        step!(s, int_equil)
    end

    for i in 1:N_prod
        step!(s, int_prod)

        i % output_skip == 0 || continue

        println(com_r(CN1, CN2, CJ, s))
    end

    s
end


using PdfFraud
using GenerativeModels
using Flux
using ValueHistories
using ArgParse
using BSON
using GMExtensions
using CuArrays
using DelimitedFiles, DataFrames, CSV
using PyPlot

# this is for running the script from REPL - default values
if length(ARGS) == 0
	push!(ARGS, "vae_runs/2")
end
# parse arguments
s = ArgParseSettings()
@add_arg_table s begin
    "modelpath"
    	default = ""
    	arg_type = String
    	help = "a master dir of where the models are saved"
    "--imodels"
    	default = [0]
    	help = "only compute models given by indices"
    	arg_type = Int
    	nargs = '+'
end
parsed_args = parse_args(ARGS, s)

# extract settings from the parsed arguments
modelpath = parsed_args["modelpath"]
modelpath = (length(modelpath) > 0 && modelpath[1] == "/") ? modelpath :
	joinpath(PdfFraud.experiment_path, modelpath, "models")
reprpath = joinpath(dirname(modelpath), "representations")
mkpath(reprpath)
imodels = parsed_args["imodels"]

# get labeled data
data, labels, sha256, page_nums = PdfFraud.load_pdf_data()
data = data |> gpu;

# save the page info
lf = joinpath(reprpath, "index.csv")
index = DataFrame()
index[!,:sha256] = vcat(map(x->repeat([x[1]], x[2]), zip(sha256, page_nums))...)
index[!,:page] = vcat(map(x->1:x, page_nums)...)
CSV.write(lf, index)

# load model(s)
model_files = joinpath.(modelpath, readdir(modelpath))
@info "Found $(length(model_files)) saved models."
if imodels[1] != 0
	model_files = model_files[imodels]
end
@info "Continuing with $(length(model_files)) models."

# now get the representations

# now lets check the encodings
zfs = joinpath.(reprpath, filter(x->x!="index.csv", readdir(reprpath)))
Zs = map(zf -> readdlm(zf, ',') ,zfs)

linds = map(x->x!=nothing,labels)
Y = labels[linds];

figure(figsize=(10,25))
for iz in 41:50#min(10,length(Zs))
	subplot(5,2,iz%10+1)
	Z = Zs[iz][linds,:];
	for y in unique(Y)
		inds = Y .== y
		scatter(Z[inds,1], Z[inds,2])
	end
	title(basename(zfs[iz]), fontsize=8)
end
tight_layout()


# This file is a part of Julia. License is MIT: https://julialang.org/license

module FieldValues

using ...VersionWeights

export FieldValue, Field, validmax, secondmax

# FieldValue is a hierarchical numeric type with 5 levels.
# When summing two FieldValues, the levels are summed independently.
# When comparing them, lower levels take precedence.
# The levels are used as such:
#  l0 : for hard constraints (dependencies and requirements)
#  l1 : for favoring higher versions of the explicitly required
#       packages
#  l2 : for favoring higher versions of all other packages
#  l3 : for favoring uninstallation of non-needed packages
#
struct FieldValue
    l0::Int64
    l1::VersionWeight
    l2::VersionWeight
    l3::Int64
    FieldValue(l0::Integer = 0,
               l1::VersionWeight = zero(VersionWeight),
               l2::VersionWeight = zero(VersionWeight),
               l3::Integer = 0) = new(l0, l1, l2, l3)
end

# This isn't nice, but it's for debugging only anyway
function Base.show(io::IO, a::FieldValue)
    print(io, a.l0)
    a == FieldValue(a.l0) && return
    print(io, ".", a.l1)
    a == FieldValue(a.l0, a.l1) && return
    print(io, ".", a.l2)
    a == FieldValue(a.l0, a.l1, a.l2) && return
    print(io, ".", a.l3)
    return
end

const Field = Vector{FieldValue}

Base.zero(::Type{FieldValue}) = FieldValue()

Base.typemin(::Type{FieldValue}) = (x=typemin(Int64); y=typemin(VersionWeight); FieldValue(x, y, y, x))

Base.:-(a::FieldValue, b::FieldValue) = FieldValue(a.l0-b.l0, a.l1-b.l1, a.l2-b.l2, a.l3-b.l3)
Base.:+(a::FieldValue, b::FieldValue) = FieldValue(a.l0+b.l0, a.l1+b.l1, a.l2+b.l2, a.l3+b.l3)

function Base.isless(a::FieldValue, b::FieldValue)
    a.l0 < b.l0 && return true
    a.l0 > b.l0 && return false
    c = cmp(a.l1, b.l1)
    c < 0 && return true
    c > 0 && return false
    c = cmp(a.l2, b.l2)
    c < 0 && return true
    c > 0 && return false
    a.l3 < b.l3 && return true
    return false
end

Base.:(==)(a::FieldValue, b::FieldValue) =
    a.l0 == b.l0 && a.l1 == b.l1 && a.l2 == b.l2 && a.l3 == b.l3

Base.abs(a::FieldValue) = FieldValue(abs(a.l0), abs(a.l1), abs(a.l2), abs(a.l3))

Base.copy(a::FieldValue) = FieldValue(a.l0, copy(a.l1), copy(a.l2), a.l3)

# if the maximum field has l0 < 0, it means that
# some hard constraint is being violated
validmax(a::FieldValue) = a.l0 >= 0

# like usual indmax, but favors the highest indices
# in case of a tie
function Base.indmax(f::Field)
    m = typemin(FieldValue)
    mi = 0
    for j = length(f):-1:1
        if f[j] > m
            m = f[j]
            mi = j
        end
    end
    @assert mi != 0
    return mi
end

# secondmax returns the (normalized) value of the second maximum in a
# field. It's used to determine the most polarized field.
function secondmax(f::Field, msk::BitVector = trues(length(f)))
    m = typemin(FieldValue)
    m2 = typemin(FieldValue)
    for i = 1:length(f)
        msk[i] || continue
        a = f[i]
        if a > m
            m2 = m
            m = a
        elseif a > m2
            m2 = a
        end
    end
    return m2 - m
end

# Support broadcasting like a scalar by default
Base.Broadcast.broadcastable(a::FieldValue) = Ref(a)

end


export clip_segment, put_together

function bandpass_filter(gui,inds,wn1,wn2)

    responsetype = Bandpass(wn1,wn2; fs=30000)
    designmethod=Butterworth(4)
    df1=digitalfilter(responsetype,designmethod)
    myfilt=DF2TFilter(df1)
    test_data=gui.y_data[inds,1];
    filt!(test_data,myfilt,test_data);
    test_data
end

function thres_ts(data,thres)

    test_data=data./maximum(data)
    spike_times=falses(length(test_data))
    spike_inds=zeros(Int64,0)
    for i=1:length(test_data)

        if abs(test_data[i])>thres
            spike_times[i]=true
            i=i+30
        end
    end
    spike_times
end

#Convolve spike timestamps with fake spike waveform of 1 ms duration
function spike_audio(spike_times)
    conv(sin.(0:pi/15:2*pi),spike_times)
end

function sort_spikes(gui,t)

    gui.s.index=0
    SpikeSorting.onlinesort!(gui.s,gui.y_data[(t-30000)+1:t,1],gui.buf,gui.spike_nums)

    gui.spikes_ts=[]
    for i=1:gui.spike_nums[1]
        push!(gui.spikes_ts,t-30000+1+gui.buf[i].inds[1])
    end
    gui.spike_nums[1]=0

    nothing
end

function get_sorted_ts(gui,t)

    gui.spikes_ts=gui.pre_sorted_ts[(gui.pre_sorted_ts.>(t-30000+1))&(gui.pre_sorted_ts.<t)]

end
function event_triggered(gui,t)

    my_events=gui.event_ts[2][((gui.event_ts[2].>(t-30000)).&(gui.event_ts[2].<t))]

    myrange=-3000:300:3000

    for k=1:length(gui.myhist)
        gui.myhist[k]=0
    end

    for i=1:length(my_events)
        for j=1:length(gui.spikes_ts)
            for k=1:length(myrange)-1
               ttt = gui.spikes_ts[j]-my_events[i]
                if (ttt > myrange[k]).&(ttt<myrange[k+1])
                   gui.myhist[k] += 1
                end
            end
        end
    end

    nothing
end

#Get the video times and analog indexes that are closest
#to t1 and t2 while also being equal to a value the slider adopts
#and being a whole second value of the mp4 video
function get_clip_times(gui,t1,t2)

    slider_step = round(Int64,gui.max_time / ((gui.max_time-100) / 1000 * 10))
    total_slider_values = 30000:slider_step:gui.max_time

    v_frame_rate = 25

    #Video Frame that is equal to a second in save video nearest to desired time
    full_frame = round(Int,gui.video_ts[t1] / v_frame_rate)

    #Corresponding index of analog channel that corresponds to video
    first_ind=findfirst(gui.video_ts./v_frame_rate .== full_frame)

    #First value of the slider that corresponds to that frame
    first_slider_value = findfirst(total_slider_values.>first_ind)

    #Video time in seconds that is closest to t2
    end_frame = round(Int,gui.video_ts[t2] / v_frame_rate)

    end_ind = findfirst(gui.video_ts./v_frame_rate .== end_frame)

    last_slider_value = findfirst(total_slider_values.>end_ind)

    clip_times(t1,t2,full_frame,end_frame,first_ind,end_ind,first_slider_value,last_slider_value)
end

function clip_segment(gui,t1,t2)

    tt = get_clip_times(gui,t1,t2)

    num_frames = tt.ts2 - tt.ts1

    #Create video clip of desired length
    run(`ffmpeg -ss $(tt.ts1) -i $(gui.vid_path) -c copy -t $(num_frames) short.mp4`)

    #Cut out corresponding audio for that second

    my_voltage = gui.y_data[tt.ind1:(tt.ind2),1]

    gui.s.index=0
    #Sort so that only spikes are left to remove crumby audio
    SpikeSorting.onlinesort!(gui.s,my_voltage,gui.buf,gui.spike_nums)

    my_audio=zeros(Float32,length(my_voltage))

    for i=1:gui.spike_nums[1]
        my_audio[gui.buf[i].inds]=my_voltage[gui.buf[1].inds]
    end

    gui.spike_nums[1]=0

    #We want the duration to be the same as the length of the video.
    #so num_frames * 100
    WAV.wavwrite(my_audio,"test.wav",Fs=div(t2-t1,num_frames))

    output_name = string(t1,"_with_audio.mp4")

    #Combine audio and video
    run(`ffmpeg -i short.mp4 -i test.wav -c:v copy -c:a aac -strict experimental $(output_name)`)

    #The above video has audio and video synchronized, but the length of the video is still a
    #Function of the length of time (t2-t1), as well as how many video frames actually represent
    #one second. to be played at 1/10 of normal speed, we want a 25 fps video to represent
    # 250 camera frames
    #If the camera is higher or lower FPS, it should be adjusted accordingly

    total_time = (t2-t1)/30000

    #How do we determine frame rate of camera?
    #Probably should calculate it roughly when we load in video data
    total_camera_time = (num_frames * 25)

    #run(`ffmpeg -i $(output_name) -filter_complex "[0:v]setpts=0.5*PTS[v];[0:a]atempo=2.0[a]" -map "[v]" -map "[a]" out2.mp4`)

    nothing
end

function record_frames(gui,t1,t2)

    tt = get_clip_times(gui,t1,t2)

    io=GLVisualize.create_video_stream("analog.mp4",gui.win)

    total_frames = tt.s2 - tt.s1 - 1
    Reactive.set_value!(gui.p_slider.parents[1],tt.s1)
    Reactive.activate!(gui.p_slider.parents[1])

    last_time = gui.t
    GLVisualize.add_frame!(io)

    i=0
    while (i<total_frames)

        yield()

        if last_time != gui.t
            GLVisualize.add_frame!(io)
            i = i + 1
            last_time = gui.t
        end
    end

    close(io.io)

    sleep(5.0)
    #Because the video is saved at 35 fps, but the video is actually being
    #played at 30 fps, we need to change the video to be 30 fps
    run(`ffmpeg -y -i analog.mp4 -vf "setpts=0.833*PTS" -r 30 analog2.mp4`)

end

function put_together(gui,t1,t2)

    mkdir(string(t1))
    cd(string(t1))

    clip_segment(gui,t1,t2)
    resize!(gui.win,640,480)
    sleep(2.0)
    record_frames(gui,t1,t2)

    out_name = string(t1,"_with_audio.mp4")
    final_name = string(t1,".mp4")

    run(`ffmpeg -i analog2.mp4 -i $(out_name) -filter_complex hstack $(final_name)`)

    run(`rm short.mp4`)
    run(`rm $(out_name)`)
    run(`rm analog2.mp4`)
    run(`rm analog.mp4`)
    run(`rm test.wav`)
    cd("..")
    nothing
end


isnormal(c::SGRColor) = isempty(c.class)

is216color(c::SGRColor) = is216color(c.hex)

function is216color(hex::AbstractString)
    hex == "000" || hex == "fff" || occursin(r"(?:00|5f|87|af|d7|ff){3}$", hex)
end

function codes(c::SGRColor)
    m = match(r"^([345]8)_([25])$", c.class)
    m === nothing && return (parse(Int, c.class),)
    code, sub = m.captures
    codei = parse(Int, code)
    if sub == "5"
        if is216color(c)
            h = parse(UInt32, c.hex, base=16)
            h === 0x00000fff && return (codei, 5, 231)
            r = (h >> 0x10) % UInt8 ÷ 0x30
            g = (h >> 0x08) % UInt8 ÷ 0x30
            b = (h >> 0x00) % UInt8 ÷ 0x30
            return (codei, 5, (r * 0x24 + g * 0x6 + b) + 16)
        else
            h = parse(UInt8, c.hex[1:2], base=16)
            g = (h - 0x8) ÷ 0xa
            return (codei, 5, g + 232)
        end
    else
        if length(c.hex) == 3
            h = parse(UInt16, c.hex, base=16)
            r = (h >> 0x8)
            g = (h >> 0x4) & 0xf
            b = h & 0xf
            return (codei, 2, r * 17, g * 17, b * 17)
        else
            h = parse(UInt32, c.hex, base=16)
            r = (h >> 0x10)
            g = (h >> 0x8) & 0xff
            b = h & 0xff
            return (codei, 2, Int(r), Int(g), Int(b))
        end
    end
end

function short_hex(r::UInt8, g::UInt8, b::UInt8)
    rgb6 = UInt32(r) << 0x10 + UInt32(g) << 0x8 + b
    rgb6 === (rgb6 & 0x0f0f0f) * 0x11 || return string(rgb6, pad=6, base=16)
    string(UInt16(r >> 0x4) << 0x8 + UInt16(g >> 0x4) << 0x4 + b >> 0x4, pad=3, base=16)
end

function set_16colors!(ctx::SGRContext, d::AbstractString)
    if d[1] === '3' || d[1] === '9'
        ctx.fg = SGRColor(d)
    else
        ctx.bg = SGRColor(d)
    end
end

const SCALE_216 = UInt8[0x00, 0x5f, 0x87, 0xaf, 0xd7, 0xff]

function set_256colors!(ctx::SGRContext, d::AbstractString, color::AbstractString)
    fore = d[1] === '3'
    hex = ""
    colorid = isempty(color) ? 0x0 : parse(UInt8, color)
    if colorid < 0x8
        class = d[1] * string(colorid)
    elseif colorid < 0x10
        class = (fore ? "9" : "10") * string(colorid - 0x8)
    else
        if colorid < 0xe8
            c = colorid - 0x10
            r = SCALE_216[c ÷ 0x24 + 1]
            g = SCALE_216[c ÷ 0x6 % 0x6 + 1]
            b = SCALE_216[c % 0x6 + 1]
            hex = short_hex(r, g, b)
        else
            g = (colorid - 0xe8) * 0xa + 0x8
            hex = short_hex(g, g, g)
        end
        class = d * "_5"
    end

    if fore
        ctx.fg = SGRColor(class, hex)
    else
        ctx.bg = SGRColor(class, hex)
    end
end

function set_24bitcolors!(ctx::SGRContext, d::AbstractString,
                          r::AbstractString, g::AbstractString, b::AbstractString)
    r8 = isempty(r) ? 0x0 : parse(UInt8, r)
    g8 = isempty(g) ? 0x0 : parse(UInt8, g)
    b8 = isempty(b) ? 0x0 : parse(UInt8, b)
    hex = short_hex(r8, g8, b8)
    if is216color(hex) || (r8 === g8 === b8 && (r8 - 8) % 10 == 0)
        class = d * "_5"
    else
        class = d * "_2"
    end
    if d == "38"
        ctx.fg = SGRColor(class, hex)
    else
        ctx.bg = SGRColor(class, hex)
    end
end


module ShallowWaterModels

export ShallowWaterModel

using KernelAbstractions: @index, @kernel, Event, MultiEvent
using KernelAbstractions.Extras.LoopInfo: @unroll

using Oceananigans.Utils: launch!

import Oceananigans: fields, prognostic_fields
import Oceananigans.LagrangianParticleTracking: update_particle_properties!

#####
##### ShallowWaterModel definition
#####

include("shallow_water_model.jl")
include("set_shallow_water_model.jl")
include("show_shallow_water_model.jl")

#####
##### Time-stepping ShallowWaterModels
#####

"""
    fields(model::ShallowWaterModel)

Returns a flattened `NamedTuple` of the fields in `model.solution` and `model.tracers`.
"""
fields(model::ShallowWaterModel) = merge(model.solution, model.tracers)
prognostic_fields(model::ShallowWaterModel) = fields(model)

include("solution_and_tracer_tendencies.jl")
include("calculate_shallow_water_tendencies.jl")
include("update_shallow_water_state.jl")
include("shallow_water_advection_operators.jl")
include("shallow_water_cell_advection_timescale.jl")

# No support for particle advection yet.
update_particle_properties!(model::ShallowWaterModel, Δt) = nothing

end # module


module InfraFlow

include("delim.jl")

using JuMP, GLPK, Test, YAML, Logging
const MOI = JuMP.MathOptInterface

export get_data, formulate_gmcnf!, run
"""
Author: Will Usher
Date: 3rd January 2018

A linear programming new_capacity expansion generalized multi-commodity network
flow implemented using JuMP/Julia based on generalized multi-commodity network
flow as found in:

Ishimatsu, Takuto. “Generalized Multi-Commodity Network Flows : Case Studies
in Space Logistics and Complex Infrastructure Systems.”
Masssachusett Institute of Technology, 2013.
"""

"""
    get_data(file_path)

Load in model data from a YAML file
"""
function get_data(file_path::AbstractString)

    data = YAML.load(open(file_path))

    model_data = Dict{String,Any}()

    model_data["years"] = data["years"]
    years = Dict{Int,Int}([data["years"][i]=>i for i in eachindex(data["years"])])
    model_data["commodities"] = data["commodities"]
    commodities = Dict{String,Int}([data["commodities"][i]=>i for i in eachindex(data["commodities"])])

    model_data["discount_rate"] = data["discount_rate"]

    nodes = Dict{String,Int}()
    node_names = []
    self_loops = []
    for index in eachindex(data["nodes"])
        node = data["nodes"][index]
        nodes[node["name"]] = index
        push!(node_names, node["name"])
        if haskey(node, "requirements")
            push!(self_loops, index)
        end

    end
    model_data["nodes"] = node_names

    num_nodes = length(node_names)
    num_comm = length(commodities)
    num_years = length(years)

    cap2act = zeros(Float64, (num_nodes, num_nodes, num_comm))
    outflow_cost = zeros(Float64, (num_nodes, num_nodes, num_comm))
    inflow_cost = zeros(Float64,(num_nodes, num_nodes, num_comm))
    residual_capacity = zeros(Float64, (num_nodes, num_nodes, num_comm, num_years))
    maximum_new_capacity = fill(99999999999999999999.0, (num_nodes, num_nodes, num_comm, num_years))
    transformation = zeros(Float64, (num_nodes, num_nodes, num_comm, num_comm))

    requirements_outflow = zeros(Float64, (num_nodes, num_nodes, num_comm, num_comm))
    requirements_inflow = zeros(Float64, (num_nodes, num_nodes, num_comm, num_comm))

    edges = []
    for edge in data["edges"]
        source = nodes[edge["source"]]
        sink = nodes[edge["sink"]]
        push!(edges, source, sink)

        for flow in edge["flow"]
            comm_idx = commodities[flow["name"]]
            requirements_inflow[source, sink, comm_idx, comm_idx] = 1
            requirements_outflow[source, sink, comm_idx, comm_idx] = 1
        end

        if haskey(edge, "cap2act")
            for (comm, value) in edge["cap2act"]
                comm_idx = commodities[comm]
                cap2act[source, sink, comm_idx] = value
            end
        end

        if haskey(edge, "operational_cost")
            for (comm, value) in edge["operational_cost"]
                comm_idx = commodities[comm]
                outflow_cost[source, sink, comm_idx] = value
            end
        end

        if haskey(edge, "losses")
            for (comm, value) in edge["losses"]
                comm_idx = commodities[comm]
                transformation[source, sink, comm_idx, comm_idx] = (1.0 - value)
            end
        end

    end

    for loop in self_loops
        push!(edges, loop, loop)
    end

    edges = transpose(reshape(edges, 2, :))
    model_data["edges"] = edges

    demand = zeros((num_nodes, num_comm, num_years))
    capacity_cost = zeros((num_nodes, num_nodes, num_comm))

    for index in eachindex(data["nodes"])
        node = data["nodes"][index]
        source = node["name"]
        node_idx = nodes[source]
        if haskey(node, "demand")
            for (commodity, yearly_data) in node["demand"]
                for (year, value) in yearly_data
                    node_idx = nodes[source]
                    comm_index = commodities[commodity]
                    year_idx = years[year]
                    demand[node_idx, comm_index, year_idx] = value
                end
            end
        end

        if haskey(node, "investment_cost")
            output_commodity = node["output"]
            output_comm_idx = commodities[output_commodity]
            capacity_cost[node_idx, node_idx, output_comm_idx] = node["investment_cost"]
        end

        if haskey(node, "cap2act")
            output_commodity = node["output"]
            output_comm_idx = commodities[output_commodity]
            cap2act[node_idx, node_idx, output_comm_idx] = node["cap2act"]
        end

        if haskey(node, "requirements")
            for requirement in node["requirements"]
                comm = node["output"]
                comm_idx = commodities[comm]
                r_comm = requirement["name"]
                r_comm_idx = commodities[r_comm]
                cap2act[node_idx, node_idx, r_comm_idx] = 1
                transformation[node_idx, node_idx, comm_idx, r_comm_idx] = 1
                requirements_outflow[node_idx, node_idx, r_comm_idx, comm_idx] = requirement["value"]
                requirements_inflow[node_idx, node_idx, comm_idx, comm_idx] = 1
                requirements_outflow[node_idx, node_idx, comm_idx, comm_idx] = 1
                if haskey(node, "residual_capacity")
                    for (year, value) in node["residual_capacity"]
                        year_idx = years[year]
                        residual_capacity[node_idx, node_idx, comm_idx, year_idx] = value
                    end
                end
                if haskey(node, "maximum_new_capacity")
                    for (year, value) in node["maximum_new_capacity"]
                        year_idx = years[year]
                        maximum_new_capacity[node_idx, node_idx, comm_idx, year_idx] = value
                    end
                end
            end

        end

    end

    model_data["demand"] = demand
    model_data["capacity_cost"] = capacity_cost
    model_data["cap2act"] = cap2act
    model_data["outflow_cost"] = outflow_cost
    model_data["inflow_cost"] = inflow_cost
    model_data["residual_capacity"] = residual_capacity
    model_data["maximum_new_capacity"] = maximum_new_capacity
    model_data["transformation"] = transformation
    model_data["requirements_inflow"] = requirements_inflow
    model_data["requirements_outflow"] = requirements_outflow

    source_nodes = view(edges, :, 1)
    sink_nodes = view(edges, :, 2)

    outflow_edges = make_edge_dict(source_nodes, sink_nodes)
    inflow_edges = make_edge_dict(sink_nodes, source_nodes)

    model_data["outflow_edges"] = outflow_edges
    model_data["inflow_edges"] = inflow_edges

    return model_data
end

"""
    make_edge_dict(edge_nodes, other_nodes)

"""
function make_edge_dict(edge_nodes::AbstractArray, other_nodes::AbstractArray)
    edges = Dict{Int8,Array{Int8}}()
    for node in Set(edge_nodes)
        for i in eachindex(edge_nodes)
            if edge_nodes[i] == node
                if haskey(edges, node)
                    push!(edges[node], other_nodes[i])
                else
                    push!(edges, node=>[other_nodes[i]])
                end
            end
        end
    end
    return edges
end

"""
    formulate_gmcnf()

``\\sum_{ijky} c_{ijky}x_{ijky}``

"""
function formulate_gmcnf!(model_data::Dict{String,Any}, model; verbose = true)

    nodes = model_data["nodes"]
    edges = model_data["edges"]
    commodities = model_data["commodities"]
    discount_rate = model_data["discount_rate"]
    years = model_data["years"]

    outflow_edges = model_data["outflow_edges"]
    inflow_edges = model_data["inflow_edges"]

    num_nodes = length(nodes)
    num_edges = length(edges)
    num_comm = length(commodities)
    num_years = length(years)

    # demand at node by commodity
    demand = model_data["demand"]

    capacity_cost = model_data["capacity_cost"]
    cap2act = model_data["cap2act"]

    inflow_cost = model_data["inflow_cost"]
    outflow_cost = model_data["outflow_cost"]

    # describe the commodity requirements for an in- or out-flow
    requirements_outflow = model_data["requirements_outflow"]
    requirements_inflow = model_data["requirements_inflow"]

    # describe the flow gain/loss or transformation between commodities
    transformation = model_data["transformation"]

    # upper bound on operational decision variables (new_capacity)
    residual_capacity = model_data["residual_capacity"]

    # upper bound on new capacity
    maximum_new_capacity = model_data["maximum_new_capacity"]

    new_capacity = @variable(model,
                         new_capacity[i=1:num_nodes, j=1:num_nodes, k=1:num_comm, y=1:num_years],
                         lower_bound = 0)

    outflow = @variable(model,
                        outflow[i=1:num_nodes, j=1:num_nodes, k=1:num_comm, y=1:num_years],
                        lower_bound = 0)
    inflow = @variable(model,
                       inflow[i=1:num_nodes, j=1:num_nodes, k=1:num_comm, y=1:num_years],
                       lower_bound = 0)

    @constraint(
        model,
        max_new_capacity[i=1:num_nodes, j=1:num_nodes, k=1:num_comm, y=1:num_years],
        new_capacity[i, j, k, y] <= maximum_new_capacity[i, j, k, y]
    )

    @variable(
        model,
        total_annual_capacity[i in keys(outflow_edges), j in outflow_edges[i], k=1:num_comm, y=1:num_years],
        lower_bound = 0
    )

    @constraint(
        model,
        accumulate_capacity[i in keys(outflow_edges), j in outflow_edges[i], k=1:num_comm, y=1:num_years],
        total_annual_capacity[i, j, k, y] == residual_capacity[i, j, k, y] + sum(new_capacity[i, j, k, z] for z in 1:y)
    )

    @constraint(
        model,
        capacity_exp_outflow[i in keys(outflow_edges), j in outflow_edges[i], k=1:num_comm, y=1:num_years],
        outflow[i, j, k, y] <= total_annual_capacity[i, j, k, y] * cap2act[i, j, k])

    @constraint(
        model,
        capacity_exp_inflow[i in keys(outflow_edges), j in outflow_edges[i], k=1:num_comm, y=1:num_years],
        inflow[i, j, k, y] <= total_annual_capacity[i, j, k, y] * cap2act[i, j, k])

    function discount_factor(year::Int)
        return ((1 + discount_rate) ^ (years[year] - years[1]))
    end

    discounted_capital_cost = @expression(
        model,
        [i in keys(outflow_edges), j in outflow_edges[i], k=1:num_comm, y=1:num_years],
        new_capacity[i, j, k, y] * capacity_cost[i, j, k] / discount_factor(y)
    )

    discounted_operational_cost = @expression(
        model,
        [i in keys(outflow_edges), j in outflow_edges[i], k=1:num_comm, y=1:num_years],
        outflow_cost[i, j, k] * outflow[i, j, k, y]
        + inflow_cost[i, j, k] * inflow[i, j, k, y]
        / discount_factor(y)
    )

    @objective(
        model,
        Min,
        sum(discounted_operational_cost[i, j, k, y]
            + discounted_capital_cost[i, j, k, y]
            for i in keys(outflow_edges), j in outflow_edges[i], k in 1:num_comm, y=1:num_years)
        )

    requirements_outflow_const = @expression(
        model,
        requirements_outflow_const[i=1:num_nodes, k=1:num_comm, y=1:num_years],
        if haskey(outflow_edges, i)
            sum(
                sum(requirements_outflow[i, j, k, l] for l in 1:num_comm) * outflow[i, j, k, y]
                for j in outflow_edges[i]
            )

        else
            @info "No outflow edge found for ($(nodes[i]), $(commodities[k]), $(years[y]))"
            0
        end
    )

    requirements_inflow_const = @expression(
        model,
        requirements_inflow_const[i=1:num_nodes, k=1:num_comm, y=1:num_years],
        if haskey(inflow_edges, i)
            sum(
                sum(requirements_inflow[h, i, k, m] for m in 1:num_comm) * inflow[h, i, k, y]
                for h in inflow_edges[i]
            )

        else
            @info "No inflow edge found for ($(nodes[i]), $(commodities[k]), $(years[y]))"
            0
        end
    )

    mass_balance = @constraint(
        model,
        mass_balance[i=1:num_nodes, k=1:num_comm, y=1:num_years],
        requirements_outflow_const[i, k, y] - requirements_inflow_const[i, k, y]
        <= demand[i, k, y])

    flow_transformation = @constraint(
        model,
        flow_transformation[i in keys(outflow_edges), j in outflow_edges[i], k in 1:num_comm, l in 1:num_comm, y=1:num_years; transformation[i, j, k, l] != 0],
        transformation[i, j, k, l] * outflow[i, j, l, y] == inflow[i, j, k, y]
        )

    return model
end

function print_vars(var_object)
    for var in var_object
        if JuMP.value(var) != 0.0
            println("$(var): $(JuMP.value(var))")
        end
    end
end

function print_duals(con_object)
    for con in con_object
        if JuMP.shadow_price(con) != 0.0
            println("$(con): $(JuMP.shadow_price(con))")
        end
    end
end

function print_constraint(con_object)
    for con in con_object
        println("$(con)")
    end
end

function run(file_path::String)

    model_data = get_data(file_path)

    model = Model(with_optimizer(GLPK.Optimizer))

    @time model = formulate_gmcnf!(model_data, model, verbose = true)

    println("Compiled model, now running")
    @time JuMP.optimize!(model)
    println("Finished running, objective: £$(JuMP.objective_value(model))")

    status = JuMP.termination_status(model) == MOI.OPTIMAL
    feasible = JuMP.primal_status(model) == MOI.FEASIBLE_POINT

    if status
        println("\nSolution optimal\n")
        outflow = model[:outflow]
        inflow = model[:inflow]
        total_capacity = model[:total_annual_capacity]
        new_capacity = model[:new_capacity]

        write_results(outflow, "outflow", model_data)
        write_results(inflow, "inflow", model_data)
        write_results(total_capacity, "total_capacity", model_data)
        write_results(new_capacity, "new_capacity", model_data)

    else
        println("\nSolution not optimal\n")
        println("i,j=> $(model_data["nodes"])")
        println("k=> $(model_data["commodities"])")
        println("y=> $(model_data["years"])")
        println("mass_balance[i, k, y]")
        print_constraint(model[:mass_balance])
        println("flow_transformation[i, j, k, y]")
        print_constraint(model[:flow_transformation])
        print_constraint(model[:max_new_capacity])
        print_constraint(model[:accumulate_capacity])
    end

end

if length(ARGS) > 0
    run(ARGS[1])
end

end


#Additional constants for repeated structs
const TGOV1DU = SteamTurbineGov1

"""
Parse .dyr file into a dictionary indexed by bus number.
Each bus number key has a dictionary indexed by component type and id.

Comments in .dyr files are not supported (beginning of lines with //).

"""
function _parse_dyr_file(file::AbstractString)
    dyr_text = read(file, String)
    start = 1
    parsed_values = Dict{Int, Dict}()
    models = Vector{String}()
    while start < length(dyr_text)
        val = findnext('/', dyr_text, start)
        if isnothing(val)
            break
        end
        text = strip(dyr_text[start:(val - 1)])
        if !isempty(text)
            line = replace(text, "\'" => "")
            line = replace(line, "," => " ")
            val_array = strip.(split(line))
            bus = parse(Int, val_array[1])
            model = string(val_array[2])
            id = string(val_array[3])
            component_dict = get!(parsed_values, bus, Dict{Tuple{String, String}, Array}())
            component_dict[(model, id)] = parse.(Float64, val_array[4:end])
        end
        start = val + 1
    end
    return parsed_values
end

function _parse_input_types(_v, val)
    #If the parameter is a Float64, then use the value directly as the argument.
    #Typically uses for the resistance that is not available in .dyr files.
    if isa(_v, Float64)
        if isnan(_v)
            error("nan for $(_v)")
        end
        return _v
        #If the parameter is an Int, then use the integer as the key of the value in the dictionary.
    elseif isa(_v, Int)
        return val[_v]
    elseif _v == "NaN"
        return NaN
        #If the parameter is a tuple (as a string), then construct the tuple directly.
    elseif isa(_v, String)
        m = match(r"^\((\d+)\s*,\s*(\d+)\)$", _v)
        if m !== nothing
            _tuple_ix = parse.(Int, m.captures)
            return Tuple(val[_tuple_ix])
        else
            error("String $(_v) not recognized for parsing")
        end
    else
        error("invalid input value $val")
    end
    return
end

"""
Populate arguments in a vector for each dynamic component (except Shafts).
Returns a vector with the parameter values of the argument of each component.

"""
function _populate_args(param_map::Vector, val)
    struct_args = Vector{Any}(undef, length(param_map))
    _populate_args!(struct_args, param_map, val, "")
    return struct_args
end

function _populate_args!(struct_args, param_map::Vector, val, ::String)
    for (ix, _v) in enumerate(param_map)
        parameter_value = _parse_input_types(_v, val)
        if !all(isnan.(parameter_value))
            struct_args[ix] = parameter_value
        end
    end
    return
end

function _populate_args!(struct_args, param_dic::Dict, val, id::String)
    param_map = param_dic[id]
    _populate_args!(struct_args, param_map, val, id)
end

"""
Create a SingleMass shaft struct directly using the parameter mapping.

"""
function _make_shaft(param_map, val)
    constructor_shaft =
        (args...) -> InteractiveUtils.getfield(PowerSystems, :SingleMass)(args...)
    shaft_args = _populate_args(param_map, val)
    return constructor_shaft(shaft_args...)
end

function _make_source(g::StaticInjection, r::Float64, x::Float64)
    return Source(
        name = get_name(g),
        available = true,
        bus = get_bus(g),
        active_power = get_active_power(g),
        reactive_power = get_reactive_power(g),
        R_th = r,
        X_th = x,
    )
end

function _assign_missing_components!(
    bus_dict,
    component_table,
    ::Type{T},
) where {T <: DynamicGenerator}
    for va in values(bus_dict)
        if ismissing(va[component_table["AVR"]])
            va[component_table["AVR"]] = AVRFixed(1.0)
        end
        if ismissing(va[component_table["TurbineGov"]])
            va[component_table["TurbineGov"]] = TGFixed(1.0)
        end
        if ismissing(va[component_table["PSS"]])
            va[component_table["PSS"]] = PSSFixed(0.0)
        end
    end
    return
end

function _assign_missing_components!(
    va,
    component_table,
    ::Type{T},
) where {T <: DynamicInverter}
    if ismissing(va[component_table["Converter"]])
        va[component_table["Converter"]] = AverageConverter(750.0, 2.75)
    end
    if ismissing(va[component_table["Filter"]])
        va[component_table["Filter"]] = RLFilter(0.0, 0.0)
    end
    if ismissing(va[component_table["FrequencyEstimator"]])
        va[component_table["FrequencyEstimator"]] = FixedFrequency()
    end
    if ismissing(va[component_table["DCSource"]])
        va[component_table["DCSource"]] = FixedDCSource(750.0)
    end
    return
end

"""
Parse a .dyr file directly from its name by constructing its dictionary of dictionaries.

"""
function _parse_dyr_components(dyr_file::AbstractString)
    ext = splitext(dyr_file)[2]
    if lowercase(ext) in [".dyr"]
        data = _parse_dyr_file(dyr_file)
        return _parse_dyr_components(data)
    else
        throw(DataFormatError("$dyr_file is not a .dyr file type"))
    end
end

"""
Parse dictionary of dictionaries of data (from `_parse_dyr_file`) into a dictionary of struct components.
The function receives the parsed dictionary and constructs a dictionary indexed by bus, that contains a
dictionary with each dynamic generator and dynamic inverter components (indexed via its id).

For Generators, each dictionary indexed by id contains a vector with 5 of its components:
* Machine
* Shaft
* AVR
* TurbineGov
* PSS

For Inverters, each dictionary indexed by id contains a vector with 7 of its components:
* Converter
* ActivePowerControl
* ReactivePowerControl
* InnerControl
* DCSource
* FrequencyEstimator
* Filter

"""
function _parse_dyr_components(data::Dict)
    yaml_mapping = open(PSSE_DYR_MAPPING_FILE) do io
        aux = YAML.load(io)
        return aux
    end
    #param_map contains the mapping between struct params and dyr file.
    param_map = yaml_mapping["parameter_mapping"][1]
    #gen_map contains all the supported structs for generators
    gen_map = yaml_mapping["generator_mapping"][1]
    #inv_map contains al the supported structs for inverters
    inv_map = yaml_mapping["inverter_mapping"][1]
    #dic will contain the dictionary index by bus.
    #Each entry will be a dictionary, with id as keys, that contains the vector of components
    dic = Dict{Int, Any}()
    _parse_dyr_generator_components!(dic, data, gen_map, param_map)
    _parse_dyr_inverter_components!(dic, data, inv_map, param_map)
    return dic
end

"""
Parse dictionary of data (from `_parse_dyr_file`) into a dictionary of struct components.
The function receives the parsed dictionary and constructs a dictionary indexed by bus, that contains a
dictionary with each dynamic generator indexed by its id.

"""
function _parse_dyr_generator_components!(
    dic::Dict,
    data::Dict,
    gen_map::Dict,
    param_map::Dict,
)
    component_table =
        Dict("Machine" => 1, "Shaft" => 2, "AVR" => 3, "TurbineGov" => 4, "PSS" => 5)
    for (bus_num, bus_data) in data #bus_data is a dictionary with values per component (key)
        bus_dict = Dict{String, Any}()
        for (componentID, componentValues) in bus_data
            #Fill array of 5 components per generator
            temp = get!(bus_dict, componentID[2], fill!(Vector{Any}(undef, 5), missing))
            #Only create if name is in the supported keys in the mapping
            if componentID[1] in keys(gen_map)
                #Get the component dictionary
                components_dict = gen_map[componentID[1]]
                for (gen_field, struct_as_str) in components_dict
                    #Get the parameter indices w/r to componentValues
                    params_ix = param_map[struct_as_str]
                    #Construct Shaft if it appears
                    if occursin("Shaft", struct_as_str)
                        temp[2] = _make_shaft(params_ix, componentValues)
                        continue
                    end
                    #Construct Components
                    component_constructor =
                        (args...) ->
                            InteractiveUtils.getfield(PowerSystems, Symbol(struct_as_str))(
                                args...,
                            )
                    struct_args = _populate_args(params_ix, componentValues)
                    if struct_as_str == "BaseMachine"
                        struct_args = [
                            0.0,    #R
                            1e-5,   #X
                            1.0,    #eq_p
                        ]
                    end
                    temp[component_table[gen_field]] = component_constructor(struct_args...)
                end
            else
                @warn "$(componentID[1]) at bus $bus_num, id $(componentID[2]), not supported in PowerSystems.jl. Skipping data."
            end
        end
        #Assign generic components if there were not provided in Dynamic Data
        _assign_missing_components!(bus_dict, component_table, DynamicGenerator)
        #Store dictionary of components in a dictionary indexed by bus
        dic[bus_num] = bus_dict
    end
end

"""
Parse dictionary of data (from `_parse_dyr_file`) into a dictionary of struct components.
The function receives the parsed dictionary and constructs a dictionary indexed by bus, that contains a
dictionary with each dynamic inverter indexed by its id.

"""
function _parse_dyr_inverter_components!(
    dic::Dict,
    data::Dict,
    inv_map::Dict,
    param_map::Dict,
)
    component_table = Dict(
        "Converter" => 1,
        "ActivePowerControl" => 2,
        "ReactivePowerControl" => 3,
        "InnerControl" => 4,
        "DCSource" => 5,
        "FrequencyEstimator" => 6,
        "Filter" => 7,
    )
    for (bus_num, bus_data) in data #bus_data is a dictionary with values per component (key is a tuple of PSSE name and number ID)
        #bus_dict will add a vector of components structs for each generator ID on the bus_num key of the master dictionary 'dic'
        #bus_dict_values will only contain the vector of parameters that will be used to construct the structs
        bus_dict_values = Dict{String, Any}()
        for (componentID, componentValues) in bus_data
            #ComponentID is a tuple:
            #ComponentID[1] is the PSSE name
            #ComponentID[2] is the number ID of the generator/inverter
            #Fill array of 7 components per inverter
            #Only create if name is in the supported keys in the mapping
            if componentID[1] in keys(inv_map)
                #Get the component dictionary
                temp = get!(bus_dict_values, componentID[2], Dict{Any, Vector{Any}}())
                components_dict = inv_map[componentID[1]]
                for (inv_field, struct_as_str) in components_dict
                    param_vec = get!(
                        temp,
                        (inv_field, struct_as_str),
                        _instantiate_param_vector_size(struct_as_str, param_map),
                    )
                    params_ix = param_map[struct_as_str]
                    _populate_args!(param_vec, params_ix, componentValues, componentID[1])
                end
            end
        end
        #Store vector of components in a dictionary indexed by bus
        for (device_id, device_info) in bus_dict_values
            temp_vec = fill!(Vector{Any}(undef, 7), missing)
            for (k, struct_args) in device_info
                component_type, struct_as_str = k
                _convert_argument_types!(struct_as_str, struct_args)
                component_constructor =
                    (args...) ->
                        InteractiveUtils.getfield(PowerSystems, Symbol(struct_as_str))(
                            args...,
                        )
                temp_vec[component_table[component_type]] =
                    component_constructor(struct_args...)
            end
            _assign_missing_components!(temp_vec, component_table, DynamicInverter)
            dic[bus_num][device_id] = temp_vec
        end
    end
end

"""
Construct appropiate vector size for components that collect parameters from
more than 2 PSS/E components

"""
function _instantiate_param_vector_size(str::AbstractString, param_map::Dict)
    if str == "ActiveRenewableControllerAB"
        return fill!(Vector{Any}(undef, 17), missing)
    elseif str == "ReactiveRenewableControllerAB"
        return fill!(Vector{Any}(undef, 25), missing)
    elseif str == "RECurrentControlB"
        return fill!(Vector{Any}(undef, 12), missing)
    elseif str in keys(param_map)
        return fill!(Vector{Any}(undef, length(param_map[str])), missing)
    else
        error("String $(str) not supported in the parser")
    end
end

"""
Convert specific parameters to types that are not Float64 for
specific inverter components

"""
function _convert_argument_types!(str::AbstractString, struct_args::Vector)
    if str == "ActiveRenewableControllerAB"
        struct_args[1:5] .= Int.(struct_args[1:5])
        struct_args[4] = string(struct_args[4])
    elseif str == "ReactiveRenewableControllerAB"
        struct_args[1:8] .= Int.(struct_args[1:8])
        struct_args[4] = string(struct_args[4])
    elseif str == "RECurrentControlB"
        struct_args[1:2] .= Int.(struct_args[1:2])
    elseif str == "RenewableEnergyConverterTypeA"
        #No changes to struct_args
    else
        error("$str not defined for dynamic component arguments")
    end
end

"""
Add to a system already created the dynamic components.
The system should already be parsed from a .raw file.

## Examples:
```julia
dyr_file = "Example.dyr"
add_dyn_injectors!(sys, dyr_file)
```

"""
function add_dyn_injectors!(sys::System, dyr_file::AbstractString)
    bus_dict_gen = _parse_dyr_components(dyr_file)
    add_dyn_injectors!(sys, bus_dict_gen)
end

function add_dyn_injectors!(sys::System, bus_dict_gen::Dict)
    @info "Generators provided in .dyr, without a generator in .raw file will be skipped."
    for g in collect(get_components(ThermalStandard, sys))
        _num = get_number(get_bus(g))
        _name = get_name(g)
        _id = split(_name, "-")[end]
        temp_dict = get(bus_dict_gen, _num, nothing)
        if temp_dict === nothing
            @warn "Generator at bus $(_num), id $(_id), not found in Dynamic Data.\nVoltage Source will be used to model it."
            r, x = get_ext(g)["z_source"]
            if x == 0.0
                @warn "No series reactance found. Setting it to 1e-6"
                x = 1e-6
            end
            s = _make_source(g, r, x)
            remove_component!(typeof(g), sys, _name)
            add_component!(sys, s)
        elseif all(.!ismissing.(temp_dict[_id]))
            if length(temp_dict[_id]) == 5 #Generator has 5 components
                #Obtain Machine from Dictionary
                machine = temp_dict[_id][1]
                #Update R,X from RSORCE and XSORCE from RAW file
                r, x = get_ext(g)["z_source"]
                set_R!(machine, r)
                #Obtain Shaft from dictionary
                shaft = temp_dict[_id][2]
                if typeof(machine) == BaseMachine
                    if x == 0.0
                        @warn "No series reactance found. Setting it to 1e-6"
                        x = 1e-6
                    end
                    set_Xd_p!(machine, x)
                    if get_H(shaft) == 0.0
                        @info "Machine at bus $(_num), id $(_id) has zero inertia. Modeling it as Voltage Source"
                        s = _make_source(g, r, x)
                        remove_component!(typeof(g), sys, _name)
                        add_component!(sys, s)
                        #Don't add DynamicComponent in case of adding Source
                        continue
                    end
                end
                #Add Dynamic Generator
                dyn_gen = DynamicGenerator(get_name(g), 1.0, temp_dict[_id]...)
                add_component!(sys, dyn_gen, g)
            elseif length(temp_dict[_id]) == 7 #Inverter has 7 components (6 if Outer is put together)
                dyn_inv = DynamicInverter(get_name(g), 1.0, temp_dict[_id]...)
                add_component!(sys, dyn_inv, g)
            else
                @assert false
            end
        else
            error("Generator at bus $(_num), id $(_id), name $(_name), not supported")
        end
    end
end