# -------- Auxiliary Functions ----------------------------------------------- #

# Set an optimizer for all parameters of a model
function set_optimizer!(model, opt = Knet.Adam; kwargs...)

  for param in Knet.params(model)

    # Feature heads that are not used have length 0. They do not get an
    # optimizer.
    if length(param) > 0

      # If opt is given, overwrite every optimizer in param.opt
      if !isnothing(opt)

        param.opt = opt(; kwargs...)

      # If opt is not given, overwrite only non-existing param.opt with Adam
      elseif isnothing(opt) && isnothing(param.opt)

        param.opt = Knet.Adam(; kwargs...)

      end

    end

  end

end

# A single training step, the loss is returned
function train_step!(l :: Loss, model, cache :: DataCache)

  tape = Knet.@diff sum(loss(l, model, cache))

  for param in Knet.params(model)
    Knet.update!(Knet.value(param), Knet.grad(tape, param), param.opt)
  end

  Knet.value(tape)

end

# -------- Training on Datasets ---------------------------------------------- #

"""
    train!(model/player, dataset; <keyword arguments>)

Train `model`, or the training model of `player`, on `dataset`.

# Arguments
' `loss = Loss()`: Loss used for training.
- `epochs = 10`: Number of iterations through `dataset`.
- `batchsize = 50`: Batchsize for the update steps.
- `callback_epoch`: Function called after each epoch.
- `callback_step`: Function called after each update step.
- `optimizer = nothing`: Optimizer initialized for each weight in `model`
- `kwargs...`: Keyword arguments for `optimizer`

# Examples
```julia
set = record_self(MCTSPlayer(), 10, game = TicTacToe())
model = NeuralModel(TicTacToe, @chain TicTacToe Dense(50))
loss = Loss(policy = 0.15, regularization=1e-4)
train!(model, set, loss = loss, epochs = 15)
```

"""
function train!( player  :: Union{Player, Model}
               , trainset :: DataSet
               ; loss = Loss()
               , epochs = 10
               , batchsize = 50
               , callback_step = (_) -> nothing
               , callback_epoch = (_) -> nothing
               , optimizer = nothing
               , kwargs... )

  # Get basic info about the player's model
  model = training_model(player)
  gpu = on_gpu(model)

  # Check if features can be enabled
  use_features = check_features(loss, model, trainset)

  # Set/overwrite optimizers
  set_optimizer!(model, optimizer; kwargs...)

  # Generate the batch iterator
  batches = Batches( trainset
                   , batchsize
                   , shuffle = true
                   , partial = false
                   , gpu = gpu )

  # Iterate through the trainset
  for j in 1:epochs

    for (i, cache) in enumerate(batches)

      train_step!(loss, model, cache)
      callback_step(i)

    end

    callback_epoch(j)

  end

end

# -------- Training by Playing ----------------------------------------------- #

function _train!( player :: Player{G}
                , gen_data :: Function
                ; loss = Loss()
                , epochs = 10
                , playings = 20
                , iterations = 10
                , batchsize = 50
                , testfrac = 0.1
                , optimizer = Knet.Adam
                , replays = 0
                , quiet = false
                , callback_epoch = (_) -> nothing
                , callback_iter = (_) -> nothing
                , kwargs...
                ) where {G <: Game}

  @assert playings > 0 "Number of playings for training must be positive"

  # If we do not print results, it is not necessary to test
  quiet || testfrac < 0 && (testfrac = 0.)

  # Get number of playings used for training
  train_playings = ceil(Int, playings * (1-testfrac))

  # Print the loss header if not quiet
  !quiet && print_loss_header(loss, check_features(loss, player))

  # Set the optimizer for the player's model
  set_optimizer!(training_model(player), optimizer; kwargs...)

  # Save the last `replay` generated datasets and use them for training
  replay_buffer = []

  for i in 1:epochs

    # Generate callback functions for the progress meter
    step, finish = stepper("# Playing...", playings)
    cb = quiet ? () -> nothing : step

    # Generate new datasets for this generation
    datasets = gen_data(cb, playings)

    # Create the training set by merging with the last `playing` generations of
    # datasets that are stored in the replay_buffer
    trainset = merge(datasets[1:train_playings]...)
    for set in replay_buffer
      trainset = merge(trainset, set...)
    end

    # Create the testing set by collecting the remaining datasets generated
    # in this epoche
    if testfrac > 0
      testset = merge(datasets[train_playings+1:playings]...)
    else
      testset = DataSet{G}()
    end

    # Clear the progress bar (if it was printed)
    finish()

    # Prepare the next progress bar
    steps = iterations * div(length(trainset), batchsize)
    step, finish = stepper("# Learning...", steps)
    cb = (_) -> quiet ? nothing : step()

    # The Knet allocator works better for training but worse for playing
    switch_knet_allocator()

    # Train the player with the generated dataset
    train!( player
          , trainset
          , loss = loss
          , epochs = iterations
          , batchsize = batchsize
          , callback_step = cb
          , callback_epoch = callback_iter )

    # Clear the progress bar
    finish()

    # Calculate and print loss for this epoch if not quiet
    !quiet && print_loss(loss, player, i, trainset, testset)

    # Undo the changes in the Knet allocator
    switch_knet_allocator()

    # Bring the full dataset from this epoche in the replay buffer
    push!(replay_buffer, datasets)

    # Remove the oldest item from the replay buffer if it is to large
    length(replay_buffer) >= replays+1 && deleteat!(replay_buffer, 1)

    # Call callback function
    callback_epoch(i)

  end

end

"""
    train_self!(player; <keyword arguments>)

Train an MCTS `player` via playing against itself.

The training model of `player` learns to predict the MCTS policy, the value of
game states, and possible features (if `player` supports the same features as
the keyword argument `loss`). Note that only players with NeuralModel-based
training models can be trained currently.

# Arguments
- `loss = Loss()`: Loss used for training.
- `epochs = 10`: Number of epochs.
- `playings = 20`: Games played per `epoch` for training-set generation.
- `iterations = 10`: Number of training epochs per training-set.
- `batchsize = 50`: Batchsize during training from the training-set.
- `testfrac = 0.1`: Fraction of `playings` used to create test-sets.
- `branching = 0.`: Random branching probability.
- `augment = true`: Whether to use augmentation on the created data sets.
- `replays = 0`: Add datasets from last `replay` epochs to the trainset.
- `quiet = false`: Whether to suppress logging of training progress.
- `callback_epoch`: Function called after every epoch.
- `callback_iter`: Function called after every iteration.
- `distributed = false`: Shall workers be used for self-playings?
- `tickets = nothing`: Number of tickets if distributed.
- `optimizer = Adam`: Optimizer for each weight in the training model.
- `kwargs...`: Keyword arguments for `optimizer`

# Examples
```julia
# Self-train a simple neural network model for 5 epochs
model = NeuralModel(TicTacToe, @chain TicTacToe Conv(64, relu) Dense(32, relu))
player = MCTSPlayer(model, power = 50, temperature = 0.75, exploration = 2.)
train_self!(player, epochs = 5, playings = 100)
```
"""
function train_self!( player :: MCTSPlayer
                    ; loss = Loss()
                    , prepare = prepare(steps = 0)
                    , branch = branch(prob = 0.)
                    , augment = true
                    , distributed = false
                    , tickets = nothing
                    , kwargs... )

  # Only use player-enabled features for recording if they are compatible with
  # the loss
  features = check_features(loss, player) ? Jtac.features(player) : Feature[]

  # Function to generate datasets through selfplays
  gen_data = (cb, n) -> record_self( player
                                   , n
                                   , prepare = prepare
                                   , branch = branch
                                   , augment = augment
                                   , features = features
                                   , merge = false
                                   , callback = cb
                                   , distributed = distributed
                                   , tickets = tickets )

  _train!(player, gen_data; loss = loss, kwargs...)

end

"""
    train_against!(player, enemy; <keyword arguments>)

Train an MCTS `player` under `loss` through playing against `enemy`.

The training model of `player` learns to predict the MCTS policy, the value of
game states, and possible features (if `player` supports the same features as
the keyword argument `loss`) when playing against `enemy`. If the enemy is too
good, this may turn out to be a bad learning mode: a player that loses all the
time will produce very pessimistic value predictions for each single game state,
which will harm the MCTS algorithm for improved policies. Note that only players
with NeuralModel-based training models can be trained currently.

# Arguments
- `loss = Loss()`: Loss used for training.
- `start`: Function that (randomly) yields -1 or 1 to fix the starting player.
- `epochs = 10`: Number of epochs.
- `playings = 20`: Games played per `epoch` for training-set generation.
- `iterations = 10`: Number of training epochs per training-set.
- `batchsize = 50`: Batchsize during training from the training-set.
- `testfrac = 0.1`: Fraction of `playings` used to create test-sets.
- `branching = 0.`: Random branching probability.
- `augment = true`: Whether to use augmentation on the created data sets.
- `replays = 0`: Add datasets from last `replay` epochs to the trainset.
- `quiet = false`: Whether to suppress logging of training progress.
- `callback_epoch`: Function called after every epoch.
- `callback_iter`: Function called after every iteration.
- `distributed = false`: Shall workers be used for self-playings?
- `tickets = nothing`: Number of tickets if distributed.
- `optimizer = Adam`: Optimizer for each weight in the training model.
- `kwargs...`: Keyword arguments for `optimizer`

# Examples
```julia
# Train a simple neural network model against an MCTS player for 5 epochs
model = NeuralModel(TicTacToe, @chain TicTacToe Conv(64, relu) Dense(32, relu))
player = MCTSPlayer(model, power = 50, temperature = 0.75, exploration = 2.)
enemy = MCTSPlayer(power = 250)
train_against!(player, enemy, epochs = 5, playings = 100)
```
"""
function train_against!( player :: MCTSPlayer
                       , enemy
                       ; loss = Loss()
                       , start :: Function = () -> rand([-1, 1])
                       , branching = 0.
                       , augment = true
                       , distributed = false
                       , tickets = nothing
                       , kwargs... )

  features = check_features(loss, player) ? features(player) : Feature[]

  gen_data = (cb, n) -> record_against( player
                                      , enemy
                                      , n
                                      , start = start
                                      , augment = augment
                                      , branching = branching
                                      , features = features
                                      , merge = false
                                      , callback = cb
                                      , distributed = distributed
                                      , tickets = tickets )

  _train!(player, gen_data; loss = loss, kwargs...)

end

"""
    train_from_model!(pupil, teacher [, players]; <keyword arguments>)

Train a `pupil` by letting it approximate the predictions of
`teacher`'s model on game states created by `players`.

In each epoch, two random `players` are chosen (by default, these are the pupil
and the teacher) to generate a set of game states by playing the game. These
games are used to create a training set by applying the teachers training model.
Note that only players with NeuralModel-based training models can be trained
currently.

# Arguments
- `loss = Loss()`: Loss used for training.
- `epochs = 10`: Number of epochs.
- `playings = 20`: Games played per `epoch` for training-set generation.
- `iterations = 10`: Number of training epochs per training-set.
- `batchsize = 50`: Batchsize during training from the training-set.
- `testfrac = 0.1`: Fraction of `playings` used to create test-sets.
- `branching = 0.`: Random branching probability.
- `augment = true`: Whether to use augmentation on the created data sets.
- `replays = 0`: Add datasets from last `replay` epochs to the trainset.
- `quiet = false`: Whether to suppress logging of training progress.
- `callback_epoch`: Function called after every epoch.
- `callback_iter`: Function called after every iteration.
- `optimizer = Adam`: Optimizer for each weight in the training model.
- `kwargs...`: Keyword arguments for `optimizer`
"""
function train_from_model!( pupil :: Player{G}
                          , teacher :: Union{Model{H}, Player{H}}
                          , players = [pupil]
                          ; loss = Loss()
                          , branching = 0.
                          , augment = true
                          , kwargs...
                          ) where {H <: Game, G <: H}

  # Complement the branching options (before, during, steps)
  branching = branch_options(branching)

  # Check if pupil and teacher are compatible featurewise
  use_features = features(pupil) == features(teacher) != Feature[]

  # Function that generates datasets
  gen_data = (cb, n) -> begin

    # TODO: Async?

    # Construct a list of games by letting two players compete
    datasets = asyncmap(1:n) do _

      # Get two players at random
      p1, p2 = rand(players, 2)

      # Watch them playing
      games = pvp_games(p1, p2, game = branch_root(G(), branching))[1:end-1]

      # Force the players to 'make errors' sometimes
      branchgames = branch(games, branching)
      branches = map(g -> pvp_games(p1, p2, g), branchgames)

      # Let the teacher model look at all games to generate a dataset
      ds = record_model( training_model(teacher)
                       , vcat(games, branches...)
                       , use_features = use_features
                       , augment = augment )

      # Give the signal that one playing is complete
      cb()

      ds

    end

    datasets

  end

  _train!(pupil, gen_data; loss = loss, kwargs...)

end

# -------- Training With Contests -------------------------------------------- #

"""
    with_contest(train_function, player, <arguments>; <keyword arguments>)

Train `player` according to `train_function` while conducting regular contests
during the training process.

The argument `train_function` must be one of `train_self!`, `train_against!`, or
`train_from!`. The function `with_contest` will always at least print the
results of a contest before training and after training.

# Arguments
The non-keyword arguments supported are the non-keyword arguments that are also
accepted / needed by `train_function`. As keyword arguments, all keyword
arguments of `train_function` are supported. Additionally, the following
arguments can be used to adapt the behavior of `with_contest`:

- `interval = 10`: After how many epochs do contests take place.
- `length = 250`: Number of playings during one contest.
- `opponents`: List of opponents for the competition. To this list the
   two players `current` and `initial` are added, which are IntuitionPlayers
   with the model (or a copy of the initial model) of `player`.
- `temperature`: Temperature of the intuition players `current` and `initial`.
- `cache = 0`: Number of games to be cached. If `cache > 0`, a number
   of `cache` games between all players that are independent of `player`'s model
   are conducted and cached for all competitions that follow.

# Examples
```julia

G = TicTacToe
loss = Loss(policy = 0.25)
opponents = [MCTSPlayer(power = 50), MCTSPlayer(power = 500)]

model = NeuralModel(G, @chain G Conv(100, relu) Dense(32, relu))
player = MCTSPlayer(model, power = 25)

with_contest( train_self!
            , player
            , loss = loss
            , cache = 1000
            , length = 500
            , opponents = opponents
            , interval = 5
            , epochs = 20
            , playings = 150 )
```

"""
function with_contest( trainf!     # the training function
                     , player :: MCTSPlayer
                     , args...
                     ; loss = Loss()
                     , opponents = Player[]
                     , length :: Int = 250
                     , cache :: Int = 0
                     , interval :: Int = 10
                     , temperature = player.temperature
                     , distributed = false
                     , epochs = 10
                     , kwargs... )

  # One can set length == 0 to access trainf! without contests
  if length <= 0

    @info "Contests are disabled."
    return trainf!( player
                  , args...
                  ; loss = loss
                  , epochs = epochs
                  , distributed = distributed
                  , kwargs... )

  end

  # Rename the length keyword argument
  len = length
  length = Base.length

  # List of all players that will compete in the contest
  players = [

    IntuitionPlayer( player.model
                   , temperature = temperature
                   , name = "current" );
    IntuitionPlayer( copy(player.model)
                   , temperature = temperature
                   , name = "initial" );
    opponents

  ]

  # Creating a cache of contest games for all players that do not depend on the
  # model that will be trained
  if cache > 0

    # Get the training model
    model = training_model(player)

    # Reorder players such that the active (i.e., learning) players come last
    # and passive ones first
    active = filter(p -> training_model(p) == model, players)
    passive = filter(p -> training_model(p) != model, players)
    players = [passive; active]

    # Get the respective numbers of players
    n  = length(players)
    np = length(passive)
    na = length(active)

    # Get the indices of all active models
    aidx = collect(1:na) .+ np

    # Get the progress-meter going
    step, finish = stepper("# Caching...", cache)

    # Create the cache for games between passive players
    cache_results = zeros(Int, n, n, 3)
    cache_results[1:np,1:np,:] = compete( passive
                                        , cache
                                        , distributed = distributed
                                        , callback = step )

    # Remove the progress bar
    finish()

    #  Leave a message that confirms caching.
    print(gray_crayon)
    println("# Cached $cache matches by $n players")

  else

    # If caching is not activated, regard each player as active
    aidx = 1:length(players)
    cache_results = zeros(Int, length(players), length(players), 3)

  end

  # Create the callback function
  cb = epoch -> begin
    if epoch % interval == 0

      step, finish = stepper("# Contest...", len)

      results = compete( players
                       , len
                       , aidx
                       , distributed = distributed
                       , callback = step )

      finish()

      print_ranking(Ranking(players, results + cache_results))

      0 < epoch < epochs && print_loss_header(loss, check_features(loss, player))

    end
  end

  # Run contest before training
  cb(0)

  # Train with a contest every interval epochs
  trainf!( player
         , args...
         ; loss = loss
         , callback_epoch = cb
         , epochs = epochs
         , distributed = distributed
         , kwargs... )

  # Run contest after training, if it was not run in trainf! already
  epochs % interval == 0 ? nothing : cb(epochs)

end


@testset "SimResource FIFO/LIFO" begin
    function rfifo(fr)
        tmp = []
        r = SimResource{Int}(quantity=5, lo=0, hi=5,
                             max_requests=3, fifo_requests=fr)
        s = Scheduler()
        @test_throws ErrorException SimResource{Int}(quantity=0, lo=1, hi=5)
        @test_throws ErrorException SimResource{Int}(quantity=3, lo=3, hi=2)

        @test request!(s, r, 2, x -> push!(tmp, 1))[1]
        s.now = 1.0
        @test request!(s, r, 2, x -> push!(tmp, 2))[1]
        @test request!(s, r, 2, x -> push!(tmp, 3))[1]
        @test request!(s, r, 2, x -> push!(tmp, 4))[1]
        @test request!(s, r, 2, x -> push!(tmp, 5))[1]
        @test !request!(s, r, 2, x -> push!(tmp, 6))[1]
        @test length(r.requests) == 3
        s.now = 2.0
        @test provide!(s, r, 1) == 1
        @test request!(s, r, 2, x -> push!(tmp, 7))[1]
        s.now = 3.0
        @test provide!(s, r, 10) == 5
        go!(s, 6.0)
        @test length(r.requests) == 1
        r.requests[1].request(s)
    end

    @test rfifo(true) == [1,2,3,4,5,7]
    @test rfifo(false) == [1,2,5,7,4,3]
end

@testset "SimResource waive! test" begin
    s = Scheduler()
    r = SimResource{Int}()
    @test !waive!(r, ResourceRequest(1, identity))

    rr1 = request!(s, r, 2, x -> error(2))[2]
    rr2 = request!(s, r, 2, x -> error(3))[2]
    rr3 = request!(s, r, 2, x -> error(4))[2]

    @test length(r.requests) == 3
    @test waive!(r, rr2)
    @test length(r.requests) == 2
    @test !waive!(r, rr2)
    @test waive!(r, rr1)
    @test waive!(r, rr3)
    @test isempty(r.requests)
end


# Boolean Operators
struct Unify <: Group
    elements::AbstractArray{Op, 1}
end
struct Difference <: Group
    elements::AbstractArray{Op, 1}
end

function toString(t::Unify)
    output = "union(){"
    for c in t.elements
        output = output * toString(c)
    end
    output * "};"
end


let
    pomdp = BabyPOMDP()
    up = updater(pomdp)

    bmdp = GenerativeBeliefMDP(pomdp, up)
    b = initialstate(bmdp, Random.GLOBAL_RNG)
end


# direct integration algorithms
export Bathe,
       CentralDifference,
       Houbolt,
       Linear,
       Newmark,
       BackwardEuler,
       Trapezoidal

# solution API
export solve,
       displacements,
       velocities,
       accelerations,
       times,
       dim


#=
A place for general utilities that don't fit in elsewhere.

=#

# Traverses an expression, replacing variable symbols with the value arrays
function replace_var_symbols_with_values(ex)
    if typeof(ex) == Expr
        if ex.head === :call # function symbols (like :sin in sin(x)) don't need to be processed
            for i=2:length(ex.args)
                ex.args[i] = replace_var_symbols_with_values(ex.args[i]);
            end
        else
            for i=1:length(ex.args)
                ex.args[i] = replace_var_symbols_with_values(ex.args[i]);
            end
        end

    elseif typeof(ex) == Symbol
        # Check if it is a variable and substitute if needed
        var_index = 0;
        for v in variables
            if ex === v.symbol
                var_index = v.index;
            end
        end
        if var_index > 0
            if variables[var_index].type == SCALAR
                newex = :(Finch.variables[$var_index].values[node_index]);
            else
                newex = :(Finch.variables[$var_index].values[:,node_index]);
            end
            ex = newex;
        end
    end

    return ex;
end


using ITensors,
      HDF5,
      Test

@testset "HDF5 Read and Write" begin

  i = Index(2,"i")
  j = Index(3,"j")
  k = Index(4,"k")

  @testset "TagSet" begin
    ts = TagSet("A,Site,n=2")
    fo = h5open("data.h5","w")
    write(fo,"tags",ts)
    close(fo)

    fi = h5open("data.h5","r")
    rts = read(fi,"tags",TagSet)
    close(fi)
    @test rts == ts
  end

  @testset "Index" begin
    i = Index(3,"Site,S=1")
    fo = h5open("data.h5","w")
    write(fo,"index",i)
    close(fo)

    fi = h5open("data.h5","r")
    ri = read(fi,"index",Index)
    close(fi)
    @test ri == i

    # primed Index
    i = Index(3,"Site,S=1")
    i = prime(i,2)
    fo = h5open("data.h5","w")
    write(fo,"index",i)
    close(fo)

    fi = h5open("data.h5","r")
    ri = read(fi,"index",Index)
    close(fi)
    @test ri == i
  end

  @testset "IndexSet" begin
    is = IndexSet(i,j,k)

    fo = h5open("data.h5","w")
    write(fo,"inds",is)
    close(fo)

    fi = h5open("data.h5","r")
    ris = read(fi,"inds",IndexSet)
    close(fi)
    @test ris == is
  end

  @testset "ITensor" begin

    # default constructed case
    T = ITensor()

    fo = h5open("data.h5","w")
    write(fo,"defaultT",T)
    close(fo)

    fi = h5open("data.h5","r")
    rT = read(fi,"defaultT",ITensor)
    close(fi)
    @test typeof(store(T)) == typeof(store(ITensor()))

    # real case
    T = randomITensor(i,j,k)

    fo = h5open("data.h5","w")
    write(fo,"T",T)
    close(fo)

    fi = h5open("data.h5","r")
    rT = read(fi,"T",ITensor)
    close(fi)
    @test norm(rT-T)/norm(T) < 1E-10

    # complex case
    T = randomITensor(ComplexF64,i,j,k)

    fo = h5open("data.h5","w")
    write(fo,"complexT",T)
    close(fo)

    fi = h5open("data.h5","r")
    rT = read(fi,"complexT",ITensor)
    close(fi)
    @test norm(rT-T)/norm(T) < 1E-10
  end

  #
  # Clean up the test hdf5 file
  #
  rm("data.h5",force=true)

end

nothing
nothing


# ---
# jupyter:
#   jupytext:
#     formats: ipynb,jl:light
#     text_representation:
#       extension: .jl
#       format_name: light
#       format_version: '1.4'
#       jupytext_version: 1.2.4
#   kernelspec:
#     display_name: Julia 1.3.0-rc4
#     language: julia
#     name: julia-1.3
# ---

# # Access Argo data using [ArgoVis](https://argovis.colorado.edu/ng/home) in Julia
#
# Citation for the Argovis web application and the Argovis database:
# Tucker, T., D. Giglio, M. Scanderbeg, and S.S.P. Shen, 0: Argovis: A Web Application for Fast Delivery, Visualization, and Analysis of Argo Data. J. Atmos. Oceanic Technol., 0, https://doi.org/10.1175/JTECH-D-19-0041.1
#
# Argo data reference
# " These data were collected and made freely available by the International Argo Program and the national programs that contribute to it. (http://www.argo.ucsd.edu, http://argo.jcommops.org). The Argo Program is part of the Global Ocean Observing System. "
# Argo (2000). Argo float data and metadata from Global Data Assembly Centre (Argo GDAC). SEANOE. http://doi.org/10.17882/42182

# +
#run(pipeline(`which python`,"whichpython.txt")) #external python path
#ENV["PYTHON"]=readline("whichpython.txt")
#import Pkg; Pkg.build("PyCall")

using PyCall

requests = pyimport("requests")
pd = pyimport("pandas")
np = pyimport("numpy")
sp = pyimport("scipy.interpolate")
griddata=sp.griddata
datetime = pyimport("datetime")
pdb = pyimport("pdb")
os = pyimport("os")
netCDF4 = pyimport("netCDF4")
netcdf_dataset=netCDF4.Dataset

# +
function get_monthly_profile_pos(month, year)
    baseURL = "https://argovis.colorado.edu/selection/profiles"
    url = baseURL*"/"*string(month)*"/"*string(year)
    resp = requests.get(url)
    monthlyProfilePos = resp.json()
    return monthlyProfilePos
end

function parse_meta_into_df(profiles)
    #initialize dict
    df = pd.DataFrame(profiles)
    return df
end

# set month and year for metadata
month = 1
year = 2004
metaProfiles = get_monthly_profile_pos(month, year)
metaDf = parse_meta_into_df(metaProfiles)
# -


export PlotShd

"""

    PlotShd(filename)

Plot transmission loss field

# Arguments
- `filename`: *.shd

# Output
- Heatmap transmission loss

# Example
```julia-repl
julia> PlotShd("Case1.shd")
```

"""
function PlotShd(filename::String;
        xlabs::String = "Distance, m",
        ylabs::String = "Transmission loss, dB",
        cblabs::String = "TL, dB",
        climb = (40,80))

        pressure, Pos_r_r, Pos_r_z = read_shd(filename)

        tlt = abs.(pressure)
        tlt = -20.0f0 * log10.(tlt)

        #seekend(io)

        heatmap(Pos_r_r[:,1],Pos_r_z[:,1],  tlt,
                clim= climb,
                c=cgrad(:Paired_6,rev = true),
                colorbartitle = cblabs,
                xlabel=xlabs, ylabel=ylabs)
        yflip!(true)

end # PlotShd function


using Abaco
using Test

interval=10
ts = nowts()

sn1 = "tn01"
sn2 = "tn02"
sn3 = "tn03"

count = 0
function onresult(ts, ne, name, value, inputs)
    global count
    count += 1
    @debug "timestamp [$ts]: ne: [$ne] $name = $value"
    if count == 1
        @test value.contribs == 1
        @test value.value == 250
    elseif count == 2
        @test value.contribs == 2
        @test value.value == 950
        @test value.expected == 3
    elseif count == 3
        @test value.contribs == 3
        @test value.value == 1950
    end
end

abaco = abaco_init(onresult, interval=interval)

@debug "starting demo with time span: $interval, values timestamp: $ts"

#setup_settings(abaco, "sensor", oncomplete=onresult)
setup_settings(abaco, "hub", oncomplete=onresult)

## If you have a lot of formulas use a csv file
#using DelimitedFiles
#datadir=joinpath(@__DIR__,"data")
#df = readdlm(joinpath(datadir,"formulas_co2.csv"), ';', header=true)
#add_formulas(abaco, df)

formula(abaco, "total_footprint", "sum(sensor.footprint)", "hub")

city = node(abaco, "trento", "hub")

node(abaco, city, sn1, "sensor")
node(abaco, city, sn2, "sensor")
node(abaco, city, sn3, "sensor")

current_footprint = get_collected(abaco, city.ne, "sensor.footprint")
@debug "initial carbon footprints: $current_footprint"

ingest(abaco, ts, sn1, Dict("footprint" => 250.0))
ingest(abaco, ts, sn2, Dict("footprint" => 700.0))

current_footprint = get_collected(abaco, city.ne, "sensor.footprint", ts)
@debug "current carbon footprint: $current_footprint"

footprint_sum = sum_collected(abaco, city.ne, "sensor.footprint", ts)
@debug "footprint sum: $footprint_sum"
@test footprint_sum.contribs == 2
@test footprint_sum.expected == 3
@test footprint_sum.value == 950

ingest(abaco, ts, sn3, Dict("footprint" => 1000.0))

footprint_sum = sum_collected(abaco, city.ne, "sensor.footprint", ts)
@debug "footprint sum: $footprint_sum"
@test footprint_sum.contribs == 3
@test footprint_sum.value == 1950

@test count == 3


abstract type SpecialArray{T,n} <: AbstractArray{T,n} end

Base.size(a::SpecialArray) = size(a.values)
Base.IndexStyle(::Type{<:SpecialArray}) = IndexLinear()
#Base.linearindexing{T<:SpecialArray}(::Type{T}) = Base.LinearFast()
Base.getindex(a::SpecialArray, i::Int) = a.values[i]
Base.convert(::Type{Array{T,n}}, x::SpecialArray) where {T,n} = x.values
Base.promote_rule(::Type{AbstractArray{T,n}},::SpecialArray{T,n}) where {T,n} =
  Type{Array{T,n}}

"""
    ParameterValuesArray{T,n}

Same as Array, but tracks if user has modified values.
"""
mutable struct ParameterValuesArray{T,n} <: SpecialArray{T,n}
  values :: Array{T,n}
  ismodified :: Bool
end
function Base.setindex!(a::ParameterValuesArray, v, i::Int)
  if ~isfinite(v)
    throw(DomainError("$v is not valid LTspice parameter value"))
  end
  a.ismodified = true
  a.values[i] = v
end
Base.convert(::Type{ParameterValuesArray{T,n}}, x::AbstractArray{S,n}) where {T,n,S} =
  ParameterValuesArray{T,n}(Array{T,n}(x),true)
Base.convert(::Type{LTspice.ParameterValuesArray{T,n}}, x::LTspice.ParameterValuesArray{T,n}) where {T,n} =
  ParameterValuesArray{T,n}(Array{T,n}(x),true)
"""
    MeasurementValuesArray{T,n}

Same as Array, but setindex! returns error.
"""
mutable struct MeasurementValuesArray{T,n} <: SpecialArray{T,n}
  values :: Array{T,n}
end
function Base.setindex!(a::MeasurementValuesArray, ::Any, ::Any)
  throw(ErrorException("Measurments may not be set.  Run simulation to update."))
end


import Base: copy, copyto!, ==
import Random: rand, seed!
import RandomNumbers: gen_seed, union_uint, seed_type, unsafe_copyto!, unsafe_compare

"""
```julia
ARS1x{R} <: R123Generator1x{UInt128}
ARS1x([seed, R=7])
```

ARS1x is one kind of ARS Counter-Based RNGs. It generates one `UInt128` number at a time.

`seed` is an `Integer` which will be automatically converted to `UInt128`.

`R` denotes to the Rounds which should be at least 1 and no more than 10. With 7 rounds (by default), it has
a considerable safety margin over the minimum number of rounds with no known statistical flaws, but still has
excellent performance.

Only available when [`R123_USE_AESNI`](@ref).
"""
mutable struct ARS1x{R} <: R123Generator1x{UInt128}
    x::UInt128
    ctr::UInt128
    key::UInt128
end

function ARS1x(seed::Integer=gen_seed(UInt128), R::Integer=7)
    @assert 1 <= R <= 10
    r = ARS1x{Int(R)}(0, 0, 0)
    seed!(r, seed)
end

function seed!(r::ARS1x, seed::Integer=gen_seed(UInt128))
    r.key = seed % UInt128
    r.ctr = 0
    random123_r(r)
    r
end

@inline seed_type(::Type{ARS1x{R}}) where R = UInt128

for R = 1:10
    @eval @inline function ars1xm128i(r, ::Type{Val{$R}})
        p = Ptr{UInt128}(pointer_from_objref(r))
        ccall(($("ars1xm128i$R"), librandom123), Nothing, (
        Ptr{UInt128}, Ptr{UInt128}, Ptr{UInt128}
        ), p + 16, p + 32, p)
        unsafe_load(p, 1)
    end
end

copyto!(dest::ARS1x{R}, src::ARS1x{R}) where R = unsafe_copyto!(dest, src, UInt128, 3)

copy(src::ARS1x{R}) where R = ARS1x{R}(src.x, src.key, src.ctr)

==(r1::ARS1x{R}, r2::ARS1x{R}) where R = unsafe_compare(r1, r2, UInt128, 3)

@inline function random123_r(r::ARS1x{R}) where R
    ars1xm128i(r, Val{R})
    (r.x,)
end

"""
```julia
ARS4x{R} <: R123Generator4x{UInt32}
ARS4x([seed, R=7])
```

ARS4x is one kind of ARS Counter-Based RNGs. It generates four `UInt32` numbers at a time.

`seed` is a `Tuple` of four `Integer`s which will all be automatically converted to `UInt32`.

`R` denotes to the Rounds which must be at least 1 and no more than 10. With 7 rounds (by default), it has a
considerable safety margin over the minimum number of rounds with no known statistical flaws, but still has
excellent performance.

Only available when [`R123_USE_AESNI`](@ref).
"""
mutable struct ARS4x{R} <: R123Generator4x{UInt32}
    x1::UInt32
    x2::UInt32
    x3::UInt32
    x4::UInt32
    ctr1::UInt128
    key::UInt128
    p::Int
end

function ARS4x(seed::NTuple{4, Integer}=gen_seed(UInt32, 4), R::Integer=7)
    @assert 1 <= R <= 10
    r = ARS4x{Int(R)}(0, 0, 0, 0, 0, 0, 0)
    seed!(r, seed)
end

function seed!(r::ARS4x, seed::NTuple{4, Integer}=gen_seed(UInt32, 4))
    r.key = union_uint(map(x -> x % UInt32, seed))
    r.ctr1 = 0
    p = 0
    random123_r(r)
    r
end

@inline seed_type(::Type{ARS4x{R}}) where R = NTuple{4, UInt32}

@inline function random123_r(r::ARS4x{R}) where R
    ars1xm128i(r, Val{R})
    (r.x1, r.x2, r.x3, r.x4)
end

function copyto!(dest::ARS4x{R}, src::ARS4x{R}) where R
    unsafe_copyto!(dest, src, UInt128, 3)
    dest.p = src.p
    dest
end

copy(src::ARS4x{R}) where R = ARS4x{R}(src.x1, src.x2, src.x3, src.x4, src.key, src.ctr1, src.p)

==(r1::ARS4x{R}, r2::ARS4x{R}) where R = unsafe_compare(r1, r2, UInt128, 3) && r1.p == r2.p


# This file is a part of Julia. License is MIT: https://julialang.org/license

make_value(::Type{T}, i::Integer) where {T<:Integer} = 3*i%T
make_value(::Type{T},i::Integer) where {T<:AbstractFloat} = T(3*i)

Vec{N,T} = NTuple{N,Base.VecElement{T}}

# Crash report for #15244 motivated this test.
@generated function thrice_iota(::Type{Vec{N,T}}) where {N,T}
    :(tuple($([:(Base.VecElement(make_value($T,$i))) for i in 1:N]...)))
end

function call_iota(n::Integer,t::DataType)
    x = thrice_iota(Vec{n,t})
    @test x[1].value === make_value(t,1)
    @test x[n].value === make_value(t,n)
end

# Try various tuple lengths and element types
for i=1:20
    for t in [Bool, Int8, Int16, Int32, Int64, Float32, Float64]
        call_iota(i,t)
    end
end

# Try various large tuple lengths and element types #20961
for i in (34, 36, 48, 64, 72, 80, 96)
    for t in [Bool, Int8, Int16, Int32, Int64, Float32, Float64]
        call_iota(i,t)
    end
end

# Another crash report for #15244 motivated this test.
struct Bunch{N,T}
    elts::NTuple{N,Base.VecElement{T}}
end

unpeel(x) = x.elts[1].value

@test unpeel(Bunch{2,Float64}((Base.VecElement(5.0),
                               Base.VecElement(4.0)))) === 5.0

rewrap(x) = VecElement(x.elts[1].value+0)
b = Bunch((VecElement(1.0), VecElement(2.0)))

@test rewrap(b)===VecElement(1.0)

struct Herd{N,T}
    elts::NTuple{N,Base.VecElement{T}}
    Herd{N,T}(elts::NTuple{N,T}) where {N,T} = new(ntuple(i->Base.VecElement{T}(elts[i]), N))
end

function check(x::Herd{N,T}) where {N,T}
    for i=1:N
        @test x.elts[i].value === N*N+i-1
    end
end

check(Herd{1,Int}((1,)))
check(Herd{2,Int}((4,5)))
check(Herd{4,Int}((16,17,18,19)))

struct Gr{N, T}
    u::T
    v::Bunch{N,T}
    w::T
end

a = Vector{Gr{2,Float64}}(2)
a[2] = Gr(1.0, Bunch((VecElement(2.0), VecElement(3.0))), 4.0)
a[1] = Gr(5.0, Bunch((VecElement(6.0), VecElement(7.0))), 8.0)
@test a[2] == Gr(1.0, Bunch((VecElement(2.0), VecElement(3.0))), 4.0)

@test isa(VecElement((1,2)), VecElement{Tuple{Int,Int}})

# The following test mimic SIMD.jl
const _llvmtypes = Dict{DataType, String}(
    Float64 => "double",
    Float32 => "float",
    Int32 => "i32",
    Int64 => "i64"
)

@generated function vecadd(x::Vec{N, T}, y::Vec{N, T}) where {N, T}
    llvmT = _llvmtypes[T]
    func = T <: AbstractFloat ? "fadd" : "add"
    exp = """
    %3 = $(func) <$(N) x $(llvmT)> %0, %1
    ret <$(N) x $(llvmT)> %3
    """
    return quote
        Base.@_inline_meta
        Base.llvmcall($exp, Vec{$N, $T}, Tuple{Vec{$N, $T}, Vec{$N, $T}}, x, y)
    end
end

function f20961(x::Vector{Vec{N, T}}, y::Vector{Vec{N, T}}) where{N, T}
    @inbounds begin
        a = x[1]
        b = y[1]
        return vecadd(a, b)
    end
end

# Test various SIMD Vectors with known good sizes
for T in (Float64, Float32, Int64, Int32)
    for N in 1:36
        # For some vectortypes Julia emits llvm arrays instead of vectors
        if N %  7 == 0 || N % 11 == 0 || N % 13 == 0 || N % 15 == 0 ||
           N % 19 == 0 || N % 23 == 0 || N % 25 == 0 || N % 27 == 0 ||
           N % 29 == 0 || N % 31 == 0
            continue
        end
        a = ntuple(i->VecElement(T(i)), N)
        result = ntuple(i-> VecElement(T(i+i)), N)
        b = vecadd(a, a)
        @test b == result
        b = f20961([a], [a])
        @test b == result
    end
end


function test_deprecated()
  @testset ExtendedTestSet "Deprecated methods" begin
    n = 10
    mem = 3
    T = Float16
    @test_deprecated LBFGSOperator(n, mem, scaling = false)
    @test_deprecated InverseLBFGSOperator(n, mem, scaling = false)
    @test_deprecated LBFGSOperator(T, n, mem, scaling = false)
    @test_deprecated InverseLBFGSOperator(T, n, mem, scaling = false)
    @test_deprecated LSR1Operator(n, mem, scaling = false)
    @test_deprecated LSR1Operator(T, n, mem, scaling = false)
  end
end

test_deprecated()


#===============================================================================

                        Definitions of Workspaces
        Defines *structs*:
        - StandardGlobalSubworkspace - convenience container that is most likely
                             going to be used even with custom global Workspaces
        - StandardLocalSubworkspace - convenience container that is most likely
                             going to be used even with custom local Workspaces
        - GenericGlobalWorkspace - simple global Workspace for simple problems
        - GenericLocalWorkspace - simple local Workspace for simple problems
        As well as functions:
        - init_global_workspace
        - create_workspaces

===============================================================================#

#                              GLOBAL WORKSPACE
#-------------------------------------------------------------------------------

#=        |-------------------------------------------------------|
          |    each custom GlobalWorkspace MUST overwrite these:  |
          |-------------------------------------------------------|           =#
"""
    init_global_workspace(
        ::MCMCBackend,
        num_mcmc_steps,
        updates::Vector{<:MCMCUpdate},
        data,
        θinit::Vector{T};
        kwargs...
    ) where T

Initialize the `<custom>GlobalWorkspace`. `<custom>MCMCBackend` points to which
`GlobalWorkspace` constructors to use, `updates` is a list of MCMC updates,
`θinit` is the initial guess for the parameter and `kwargs` are the named
arguments passed to `run!`.
"""
function init_global_workspace(
        ::MCMCBackend,
        num_mcmc_steps,
        updates::Vector{<:MCMCUpdate},
        data,
        θinit::Vector{T};
        kwargs...
    ) where T
    error("init_global_workspace not implemented")
end

"""
    loglikelihood(ws::GlobalWorkspace, ::Proposal)

Evaluate the loglikelihood for the proposal Law and observations stored in a
global workspace.
"""
function Distributions.loglikelihood(ws::GlobalWorkspace, ::Proposal)
    error(
        string(
            "loglikelihood(ws::GlobalWorkspace, ::Proposal) for a custom ",
            "GlobalWorkspace is not implemented"
        )
    )
end

"""
    loglikelihood(ws::GlobalWorkspace, ::Previous)

Evaluate the loglikelihood for the accepted Law and observations stored in a
global workspace.
"""
function Distributions.loglikelihood(ws::GlobalWorkspace, ::Previous)
    error(
        string(
            "loglikelihood(ws::GlobalWorkspace, ::Previous) for a custom ",
            "GlobalWorkspace is not implemented"
        )
    )
end

#=
        for custom GlobalWorkspaces these methods will work so long as
        they contain sub_ws::StandardGlobalSubworkspace, otherwise they
        need to be overwritten.
                                                                              =#

"""
    num_mcmc_steps(ws::GlobalWorkspace)

Return the total set number of MCMC iterations.
"""
num_mcmc_steps(ws::GlobalWorkspace) = num_mcmc_steps(ws.sub_ws)

"""
    state(ws::GlobalWorkspace)

Return currently accepted state of the chain.
"""
state(ws::GlobalWorkspace) = state(ws.sub_ws)

"""
    state(ws::GlobalWorkspace, step)

Return the state of the chain accepted at the the `step` iteration of the Markov
chain.
"""
state(ws::GlobalWorkspace, step) = state(ws.sub_ws, step)

"""
    state°(ws::GlobalWorkspace, step)

Return the state of the chain proposed at the the `step` iteration of the Markov
chain.
"""
state°(ws::GlobalWorkspace, step) = state°(ws.sub_ws, step)

"""
    num_updt(ws::GlobalWorkspace)

Return the total set number of MCMC updates that may be performed at each MCMC
iteration.
"""
num_updt(ws::GlobalWorkspace) = num_updt(ws.sub_ws)

"""
    estim_mean(ws::GlobalWorkspace)

Return the empirical mean of the parameter.
"""
estim_mean(ws::GlobalWorkspace) = estim_mean(ws.sub_ws)

"""
    estim_cov(ws::GlobalWorkspace)

Return the empirical covariance of the parameter.
"""
estim_cov(ws::GlobalWorkspace) = estim_cov(ws.sub_ws)

#=
                            standard sub-Workspace
                                                                              =#

"""
    struct StandardGlobalSubworkspace{T,TD} <: GlobalWorkspace{T}
        state::Vector{T}
        state_history::Vector{Vector{Vector{T}}}
        state_proposal_history::Vector{Vector{Vector{T}}}
        data::TD
        stats::GenericChainStats{T}
    end

Standard containers expected to be present in every global workspace. `state` is
the currently accepted parameter θ, `state_history` is a chain of `state`s that
have been accepted and `state_proposal_history` is a chain of `state`s that have
been proposed. `data` are the data passed to an MCMC sampler (usually just a
pointer) and `stats` gathers some basic online information about the chain.
"""
struct StandardGlobalSubworkspace{T,TD} <: GlobalWorkspace{T}
    state::Vector{T}
    state_history::Vector{Vector{Vector{T}}}
    state_proposal_history::Vector{Vector{Vector{T}}}
    data::TD
    stats::GenericChainStats{T}

    function StandardGlobalSubworkspace(
            num_mcmc_steps,
            num_updates,
            data::TD,
            θinit::Vector{T},
        ) where {T,TD}
        M, NU = num_mcmc_steps, num_updates # name alias
        ∅s = [Vector{T}(undef, length(θinit)) for _=1:NU]
        new{T,TD}(
            deepcopy(θinit),
            [deepcopy(∅s) for _=1:M],
            [deepcopy(∅s) for _=1:M],
            data,
            GenericChainStats(θinit, NU, M),
        )
    end
end

num_mcmc_steps(ws::StandardGlobalSubworkspace) = length(ws.state_history)
state(ws::StandardGlobalSubworkspace) = ws.state
function state(ws::StandardGlobalSubworkspace, step)
    ws.state_history[step.mcmciter][step.pidx]
end
function state°(ws::StandardGlobalSubworkspace, step)
    ws.state_proposal_history[step.mcmciter][step.pidx]
end
num_updt(ws::StandardGlobalSubworkspace) = length(first(ws.state_history))
estim_mean(ws::StandardGlobalSubworkspace) = ws.stats.mean
estim_cov(ws::StandardGlobalSubworkspace) = ws.stats.cov

#=
                        simple, custom GlobalWorkspace
                                                                              =#

"""
    struct GenericGlobalWorkspace{T,TD,TL} <: GlobalWorkspace{T}
        sub_ws::StandardGlobalSubworkspace{T,TD}
        P::TL
        P°::TL
    end

Generic global workspace with `sub_ws` containing current `state` and keeping
track of its history and some basic statistics. `P` and `P°` are the target laws
with accepted and proposal `state` set as parameters.
"""
struct GenericGlobalWorkspace{T,TD,TL} <: GlobalWorkspace{T}
    sub_ws::StandardGlobalSubworkspace{T,TD}
    P::TL
    P°::TL
end

function init_global_workspace(
        ::GenericMCMCBackend,
        num_mcmc_steps,
        updates::Vector{<:MCMCUpdate},
        data,
        θinit::Vector{T};
        kwargs...
    ) where T
    sub_ws = StandardGlobalSubworkspace(
        num_mcmc_steps,
        length(updates),
        data,
        θinit,
    )
    GenericGlobalWorkspace{T,typeof(data),typeof(data.P)}(
        sub_ws,
        deepcopy(data.P),
        deepcopy(data.P),
    )
end

function Distributions.loglikelihood(ws::GenericGlobalWorkspace, ::Proposal)
    loglikelihood(ws.P°, ws.sub_ws.data.obs)
end

#=
                generic methods for all GlobalWorkspaces
                                                                              =#

function Base.summary(ws::GlobalWorkspace; init=false)
    summary(Base.stdout, ws, Val(init))
end

Base.summary(io::IO, ws::GlobalWorkspace; init=false) = summary(io, ws, Val(init))

function Base.summary(io::IO, ws::GlobalWorkspace, ::Val{true})
    println(io, "Number of MCMC iterations: ", num_mcmc_steps(ws))
    println(io, "Initial θ guess: ", state(ws))
    println(io, "Number of updates at each MCMC iteration: ", num_updt(ws))
    println(io, "Chosen `GlobalWorkspace:` ", remove_curly(typeof(ws)))
end

function Base.summary(io::IO, ws::GlobalWorkspace, ::Val{false})
    println(io, "Number of MCMC iterations: ", num_mcmc_steps(ws))
    println(io, "Estimated E[θ]: ", estim_mean(ws))
    println(io, "Estimated Cov[θ]: ")
    show(io, "text/plain", estim_cov(ws))
end

#                              LOCAL WORKSPACE
#-------------------------------------------------------------------------------

#=
        for custom LocalWorkspaces these methods MUST be overwritten:
                                                                              =#
"""
    create_workspace(
        ::MCMCBackend,
        mcmcupdate,
        global_ws::GlobalWorkspace,
        num_mcmc_steps
    ) where {T}

Create a local workspace for a given `mcmcupdate`.
"""
function create_workspace(
        ::MCMCBackend,
        mcmcupdate,
        global_ws::GlobalWorkspace,
        num_mcmc_steps
    ) where {T}
    error("create_workspace (localworkspace) is not defined.")
end

"""
    accepted(ws::LocalWorkspace, i::Int)

Return boolean for whether the ith update has been accepted.
"""
function accepted(ws::LocalWorkspace, i::Int)
    error("accepted(ws::LocalWorkspace, i) is not implemented")
end

"""
    set_accepted!(ws::LocalWorkspace, i::Int, v)

Set boolean for whether the ith update has been accepted.
"""
set_accepted!(ws::LocalWorkspace, i::Int, v) = (ws.acceptance_history[i] = v)

#=
        for custom LocalWorkspaces these methods will work so long as
        they contain sub_ws::StandardGlobalSubworkspace and
        sub_ws°::StandardGlobalSubworkspace, otherwise they need to be
        overwritten.
                                                                              =#
"""
    ll(ws::LocalWorkspace)

Return log-likelihood of the currently accepted parameter.
"""
ll(ws::LocalWorkspace) = ll(ws.sub_ws)

"""
    ll°(ws::LocalWorkspace)

Return log-likelihood of the currently proposed parameter.
"""
ll°(ws::LocalWorkspace) = ll(ws.sub_ws°)

"""
    ll(ws::LocalWorkspace, i::Int)

Return log-likelihood of the ith accepted parameter.
"""
ll(ws::LocalWorkspace, i::Int) = ll(ws.sub_ws, i)

"""
    ll°(ws::LocalWorkspace, i::Int)

Return log-likelihood of the ith proposed parameter.
"""
ll°(ws::LocalWorkspace, i::Int) = ll(ws.sub_ws°)

"""
    state(ws::LocalWorkspace)

Return the last accepted state.
"""
state(ws::LocalWorkspace) = state(ws.sub_ws)

"""
    state°(ws::LocalWorkspace)

Return the last proposed state.
"""
state°(ws::LocalWorkspace) = state(ws.sub_ws°)

#=
        for custom LocalWorkspaces these are well defined, but may
        not make sense:
                                                                              =#
"""
    create_workspaces(v::MCMCBackend, mcmc::MCMC)

Create local workspaces, one for each update.
"""
function create_workspaces(backend::MCMCBackend, mcmc::MCMC)
    [
        create_workspace(
            backend,
            mcmc.updates[i],
            mcmc.workspace,
            mcmc.schedule.num_mcmc_steps
        ) for i in 1:mcmc.schedule.num_updates
    ]
end

"""
    llr(ws::LocalWorkspace, i::Int)

Compute log-likelihood ratio at ith mcmc iteration.
"""
llr(ws::LocalWorkspace, i::Int) = sum(ll°(ws, i) .- ll(ws, i))

"""
    name_of_update(ws::LocalWorkspace)

Return the name of the update.
"""
name_of_update(ws::LocalWorkspace) = "unknown name"

#=
                        standard local sub-Workspace
                                                                              =#

"""
    struct StandardLocalSubworkspace{T} <: LocalWorkspace{T}
        state::Vector{T}
        ll::Vector{Float64}
        ll_history::Vector{Vector{Float64}}
        ∇ll::Vector{Vector{Float64}}
        momentum::Vector{Float64}
    end

Standard containers likely to be present in every local workspace. `state` is
the currently accepted *subset* of parameter θ that the corresponding update
operates on, `ll` is the corresponding log-likelihood and `ll_history` is the
chain of log-likelihoods. A single `ll` is of the type `Vector{Float64}` to
reflect the fact that a problem might admit a natural factorisation into
independent components that may be operated on independently, in parallel with
each entry in `ll` corresponding to a separate component. `∇ll` is the gradient
of log-likelihood (needed by gradient-based algorithms) and `momentum` is the
variable needed for the Hamiltionan dynamics. `∇ll` and `momentum` may be simply
left untouched if the problem does not need them.
"""
struct StandardLocalSubworkspace{T} <: LocalWorkspace{T}
    state::Vector{T}
    ll::Vector{Float64}
    ll_history::Vector{Vector{Float64}}
    ∇ll::Vector{Vector{Float64}}
    momentum::Vector{Float64}

    function StandardLocalSubworkspace(
            state::Vector{T}, M; num_indep_blocks=1
        ) where T
        new{T}(
            deepcopy(state),
            Float64[-Inf for _=1:num_indep_blocks],
            [zeros(Float64, num_indep_blocks) for _=1:M],
            [zeros(Float64, length(state)) for _=1:num_indep_blocks],
            zero(state),
        )
    end
end

ll(ws::StandardLocalSubworkspace) = ws.ll
ll(ws::StandardLocalSubworkspace, i::Int) = ws.ll_history[i]
state(ws::StandardLocalSubworkspace) = ws.state

#=
                        simple, custom LocalWorkspace
                                                                              =#

"""
    struct GenericLocalWorkspace{T} <: LocalWorkspace{T}
        sub_ws::StandardLocalSubworkspace{T}
        sub_ws°::StandardLocalSubworkspace{T}
        acceptance_history::Vector{Bool}
    end

Generic local workspace with `sub_ws` containing a subset of `state` that the
corresponding updates operates on, with currently accepted value of `state` as
well as its log-likelihood. `sub_ws°` corresponds to a proposal `state`.
`acceptance_history` keeps track of accept/reject decisions.
"""
struct GenericLocalWorkspace{T} <: LocalWorkspace{T}
    sub_ws::StandardLocalSubworkspace{T}
    sub_ws°::StandardLocalSubworkspace{T}
    acceptance_history::Vector{Bool}
    updt_name::String
end

function create_workspace(
        ::GenericMCMCBackend,
        mcmcupdate,
        global_ws::GenericGlobalWorkspace{T},
        num_mcmc_steps
    ) where {T}
    _state = state(global_ws, mcmcupdate)[:]
    GenericLocalWorkspace{T}(
        StandardLocalSubworkspace(_state, num_mcmc_steps),
        StandardLocalSubworkspace(_state, num_mcmc_steps),
        Vector{Bool}(undef, num_mcmc_steps),
        string(remove_curly(typeof(mcmcupdate))),
    )
end

accepted(ws::GenericLocalWorkspace, i::Int) = ws.acceptance_history[i]
name_of_update(ws::GenericLocalWorkspace) = ws.updt_name


using CSV
using DataFrames
using YAML

using SLiDE  # see src/SLiDE.jl

"""
# 3: Edit data
Standardize data read from csv file into DataFrame and manipulated using the
`edit_with` function from the SLiDE module.

This example will most-likely eventually be incorporated into some build stream method.
Relevant functions can be found in the associated files:
- load_from() - src/parse/load_data.jl
- edit_with() - src/parse/edit_data.jl
- read_file() - src/parse/load_data.jl
"""

DATA_DIR = joinpath("tests", "data")

y = read_file(joinpath(DATA_DIR, "test_datastream.yml"))
df = read_file(joinpath(DATA_DIR, "test_datastream.csv"))

# TERNARY OPERATORS are a type of CONDITIONAL EVALUATION and can be used to write
# single-line if-then-else statements:
#   <if> ? <then> : <else>
# && is a type of SHORT-CIRCUIT EVALUATION that can be used to write
# single-line if-then statements:
#   <if> && (<else>)
# See:
# https://discourse.julialang.org/t/style-question-ternary-operator-or-short-circuit-operator-or-if-end/34224/2
# https://docs.julialang.org/en/v1/manual/control-flow/#man-conditional-evaluation-1
# https://docs.julialang.org/en/v1/manual/control-flow/#Short-Circuit-Evaluation-1
"Drop"     in keys(y) && (df = edit_with(df, y["Drop"]))
"Rename"   in keys(y) && (df = edit_with(df, y["Rename"]))
"Group"    in keys(y) && (df = edit_with(df, y["Group"]))
"Match"    in keys(y) && (df = edit_with(df, y["Match"]))
"Melt"     in keys(y) && (df = edit_with(df, y["Melt"]))
"Add"      in keys(y) && (df = edit_with(df, y["Add"]))
"Map"      in keys(y) && (df = edit_with(df, y["Map"]))
"Replace"  in keys(y) && (df = edit_with(df, y["Replace"]))
"Drop"     in keys(y) && (df = edit_with(df, y["Drop"]))
"Operate"  in keys(y) && (df = edit_with(df, y["Operate"]))
"Describe" in keys(y) && (df = edit_with(df, y["Describe"], y["CSVInput"]))
"Order"    in keys(y) && (df = edit_with(df, y["Order"]))

show(df)


function tsort(nvertices, egress)
    # TODO could output list of independent nodes, such that parallel eval of
    # the lists are possible.
    color = zeros(Int, nvertices)
    order = Vector{Int}()
    stack = Vector{Int}()
    for v in 1:nvertices
        color[v] != 0 && continue
        push!(stack, v)
        while !isempty(stack)
            cur = stack[end]
            color[cur] = 1
            for child in egress(cur)
                assert(color[child] != 1)  # cycle (not a DAG)
                color[child] != 0 && continue
                push!(stack, child)
            end
            if cur == stack[end]
                color[cur] = 2
                push!(order, cur)
                pop!(stack)
            end
        end
    end
    reverse(order)
end

struct EvalTSort end
# TODO maybe pass Dag, assume static or handle Dag version number
# TODO this also assume Dag node are number 1 to N (start at 1)

# TODO fire interface should accept multiple nodes at once (event group)
function fire(eve::EvalTSort, d::Dag, src_nid, src_v)
    # TODO assume Dag is static (or at least no new node can fire during fire)
    #      SimpleTrait could help here
    # TODO caching of tsort result, reuse of `firing` memory etc
    t_order = tsort(length(d.nodes), nid -> [n for (n,_) in d.links[nid] if n != 0])

    # TODO maybe have array of nullable instead of dict, and iterate in zip
    firing = Dict{Int, Any}(src_nid=>src_v)
    for nid in t_order
        !haskey(firing, nid) && continue

        nid_v = firing[nid]
        # TODO could remove from firing (and stop when firing is empty)

        targets = Base.get(d.links, nid, [])
        for (dst_nid, f) in targets
            # TODO ugly hack to register external callback
            if dst_nid == 0
                f(nid_v)
                continue
            end

            # TODO store dst directly in targets?
            dst = d.nodes[dst_nid]
            # assert dst must be in rest of t_order

            tv = f(nid_v)  # feed builder edge buffer
            if !isnull(tv)
                # nodes are evaluated in sequence, eliding previous values
                # i.e. if a node produce multiple values (triggered by multiple
                # parents) only the latest produced value is kept.
                firing[dst.nid] = get(tv)
            end
        end
    end
end


# tests to be run on a computer where LTspice is installed.
using LTspice
using Test

function localtests()

  test1 = LTspiceSimulation("test1.asc",tempdir=true)
  v = 20.0
  r = 2.0
  i = test1(v,r)[1]
  @test i == 10.0

  testinc1 = LTspiceSimulation("testInc1.asc",tempdir=true)
  @test testinc1["incA"] == 1.0
  @test testinc1["incB"] == 2.0
  @test testinc1["incC"] == 3.0
  @test testinc1["incD"] == 4.0
  @test testinc1["incE"] == 5.0
  @test testinc1["incF"] == 6.0

  test14 = LTspiceSimulation("test14.asc",tempdir=true)
  testarray = [1.0,2.0]
  for a in testarray
    for b in 10*testarray
      for d in 100*testarray
        for e in 1000*testarray
          for this_is_a_long_name in 10000*testarray
            test14["a"] = a
            test14["b"] = b
            test14["d"] = d
            test14["e"] = e
            test14["this_is_a_long_name"] = this_is_a_long_name
            @test test14["m1"] == a + b + d + e + this_is_a_long_name
          end
        end
      end
    end
  end

end
localtests()


struct Environments
    I::MPS
    H::MPS
    InProgress::Matrix{ITensor}
end

function buildEdgeEnvironment(A::PEPS, H, left_H_terms, next_combiners, side::Symbol, col::Int; kwargs...)::Environments
    Ny, Nx              = size(A)
    is_cu::Bool         = is_gpu(A)
    up_combiners        = Vector{ITensor}(undef, Ny-1)
    fake_next_combiners = Vector{ITensor}(undef, Ny)
    fake_prev_combiners = is_cu ? fill(cuITensor(1.0), Ny) : fill(ITensor(1.0), Ny)
    I_mpo, fake_next_combiners, up_combiners = buildNewI(A, col, fake_prev_combiners, side)
    I_mps          = MPS(Ny, tensors(I_mpo), 0, Ny+1)
    @debug "Built new I"
    copyto!(next_combiners, fake_next_combiners)
    field_H_terms  = getDirectional(vcat(H[:, col]...), Field)
    vert_H_terms   = getDirectional(vcat(H[:, col]...), Vertical)
    vHs            = [buildNewVerticals(A, fake_prev_combiners, fake_next_combiners, up_combiners, vert_H_terms[vert_op], col) for vert_op in 1:length(vert_H_terms)]
    @debug "Built new Vs"
    fHs            = [buildNewFields(A, fake_prev_combiners, fake_next_combiners, up_combiners, field_H_terms[field_op], col) for field_op in 1:length(field_H_terms)]
    Hs             = MPS[MPS(Ny, tensors(H_term), 0, Ny+1) for H_term in vcat(vHs, fHs)]
    @inbounds for row in 1:Ny-1
        ci = linkindex(I_mps, row)
        ni = Index(ITensors.dim(ci), "u,Link,c$col,r$row")
        replaceindex!(I_mps[row], ci, ni)
        replaceindex!(I_mps[row+1], ci, ni)
    end
    maxdim::Int     = get(kwargs, :maxdim, 1)
    cutoff::Float64 = get(kwargs, :cutoff, 0.0)
    env_maxdim::Int = get(kwargs, :env_maxdim, maxdim)
    H_overall       = sum(Hs; cutoff=cutoff, maxdim=env_maxdim)
    @debug "Summed Hs, maxdim=$maxdim"
    side_H       = side == :left ? H[:, col] : H[:, col - 1]
    side_H_terms = getDirectional(vcat(side_H...), Horizontal)
    @debug "Trying to alloc $(Ny*length(side_H_terms))"
    in_progress  = Matrix{ITensor}(undef, Ny, length(side_H_terms))
    @inbounds for side_term in 1:length(side_H_terms)
        @debug "Generating edge bonds for term $side_term"
        in_progress[1:Ny, side_term] = generateEdgeDanglingBonds(A, up_combiners, side_H_terms[side_term], side, col)
    end
    @debug "Generated edge bonds"
    return Environments(I_mps, H_overall, in_progress)
end

function buildNextEnvironment(A::PEPS, prev_Env::Environments, H, previous_combiners::Vector{ITensor}, next_combiners::Vector{ITensor}, side::Symbol, col::Int; kwargs...)
    Ny, Nx = size(A)
    working_combiner = Vector{ITensor}(undef, Ny)
    up_combiners     = Vector{ITensor}(undef, Ny-1)
    @timeit "build I" begin
        I_mpo, working_combiner, up_combiners = buildNewI(A, col, previous_combiners, side)
    end
    @debug "Built I_mpo"
    copyto!(next_combiners, working_combiner)
    @inbounds for row in 1:Ny-1
        ci = linkindex(I_mpo, row)
        ni = Index(ITensors.dim(ci), "u,Link,c$col,r$row")
        replaceindex!(I_mpo[row], ci, ni)
        replaceindex!(I_mpo[row+1], ci, ni)
    end
    cutoff::Float64 = get(kwargs, :cutoff, 1e-13)
    maxdim::Int     = get(kwargs, :maxdim, 1)
    env_maxdim::Int = get(kwargs, :env_maxdim, maxdim)
    @timeit "build new_I and new_H" begin
        new_I     = applyMPO(I_mpo, prev_Env.I; cutoff=cutoff, maxdim=env_maxdim)
        new_H     = applyMPO(I_mpo, prev_Env.H; cutoff=cutoff, maxdim=env_maxdim)
    end
    @debug "Built new I and H"
    field_H_terms = getDirectional(vcat(H[:, col]...), Field)
    vert_H_terms  = getDirectional(vcat(H[:, col]...), Vertical)
    hori_H_terms  = getDirectional(vcat(H[:, col]...), Horizontal)
    side_H        = side == :left ? H[:, col] : H[:, col - 1]
    side_H_terms  = getDirectional(vcat(side_H...), Horizontal)
    H_term_count  = 1 + length(field_H_terms) + length(vert_H_terms)
    H_term_count += (side == :left ? length(side_H_terms) : length(hori_H_terms))
    final_H       = deepcopy(new_H)
    new_H_mps     = Vector{MPS}(undef, H_term_count)
    new_H_mps[1]  = deepcopy(new_H)
    @timeit "build new verts" begin
        vHs = [buildNewVerticals(A, previous_combiners, next_combiners, up_combiners, vert_H_terms[vert_op], col) for vert_op in 1:length(vert_H_terms)]
    end
    @debug "Built new Vs"
    @timeit "build new fields" begin
        fHs = [buildNewFields(A, previous_combiners, next_combiners, up_combiners, field_H_terms[field_op], col) for field_op in 1:length(field_H_terms)]
    end
    @timeit "build new H array" begin
        for (vv, vH) in enumerate(vHs)
            new_H_mps[1+vv] = applyMPO(vH, prev_Env.I; cutoff=cutoff, maxdim=env_maxdim)
        end
        for (ff, fH) in enumerate(fHs)
            new_H_mps[1+length(vHs)+ff] = applyMPO(fH, prev_Env.I; cutoff=cutoff, maxdim=env_maxdim)
        end
    end
    connect_H    = side == :left ? side_H_terms : hori_H_terms
    @timeit "connect dangling bonds" begin
        for (cc, cH) in enumerate(connect_H)
            new_H = connectDanglingBonds(A, next_combiners, up_combiners, cH, prev_Env.InProgress[:, cc], side, -1, col; kwargs...)
            new_H_mps[length(vert_H_terms) + length(field_H_terms) + 1 + cc] = MPS(Ny, new_H, 0, Ny+1)
        end
    end
    @debug "Connected dangling bonds"

    @timeit "sum H mps" begin
        H_overall    = sum(new_H_mps; cutoff=cutoff, maxdim=env_maxdim)
    end
    @debug "Summed Hs"
    gen_H_terms  = side == :left ? hori_H_terms : side_H_terms
    in_progress  = Matrix{ITensor}(undef, Ny, length(side_H_terms))
    @timeit "gen dangling bonds" begin
        @inbounds for side_term in 1:length(gen_H_terms)
            in_progress[1:Ny, side_term] = generateNextDanglingBonds(A, previous_combiners, next_combiners, up_combiners, gen_H_terms[side_term], prev_Env.I, side, col; kwargs...)
        end
    end
    @debug "Generated next dangling bonds"
    return Environments(new_I, H_overall, in_progress)
end

function buildNewVerticals(A::PEPS, previous_combiners::Vector, next_combiners::Vector{ITensor}, up_combiners::Vector{ITensor}, H, col::Int)::MPO
    Ny, Nx = size(A)
    is_cu               = is_gpu(A)
    col_site_inds       = [findindex(A[row, col], "Site") for row in 1:Ny]
    ops                 = ITensor[spinI(spin_ind; is_gpu=is_cu) for spin_ind in col_site_inds]
    vertical_row_a      = H.sites[1][1]
    vertical_row_b      = H.sites[2][1]
    ops[vertical_row_a] = replaceindex!(copy(H.ops[1]), H.site_ind, col_site_inds[vertical_row_a])
    ops[vertical_row_a] = replaceindex!(ops[vertical_row_a], H.site_ind', col_site_inds[vertical_row_a]')
    ops[vertical_row_b] = replaceindex!(copy(H.ops[2]), H.site_ind, col_site_inds[vertical_row_b])
    ops[vertical_row_b] = replaceindex!(ops[vertical_row_b], H.site_ind', col_site_inds[vertical_row_b]')
    internal_cmb_u      = is_cu ? vcat(cuITensor(1.0), up_combiners, cuITensor(1.0)) : vcat(ITensor(1.0), up_combiners, ITensor(1.0))
    AAs            = Vector{ITensor}(undef, Ny)
    AAs[1] = A[1, col] * ops[1] * prime(dag(A[1, col])) * next_combiners[1] * up_combiners[1] * previous_combiners[1]
    @inbounds for row in 2:Ny-1
        AAs[row] = A[row, col] * ops[row] * prime(dag(A[row, col])) * next_combiners[row] * previous_combiners[row] * up_combiners[row-1] * up_combiners[row]
    end
    AAs[Ny] = A[Ny, col] * ops[Ny] * prime(dag(A[Ny, col])) * next_combiners[Ny] * previous_combiners[Ny] * up_combiners[Ny-1]
    return MPO(Ny, AAs, 0, Ny+1)
end

function buildNewFields(A::PEPS, previous_combiners::Vector, next_combiners::Vector{ITensor}, up_combiners::Vector{ITensor}, H, col::Int)::MPO
    Ny, Nx = size(A)
    col_site_inds  = [findindex(A[row, col], "Site") for row in 1:Ny]
    is_cu         = is_gpu(A)
    ops            = ITensor[spinI(spin_ind; is_gpu=is_cu) for spin_ind in col_site_inds]
    field_row      = H.sites[1][1]
    ops[field_row] = replaceindex!(copy(H.ops[1]), H.site_ind, col_site_inds[field_row])
    ops[field_row] = replaceindex!(ops[field_row], H.site_ind', col_site_inds[field_row]')
    #internal_cmb_u = is_cu ? vcat(cuITensor(1.0), up_combiners, cuITensor(1.0)) : vcat(ITensor(1.0), up_combiners, ITensor(1.0))
    AAs            = Vector{ITensor}(undef, Ny)
    AAs[1] = A[1, col] * ops[1] * prime(dag(A[1, col])) * next_combiners[1] * up_combiners[1] * previous_combiners[1]
    @inbounds for row in 2:Ny-1
        AAs[row] = A[row, col] * ops[row] * prime(dag(A[row, col])) * next_combiners[row] * previous_combiners[row] * up_combiners[row-1] * up_combiners[row]
    end
    AAs[Ny] = A[Ny, col] * ops[Ny] * prime(dag(A[Ny, col])) * next_combiners[Ny] * previous_combiners[Ny] * up_combiners[Ny-1]
    #AAs            = [ A[row, col] * ops[row] * prime(dag(A[row, col])) * next_combiners[row] * previous_combiners[row] * internal_cmb_u[row] * internal_cmb_u[row+1] for row in 1:Ny ]
    return MPO(Ny, AAs, 0, Ny+1)
end

function buildNewI(A::PEPS, col::Int, previous_combiners::Vector, side::Symbol)::Tuple{MPO, Vector{ITensor}, Vector{ITensor}}
    Ny, Nx         = size(A)
    next_col       = side == :left ? col + 1 : col - 1 # side is handedness of environment
    AA             = [A[row, col] * prime(dag(A[row, col]), "Link") for row in 1:Ny]
    next_combiners = [combine(A[row, col], A[row, next_col], "Site,r$row,c$col") for row in 1:Ny]
    up_combiners   = [combine(A[row, col], A[row+1, col], "Link,CMB,c$col,r$row") for row in 1:Ny-1]
    @inbounds for row in 1:Ny
        AA[row] *= previous_combiners[row]
        AA[row] *= next_combiners[row]
        if row > 1
            AA[row] *= up_combiners[row-1]
        end
        if row < Ny
            AA[row] *= up_combiners[row]
        end
    end
    Iapp   = MPO(Ny, AA, 0, Ny+1)
    return Iapp, next_combiners, up_combiners
end

function generateEdgeDanglingBonds(A::PEPS, up_combiners::Vector{ITensor}, H, side::Symbol, col::Int)::Vector{ITensor}
    Ny, Nx         = size(A)
    is_cu         = is_gpu(A)
    op_row         = side == :left ? H.sites[1][1] : H.sites[2][1];
    H_op           = side == :left ? H.ops[1] : H.ops[2];
    col_site_inds  = [findindex(A[row, col], "Site") for row in 1:Ny]
    ops            = ITensor[spinI(spin_ind; is_gpu=is_cu) for spin_ind in col_site_inds]
    ops[op_row]    = replaceindex!(copy(H_op), H.site_ind, col_site_inds[op_row])
    ops[op_row]    = replaceindex!(ops[op_row], H.site_ind', col_site_inds[op_row]')
    #internal_cmb_u = is_cu ? vcat(cuITensor(1.0), up_combiners, cuITensor(1.0)) : vcat(ITensor(1.0), up_combiners, ITensor(1.0))
    #this_IP        = [A[row, col] * ops[row] * prime(dag(A[row, col])) * internal_cmb_u[row] * internal_cmb_u[row+1] for row in 1:Ny]
    this_IP        = Vector{ITensor}(undef, Ny)
    this_IP[1] = A[1, col] * ops[1] * prime(dag(A[1, col])) * up_combiners[1]
    for row in 2:Ny-1
        this_IP[row] = A[row, col] * ops[row] * prime(dag(A[row, col])) * up_combiners[row-1] * up_combiners[row]
    end
    this_IP[Ny] = A[Ny, col] * ops[Ny] * prime(dag(A[Ny, col])) * up_combiners[Ny-1]
    return this_IP
end

function generateNextDanglingBonds(A::PEPS, previous_combiners::Vector{ITensor}, next_combiners::Vector{ITensor}, up_combiners::Vector{ITensor}, H, Ident::MPS, side::Symbol, col::Int; kwargs...)::Vector{ITensor}
    Ny, Nx          = size(A)
    is_cu          = is_gpu(A)
    op_row          = side == :left ? H.sites[1][1] : H.sites[2][1]
    H_op            = side == :left ? H.ops[1] : H.ops[2]
    col_site_inds   = [findindex(A[row, col], "Site") for row in 1:Ny]
    ops             = ITensor[spinI(spin_ind; is_gpu=is_cu) for spin_ind in col_site_inds]
    ops[op_row]     = replaceindex!(copy(H_op), H.site_ind, col_site_inds[op_row])
    ops[op_row]     = replaceindex!(ops[op_row], H.site_ind', col_site_inds[op_row]')
    #internal_cmb_u  = is_cu ? vcat(cuITensor(1.0), up_combiners, cuITensor(1.0)) : vcat(ITensor(1.0), up_combiners, ITensor(1.0))
    #this_IP         = [A[row, col] * ops[row] * prime(dag(A[row, col])) * previous_combiners[row] * next_combiners[row] * internal_cmb_u[row] * internal_cmb_u[row+1] for row in 1:Ny]
    this_IP         = Vector{ITensor}(undef, Ny)
    this_IP[1] = A[1, col] * ops[1] * prime(dag(A[1, col])) * previous_combiners[1] * next_combiners[1] * up_combiners[1]
    for row in 2:Ny-1
        this_IP[row] = A[row, col] * ops[row] * prime(dag(A[row, col])) * previous_combiners[row] * next_combiners[row] * up_combiners[row-1] * up_combiners[row]
    end
    this_IP[Ny] = A[Ny, col] * ops[Ny] * prime(dag(A[Ny, col])) * previous_combiners[Ny] * next_combiners[Ny] * up_combiners[Ny-1]
    in_progress_MPO = MPO(Ny, this_IP, 0, Ny+1)
    cutoff::Float64 = get(kwargs, :cutoff, 1e-13)
    maxdim::Int     = get(kwargs, :maxdim, 1)
    env_maxdim::Int = get(kwargs, :env_maxdim, maxdim)
    result          = applyMPO(in_progress_MPO, Ident; cutoff=cutoff, maxdim=env_maxdim)
    return ITensor[result[row]*next_combiners[row] for row in 1:Ny]
end

function connectDanglingBonds(A::PEPS, next_combiners::Vector{ITensor}, up_combiners::Vector{ITensor}, oldH, in_progress::Vector{ITensor}, side::Symbol, work_row::Int, col::Int; kwargs...)::Vector{ITensor}
    Ny, Nx   = size(A)
    is_cu    = is_gpu(A)
    op_row_a = oldH.sites[1][1]
    op_row_b = oldH.sites[2][1]
    op       = side == :left ? oldH.ops[2] : oldH.ops[1]
    application_row = side == :left ? op_row_b : op_row_a
    col_site_inds  = [findindex(A[row, col], "Site") for row in 1:Ny]
    ops            = ITensor[spinI(spin_ind; is_gpu=is_cu) for spin_ind in col_site_inds]
    ops[application_row] = replaceindex!(copy(op), oldH.site_ind, col_site_inds[application_row])
    ops[application_row] = replaceindex!(ops[application_row], oldH.site_ind', col_site_inds[application_row]')
    #internal_cmb_u = is_cu ? vcat(cuITensor(1.0), up_combiners, cuITensor(1.0)) : vcat(ITensor(1.0), up_combiners, ITensor(1.0))
    in_prog_mps    = MPS(Ny, in_progress, 0, Ny + 1)
    this_IP        = Vector{ITensor}(undef, Ny)
    this_IP[1] = A[1, col] * ops[1] * prime(dag(A[1, col])) * next_combiners[1] * up_combiners[1]
    for row in 2:Ny-1
        this_IP[row] = A[row, col] * ops[row] * prime(dag(A[row, col])) * next_combiners[row] * up_combiners[row-1] * up_combiners[row]
    end
    this_IP[Ny] = A[Ny, col] * ops[Ny] * prime(dag(A[Ny, col])) * next_combiners[Ny] * up_combiners[Ny-1]
    #this_IP        = [A[row, col] * ops[row] * prime(dag(A[row, col])) * next_combiners[row] * internal_cmb_u[row] * internal_cmb_u[row+1] for row in 1:Ny]
    if 0 < work_row < Ny + 1
        this_IP[work_row] = ops[work_row]
    end
    completed_H = MPO(Ny, this_IP, 0, Ny+1)
    if work_row == -1
        dummy_cmbs      = [combiner(commoninds(completed_H[row], in_prog_mps[row]), tags="Site,r$row")[1] for row in 1:Ny]
        completed_H.A_  = dummy_cmbs .* tensors(completed_H)
        in_prog_mps.A_  = dummy_cmbs .* tensors(in_prog_mps)
        cutoff::Float64 = get(kwargs, :cutoff, 1e-13)
        maxdim::Int     = get(kwargs, :maxdim, 1)
        env_maxdim::Int = get(kwargs, :env_maxdim, maxdim)
        result          = applyMPO(completed_H, in_prog_mps; cutoff=cutoff, maxdim=env_maxdim)
        return tensors(result)
    else
        @inbounds for row in 1:Ny-1
            ci = linkIndex(completed_H, row)
            ni = Index(ITensors.dim(ci), "u,Link,c$col,r$row")
            replaceIndex!(completed_H[row],   ci, ni)
            replaceIndex!(completed_H[row+1], ci, ni)
        end
        return tensors(completed_H) .* tensors(in_prog_mps)
    end
end

function buildLs(A::PEPS, H; kwargs...)
    Ny, Nx = size(A)
    previous_combiners = Vector{ITensor}(undef, Ny)
    next_combiners     = Vector{ITensor}(undef, Ny)
    Ls                 = Vector{Environments}(undef, Nx)
    start_col::Int     = get(kwargs, :start_col, 1)
    if start_col == 1
        left_H_terms = getDirectional(H[1], Horizontal)
        @debug "Building left col $start_col"
        Ls[1] = buildEdgeEnvironment(A, H, left_H_terms, previous_combiners, :left, 1; kwargs...)
    elseif start_col - 1 > 1
        previous_combiners = [reconnect(commonindex(A[row, start_col], A[row, start_col - 1]), Ls[start_col-1].I[row]) for row in 1:Ny]
    end
    loop_col = start_col == 1 ? 2 : start_col
    @inbounds for col in loop_col:(Nx-1)
        @debug "Building left col $col"
        Ls[col] = buildNextEnvironment(A, Ls[col-1], H, previous_combiners, next_combiners, :left, col; kwargs...)
        previous_combiners = deepcopy(next_combiners)
    end
    return Ls
end

function buildRs(A::PEPS, H; kwargs...)
    Ny, Nx = size(A)
    previous_combiners = Vector{ITensor}(undef, Ny)
    next_combiners     = Vector{ITensor}(undef, Ny)
    start_col::Int     = get(kwargs, :start_col, Nx)
    Rs                 = Vector{Environments}(undef, Nx)
    if start_col == Nx
        right_H_terms = getDirectional(H[Nx-1], Horizontal)
        Rs[Nx]        = buildEdgeEnvironment(A, H, right_H_terms, previous_combiners, :right, Nx; kwargs...)
    elseif start_col + 1 < Nx
        previous_combiners = [reconnect(commonindex(A[row, start_col], A[row, start_col + 1]), Rs[start_col+1].I[row]) for row in 1:Ny]
    end
    loop_col = start_col == Nx ? Nx - 1 : start_col
    @inbounds for col in reverse(2:loop_col)
        @debug "Building right col $col"
        Rs[col] = buildNextEnvironment(A, Rs[col+1], H, previous_combiners, next_combiners, :right, col; kwargs...)
        previous_combiners = deepcopy(next_combiners)
    end
    return Rs
end