# This file is part of Kpax3. License is MIT.

function gibbs!(data::Matrix{UInt8},
                priorR::PriorRowPartition,
                priorC::PriorColPartition,
                settings::KSettings,
                support::MCMCSupport,
                state::AminoAcidState)
  shuffleunits!(support.u)

  for a in support.u
    resizesupport!(support, state.k + 1)
    gibbsallocate!(a, data, priorR, priorC, settings, support, state)
  end

  nothing
end

function gibbsallocate!(i::Int,
                        data::Matrix{UInt8},
                        priorR::PriorRowPartition,
                        priorC::PriorColPartition,
                        settings::KSettings,
                        support::MCMCSupport,
                        state::AminoAcidState)
  hi = state.R[i]

  initsupportgibbs!(hi, priorR, state, support)

  hj = 0

  # normalizing constant
  logc = -Inf

  # temporary variables
  li = 0
  lj = 0

  g = 0
  y = 0.0
  v = 0

  if state.v[hi] > 1
    support.vi = state.v[hi] - 1

    li = support.k + 1
    lj = support.k + 2

    support.t[li] = clusterweight(support.vi, priorR)
    support.t[lj] = clusterweight(1, state.k, priorR)

    # there are three options:
    # 1) i put back into hi (likelihood does not change)
    # 2) i put into hj (remove i from hi and compute the singleton cluster)
    # 3) i put into another existing cluster (remove i from hi and compute the
    #                                         new likelihood)

    for b in 1:support.m
      # remove i from its cluster
      support.ni[b] = state.n1s[hi, b] - float(data[b, i])

      # support.lpi and support.lpj are large enough to be used as a storage
      # support.lpi will contain the global sum of clusters not containing i
      # support.lpj will contain sums associated with hi and hj
      support.lpi[1, b] = priorC.logγ[1]
      support.lpi[2, b] = priorC.logγ[2]
      support.lpi[3, b] = priorC.logγ[3]
      support.lpi[4, b] = priorC.logγ[3]

      for l in 1:support.k
        g = support.cl[l]
        y = state.n1s[g, b] + float(data[b, i])
        v = state.v[g] + 1
        gibbsupdateclustergmove!(y, v, b, l, g, state.k, priorC, support)
      end
      gibbsupdateclusteri!(state.k, b, li, priorC, support)
      gibbsupdateclusterj!(data[b, i], state.k + 1, b, li, lj, priorC, support)
    end

    gibbscomputeprobi!(li, support)
    for b in 1:support.m
      for l in 1:support.k
        gibbscomputeprobg!(b, l, li, support)
      end
      gibbscomputeprobj!(b, li, lj, support)
    end

    logc = gibbscomputenormconst(lj, support)

    l = gibbssamplecluster(logc, lj, support)

    # if l == li do nothing
    if l <= support.k
      gibbsmove!(i, hi, support.cl[l], li, l, data, priorR, support, state)
    elseif l == lj
      gibbssplit!(i, hi, li, lj, data, priorR, settings, support, state)
    end
  else
    # li and lj are the same
    lj = support.k + 1

    support.t[lj] = clusterweight(1, support.k, priorR)

    for b in 1:support.m
      # support.lpi and support.lpj are large enough to be used as a storage
      # support.lpi will contain the global sum of clusters not containing i
      # support.lpj will contain sums associated with hi and hj
      support.lpi[1, b] = priorC.logγ[1]
      support.lpi[2, b] = priorC.logγ[2]
      support.lpi[3, b] = priorC.logγ[3]

      for l in 1:support.k
        g = support.cl[l]
        y = state.n1s[g, b] + float(data[b, i])
        v = state.v[g] + 1
        gibbsupdateclustergmerge!(y, v, b, l, g, support.k, priorC, support)
      end
    end

    gibbscomputeprobi!(lj, support)
    for b in 1:support.m, l in 1:support.k
      gibbscomputeprobg!(b, l, support)
    end

    logc = gibbscomputenormconst(lj, support)

    l = gibbssamplecluster(logc, lj, support)

    # if l == lj do nothing
    if l <= support.k
      gibbsmerge!(i, hi, support.cl[l], l, data, priorR, support, state)
    end
  end

  nothing
end

function initsupportgibbs!(hi::Int,
                           priorR::PriorRowPartition,
                           state::AminoAcidState,
                           support::MCMCSupport)
  # initialize clusters other than hi
  g = 0
  support.k = 0
  for l in 1:state.k
    g = state.cl[l]
    if g != hi
      support.k += 1
      support.cl[support.k] = g
      support.t[support.k] = clusterweight(state.v[g], priorR)
    end
  end

  nothing
end

function gibbsupdateclustergmove!(y::Float64,
                                  v::Int,
                                  b::Int,
                                  l::Int,
                                  g::Int,
                                  k::Int,
                                  priorC::PriorColPartition,
                                  support::MCMCSupport)
  support.lpi[1, b] += support.lp[1, g, b]
  support.lpi[2, b] += support.lp[2, g, b]

  support.tmp[3] = priorC.logω[k][1] + support.lp[3, g, b]
  support.tmp[4] = priorC.logω[k][2] + support.lp[4, g, b]

  support.tmp[1] = if support.tmp[3] > support.tmp[4]
    support.tmp[3] + log1p(exp(support.tmp[4] - support.tmp[3]))
  else
    support.tmp[4] + log1p(exp(support.tmp[3] - support.tmp[4]))
  end

  support.lpi[3, b] += support.tmp[1]

  # when we move unit i to the singleton cluster, the total number of clusters
  # increases by one
  support.tmp[3] = priorC.logω[k + 1][1] + support.lp[3, g, b]
  support.tmp[4] = priorC.logω[k + 1][2] + support.lp[4, g, b]

  support.lpi[4, b] += if support.tmp[3] > support.tmp[4]
    support.tmp[3] + log1p(exp(support.tmp[4] - support.tmp[3]))
  else
    support.tmp[4] + log1p(exp(support.tmp[3] - support.tmp[4]))
  end

  support.lq[1, l, b] = logmarglik(y, v, priorC.A[1, b], priorC.B[1, b])
  support.lq[2, l, b] = logmarglik(y, v, priorC.A[2, b], priorC.B[2, b])
  support.lq[3, l, b] = logmarglik(y, v, priorC.A[3, b], priorC.B[3, b])
  support.lq[4, l, b] = logmarglik(y, v, priorC.A[4, b], priorC.B[4, b])

  support.tmp[3] = priorC.logω[k][1] + support.lq[3, l, b]
  support.tmp[4] = priorC.logω[k][2] + support.lq[4, l, b]

  support.tmp[2] = if support.tmp[3] > support.tmp[4]
    support.tmp[3] + log1p(exp(support.tmp[4] - support.tmp[3]))
  else
    support.tmp[4] + log1p(exp(support.tmp[3] - support.tmp[4]))
  end

  # we added extra elements to support.lpi and they should be subtracted
  support.lr[1, l, b] = support.lq[1, l, b] - support.lp[1, g, b]
  support.lr[2, l, b] = support.lq[2, l, b] - support.lp[2, g, b]
  support.lr[3, l, b] = support.tmp[2] - support.tmp[1]

  nothing
end

function gibbsupdateclustergmerge!(y::Float64,
                                   v::Int,
                                   b::Int,
                                   l::Int,
                                   g::Int,
                                   k::Int,
                                   priorC::PriorColPartition,
                                   support::MCMCSupport)
  support.lpi[1, b] += support.lp[1, g, b]
  support.lpi[2, b] += support.lp[2, g, b]

  support.tmp[3] = priorC.logω[k][1] + support.lp[3, g, b]
  support.tmp[4] = priorC.logω[k][2] + support.lp[4, g, b]

  support.tmp[1] = if support.tmp[3] > support.tmp[4]
    support.tmp[3] + log1p(exp(support.tmp[4] - support.tmp[3]))
  else
    support.tmp[4] + log1p(exp(support.tmp[3] - support.tmp[4]))
  end

  support.lpi[3, b] += support.tmp[1]

  support.lq[1, l, b] = logmarglik(y, v, priorC.A[1, b], priorC.B[1, b])
  support.lq[2, l, b] = logmarglik(y, v, priorC.A[2, b], priorC.B[2, b])
  support.lq[3, l, b] = logmarglik(y, v, priorC.A[3, b], priorC.B[3, b])
  support.lq[4, l, b] = logmarglik(y, v, priorC.A[4, b], priorC.B[4, b])

  support.tmp[3] = priorC.logω[k][1] + support.lq[3, l, b]
  support.tmp[4] = priorC.logω[k][2] + support.lq[4, l, b]

  support.tmp[2] = if support.tmp[3] > support.tmp[4]
    support.tmp[3] + log1p(exp(support.tmp[4] - support.tmp[3]))
  else
    support.tmp[4] + log1p(exp(support.tmp[3] - support.tmp[4]))
  end

  # we added extra elements to support.lpi and they should be subtracted
  support.lr[1, l, b] = support.lq[1, l, b] - support.lp[1, g, b]
  support.lr[2, l, b] = support.lq[2, l, b] - support.lp[2, g, b]
  support.lr[3, l, b] = support.tmp[2] - support.tmp[1]

  nothing
end

function gibbsupdateclusteri!(k::Int,
                              b::Int,
                              li::Int,
                              priorC::PriorColPartition,
                              support::MCMCSupport)
  y = support.ni[b]
  v = support.vi

  support.lq[1, li, b] = logmarglik(y, v, priorC.A[1, b], priorC.B[1, b])
  support.lq[2, li, b] = logmarglik(y, v, priorC.A[2, b], priorC.B[2, b])
  support.lq[3, li, b] = logmarglik(y, v, priorC.A[3, b], priorC.B[3, b])
  support.lq[4, li, b] = logmarglik(y, v, priorC.A[4, b], priorC.B[4, b])

  support.tmp[3] = priorC.logω[k][1] + support.lq[3, li, b]
  support.tmp[4] = priorC.logω[k][2] + support.lq[4, li, b]

  support.lpj[1, b] = if support.tmp[3] > support.tmp[4]
    support.tmp[3] + log1p(exp(support.tmp[4] - support.tmp[3]))
  else
    support.tmp[4] + log1p(exp(support.tmp[3] - support.tmp[4]))
  end

  nothing
end

function gibbsupdateclusterj!(y::UInt8,
                              k::Int,
                              b::Int,
                              li::Int,
                              lj::Int,
                              priorC::PriorColPartition,
                              support::MCMCSupport)
  # add i to the singleton cluster (increases k by one)
  support.tmp[3] = priorC.logω[k][1] + support.lq[3, li, b]
  support.tmp[4] = priorC.logω[k][2] + support.lq[4, li, b]

  support.lpj[2, b] = if support.tmp[3] > support.tmp[4]
    support.tmp[3] + log1p(exp(support.tmp[4] - support.tmp[3]))
  else
    support.tmp[4] + log1p(exp(support.tmp[3] - support.tmp[4]))
  end

  support.lq[1, lj, b] = logmarglik(y, 1, priorC.A[1, b], priorC.B[1, b])
  support.lq[2, lj, b] = logmarglik(y, 1, priorC.A[2, b], priorC.B[2, b])
  support.lq[3, lj, b] = logmarglik(y, 1, priorC.A[3, b], priorC.B[3, b])
  support.lq[4, lj, b] = logmarglik(y, 1, priorC.A[4, b], priorC.B[4, b])

  support.tmp[3] = priorC.logω[k][1] + support.lq[3, lj, b]
  support.tmp[4] = priorC.logω[k][2] + support.lq[4, lj, b]

  support.lpj[3, b] = if support.tmp[3] > support.tmp[4]
    support.tmp[3] + log1p(exp(support.tmp[4] - support.tmp[3]))
  else
    support.tmp[4] + log1p(exp(support.tmp[3] - support.tmp[4]))
  end

  nothing
end

function gibbscomputeprobg!(b::Int,
                            l::Int,
                            li::Int,
                            support::MCMCSupport)
  # take into consideration that cluster hi has changed
  support.tmp[1] = support.lpi[1, b] + support.lr[1, l, b] +
                   support.lq[1, li, b]
  support.tmp[2] = support.lpi[2, b] + support.lr[2, l, b] +
                   support.lq[2, li, b]
  support.tmp[3] = support.lpi[3, b] + support.lr[3, l, b] +
                   support.lpj[1, b]

  support.t[l] += if (support.tmp[1] >= support.tmp[2]) &&
                     (support.tmp[1] >= support.tmp[3])
                    support.tmp[1] +
                    log1p(exp(support.tmp[2] - support.tmp[1]) +
                          exp(support.tmp[3] - support.tmp[1]))
                  elseif (support.tmp[2] >= support.tmp[1]) &&
                         (support.tmp[2] >= support.tmp[3])
                    support.tmp[2] +
                    log1p(exp(support.tmp[1] - support.tmp[2]) +
                          exp(support.tmp[3] - support.tmp[2]))
                  else
                    support.tmp[3] +
                    log1p(exp(support.tmp[1] - support.tmp[3]) +
                          exp(support.tmp[2] - support.tmp[3]))
                  end

  nothing
end

function gibbscomputeprobg!(b::Int,
                            l::Int,
                            support::MCMCSupport)
  # when merging, cluster hi disappears
  support.tmp[1] = support.lpi[1, b] + support.lr[1, l, b]
  support.tmp[2] = support.lpi[2, b] + support.lr[2, l, b]
  support.tmp[3] = support.lpi[3, b] + support.lr[3, l, b]

  support.t[l] += if (support.tmp[1] >= support.tmp[2]) &&
                     (support.tmp[1] >= support.tmp[3])
                    support.tmp[1] +
                    log1p(exp(support.tmp[2] - support.tmp[1]) +
                          exp(support.tmp[3] - support.tmp[1]))
                  elseif (support.tmp[2] >= support.tmp[1]) &&
                         (support.tmp[2] >= support.tmp[3])
                    support.tmp[2] +
                    log1p(exp(support.tmp[1] - support.tmp[2]) +
                          exp(support.tmp[3] - support.tmp[2]))
                  else
                    support.tmp[3] +
                    log1p(exp(support.tmp[1] - support.tmp[3]) +
                          exp(support.tmp[2] - support.tmp[3]))
                  end
  nothing
end

function gibbscomputeprobi!(li::Int,
                            support::MCMCSupport)
  # cluster hi does not change => marginal likelihood does not change
  support.t[li] += support.logmlik
  nothing
end

function gibbscomputeprobj!(b::Int,
                            li::Int,
                            lj::Int,
                            support::MCMCSupport)
  support.tmp[1] = support.lpi[1, b] + support.lq[1, li, b] +
                   support.lq[1, lj, b]
  support.tmp[2] = support.lpi[2, b] + support.lq[2, li, b] +
                   support.lq[2, lj, b]
  support.tmp[3] = support.lpi[4, b] + support.lpj[2, b] + support.lpj[3, b]

  support.t[lj] += if (support.tmp[1] >= support.tmp[2]) &&
                      (support.tmp[1] >= support.tmp[3])
                     support.tmp[1] +
                     log1p(exp(support.tmp[2] - support.tmp[1]) +
                           exp(support.tmp[3] - support.tmp[1]))
                   elseif (support.tmp[2] >= support.tmp[1]) &&
                          (support.tmp[2] >= support.tmp[3])
                     support.tmp[2] +
                     log1p(exp(support.tmp[1] - support.tmp[2]) +
                           exp(support.tmp[3] - support.tmp[2]))
                   else
                     support.tmp[3] +
                     log1p(exp(support.tmp[1] - support.tmp[3]) +
                           exp(support.tmp[2] - support.tmp[3]))
                   end

  nothing
end

function gibbscomputenormconst(k::Int,
                               support::MCMCSupport)
  logc = -Inf
  y = 0.0

  for l in 1:k
    if support.t[l] > logc
      logc = support.t[l]
    end
  end

  for l in 1:k
    y += exp(support.t[l] - logc)
  end

  logc += log(y)

  logc
end

function gibbssamplecluster(logc::Float64,
                            k::Int,
                            support::MCMCSupport)
  # find the maximum index for which the following holds
  # c * p <= sum(t[1:l])
  q = logc + log(rand())

  g = 1
  y = support.t[g]

  while (q > y) && (g < k)
    g += 1
    y = support.t[g] + log1p(exp(y - support.t[g]))
  end

  g
end

function gibbsmerge!(i::Int,
                     hi::Int,
                     hj::Int,
                     lg::Int,
                     data::Matrix{UInt8},
                     priorR::PriorRowPartition,
                     support::MCMCSupport,
                     state::AminoAcidState)
  support.logmlik = support.t[lg] - clusterweight(state.v[hj], priorR)

  state.R[i] = hj

  state.emptycluster[hi] = true

  support.k = 0
  for a in 1:length(state.emptycluster)
    if !state.emptycluster[a]
      support.k += 1
      state.cl[support.k] = a
    end
  end

  state.k = support.k

  state.logpR += logratiopriorrowmerge(state.k, state.v[hj], priorR)

  state.v[hj] += 1

  resize!(state.unit[hj], state.v[hj])
  state.unit[hj][state.v[hj]] = i

  for b in 1:support.m
    state.n1s[hj, b] += float(data[b, i])

    support.lp[1, hj, b] = support.lq[1, lg, b]
    support.lp[2, hj, b] = support.lq[2, lg, b]
    support.lp[3, hj, b] = support.lq[3, lg, b]
    support.lp[4, hj, b] = support.lq[4, lg, b]
  end

  nothing
end

function gibbsmove!(i::Int,
                    hi::Int,
                    hj::Int,
                    li::Int,
                    lg::Int,
                    data::Matrix{UInt8},
                    priorR::PriorRowPartition,
                    support::MCMCSupport,
                    state::AminoAcidState)
  support.logmlik = support.t[lg] - clusterweight(state.v[hj], priorR)

  state.logpR += logratiopriorrowmove(state.v[hi], state.v[hj], priorR)

  state.R[i] = hj

  h = 0
  for a in 1:state.v[hi]
    if state.unit[hi][a] != i
      h += 1
      state.unit[hi][h] = state.unit[hi][a]
    end
  end

  state.v[hi] = support.vi
  state.v[hj] += 1

  resize!(state.unit[hj], state.v[hj])
  state.unit[hj][state.v[hj]] = i

  for b in 1:support.m
    state.n1s[hi, b] = support.ni[b]
    state.n1s[hj, b] += float(data[b, i])

    support.lp[1, hi, b] = support.lq[1, li, b]
    support.lp[2, hi, b] = support.lq[2, li, b]
    support.lp[3, hi, b] = support.lq[3, li, b]
    support.lp[4, hi, b] = support.lq[4, li, b]

    support.lp[1, hj, b] = support.lq[1, lg, b]
    support.lp[2, hj, b] = support.lq[2, lg, b]
    support.lp[3, hj, b] = support.lq[3, lg, b]
    support.lp[4, hj, b] = support.lq[4, lg, b]
  end

  nothing
end

function gibbssplit!(i::Int,
                     hi::Int,
                     li::Int,
                     lj::Int,
                     data::Matrix{UInt8},
                     priorR::PriorRowPartition,
                     settings::KSettings,
                     support::MCMCSupport,
                     state::AminoAcidState)
  resizestate!(state, state.k + 1, settings)

  support.logmlik = support.t[lj] - clusterweight(1, state.k, priorR)

  hj = findfirst(state.emptycluster)

  state.R[i] = hj

  state.emptycluster[hj] = false

  support.k = 0
  for a in 1:length(state.emptycluster)
    if !state.emptycluster[a]
      support.k += 1
      state.cl[support.k] = a
    end
  end

  state.k = support.k

  h = 0
  for a in 1:state.v[hi]
    if state.unit[hi][a] != i
      h += 1
      state.unit[hi][h] = state.unit[hi][a]
    end
  end

  state.logpR += logratiopriorrowsplit(state.k, state.v[hi], priorR)

  state.v[hi] = support.vi
  state.v[hj] = 1

  resize!(state.unit[hj], 1)
  state.unit[hj][1] = i

  for b in 1:support.m
    state.n1s[hi, b] = support.ni[b]
    state.n1s[hj, b] = float(data[b, i])

    support.lp[1, hi, b] = support.lq[1, li, b]
    support.lp[2, hi, b] = support.lq[2, li, b]
    support.lp[3, hi, b] = support.lq[3, li, b]
    support.lp[4, hi, b] = support.lq[4, li, b]

    support.lp[1, hj, b] = support.lq[1, lj, b]
    support.lp[2, hj, b] = support.lq[2, lj, b]
    support.lp[3, hj, b] = support.lq[3, lj, b]
    support.lp[4, hj, b] = support.lq[4, lj, b]
  end

  nothing
end


# *********************************************************************************
# REopt, Copyright (c) 2019-2020, Alliance for Sustainable Energy, LLC.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
# Redistributions of source code must retain the above copyright notice, this list
# of conditions and the following disclaimer.
#
# Redistributions in binary form must reproduce the above copyright notice, this
# list of conditions and the following disclaimer in the documentation and/or other
# materials provided with the distribution.
#
# Neither the name of the copyright holder nor the names of its contributors may be
# used to endorse or promote products derived from this software without specific
# prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
# OF THE POSSIBILITY OF SUCH DAMAGE.
# *********************************************************************************
"""
    MPCElectricLoad

    Base.@kwdef struct MPCElectricLoad
        loads_kw::Array{Real,1}
        critical_loads_kw::Union{Missing, Array{Real,1}} = missing
    end
"""
Base.@kwdef struct MPCElectricLoad
    loads_kw::Array{Real,1}
    critical_loads_kw::Union{Missing, Array{Real,1}} = missing
end

"""
    MPCFinancial

    Base.@kwdef struct MPCFinancial
        VoLL::Union{Array{R,1}, R} where R<:Real = 1.00
    end
"""
Base.@kwdef struct MPCFinancial
    VoLL::Union{Array{R,1}, R} where R<:Real = 1.00
end

"""
    MPCPV
```julia
Base.@kwdef struct MPCPV
    name::String="PV"
    size_kw::Real = 0
    prod_factor_series_kw::Union{Missing, Array{Real,1}} = missing
end
```
"""
Base.@kwdef struct MPCPV
    name::String="PV"
    size_kw::Real = 0
    prod_factor_series_kw::Union{Missing, Array{Real,1}} = missing
end

struct MPCElectricTariff
    energy_rates::AbstractVector{Float64}

    monthly_demand_rates::AbstractVector{Float64}
    time_steps_monthly::Array{Array{Int64,1},1}  # length = 0 or 12
    monthly_previous_peak_demands::AbstractVector{Float64}

    tou_demand_rates::AbstractVector{Float64}
    tou_demand_ratchet_timesteps::Array{Array{Int64,1},1}  # length = n_tou_demand_ratchets
    tou_previous_peak_demands::AbstractVector{Float64}

    fixed_monthly_charge::Float64
    annual_min_charge::Float64
    min_monthly_charge::Float64

    export_rates::DenseAxisArray{Array{Float64,1}}
    export_bins::Array{Symbol,1}
end

"""
    MPCElectricTariff(d::Dict)

function for parsing user inputs into
```julia
    struct MPCElectricTariff
        monthly_previous_peak_demands::Array{Float64,1}
        energy_rates::Array{Float64,1}

        monthly_demand_rates::Array{Float64,1}
        time_steps_monthly::Array{Array{Int64,1},1}  # length = 0 or 12

        tou_demand_rates::Array{Float64,1}
        tou_demand_ratchet_timesteps::Array{Array{Int64,1},1}  # length = n_tou_demand_ratchets
        tou_previous_peak_demands::Array{Float64,1}

        fixed_monthly_charge::Float64
        annual_min_charge::Float64
        min_monthly_charge::Float64

        export_rates::DenseAxisArray{Array{Float64,1}}
        export_bins::Array{Symbol,1}
    end
```

Keys for `d` include:

  - `energy_rates`

    - REQUIRED
    - must have length equal to `ElectricLoad.loads_kw`

  - `monthly_demand_rates`

    - default = [0]

  - `time_steps_monthly`

    - array of arrays for integer time steps that the `monthly_demand_rates` apply to
    - default = [collect(1:length(energy_rates))]

  - `monthly_previous_peak_demands`

    - default = [0]

  - `tou_demand_rates`

    - an array of time-of-use demand rates
    - must have length equal to `tou_demand_timesteps`
    - default = []

  - `tou_demand_timesteps`

    - an array of arrays for the integer time steps that apply to the `tou_demand_rates`
    - default = []

  - `tou_previous_peak_demands`

    - an array of the previous peak demands set in each time-of-use demand period
    - must have length equal to `tou_demand_timesteps`
    - default = []

  - `net_metering`

    - boolean, if `true` then customer DER export is compensated at the `energy_rates`

  - `export_rates`

    - can be a <:Real or Array{<:Real, 1}, or not provided
    - if provided, customer DER export is compensated at the `export_rates`

NOTE: if both `net_metering=true` and `export_rates` are provided then the model can choose from either option.
"""
function MPCElectricTariff(d::Dict)

    energy_rates = d["energy_rates"]

    monthly_demand_rates = get(d, "monthly_demand_rates", [0.0])
    time_steps_monthly = get(d, "time_steps_monthly", [collect(1:length(energy_rates))])
    monthly_previous_peak_demands = get(d, "monthly_previous_peak_demands", [0.0])

    tou_demand_rates = get(d, "tou_demand_rates", Float64[])
    tou_demand_timesteps = get(d, "tou_demand_timesteps", [])
    tou_previous_peak_demands = get(d, "tou_previous_peak_demands", Float64[])
    @assert length(tou_demand_rates) == length(tou_demand_timesteps) == length(tou_previous_peak_demands)

    # TODO can remove these inputs?
    fixed_monthly_charge = 0.0
    annual_min_charge = 0.0
    min_monthly_charge = 0.0

    # TODO handle tiered rates
    export_bins = [:NEM, :WHL]
    nem_rate = []
    NEM = get(d, "net_metering", false)
    if NEM
        nem_rate = [-0.999 * x for x in energy_rates]
    end
    # export_rates can be a <:Real or Array{<:Real, 1}, or not provided
    export_rates = get(d, "export_rates", nothing)
    if !isnothing(export_rates)
        export_rates = convert(Vector{Real}, export_rates)
    end
    whl_rate = create_export_rate(export_rates, length(energy_rates), 1)

    if !NEM & (sum(whl_rate) >= 0)
        export_rates = DenseAxisArray{Array{Float64,1}}(undef, [])
        export_bins = Symbol[]
    elseif !NEM
        export_bins = [:WHL]
        export_rates = DenseAxisArray([whl_rate], export_bins)
    elseif (sum(whl_rate) >= 0)
        export_bins = [:NEM]
        export_rates = DenseAxisArray([nem_rate], export_bins)
    else
        export_bins = [:NEM, :WHL]  # TODO add :EXC to align with REopt Lite API
        export_rates = DenseAxisArray([nem_rate, whl_rate], export_bins)
    end

    MPCElectricTariff(
        energy_rates,
        monthly_demand_rates,
        time_steps_monthly,
        monthly_previous_peak_demands,
        tou_demand_rates,
        tou_demand_timesteps,
        tou_previous_peak_demands,
        fixed_monthly_charge,
        annual_min_charge,
        min_monthly_charge,
        export_rates,
        export_bins
    )
end

"""
    MPCElecStorage
```julia
Base.@kwdef struct MPCElecStorage
    size_kw::Float64
    size_kwh::Float64
    charge_efficiency::Float64
    discharge_efficiency::Float64
    soc_min_pct::Float64
    soc_init_pct::Float64
    can_grid_charge::Bool = true
    grid_charge_efficiency::Float64
end
```
"""
Base.@kwdef struct MPCElecStorage
    size_kw::Float64
    size_kwh::Float64
    charge_efficiency::Float64
    discharge_efficiency::Float64
    soc_min_pct::Float64
    soc_init_pct::Float64
    can_grid_charge::Bool = true
    grid_charge_efficiency::Float64
end

Base.@kwdef struct MPCStorage
    types::Array{Symbol,1} = [:elec]
    size_kw::Dict{Symbol, Float64}
    size_kwh::Dict{Symbol, Float64}
    charge_efficiency::Dict{Symbol, Float64} = Dict(:elec => 0.96 * 0.975^2)
    discharge_efficiency::Dict{Symbol, Float64} = Dict(:elec => 0.96 * 0.975^2)
    soc_min_pct::Dict{Symbol, Float64} = Dict(:elec => 0.2)
    soc_init_pct::Dict{Symbol, Float64} = Dict(:elec => 0.5)
    can_grid_charge::Array{Symbol,1} = [:elec]
    grid_charge_efficiency::Dict{Symbol, Float64} = Dict(:elec => 0.96 * 0.975^2)
end

"""

NOTE: d must have symbolic keys
"""
function MPCStorage(d::Dict)
    d2 = Dict()
    d2[:can_grid_charge] = get(d, :can_grid_charge, false) ? [:elec] : Symbol[]
    if haskey(d, :can_grid_charge)
        pop!(d, :can_grid_charge)
    end
    # have to convert to all d values to DenseAxisArray's with storage type as Axis
    # (only modeling elec storage in MPC for now)
    for (k,v) in d
        d2[k] = Dict(:elec => convert(Float64, v))
    end

    return MPCStorage(; d2...)
end

"""
    MPCGenerator

struct with inner constructor:
```julia
function MPCGenerator(;
    size_kw::Real,
    fuel_cost_per_gallon::Float64 = 3.0,
    fuel_slope_gal_per_kwh::Float64 = 0.076,
    fuel_intercept_gal_per_hr::Float64 = 0.0,
    fuel_avail_gal::Float64 = 660.0,
    min_turn_down_pct::Float64 = 0.0,  # TODO change this to non-zero value
    only_runs_during_grid_outage::Bool = true,
    sells_energy_back_to_grid::Bool = false,
    om_cost_per_kwh::Float64=0.0,
    )
```
"""
struct MPCGenerator <: AbstractGenerator
    size_kw
    max_kw
    fuel_cost_per_gallon
    fuel_slope_gal_per_kwh
    fuel_intercept_gal_per_hr
    fuel_avail_gal
    min_turn_down_pct
    only_runs_during_grid_outage
    sells_energy_back_to_grid
    om_cost_per_kwh

    function MPCGenerator(;
        size_kw::Real,
        fuel_cost_per_gallon::Float64 = 3.0,
        fuel_slope_gal_per_kwh::Float64 = 0.076,
        fuel_intercept_gal_per_hr::Float64 = 0.0,
        fuel_avail_gal::Float64 = 660.0,
        min_turn_down_pct::Float64 = 0.0,  # TODO change this to non-zero value
        only_runs_during_grid_outage::Bool = true,
        sells_energy_back_to_grid::Bool = false,
        om_cost_per_kwh::Float64=0.0,
        )

        max_kw = size_kw

        new(
            size_kw,
            max_kw,
            fuel_cost_per_gallon,
            fuel_slope_gal_per_kwh,
            fuel_intercept_gal_per_hr,
            fuel_avail_gal,
            min_turn_down_pct,
            only_runs_during_grid_outage,
            sells_energy_back_to_grid,
            om_cost_per_kwh,
        )
    end
end


module CTScan

Base.Experimental.@optlevel 0

export AbstractCTScanner, CTScanner, FBPScanner
export rename, rename!, transform_data, transform_data!
export compute_gray_scale
export make_phantom, create_image!
export project_image!, reconstruct_image!
export para2fan!, fan2para!
export compute_rmse, compute_χ2, fit_gray_scale
export load_image!, load_sinogram!, load_tomogram!
export sample_sinogram!

import Base: show, copy, similar, getproperty
using IntervalSets
import Statistics; const Stat = Statistics
import CurveFit; const Fit = CurveFit
import ..Utils: _atype
using ..Applicative
using ..Monads
using ..Geometry
import ..AbstractAlgorithms: reconstruct_image, project_image
using ..AbstractAlgorithms
using ..RadonAlgorithm
import ..CTImages: ctimage, ctsinogram, cttomogram
using ..CTImages
import ..FanBeam: para2fan, fan2para
import ..CalibrationBase:
    calibrate_image,
    calibrate_image!,
    calibrate_tomogram,
    calibrate_tomogram!
using ..Calibration
using ..TestImages
import ..CTIO: load_image, write_image
import ..CTIO: load_sinogram, write_sinogram
import ..CTIO: load_tomogram, write_tomogram

include("CTImageData.jl")
using .CTImageData: AbstractCTData, CTData

"""
    abstract type AbstractCTScanner{<:AbstractArray,<:AbstractGeometry} end

Abstract type for tests in this suite.
"""
abstract type AbstractCTScanner end

const CTScannerNameType = Union{Symbol,String}

function rename(gst::AbstractCTScanner, name::AbstractString)
    typeof(gst)(gst.geometry, gst.data; name)
end

function rename!(gst::AbstractCTScanner, name::AbstractString)
    gst.name = name
end

_atype(gst::AbstractCTScanner) = _atype(gst.data)

ctimage(gst::AbstractCTScanner) = ctimage(gst.data)
ctsinogram(gst::AbstractCTScanner) = ctsinogram(gst.data)
cttomogram(gst::AbstractCTScanner) = cttomogram(gst.data)

similar(gst::AbstractCTScanner, img::AbstractCTImage...) =
    similar(gst, CTData(img...))

function similar(
    gst::AbstractCTScanner,
    g::AbstractGeometry,
    img::AbstractCTImage...,
)
    similar(gst, g, CTData(img...))
end

for (nm, ctf) ∈ pairs(CTImages.ctfn)
    v = Val{nm}
    @eval begin
        t = CTImages.ctnames.$nm
        function transform_data(f::Function, gst::AbstractCTScanner, ::$v)
            mbind($ctf(gst)) do img
                newdata = similar(gst.data, map(f, something(img)))
                similar(gst, newdata)
            end
        end
        function transform_data!(f::Function, gst::AbstractCTScanner, ::$v)
            mbind($ctf(gst)) do img
                newdata = similar(gst.data, map(f, something(img)))
                gst.data = newdata
            end
        end
        transform_data(f::Function, gst::AbstractCTScanner, ::Type{t}) =
            transform_data(f, gst, $v())
        transform_data!(f::Function, gst::AbstractCTScanner, ::Type{t}) =
            transform_data!(f, gst, $v())
    end
end

transform_data(f::Function, gst::AbstractCTScanner, s::Symbol) =
    transform_data(f, gst, Val(s))
transform_data!(f::Function, gst::AbstractCTScanner, s::Symbol) =
    transform_data!(f, gst, Val(s))

struct CTScanner{A<:AlgorithmInfo,D<:AbstractCTData} <: AbstractCTScanner
    name::CTScannerNameType
    study_id::String
    ainfo::A
    data::D

    """
    CTScanner{A,D}(
        ainfo::A[, data::D];
        name::Optional{Union{Symbol,String}} = nothing,
        study_id::Optional{String} = "Unknown"
    ) where {A <: AlgorithmInfo, D <: AbstractCTData}

    Construct `CTScanner` from an algorithm.
    """
    function CTScanner{A,D}(
        ainfo::A,
        data::D = CTData();
        name::Optional{CTScannerNameType} = nothing,
        study_id::Optional{String} = nothing,
    ) where {A <: AlgorithmInfo,D <: AbstractCTData}
        new(
            maybe(alg_name(ainfo), name),
            maybe("Unknown", study_id),
            ainfo,
            data
        )
    end
end

@inline function getproperty(ct::CTScanner, s::Symbol)
    s ≡ :geometry && return ct.ainfo.geometry
    s ≡ :params && return ct.ainfo.params
    s ≡ :algorithm && return ct.ainfo.algorithm
    getfield(ct, s)
end

function CTScanner(alg::AlgorithmInfo, data::AbstractCTData; kwargs...)
    CTScanner{typeof(alg), typeof(data)}(alg, data; kwargs...)
end

function CTScanner(
    alg::AlgorithmInfo,
    imgs::T...;
    kwargs...
) where {T<:AbstractCTImage}
    CTScanner{typeof(alg), CTData{eltype(T),_atype(T)}}(
        alg, CTData(imgs...); kwargs...)
end

function CTScanner(alg::AlgorithmInfo, gst::AbstractCTScanner; kwargs...)
    CTScanner(alg, gst.data; kwargs...)
end

function similar(gst::CTScanner, data::Optional{AbstractCTData} = nothing)
    CTScanner(
        gst.ainfo,
        maybe(similar(gst.data), data);
        gst.name,
        gst.study_id,
    )
end

const FBPScanner{G,D} = CTScanner{FBPInfo{G},D}

"""
    FBPScanner(
        geometry::AbstractGeometry,
        data::AbstractCTData;
        name::Optional{String},
        study_id::Optional{String}
    )

Construct a `FBPScanner` object.
"""
function FBPScanner(
    geometry::G,
    data::D = CTData(eltype(G));
    name::Optional{CTScannerNameType} = nothing,
    study_id::Optional{String} = nothing,
) where {G <: AbstractGeometry, D <: AbstractCTData}
    FBPScanner{G,D}(FBPInfo(geometry), data; name, study_id)
end

"""
    FBPScanner(
        geometry::AbstractGeometry,
        image::AbstractTestImage;
        name::Optional{String},
        study_id::Optional{String}
    )

Construct a `FBPScanner` object.
"""
function FBPScanner(
    geometry::AbstractGeometry,
    gs::AbstractTestImage;
    name::Optional{CTScannerNameType} = nothing,
    study_id::Optional{String} = nothing,
)
    FBPScanner(geometry, gs.image; name, study_id)
end

"""
    FBPScanner(
        image::AbstractTestImage;
        name::Optional{String},
        study_id::Optional{String}
    )

Construct a `FBPScanner` object.
"""
function FBPScanner(
    gs::AbstractTestImage;
    name::Optional{CTScannerNameType} = nothing,
    study_id::Optional{String} = nothing,
)
    FBPScanner(ParallelBeamGeometry(gs), gs; name, study_id)
end

"""
    FBPScanner(gst::AbstractCTScanner; name::Optional{String}, study_id::Optional{String})

Construct a `FBPScanner` object from another `AbstractCTScanner` object. This allows to reuse data.
"""
function FBPScanner(
    gst::AbstractCTScanner;
    name::Optional{CTScannerNameType} = nothing,
    study_id::Optional{String} = nothing,
)
    data = CTData(ctimage(gst), ctsinogram(gst), nothing)
    name = maybe(gst.name, name)
    study_id = maybe(gst.study_id, study_id)
    FBPScanner(gst.ainfo, data; name, study_id)
end

function FBPScanner(g::AbstractGeometry, imgs::AbstractCTImage...; kwargs...)
    FBPScanner(g, CTData(imgs...); kwargs...)
end

function copy(gst::FBPScanner)
    FBPScanner(gst.geometry, copy(gst.data); gst.name, gst.study_id)
end

function similar(
    gst::FBPScanner,
    g::AbstractGeometry,
    data::Optional{CTData} = nothing,
)
    FBPScanner(
        g,
        maybe(gst.data, data);
        gst.name,
        gst.study_id,
    )
end

function show(io::IO, gst::CTScanner)
    print(
        io,
        """*** $(gst.name) Scanner ***
        $(gst.algorithm)""",
    )
end

"""
    create_image!(gst::AbstractCTScanner, par::ImageParams)

Create gray scale image for the test `gst`.
"""
function create_image!(gst::AbstractCTScanner, par::ImageParams)
    #similar(gst, similar(gst.data, create_image(par)))
    gst.data.image = create_image(par)
    gst
end

"""
    project_image(gst::AbstractCTScanner, alg::Optional{A}; <keyword arguments>)
        where {A <: AbstractProjectionAlgorithm}

Compute sinogram for the test `gst`.

Keyword arguments depend on the algorithm employed, please see the relative
documentation.
"""
function project_image(
    gst::AbstractCTScanner,
    alg::Optional{A} = nothing;
    kwargs...
) where {A <: AbstractProjectionAlgorithm}
    alg = isnothing(alg) ? Radon() : alg
    sinog = project_image(ctimage(gst), RadonInfo(gst.geometry, alg); kwargs...)
    similar(gst, similar(gst.data, sinog))
end

function project_image!(
    gst::AbstractCTScanner,
    alg::Optional{A} = nothing;
    kwargs...
) where {A <: AbstractProjectionAlgorithm}
    alg = isnothing(alg) ? Radon() : alg
    sinog = project_image(ctimage(gst), gst.geometry, alg; kwargs...)
    gst.data.sinog = sinog
    gst
end

function para2fan(gst::AbstractCTScanner; kwargs...)
    fbg, sinog_fan = ctsinogram(gst) ↣ para2fan(gst.geometry; kwargs...)
    similar(gst, fbg, similar(gst.data, sinog_fan))
end

function para2fan!(gst::AbstractCTScanner; kwargs...)
    fbg, sinog_fan = ctsinogram(gst) ↣ para2fan(gst.geometry; kwargs...)
    gst.data.sinog = sinog_fan
    gst
end

function para2fan(gst::AbstractCTScanner, fbg::FanBeamGeometry; kwargs...)
    fbg, sinog_fan = ctsinogram(gst) ↣ para2fan(fbg; kwargs...)
    similar(gst, fbg, similar(gst.data, sinog_fan))
end

function fan2para(gst::AbstractCTScanner; kwargs...)
    pbg, sinog_para = ctsinogram(gst) ↣ fan2para(gst.geometry; kwargs...)
    similar(gst, pbg, similar(gst.data, sinog_para))
end

function fan2para!(gst::AbstractCTScanner; kwargs...)
    pbg, sinog_para = ctsinogram(gst) ↣ fan2para(gst.geometry; kwargs...)
    gst.data.sinog = sinog_para
    gst
end

"""
    reconstruct_image(gst::AbstractCTScanner; <keyword arguments>)

Reconstruct CT image from test `gst`.

Keyword arguments depend on the specific algorithm used, please see the relative
documentation.
"""
function reconstruct_image(gst::AbstractCTScanner; kwargs...)
    tomog = reconstruct_image(
        ctsinogram(gst), gst.ainfo; kwargs...)
    similar(gst, similar(gst.data, tomog))
end

function reconstruct_image!(gst::AbstractCTScanner; kwargs...)
    tomog = reconstruct_image(
        ctsinogram(gst), gst.ainfo; kwargs...)
    gst.data.tomog = tomog
    gst
end

"""
    load_sinogram(io::IO, gst::AbstractCTScanner)

Load sinogram from stream `io`.
"""
function load_sinogram(io::IO, gst::AbstractCTScanner)
    sinog = load_sinogram(io; gst.geometry.nd, gst.geometry.nϕ)
    similar(gst, similar(gst.data, sinog))
end

function load_sinogram!(io::IO, gst::AbstractCTScanner)
    sinog = load_sinogram(io; gst.geometry.nd, gst.geometry.nϕ)
    gst.data.sinog = sinog
    gst
end

"""
    load_sinogram(f::AbstractString, gst::AbstractCTScanner)

Load sinogram from file `f`.
"""
function load_sinogram(f::AbstractString, gst::AbstractCTScanner)
    open(f) do s
        load_sinogram(s, gst)
    end
end

function load_sinogram!(f::AbstractString, gst::AbstractCTScanner)
    open(f) do s
        load_sinogram!(s, gst)
    end
end

"""
    load_image(io::IO, gst::AbstractCTScanner)

Load image from stream `io`.
"""
function load_image(io::IO, gst::AbstractCTScanner)
    image = load_image(io; gst.geometry.rows, gst.geometry.cols)
    similar(gst, similar(gst.data, image))
end

function load_image!(io::IO, gst::AbstractCTScanner)
    image = load_image(io; gst.geometry.rows, gst.geometry.cols)
    gst.data.image = image
    gst
end

"""
    load_image(f::AbstractString, gst::AbstractCTScanner)

Load image from file `f`.
"""
function load_image(f::AbstractString, gst::AbstractCTScanner)
    open(f) do s
        load_image(s, gst)
    end
end

function load_image!(f::AbstractString, gst::AbstractCTScanner)
    open(f) do s
        load_image!(s, gst)
    end
end

"""
    write_image(io::IO, gst::AbstractCTScanner)

Write input image to stream `io`.
"""
function write_image(io::IO, gst::AbstractCTScanner)
    mbind(ctimage(gst)) do x
        write_image(io, x; header = false, row_major = true)
    end
end

"""
    write_image(f::AbstractString, gst::AbstractCTScanner)

Write input image to file `f`.
"""
function write_image(f::AbstractString, gst::AbstractCTScanner)
    open(f, "w") do s
        write_image(s, gst)
    end
end

"""
    write_sinogram(io::IO, gst::AbstractCTScanner)

Write sinogram to stream `io`.
"""
function write_sinogram(io::IO, gst::AbstractCTScanner)
    mbind(ctsinogram(gst)) do x
        write_sinogram(io, x; header = false)
    end
end

"""
    write_sinogram(f::AbstractString, gst::AbstractCTScanner)

Write sinogram to file `f`.
"""
function write_sinogram(f::AbstractString, gst::AbstractCTScanner)
    open(f, "w") do s
        write_sinogram(s, gst)
    end
end

"""
    load_tomogram(io::IO, gst::AbstractCTScanner)

Load tomogram from stream `io`.
"""
function load_tomogram(io::IO, gst::AbstractCTScanner)
    tomog = load_tomogram(io; gst.geometry.rows, gst.geometry.cols)
    similar(gst, similar(gst.data, tomog))
end

function load_tomogram!(io::IO, gst::AbstractCTScanner)
    tomog = load_tomogram(io; gst.geometry.rows, gst.geometry.cols)
    gst.data.tomog = tomog
    gst
end

"""
    load_tomogram(f::AbstractString, gst::AbstractCTScanner)

Load tomogram from file `f`.
"""
function load_tomogram(f::AbstractString, gst::AbstractCTScanner)
    open(f) do s
        load_tomogram(s, gst)
    end
end

function load_tomogram!(f::AbstractString, gst::AbstractCTScanner)
    open(f) do s
        load_tomogram!(s, gst)
    end
end

"""
    write_tomogram(io::IO, gst::AbstractCTScanner)

Write reconstructed image to stream `io`.
"""
function write_tomogram(io::IO, gst::AbstractCTScanner)
    mbind(cttomogram(gst)) do x
        write_tomogram(io, x; header = false, row_major = true)
    end
end

"""
    write_tomogram(f::AbstractString, gst::AbstractCTScanner)

Write tomogram to file `f`.
"""
function write_tomogram(f::AbstractString, gst::AbstractCTScanner)
    open(f, "w") do s
        write_tomogram(s, gst)
    end
end

"""
    sample_sinogram(gst::AbstractCTScanner; <keyword arguments>)

Returns a new test object with the resampled sinogram.
"""
function sample_sinogram(gst::AbstractCTScanner; kwargs...)
    sinog = ctsinogram(gst) ↣ sample_sinogram(; kwargs...)
    similar(gst, similar(gst.data, sinog))
end

function sample_sinogram!(gst::AbstractCTScanner; kwargs...)
    sinog = ctsinogram(gst) ↣ sample_sinogram(; kwargs...)
    gst.data.sinog = sinog
    gst
end

"""
    calibrate_image(gst::AbstractCTScanner, imp::ImageParams; interval=nothing, window=nothing)

Perform calibration of input image using image parameters as reference.
"""
function calibrate_image(
    gst::AbstractCTScanner,
    imp::AbstractImageParams;
    interval::Optional{ClosedInterval{T}} = nothing,
    window::Optional{ClosedInterval{U}} = nothing,
) where {T<:Real,U<:Real}
    cimg = calibrate_image(
        ctimage(gst),
        imp;
        interval,
        window,
    )
    similar(gst, maybe(gst.data, cimg) do x
        similar(gst.data, x)
    end)
end

function calibrate_image!(
    gst::AbstractCTScanner,
    imp::AbstractImageParams;
    interval::Optional{ClosedInterval{T}} = nothing,
    window::Optional{ClosedInterval{U}} = nothing,
) where {T<:Real,U<:Real}
    cimg = calibrate_image(
        ctimage(gst),
        imp;
        interval,
        window,
    )
    gst.data.image = cimg
    gst
end

"""
    calibrate_image(gst::AbstractCTScanner; min_pos, max_pos, interval=0..1, window=nothing)

Perform calibration of input image with reference values.
"""
function calibrate_image(
    gst::AbstractCTScanner;
    min_pos,
    max_pos,
    interval::ClosedInterval = 0..1,
    window::Optional{ClosedInterval{T}} = nothing,
) where {T<:Real}
    cimg = calibrate_image(
        ctimage(gst);
        interval,
        min_pos,
        max_pos,
        window,
    )
    similar(gst, maybe(gst.data, cimg) do x
        similar(gst.data, x)
    end)
end

function calibrate_image!(
    gst::AbstractCTScanner;
    min_pos,
    max_pos,
    interval::ClosedInterval = 0..1,
    window::Optional{ClosedInterval{T}} = nothing,
) where {T<:Real}
    cimg = calibrate_image(
        ctimage(gst);
        interval,
        min_pos,
        max_pos,
        window,
    )
    gst.data.image = cimg
    gst
end

"""
    calibrate_tomogram(gst::AbstractCTScanner, imp::ImageParams; interval=nothing, window=nothing)

Perform calibration of reconstructed image using image parameters as reference.
"""
function calibrate_tomogram(
    gst::AbstractCTScanner,
    imp::AbstractImageParams;
    interval::Optional{ClosedInterval{T}} = nothing,
    window::Optional{ClosedInterval{U}} = nothing,
) where {T<:Real,U<:Real}
    cimg = calibrate_tomogram(
        cttomogram(gst),
        imp;
        interval,
        window,
    )
    similar(gst, maybe(gst.data, cimg) do x
        similar(gst.data, x)
    end)
end

function calibrate_tomogram!(
    gst::AbstractCTScanner,
    imp::AbstractImageParams;
    interval::Optional{ClosedInterval{T}} = nothing,
    window::Optional{ClosedInterval{U}} = nothing,
) where {T<:Real,U<:Real}
    cimg = calibrate_tomogram(
        cttomogram(gst),
        imp;
        interval,
        window,
    )
    gst.data.tomog = cimg
    gst
end

"""
    calibrate_tomogram(gst::AbstractCTScanner; min_pos, max_pos, interval=0..1, window=nothing)

Perform calibration of reconstructed image with reference values.
"""
function calibrate_tomogram(
    gst::AbstractCTScanner;
    min_pos,
    max_pos,
    interval::ClosedInterval = 0..1,
    window::Optional{ClosedInterval{T}} = nothing,
) where {T<:Real}
    cimg = calibrate_tomogram(
        cttomogram(gst);
        min_pos,
        max_pos,
        interval,
        window,
    )
    similar(gst, maybe(gst.data, cimg) do x
        similar(gst.data, x)
    end)
end

function calibrate_tomogram!(
    gst::AbstractCTScanner;
    min_pos,
    max_pos,
    interval::ClosedInterval = 0..1,
    window::Optional{ClosedInterval{T}} = nothing,
) where {T<:Real}
    cimg = calibrate_tomogram(
        cttomogram(gst);
        min_pos,
        max_pos,
        interval,
        window,
    )
    gst.data.tomog = cimg
    gst
end

"""
    compute_gray_scale(image::AbstractMatrix, imp::ImageParams; mean = false)

Compute the corresponding gray scale for the test image `image`.
"""
function compute_gray_scale(image::AbstractMatrix, imp::ImageParams; mean = false)
    mean && return Stat.mean(image[gray_scale_indices(imp)...], dims = 1) |> vec
    rind, cind = gray_scale_indices(imp)
    i = sum(extrema(rind)) ÷ 2
    image[i,cind]
end

# function compute_gray_scale(
#     image::CTImageOrTomog, params::ImageParams; kwargs...)
#     mbind(image) do x
#         compute_gray_scale(x, params; kwargs...)
#     end
# end

compute_gray_scale(grsc::AbstractGrayScale; kwargs...) =
    compute_gray_scale(grsc.image, grsc.params)

compute_gray_scale(imp::ImageParams; kwargs...) = x -> compute_gray_scale(x, imp; kwargs...)

"""
    compute_gray_scale(gst::AbstractCTScanner, imp::ImageParams; mean = false)

Compute the corresponding gray scale for the reconstructed image.
"""
function compute_gray_scale(gst::AbstractCTScanner, grsc::AbstractGrayScale; kwargs...)
    cttomogram(gst) ↣ compute_gray_scale(grsc.params; kwargs...)
end

function make_phantom(f::Function, g::AbstractParallelBeamGeometry)
    g |> f |> CTImage
end

function compute_rmse(
    grimg::AbstractGrayScale,
    expected::AbstractVector,
    measured::AbstractVector;
    relative = false,
    scale::Optional{ClosedInterval} = nothing,
)
    scale = maybe(grimg.gray_scale, scale)
    rmse = Stat.mean(abs2, measured - expected) |> sqrt
    relative && return rmse / width(scale)
    rmse
end

function _fit_line(ys)
    xs = eachindex(ys)
    a, b = Fit.linear_fit(xs, ys)
    @. a + b * xs
end

function fit_gray_scale(::Union{GrayScaleLine,WhiteRect}, data::AbstractVector)
    _fit_line(data)
end

function fit_gray_scale(grimg::GrayScalePyramid, data::AbstractVector)
    plateau_len = plateau_length(grimg)
    len = (grimg.width - plateau_len) ÷ 2
    len == 0 && return _fit_line(data)
    steps = accumulate((len, len + plateau_len, grimg.width), init = 0) do x, y
        (last(x) + 1):y
    end
    lines = _fit_line.(data[l] for l ∈ steps)
    reduce(vcat, lines)
end

function compute_rmse(grimg::Union{GrayScaleLine,WhiteRect}, data::AbstractVector; relative = false, scale = nothing)
    fitted_data = fit_gray_scale(grimg, data)
    compute_rmse(grimg, fitted_data, data; relative, scale)
end

function compute_rmse(grimg::GrayScalePyramid, data::AbstractVector; relative = false, scale = nothing)
    fitted_data = fit_gray_scale(grimg, data)
    compute_rmse(grimg, fitted_data, data; relative, scale)
end

function compute_χ2(grimg::AbstractGrayScale, expected::AbstractVector, measured::Vector, variances::Vector)
    ν = length(expected) - 2
    sum(@. (measured - expected) ^ 2 / variances) / ν
end

end # module


"""
    blank(s::SSR, blank_delay::AbstractFloat)

Blank a steady state recording.

Linearly interpolate between valid data samples.
The pulse rate can be specified or is taken from type data.
"""
function blank(a::SSR, blank_delay::AbstractFloat; valid_triggers::Int=-4,
    temptrigger_rate::AbstractFloat=1/a.processing["Carrier_Frequency"], temptrigger_rate_code::Int=22,
    temptrigger::Int=33, kwargs...)

    Logging.info("Blanking SSR after $blank_delay (s)")

    # Place trigger to indicate where electrical stimulation occurs (22)
    new_triggers = extra_triggers(a.triggers, valid_triggers, temptrigger_rate_code, temptrigger_rate, samplingrate(a))

    # Place trigger where to blank from
    new_triggers = extra_triggers(new_triggers, [valid_triggers; temptrigger_rate_code], temptrigger, blank_delay, samplingrate(a), max_inserted=1)

    # Create a time vector
    a.processing["time"] = collect(1 : size(a.data, 1)) / samplingrate(a)

    # Find points to blank from
    valid_trip_idx = find(new_triggers["Code"]-252 .== temptrigger)

    # Initial values
    previous_idx = 1
    previous_value = a.data[previous_idx, :]

    # For each point to sample
    for i in 1:length(valid_trip_idx)-1

        current_valid_idx = new_triggers["Index"][valid_trip_idx[i]]
        current_value = a.data[current_valid_idx, :]
        idxs = collect(previous_idx : 1 : current_valid_idx)

        for c = 1:size(a.data, 2) # For each channel

            a.data[idxs, c] = linspace(previous_value[c], current_value[c], length(idxs))
        end

        previous_value = current_value
        previous_idx = current_valid_idx

    end

    return a
end

"""
    subsample(s::SSR)

Blank steady state recording using a set of valid samples between each pulse.

Assumes more than one valid sample per pulse. These valid samples are averaged then linearly interpolated between.
"""
function subsample(a::SSR; valid_triggers::Int=-4,
                   temptrigger_rate::AbstractFloat=1/a.processing["Carrier_Frequency"], temptrigger_rate_code::Int=22,
                   subsample_start_delay::Number=0.001, temptrigger_start_idx=33,
                   subsample_stop_delay::Number=0.0015, temptrigger_stop_idx=34,
                   plot::Bool=false, kwargs...)

    # Add a trigger (22) where each CI artifact starts
    # Add a trigger (33) where to start sampling the valid response
    # Add a trigger (34) where to stop sampling the valid response
    # Go through each (33) trigger and take mean between that and (34)
    # Place that mean value at the mean time point between them and save subsampled data to SSR type

    Logging.info("Subsampling SSR")
    Logging.debug("Pulse every $(round(temptrigger_rate, 8)) s")
    Logging.debug("Start sampling after $(round(subsample_start_delay, 8)) s")
    Logging.debug("Stop sampling after $(round(subsample_stop_delay, 8)) s")

    a.processing["time"] = collect(1:size(a.data,1)) / samplingrate(a)

    if plot
        warn("Plotting functionality has been depreciated")
    end

    # Place trigger to indicate where electrical stimulation occurs (22)
    new_triggers = extra_triggers(a.triggers, valid_triggers, temptrigger_rate_code, temptrigger_rate, samplingrate(a))

    # Place trigger to indicate where clean data starts (33)
    new_triggers = extra_triggers(new_triggers, [valid_triggers, temptrigger_rate_code], temptrigger_start_idx, subsample_start_delay, samplingrate(a), max_inserted=1)

    # Place trigger to indicate where clean data ends (34)
    new_triggers = extra_triggers(new_triggers, [valid_triggers, temptrigger_rate_code], temptrigger_stop_idx, subsample_stop_delay, samplingrate(a), max_inserted=1)

    # if length(unique(new_triggers["Index"])) != length(new_triggers["Index"])
    #     warn("Multiple triggers in the same place. Start and end of sample location might be too close together")
    # end

    # Run through all the 33s and interpolate between averaged valid values
    valid_trip_idx = find(new_triggers["Code"]-252 .== 33)

    previous_value = zeros(Float32, size(a.data,2)) # Not used, but will stop
    previous_idx = 1                                # lint errors

    cnt = 0
    for i in 1:length(valid_trip_idx)-1
        cnt += 1

        if cnt < 4
            Logging.info("Processing sample location $cnt with idx = $(valid_trip_idx[i]) and value $(new_triggers["Code"][valid_trip_idx[i]]-252)")
        end

        # Check the next index is a 34
        if new_triggers["Code"][valid_trip_idx[i]+1]-252 == temptrigger_stop_idx

            valid_range = new_triggers["Index"][valid_trip_idx[i]] : 1 : new_triggers["Index"][valid_trip_idx[i]+1]

            if i == 1
              Logging.debug("Averaging $(length(valid_range)) data points for subsample")
            end

            mean_value = mean(a.data[valid_range ,:], 1)

            mean_idx = Int(round(mean(valid_range)))

            # Only interpolate from second valid point backwards
            if cnt > 1

                idxs = collect(previous_idx : 1 : mean_idx)

                for c = 1:size(a.data,2)

                    a.data[idxs, c] = linspace(previous_value[c], mean_value[c], length(idxs))

                end
            end

            previous_value = mean_value
            previous_idx = mean_idx

        else

            error("Expected next trigger $cnt to be end of valid range. But was $(new_triggers["Code"][valid_trip_idx[i]+1]-252)")
            error(new_triggers["Code"][1:10]-252)
            error(new_triggers["Index"][1:10])
        end
    end

    return a
end


# Author: Youngjun Kim, youngjun@stanford.edu
# Date: 06/16/2014

module SimpleTCASSensorImpl

export
    initialize,
    update,

    updateSensor,

    SimpleTCASSensor,
    SimpleTCASSRState,
    SimpleTCASSensorInput,
    SimpleTCASSensorOutput

using AbstractSensorImpl
using AbstractSensorInterfaces
using CommonInterfaces

using Base.Test

import CommonInterfaces.initialize
import CommonInterfaces.update
import AbstractSensorInterfaces.updateSensor

type SimpleTCASSRState

    t::Float64
    x::Float64
    y::Float64
    h::Float64
    vx::Float64
    vy::Float64
    vh::Float64
end

type SimpleTCASSensorInput

    states::Vector{SimpleTCASSRState}
end

type SimpleTCASSensorOutput

    t::Float64

    r::Float64
    r_d::Float64

    a::Float64
    a_d::Float64

    h::Vector{Float64}
    h_d::Vector{Float64}
end

type SimpleTCASSensor <: AbstractSensor

  aircraft_number::Int
  states::Vector{SimpleTCASSRState}
  output::SimpleTCASSensorOutput

  function SimpleTCASSensor(aircraft_number::Int)
    obj = new()

    obj.aircraft_number = aircraft_number
    obj.output = SimpleTCASSensorOutput(0.0,0.0,0.0,0.0,0.0,Float64[],Float64[])

    return obj
  end
end

function updateSensor(sr::SimpleTCASSensor, input::SimpleTCASSensorInput)

  sr.states = input.states

  if sr.aircraft_number == 1
    intruder_number = 2
  else
    intruder_number = 1
  end

  t1, x1, y1, h1, vx1, vy1, vh1 = sr.states[sr.aircraft_number].t, sr.states[sr.aircraft_number].x, sr.states[sr.aircraft_number].y, sr.states[sr.aircraft_number].h, sr.states[sr.aircraft_number].vx, sr.states[sr.aircraft_number].vy, sr.states[sr.aircraft_number].vh

  t2, x2, y2, h2, vx2, vy2, vh2 = sr.states[intruder_number].t, sr.states[intruder_number].x, sr.states[intruder_number].y, sr.states[intruder_number].h, sr.states[intruder_number].vx, sr.states[intruder_number].vy, sr.states[intruder_number].vh

  #@test t1 == t2

  t = t1

  dxy = [(x2 - x1), (y2 - y1)]
  dvxy = [(vx2 - vx1), (vy2 - vy1)]
  r = norm(dxy)
  r_d = dot(dxy,dvxy) / norm(dxy)

  a = abs(h1 - h2)
  a_d = sign(h2 - h1) * (vh2 - vh1)

  h = [h1, h2]
  h_d = [vh1, vh2]

  sr.output.t = t
  sr.output.r = r
  sr.output.r_d = r_d
  sr.output.a = a
  sr.output.a_d = a_d
  sr.output.h = h
  sr.output.h_d = h_d

  return sr.output
end

update(sr::SimpleTCASSensor, input) = update(sr,convert(SimpleTCASSensorInput, input))

update(sr::SimpleTCASSensor, input::SimpleTCASSensorInput) = updateSensor(sr, input)

function initialize(sr::SimpleTCASSensor)

end

end


using DataFrames
using Gadfly

benchmarks = readtable("benchmarks.csv", names=[:language, :benchmark, :time])
cdata = benchmarks[benchmarks[:language].== "c", :]
benchmarks = join(benchmarks, cdata, on=:benchmark)
benchmarks[:time]./= benchmarks[:time_1]
benchmarks[:language] = PooledDataArray(benchmarks[:language])
benchmarks[:benchmark] = PooledDataArray(benchmarks[:benchmark])
benchmarks = benchmarks[benchmarks[:language].!= "c", :]
benchmarks[:language] = setlevels!(benchmarks[:language], Dict{UTF8String,Any}(
  [lang => (lang == "javascript" ? "JavaScript" : ucfirst(lang)) for lang in benchmarks[:language]]))

p = plot(benchmarks,
    x = :language,
    y = :time,
    color = :benchmark,
    Scale.y_log10,
    Guide.ylabel(nothing),
    Guide.xlabel(nothing),
    Theme(
        default_point_size = 1mm,
        guide_title_position = :left,
        colorkey_swatch_shape = :circle,
        minor_label_font = "Georgia",
        major_label_font = "Georgia",
    ),
)

draw(SVG("_includes/benchmarks.svg", 8inch, 8inch/golden), p)


"""
    reorder_policy(env::SupplyChainEnv, param1::Dict, param2::Dict,
                        policy_type::Union{Dict, Symbol} = :rQ,
                        review_period::Union{Int, StepRange, Vector, Dict} = 1,
                        min_order_qty::Union{Real, Dict} = 0)

Apply an inventory policy to specify the replinishment orders for each material
    throughout the `SupplyChainEnv`.

# Arguments
- `env::SupplyChainEnv`: inventory management environment
- `param1::Dict`: the `s` or `r` parameter in each node for each material in the system. The `keys` are of the form `(node, material)`.
- `param2::Dict`: the `S` or `Q` parameter in each node for each material in the system. The `keys` are of the form `(node, material)`.
- `policy_type::Union{Symbol, Dict}`: `:rQ` for an `(r,Q)` policy, or `:sS` for an `(s,S)` policy. If passing a `Dict`, the policy type should be specified for each node (keys).
- `review_period::Union{Int, StepRange, Vector, Dict}`: number of periods between each inventory review (Default = `1` for continuous review.). If a `StepRange` or `Vector` is used, the `review_period` indicates which periods the review is performed on. If a `Dict` is used, the review period should be specified for each `(node, material)` `Tuple` (keys). The values of this `Dict` can be either `Int`, `StepRange`, or `Vector`. Any missing `(node, material)` key will be assigned a default value of 1.
- `min_order_qty::Union{Real, Dict}`: minimum order quantity (MOQ) at each supply node. If a `Dict` is passed, the MOQ should be specified for each `(node, material)` `Tuple` (keys). The values should be `Real`. Any missing key will be assigned a default value of 0.
"""
function reorder_policy(env::SupplyChainEnv, param1::Dict, param2::Dict;
                        policy_variable::Union{Dict,Symbol} = :inv_position,
                        policy_type::Union{Dict, Symbol} = :rQ,
                        review_period::Union{Int, StepRange, Vector, Dict} = 1,
                        min_order_qty::Union{Real, Dict} = 0)

    #check review period
    null_action = zeros(length(env.materials)*ne(env.network))
    if review_period isa Int && !iszero(mod(env.period,review_period)) #if not in review period, send null action
        return null_action
    elseif review_period isa Union{StepRange,Vector} && !in(env.period, review_period)
        return null_action
    end

    #read parameters
    nodes = [n for n in vertices(env.network) if !isempty(inneighbors(env.network, n))] #all non-source nodes can place orders
    supply_nodes = [n for n in vertices(env.network) if !isempty(outneighbors(env.network, n))] #all non-sink nodes can place orders
    arcs = [(e.src, e.dst) for e in edges(env.network)]
    mats = env.materials

    #check inputs
    for p in mats
        for n in nodes
            for param in [param1, param2]
                if !in((n,p), keys(param))
                    param[(n,p)] = -1 #if no policy given for a node/material, set the params to -1 so that a reorder is never triggered
                end
            end
            if review_period isa Dict && !in((n,p), keys(review_period))
                review_period[(n,p)] = 1 #set to default value of 1
            end
        end
        for n in supply_nodes
            if min_order_qty isa Dict && !in((n,p), keys(min_order_qty))
                min_order_qty[(n,p)] = 0 #set to default value of 0
            end
        end
    end

    #filter data for policy
    state1_df = filter(:period => j -> j == env.period, env.inv_position, view=true) #inventory position
    state1_grp = groupby(state1_df, [:node, :material]) #group by node and material
    state2_df = filter(:period => j -> j == env.period, env.ech_position, view=true) #inventory position
    state2_grp = groupby(state2_df, [:node, :material]) #group by node and material

    #create action matrix
    action = zeros(length(mats), length(arcs)) #initialize action matrix
    for n in nodes, (k, p) in enumerate(mats)
        #if trigger level is negative, skip it
        param1[n,p] < 0 && continue
        #if not a review period, skip
        if review_period isa Dict
            review_period[n,p] isa Int && !iszero(mod(env.period, review_period[n,p])) && continue #trigger if not in review period
            review_period[n,p] isa Union{StepRange,Vector} && !in(env.period, review_period[n,p]) && continue
        end
        #get MOQ
        supplier = get_prop(env.network, n, :supplier_priority)[p][1] #get first priority supplier
        MOQ = min_order_qty isa Real ?
                min_order_qty :
                min_order_qty[(supplier,p)]
        #initialize reorder quantity
        reorder = 0
        #check if reorder is triggered & set reorder policy
        pol_var = policy_variable isa Dict ? policy_variable[n] : policy_variable
        if pol_var == :inv_position
            state = state1_grp[(node = n, material = p)].level[1]
        elseif pol_var == :ech_position
            state = state2_grp[(node = n, material = p)].level[1]
        else
            @assert pol_var in [:inv_position, :ech_position] "Policy variable must be either `:inv_position` or `:ech_position`."
        end
        if state <= param1[n,p]
            pol_type = policy_type isa Dict ? policy_type[n] : policy_type
            if pol_type == :rQ #rQ policy
                reorder = max(param2[n,p], MOQ)
            elseif pol_type == :sS #sS policy
                reorder = state < param2[n,p] ? max(param2[n,p] - state, MOQ) : 0
            else
                @assert pol_type in [:rQ, :sS] "The policy type must be either `:rQ` or `:sS`."
            end
        else
            continue
        end
        #assign reorder quantity
        j = findfirst(i -> i == (supplier, n), arcs) #find index in action matrix
        action[k, j] = reorder #sate request quantity
    end

    return collect(Iterators.flatten(action))
end

"""
    simulate_policy!(env::SupplyChainEnv, args...)

Step through a simulation using a specified reorder policy. `args` are the
arguments that are passed to the `reorder_policy` function.
"""
function simulate_policy!(env::SupplyChainEnv, args...; kwargs...)
    for t in 1:env.num_periods
        action = reorder_policy(env, args...; kwargs...)
        (env)(action)
    end
end


# This file is a part of TextClassification.jl
# License is Apache 2.0: https://www.apache.org/licenses/LICENSE-2.0.txt

using MLDataUtils, LinearAlgebra
import TextSearch: vectorize, vectorize_corpus
import StatsBase: predict
import Base: hash, isequal
export filtered_power_set, predict, predict_corpus, vectorize, vectorize_corpus,
        MicroTC, AngleDistance, CosineDistance, NormalizedAngleDistance, NormalizedCosineDistance
import Base: hash, isequal
using SparseArrays

struct MicroTC{C_<:MicroTC_Config, CLS_<:Any, TextModel_<:TextModel, LabelType<:Any}
    config::C_
    cls::CLS_
    textmodel::TextModel_
    tok::Tokenizer
    levels::Vector{LabelType}
end

function Base.show(io::IO, model::MicroTC)
    print(io, "{MicroTC")
    show(io, model.config)
    print(io, " ")
    show(io, typeof(model.cls))
    print(io, " ")
    show(io, model.textmodel)
    print(io, " ")
    show(io, model.tok)
    print(io, " ")
    show(io, model.levels)
    print(io, "}")
end

Base.copy(c::MicroTC; config=c.config, cls=c.cls, textmodel=c.textmodel) = MicroTC(config, cls, textmodel)
Base.broadcastable(tc::MicroTC) = (tc,)

"""
    create(config, train_X, train_y) # config describes a text model
    create(config, train_X, train_y, tok, dim) # config describes a classifier

Creates a new object from a configuration and a train / test datasets.

- train_X is an array of BOW objects for text models
- train_X is an array of SVEC objects for classifiers
"""
create(config::KncConfig, train_X, train_y, dim) = Knc(config, train_X, train_y)
create(config::KncProtoConfig, train_X, train_y, dim) = KncProto(config, train_X, train_y)

"""
    MicroTC(
        config::MicroTC_Config,
        train_corpus::AbstractVector,
        train_y::CategoricalArray;
        tok=Tokenizer(config.textconfig),
        verbose=true)
    MicroTC(config::MicroTC_Config, textmodel::TextModel, train_X::AbstractVector{S}, train_y::CategoricalArray; verbose=true) where {S<:SVEC}

Creates a MicroTC model on the given dataset and configuration
"""
function MicroTC(
        config::MicroTC_Config,
        train_corpus::AbstractVector,
        train_y::CategoricalArray;
        tok=Tokenizer(config.textconfig, invmap=nothing),
        verbose=true)
    train_corpus_bow = compute_bow_corpus(tok, train_corpus)
    textmodel = create(config.textmodel, train_corpus_bow, train_y)
    X = SVEC[]
    mask = Bool[]
    for bow in train_corpus_bow
        x = vectorize(textmodel, bow)
        push!(mask, false)
        length(x) == 1 && haskey(x, 0) && continue
        push!(X, x)
        mask[end] = true
    end

    m = sum(mask)
    if m != length(mask)
        @info "WARNING considering $(m) of $(length(mask)) examples after vectorization using $(config)"
    end
    MicroTC(config, textmodel, X, train_y[mask], tok=Tokenizer(tok))  # tok is a copy with isconstruction=false
end

function MicroTC(
        config::MicroTC_Config,
        textmodel::TextModel,
        train_X::AbstractVector{S},
        train_y::CategoricalArray;
        tok=Tokenizer(config.textconfig, invmap=nothing),
        verbose=true) where {S<:SVEC}
    cls = create(config.cls, train_X, train_y, textmodel.m)
    MicroTC(config, cls, textmodel, tok, copy(levels(train_y)))
end

"""
    vectorize(tc::MicroTC, text;
        bow=BOW(),
        tok=Tokenizer(tc.config.textconfig, invmap=nothing),
        normalize=true)
    vectorize(tc::MicroTC, bow::BOW)

Creates a weighted vector using the model. The input `text` can be a string or an array of strings;
it also can be an already computed bag of words.

"""
function vectorize(
        tc::MicroTC,
        text;
        bow=BOW(),
        tok=tc.tok,
        normalize=true
    )::SVEC
    vectorize(tc.textmodel, compute_bow(tok, text, bow); normalize)
end

function vectorize(
        tc::MicroTC,
        bow::BOW;
        normalize=true
    )::SVEC
    vectorize(tc.textmodel, bow; normalize=normalize)
end

function vectorize_corpus(
        tc::MicroTC,
        corpus;
        bow=BOW(),
        tok=tok=tc.tok,
        normalize=true
    )
    V = Vector{SVEC}(undef, length(corpus))

    for i in eachindex(corpus)
        empty!(bow)
        V[i] = vectorize(tc, corpus[i]; bow, tok, normalize)
    end

    V
end

"""
    predict(tc::MicroTC, text)
    predict(tc::MicroTC, vec::SVEC)

Predicts the label of the given input
"""
predict(tc::MicroTC, text) = predict(tc.cls, vectorize(tc, text))
predict(tc::MicroTC, vec::SVEC) = tc.levels[predict(tc.cls, vec)]

function predict_corpus(tc::MicroTC, corpus;
    bow=BOW(),
    tok=tc.tok,
    normalize=true)
    V = Vector{eltype(tc.levels)}(undef, length(corpus))

    for i in eachindex(corpus)
        empty!(bow)
        empty!(tok)
        v = vectorize(tc, corpus[i]; bow, tok, normalize)
        V[i] = predict(tc, v)
    end

    V
end


using SpectralVis
using Test

@testset "Spectral environment test" begin
    env = SPECENV

    @test env.λmin == 390.0
    @test env.Δλ == 1.0
    @test env.λmax == 830.0
    @test env.cmf isa cie12_10
    @test env.ex == :linear

    for i in (:none, :boundary, :linear, :parabolic)
        set_extrap(env, i)
        @test env.ex == i
    end

    @test set_extrap(env, :cubic) == DomainError

    a, b, c = 400.0, 5.0, 800.0

    set_limits(env, a, b, c)
    @test env.λmin, env.Δλ, env.λmax == 400.0, 5.0, 800.0

    @test set_colormatch(env, :cie06_2).cmf isa CIE12_2
    @test set_colormatch(:cie06_10).cmf isa CIE12_10
end

@testset "Block spectrum generator mode 0 (block_spec) test" begin
    m = 0
    λmin=380.0
    λmax=780.0
    Δλ  =  1.0
    # 1.0 @ 381.0:382.0
    λs=381.0
    λe=382.0
    refl=zeros(length(collect(λmin:Δλ:λmax)))
    refl[2:3].=1.0
    gt=i_reflectance_spec(collect(λmin:Δλ:λmax),refl)
    @test block_spec(λmin,λmax,Δλ,λs,λe,m).r == gt.r
    # 1.0 @ 781.0
    λs=778.0
    λe=779.0
    refl=zeros(length(collect(λmin:Δλ:λmax)))
    refl[end-2:end-1].=1.0
    gt=i_reflectance_spec(collect(λmin:Δλ:λmax),refl)
    @test block_spec(λmin,λmax,Δλ,λs,λe,m).r == gt.r
end

@testset "Block spectrum generator mode 1 (block_spec) test" begin
    #Inverse block spectrum
    m = 1
    λmin=380.0
    λmax=780.0
    Δλ  =  1.0
    # 1.0 @ 381.0:382.0
    λs=381.0
    λe=382.0
    # 0.0 @ 381.0
    refl=ones(length(collect(λmin:Δλ:λmax)))
    refl[2:3].=0.0
    gt=i_reflectance_spec(collect(λmin:Δλ:λmax),refl)
    @test block_spec(λmin,λmax,Δλ,λs,λe,m).r == gt.r

    # 0.0 @ 781.0
    λs=778.0
    λe=779.0
    refl=ones(length(collect(λmin:Δλ:λmax)))
    refl[end-2:end-1].=0.0
    gt=i_reflectance_spec(collect(λmin:Δλ:λmax),refl)
    @test block_spec(λmin,λmax,Δλ,λs,λe,m).r == gt.r
end

# test of convert(::Type{CIE12_2},mf::LMS06_2)
@testset "LMS06_2 to CIE12_2 conversion" begin
    test02 = cmf(CMF2012_2)
    res02  = convert(CIE12_2,cmf(LMS2006_2))

    eps=5.0E-7

    delta2_1=test02.x.-res02.x
    delta2_2=test02.y.-res02.y
    delta2_3=test02.z.-res02.z

    @test maximum(broadcast(abs,(delta2_1))) ≈ 0 atol=eps
    @test maximum(broadcast(abs,(delta2_2))) ≈ 0 atol=eps
    @test maximum(broadcast(abs,(delta2_3))) ≈ 0 atol=eps
end

# test of convert(::Type{CIE12_10},mf::LMS06_10)
@testset "LMS06_10 to CIE12_10 conversion" begin
    test10 = cmf(CMF2012_10)
    res10  = convert(CIE12_10,cmf(LMS2006_10))

    eps=5.0E-7

    delta10_1=test10.x.-res10.x
    delta10_2=test10.y.-res10.y
    delta10_3=test10.z.-res10.z

    @test maximum(broadcast(abs,(delta10_1))) ≈ 0 atol=eps
    @test maximum(broadcast(abs,(delta10_2))) ≈ 0 atol=eps
    @test maximum(broadcast(abs,(delta10_3))) ≈ 0 atol=eps
end

@testset "Spectral operators test" begin
    @test 1==1

    luminance = D65_illuminant()

end


using Unitful
using UnitfulAtomic

"""
Read the bands file

The bands file contains the eigenvalues of the

# Arguments

- reorder: If true, recover the orignial kpoint orders as defined in the cell file. Note that the
order of kpoint in other quantities, such as the optical matrix, are the same as they appear in the
bands file rather than following the internal "counter".

"""
function read_bands_castep_raw(fname::AbstractString)
    nk, ns, nelect, neign, fermi_engs, cell_mat = read_bands_header(fname)
    # Allocate the ararys
    bands = zeros(Float64, ns, nk, neign)
    kweight = zeros(nk)
    kpoints = zeros(3, nk)

    bfile = open(fname)
    ecount = 1
    is = 0
    ik = 0
    kcount = 1
    # Record the "original" kpoint index, as they defined in the input cell file
    kidx = zeros(Int64, nk)
    for (i, line) in enumerate(eachline(bfile))
        # Skip to line 11
        i < 10 && continue
        if contains(line, "K-point")
            tokens = split(line)
            ik = parse(Int, tokens[2])
            kidx[kcount] = ik
            kvec = [parse(Float64, token) for token in tokens[3:5]]
            weight = parse(Float64, tokens[6])
            kpoints[:, kcount] = kvec
            kweight[kcount] = weight
            kcount += 1
            continue
        end
        if contains(line, "Spin component")
            is = parse(Int, split(line)[3])  # Covnert unit to eV
            ecount = 1
            continue
        end
        # Read the eigen values for the corresponding kpoint
        # Because kcount was already incremeted - the current kpoint is there kcount - 1
        bands[is, kcount - 1, ecount] = parse(Float64, strip(line))
        ecount += 1
    end

    # Convert unit to eV
    bands_with_unit = uconvert.(u"eV", bands .* u"Eh_au")

    return (kpoints=kpoints,
            kweights=kweight,
            kidx=kidx,
            bands=bands_with_unit,
            nelect=nelect,
            fermi_engs=fermi_engs,
            cell_mat=cell_mat)
end

"""
Read the header for the bands file
"""
function read_bands_header(fname::AbstractString)
    bfile = open(fname)
    nk = 0
    ns = 0
    nelect = 0.
    neigen = 0
    efermi = Float64[]
    for ln in eachline(bfile)
        if contains(ln, "Number of k-points")
            nk = parse(Int, split(ln)[end])
            continue
        end

        if contains(ln, "Number of spin components")
            ns = parse(Int, split(ln)[end])
            continue
        end

        if contains(ln, "Number of electrons")
            nelect = parse(Float64, split(ln)[end])
            continue
        end

        if contains(ln, "Number of eigenvalues")
            neigen = parse(Int, split(ln)[end])
            continue
        end
        if contains(ln, "Fermi ")
            tokens = split(ln)
            efermi = [uconvert(u"eV", parse(Float64, tokens[idx]) * u"Eh_au") for idx = 6:length(tokens)]
            continue
        end
        if contains(ln, "Unit cell vectors")
            break
        end
    end
    # Read the cell vectors
    cell_vec = zeros(Float64, 3, 3)
    cell_vec[:, 1] = Float64[parse(Float64, token) for token in split(readline(bfile))]
    cell_vec[:, 2] = Float64[parse(Float64, token) for token in split(readline(bfile))]
    cell_vec[:, 3] = Float64[parse(Float64, token) for token in split(readline(bfile))]
    cell_vec_with_unit = uconvert.(u"Å", cell_vec * u"a0_au")

    close(bfile)
    @assert nk > 0
    @assert ns > 0
    @assert nelect > 0
    @assert neigen > 0
    @assert length(efermi) > 0
    return nk, ns, nelect, neigen, efermi, cell_vec_with_unit
end

precompile(read_bands_castep_raw, (String,))


# Copyright (c) 2020: Matthew Wilhelm & Matthew Stuber.
# This work is licensed under the Creative Commons Attribution-NonCommercial-
# ShareAlike 4.0 International License. To view a copy of this license, visit
# http://creativecommons.org/licenses/by-nc-sa/4.0/ or send a letter to Creative
# Commons, PO Box 1866, Mountain View, CA 94042, USA.
#############################################################################
# Dynamic Bounds Base
# The underlying abstraction layer for DynamicBounds.
# See https://github.com/PSORLab/DynamicBoundsBase.jl
#############################################################################
# src/DynamicBoundsBase.jl
# Main module code.
#############################################################################

module DynamicBoundsBase

using DocStringExtensions, ElasticArrays
using ForwardDiff: Chunk, Dual, Partials, construct_seeds, single_seed
using SciMLBase: remake, AbstractODEProblem, ODEProblem, AbstractContinuousCallback,
                 solve, ODEFunction, wrapfun_iip
using DiffEqSensitivity: extract_local_sensitivities, ODEForwardSensitivityProblem

import Base.get

export HasStateBounds, HasConstantStateBounds, HasVariableStateBounds,
       HasUserJacobian, ConstantStateBounds, VariableStateBounds,
       PolyhedralConstraint, AbstractDERelaxProblem, AbstractRelaxProblemAttribute,
       AbstractDERelaxIntegrator, AbstractIntegratorAttribute, AbstractDynamicIndex
include(joinpath(@__DIR__, "problem.jl"))

export Nominal, Lower, Upper, Undefined, IntegratorName, Value, Gradient,
       ConstantParameterValue, Subgradient, Bound, Relaxation,IsNumeric,
       IsSolutionSet, TerminationStatus, ParameterValue, ParameterBound,
       SupportSet, TerminationStatusCode, relax!,
       integrate!, make, AbstractDERelaxIntegator, AbstractODERelaxIntegrator,
       AbstractBoundLoc, TimeIndex, COMPLETED, EMPTY, RELAXATION_NOT_CALLED,
       NUMERICAL_ERROR, INVALID_OPTION, OTHER_ERROR, LIMIT_EXCEEDED,
       ParameterNumber, StateNumber, SupportNumber, LocalSensitivityOn,
       LocalIntegrator, AttachedProblem

include(joinpath(@__DIR__, "integrator.jl"))

export set!, setall!, get, getall, getall!, make, supports
include(joinpath(@__DIR__, "attribute_handlers.jl"))

export IntegratorStates
include(joinpath(@__DIR__, "utilities.jl"))

export ODERelaxProb, AbstractODERelaxProblem, ODELocalIntegrator
include(joinpath(@__DIR__, "problems", "ODERelaxProb.jl"))

end


"""
    DistributedOptimiser(optimiser)

Wrap the `optimiser` in a `DistributedOptimiser`. Before updating the
parameters, this adds the gradients across the processes using
non-blocking Allreduce

!!! note
    Remember to scale the loss function by `1 / total_workers()` to ensure
    that the gradients are correctly averaged
"""
struct DistributedOptimiser{O}
    optimiser::O
end

function apply!(o::DistributedOptimiser, state, x, y)
    y_ = allreduce!(y, +, MPI.COMM_WORLD)
    apply!(o.optimiser, state, x, y_)
end

init(o::DistributedOptimiser, x::AbstractArray) = init(o.optimiser, x)

"""
    allreduce_gradients(gs::NamedTuple; on_gpu::Bool=CUDA.functional())
"""
function allreduce_gradients(gs::NamedTuple; on_gpu::Bool=CUDA.functional())
    if on_gpu
        # Transfer data to CPU since OpenMPI Iallreduce! doesn't work for CUDA
        gs = fmap(cpu, gs)
    end
    requests = MPI.Request[]
    nonblocking_reduce_gradients(g) = g
    function nonblocking_reduce_gradients(g::AbstractArray)
        g, req = Iallreduce!(g, +, MPI.COMM_WORLD)
        push!(requests, req)
        return g
    end
    gs = fmap(nonblocking_reduce_gradients, gs)
    MPI.Waitall!(requests)
    if on_gpu
        # Transfer data back to GPU
        gs = fmap(gpu, gs)
    end
    return gs
end


module PandorasBoxColoredWater

using ..Ahorn, Maple

@mapdef Entity "pandorasBox/coloredWater" ColoredWater(x::Integer, y::Integer, width::Integer=Maple.defaultBlockWidth, height::Integer=Maple.defaultBlockHeight, hasTop::Bool=true, hasBottom::Bool=false, hasLeft::Bool=false, hasRight::Bool=false, color::String="LightSkyBlue")

function getColor(color)
    if haskey(Ahorn.XNAColors.colors, color)
        return Ahorn.XNAColors.colors[color]

    else
        try
            return ((Ahorn.argb32ToRGBATuple(parse(Int, replace(color, "#" => ""), base=16))[1:3] ./ 255)..., 1.0)

        catch

        end
    end

    return (1.0, 1.0, 1.0, 1.0)
end

const colors = sort(collect(keys(Ahorn.XNAColors.colors)))

const placements = Ahorn.PlacementDict(
    "Water (Pandora's Box)" => Ahorn.EntityPlacement(
        ColoredWater,
        "rectangle"
    ),
)

Ahorn.editingOptions(entity::ColoredWater) = Dict{String, Any}(
    "color" => colors
)

Ahorn.minimumSize(entity::ColoredWater) = 8, 8
Ahorn.resizable(entity::ColoredWater) = true, true

Ahorn.selection(entity::ColoredWater) = Ahorn.getEntityRectangle(entity)

function Ahorn.render(ctx::Ahorn.Cairo.CairoContext, entity::ColoredWater, room::Maple.Room)
    x = Int(get(entity.data, "x", 0))
    y = Int(get(entity.data, "y", 0))

    width = Int(get(entity.data, "width", 32))
    height = Int(get(entity.data, "height", 32))
    rawColor = get(entity.data, "color", "LightSkyBlue")
    color = getColor(rawColor)

    Ahorn.drawRectangle(ctx, 0, 0, width, height, color .* (1.0, 1.0, 1.0, 0.4), color)
end

end