# New factor interface, something perhaps like this

export calcFactorResidualTemporary

getFactorOperationalMemoryType(dfg::SolverParams) = CommonConvWrapper
# difficult type piracy case needing both types NoSolverParams and CommonConvWrapper.
getFactorOperationalMemoryType(dfg::NoSolverParams) = CommonConvWrapper

function _getDimensionsPartial(ccw::CommonConvWrapper)
  # @warn "_getDimensionsPartial not ready for use yet"
  ccw.partialDims
end
_getDimensionsPartial(data::GenericFunctionNodeData) = _getCCW(data) |> _getDimensionsPartial
_getDimensionsPartial(fct::DFGFactor) = _getDimensionsPartial(_getCCW(fct))
_getDimensionsPartial(fg::AbstractDFG, lbl::Symbol) = _getDimensionsPartial(getFactor(fg, lbl))

"""
    $SIGNATURES
Get `.factormetadata` for each CPT in CCW for a specific factor in `fg`.
"""
_getFMdThread(ccw::CommonConvWrapper,
              thrid::Int=Threads.threadid()) = ccw.cpt[thrid].factormetadata
#
_getFMdThread(fc::Union{GenericFunctionNodeData,DFGFactor},
              thrid::Int=Threads.threadid()) = _getFMdThread(_getCCW(fc), thrid)
#
_getFMdThread(dfg::AbstractDFG,
              lbl::Symbol,
              thrid::Int=Threads.threadid()) = _getFMdThread(_getCCW(dfg, lbl), thrid)
#

# Helper function to construct CF from a CCW
function CalcFactor(ccwl::CommonConvWrapper;
                    factor = ccwl.usrfnc!,
                    metadata = _getFMdThread(ccwl),
                    _sampleIdx = 0,
                    _measCount = length(ccwl.measurement),
                    _legacyMeas = ccwl.measurement,
                    _legacyParams = ccwl.params,
                    _allowThreads = true,
                    cache = ccwl.dummyCache )
  #
  # FIXME using ccwl.dummyCache is not thread-safe
  CalcFactor( factor,
              metadata,
              _sampleIdx,
              _measCount,
              _legacyMeas,
              _legacyParams,
              _allowThreads,
              cache )
end

"""
    $SIGNATURES

Sample the factor stochastic model `N::Int` times and store the samples in the preallocated `ccw.measurement` container.

DevNotes
- Use in place operations where possible and remember `measurement` is a `::Tuple`.
- TODO only works on `.threadid()==1` at present, see #1094
- Also see, JuliaRobotics/RoME.jl#465
"""
sampleFactor(cf::CalcFactor{<:AbstractFactor}, N::Int=1  ) = [getSample(cf) for _=1:N]

function Base.show(io::IO, x::CalcFactor)
  println(io, )
  printstyled(io, " CalcFactor:\n", color=:blue)
  println(io, "  .factor: ", typeof(x.factor))
end

Base.show(io::IO, ::MIME"text/plain", x::CalcFactor) = show(io, x)

"""
    $SIGNATURES

Function to calculate measurement dimension from factor sampling.

Notes
- Will not work in all situations, but good enough so far.
  - # TODO standardize via domain or manifold definition...??
"""
function calcZDim(cf::CalcFactor{T}) where {T <: AbstractFactor}
  #
  try
    M = getManifold(T)
    return manifold_dimension(M)
  catch
    try
      M = getManifold(cf.factor)
      return manifold_dimension(M)
    catch
      @warn "no method getManifold(::$(string(T))), calcZDim will attempt legacy length(sample) method instead"
    end
  end

  # NOTE try to make sure we get matrix back (not a vector)
  smpls = sampleFactor(cf, 2)[1]
  return length(smpls)
end

calcZDim(ccw::CommonConvWrapper) = calcZDim(CalcFactor(ccw))

calcZDim(cf::CalcFactor{<:GenericMarginal}) = 0

calcZDim(cf::CalcFactor{<:ManifoldPrior}) = manifold_dimension(cf.factor.M)

"""
    $SIGNATURES

Helper function for evaluating factor residual functions, by adding necessary `CalcFactor` wrapper.

Notes
- Factor must already be in a factor graph to work
- Will not yet properly support all multihypo nuances, more a function for testing
- Useful for debugging a factor.

Example
```julia
fg = generateGraph_Kaess()

residual = calcFactorResidual(fg, :x1x2f1, [1.0], [0.0], [0.0])
```

Related

[`calcFactorResidualTemporary`](@ref), [`_evalFactorTemporary!`](@ref), [`evalFactor`](@ref), [`approxConv`](@ref)
"""
calcFactorResidual(dfgfct::DFGFactor, args...; ccw::CommonConvWrapper=IIF._getCCW(dfgfct)) = CalcFactor(ccw)(args...)
calcFactorResidual(dfg::AbstractDFG, fctsym::Symbol, args...) = calcFactorResidual(getFactor(dfg, fctsym), args...)

"""
    $SIGNATURES

Evaluate the residual function for a single sample.

Notes
- Binary factors only at this stage, and `multihypo` does not have to be considered in this test
- Assumes calculation is for a single particle, so `meas::Tuple{Z,other}` is only a single particles value.

Example
```julia
residual = calcFactorResidualTemporary(Pose2Pose2(...), (RoME.Pose2,RoME.Pose2), (z_i,), (x1, x2))
```

Related

[`calcFactorResidual`](@ref), [`CalcResidual`](@ref), [`_evalFactorTemporary!`](@ref), [`approxConv`](@ref), [`_buildGraphByFactorAndTypes!`](@ref)
"""
function calcFactorResidualTemporary( fct::AbstractRelative,
                                      varTypes::Tuple,
                                      measurement,
                                      pts::Tuple;
                                      tfg::AbstractDFG = initfg(),
                                      _blockRecursion::Bool=false,
                                      doTime::Bool=false  )
  #

  # build a new temporary graph
  _, _dfgfct = _buildGraphByFactorAndTypes!(fct, varTypes, pts, dfg=tfg, _blockRecursion=_blockRecursion)

  # get a fresh measurement if needed
  _measurement = if measurement != [] #length(measurement) != 0
    measurement
  else
    # now use the CommonConvWrapper object in `_dfgfct`
    cfo = CalcFactor(_getCCW(_dfgfct))
    sampleFactor(cfo, 1)[1]
  end

  # assume a single sample point is being run
  res = if doTime
    @time res = calcFactorResidual(_dfgfct, _measurement, pts...)
    res
  else
    calcFactorResidual(_dfgfct, _measurement, pts...)
  end
  return res
end

## =============================================================================================
## FactorOperationalMemory helper constructors
## =============================================================================================

function ConvPerThread( X::AbstractVector{P},
                        zDim::Int,
                        factormetadata::FactorMetadata;
                        particleidx::Int=1,
                        activehypo= 1:length(params),
                        perturb=zeros(zDim),
                        res=zeros(zDim),
                        thrid_ = 0  ) where P
  #
  return ConvPerThread{typeof(res), typeof(factormetadata), Any}( thrid_,
                        particleidx,
                        factormetadata,
                        Int[activehypo;],
                        perturb,
                        X,
                        res )
end

function CommonConvWrapper( usrfnc::T,
                            X::AbstractVector{P},
                            zDim::Int,
                            varValsLink::NamedTuple,
                            factormetadata::FactorMetadata;
                            partial::Bool=false,
                            hypotheses::H=nothing,
                            certainhypo=nothing,
                            activehypo= 1:length(varValsLink),
                            nullhypo::Real=0,
                            varidx::Int=1,
                            measurement::AbstractVector=Vector(Vector{Float64}(),),
                            particleidx::Int=1,
                            xDim::Int=size(X,1),
                            partialDims::AbstractVector{<:Integer}=1:length(X),
                            perturb=zeros(zDim),
                            res::AbstractVector{<:Real}=zeros(zDim),
                            threadmodel::Type{<:_AbstractThreadModel}=MultiThreaded,
                            inflation::Real=3.0,
                            vartypes=typeof.(getVariableType.(factormetadata.fullvariables)),
                            gradients=nothing,
                            userCache::CT = nothing ) where {T<:AbstractFactor,P,H,CT}
  #

  return  CommonConvWrapper(usrfnc,
                            xDim,
                            zDim,
                            partial,
                            hypotheses,
                            certainhypo,
                            Float64(nullhypo),
                            varValsLink,
                            varidx,
                            measurement,
                            threadmodel,
                            (i->ConvPerThread(X, zDim,factormetadata, particleidx=particleidx,
                                              activehypo=activehypo,
                                              perturb=perturb, res=res )).(1:Threads.nthreads()),
                            inflation,
                            partialDims,
                            DataType[vartypes...],
                            gradients,
                            userCache )
end

function _resizePointsVector!(vecP::AbstractVector{P}, mkd::ManifoldKernelDensity, N::Int) where P
  #
  pN = length(vecP)
  resize!(vecP, N)
  for j in pN:N
    smp = AMP.sample(mkd, 1)[1]
    # @show j, smp, typeof(smp), typeof(vecP[j])
    vecP[j] = smp[1]
  end

  vecP
end

"""
    $(SIGNATURES)

Prepare the particle arrays `ARR` to be used for approximate convolution.
This function ensures that ARR has te same dimensions among all the parameters.
Function returns with ARR[sfidx] pointing at newly allocated deepcopy of the
existing values in getVal(Xi[.label==solvefor]).

Notes
- Return values `sfidx` is the element in ARR where `Xi.label==solvefor` and
- `maxlen` is length of all (possibly resampled) `ARR` contained particles.
- `Xi` is order sensitive.
- for initialization, solveFor = Nothing.
- `P = getPointType(<:InferenceVariable)`

DevNotes
- FIXME ARR internally should become a NamedTuple
"""
function _prepParamVec( Xi::Vector{<:DFGVariable},
                        solvefor::Union{Nothing, Symbol},
                        N::Int=0;
                        solveKey::Symbol=:default  ) where P
  #
  # FIXME ON FIRE, refactor to new NamedTuple instead
  varParamsAll = Vector{Vector{Any}}()

  LEN = Int[]
  maxlen = N # FIXME see #105
  count = 0
  sfidx = 0

  for xi in Xi
    vecP = getVal(xi, solveKey=solveKey)
    push!(varParamsAll, vecP)
    LEN = length.(varParamsAll)
    maxlen = maximum([N; LEN])
    count += 1
    if xi.label == solvefor
      sfidx = count #xi.index
    end
  end

  # resample variables with too few kernels (manifolds points)
  SAMP = LEN .< maxlen
  for i in 1:count
    if SAMP[i]
      Pr = getBelief(Xi[i], solveKey)
      _resizePointsVector!(varParamsAll[i], Pr, maxlen)
    end
  end

  # TODO --rather define reusable memory for the proposal
  # we are generating a proposal distribution, not direct replacement for existing memory and hence the deepcopy.
  if sfidx > 0
    varParamsAll[sfidx] = deepcopy(varParamsAll[sfidx])
  end

  # get solvefor manifolds
  # FIXME deprecate use of (:null,)
  mani = length(Xi)==0 || sfidx==0 ? (:null,) : getManifold(Xi[sfidx])

  varTypes = typeof.(getVariableType.(Xi)) # previous need to force unstable, ::Vector{DataType}

  tup = tuple(varParamsAll...)
  nms = tuple(getLabel.(Xi)...)
  ntp = NamedTuple{nms,typeof(tup)}(tup)

  # FIXME, forcing maxlen to N results in errors (see test/testVariousNSolveSize.jl) see #105
  # maxlen = N == 0 ? maxlen : N
  return ntp, maxlen, sfidx, mani, varTypes
end

"""
    $SIGNATURES
Internal method to set which dimensions should be used as the decision variables for later numerical optimization.
"""
function _setCCWDecisionDimsConv!(ccwl::Union{CommonConvWrapper{F},
                                              CommonConvWrapper{Mixture{N_,F,S,T}}} ) where {N_,F<:Union{AbstractManifoldMinimize, AbstractRelativeMinimize, AbstractRelativeRoots, AbstractPrior},S,T}
  #

  # NOTE should only be done in the constructor
  ccwl.partialDims = if ccwl.partial
    Int[ccwl.usrfnc!.partial...]
  else
    Int[1:ccwl.xDim...]
  end

  nothing
end

function _updateCPTs!(ccwl, sfidx)
  for thrid in 1:Threads.nthreads()
    cpt_ = ccwl.cpt[thrid]
    cpt_.X = ccwl.params[sfidx]
    # used in ccw functor for AbstractRelativeMinimize
    # TODO JT - Confirm it should be updated here. Testing in _prepCCW
    resize!(cpt_.res, ccwl.zDim)
    fill!(cpt_.res, 0.0)
  end
  nothing
end

function attemptGradientPrep( varTypes, usrfnc, varParamsAll, multihypo, meas_single, _blockRecursion )
  # prepare new cached gradient lambdas (attempt)
  try
    # https://github.com/JuliaRobotics/IncrementalInference.jl/blob/db7ff84225cc848c325e57b5fb9d0d85cb6c79b8/src/DispatchPackedConversions.jl#L46
    # also https://github.com/JuliaRobotics/DistributedFactorGraphs.jl/issues/590#issuecomment-891450762
    # FIXME, suppressing nested gradient propagation on GenericMarginals for the time being, see #1010
    if (!_blockRecursion) && usrfnc isa AbstractRelative && !(usrfnc isa GenericMarginal)
      # take first value from each measurement-tuple-element
      measurement_ = meas_single
      # compensate if no info available during deserialization
      # take the first value from each variable param
      pts_ = map(x->x[1], varParamsAll)
      # FIXME, only using first meas and params values at this time...
      # NOTE, must block recurions here, since FGC uses this function to calculate numerical gradients on a temp fg.
      # assume for now fractional-var in multihypo have same varType
      hypoidxs = _selectHypoVariables(pts_, multihypo)
      gradients = FactorGradientsCached!(usrfnc, tuple(varTypes[hypoidxs]...), measurement_, tuple(pts_[hypoidxs]...), _blockRecursion=true);

      return gradients
    end
  catch e
    @warn "Unable to create measurements and gradients for $usrfnc during prep of CCW, falling back on no-partial information assumption.  Enable ENV[\"JULIA_DEBUG\"] = \"IncrementalInference\" for @debug printing to see the error."
    # rethrow(e)
    @debug(e)
  end
  return nothing
end

"""
    $SIGNATURES

Notes
- Can be called with `length(Xi)==0`
"""
function _prepCCW(Xi::Vector{<:DFGVariable},
                  usrfnc::T;
                  multihypo::Union{Nothing, <:Distributions.Categorical}=nothing,
                  nullhypo::Real=0.0,
                  certainhypo = multihypo !== nothing ? collect(1:length(multihypo.p))[multihypo.p .== 0.0] : collect(1:length(Xi)),
                  inflation::Real=0.0,
                  solveKey::Symbol=:default,
                  threadmodel=MultiThreaded,
                  _blockRecursion::Bool=false,
                  userCache::CT = nothing  ) where {T <: AbstractFactor, CT}
  #
  length(Xi) !== 0 ? nothing : @debug("cannot prep ccw.param list with length(Xi)==0, see DFG #590")

  # TODO check no Anys, see #1321
  _varValsQuick, maxlen, sfidx, mani, varTypes = _prepParamVec( Xi, nothing, 0; solveKey ) # Nothing for init.

  # standard factor metadata
  sflbl = 0 == length(Xi) ? :null : getLabel(Xi[end])
  lbs = getLabel.(Xi)
  fmd = FactorMetadata(Xi, lbs, _varValsQuick, sflbl, nothing)

  # create a temporary CalcFactor object for extracting the first sample
  # TODO, deprecate this:  guess measurement points type
  # MeasType = Vector{Float64} # FIXME use `usrfnc` to get this information instead
  _cf = CalcFactor( usrfnc, fmd, 0, 1, nothing, _varValsQuick, false, userCache)

  # get a measurement sample
  meas_single = sampleFactor(_cf, 1)[1]

  #TODO preallocate measurement?
  measurement = Vector{typeof(meas_single)}()

  # partialDims are sensitive to both which solvefor variable index and whether the factor is partial
  partial = hasfield(T, :partial)
  partialDims = if partial
    Int[usrfnc.partial...]
  else
    Int[]
  end

  # as per struct CommonConvWrapper
  gradients = attemptGradientPrep( varTypes, usrfnc, _varValsQuick, multihypo, meas_single, _blockRecursion )

  # variable Types
  pttypes = getVariableType.(Xi) .|> getPointType
  PointType = 0 < length(pttypes) ? pttypes[1] : Vector{Float64}

  return CommonConvWrapper(
          usrfnc,
          PointType[],
          calcZDim(_cf),
          _varValsQuick,
          fmd;
          partial,
          measurement,
          hypotheses = multihypo,
          certainhypo,
          nullhypo,
          threadmodel,
          inflation,
          partialDims,
          vartypes = varTypes,
          gradients,
          userCache
        )
end

"""
    $(SIGNATURES)

Prepare a common functor computation object `prepareCommonConvWrapper{T}` containing
the user factor functor along with additional variables and information using during
approximate convolution computations.

DevNotes
- TODO consolidate with others, see https://github.com/JuliaRobotics/IncrementalInference.jl/projects/6
"""
function _updateCCW!( F_::Type{<:AbstractRelative},
                      ccwl::CommonConvWrapper{F},
                      Xi::AbstractVector{<:DFGVariable},
                      solvefor::Symbol,
                      N::Integer;
                      needFreshMeasurements::Bool=true,
                      solveKey::Symbol=:default  ) where {F <: AbstractFactor}
  #
  length(Xi) !== 0 ? nothing : @debug("cannot prep ccw.param list with length(Xi)==0, see DFG #590")

  # FIXME, order of fmd ccwl cf are a little weird and should be revised.
  # FIXME maxlen should parrot N (barring multi-/nullhypo issues)
  _varValsQuick, maxlen, sfidx, mani, varTypes = _prepParamVec( Xi, solvefor, N; solveKey)

  # NOTE should be selecting for the correct multihypothesis mode
  ccwl.params = _varValsQuick
  # some better consolidate is needed
  ccwl.vartypes = varTypes
  # FIXME ON FIRE, what happens if this is a partial dimension factor?  See #1246
  ccwl.xDim = getDimension(getVariableType(Xi[sfidx]))
  # TODO maybe refactor new type higher up?

  # setup the partial or complete decision variable dimensions for this ccwl object
  # NOTE perhaps deconv has changed the decision variable list, so placed here during consolidation phase
  # TODO, should this not be part of `prepareCommonConvWrapper` -- only here do we look for .partial
  _setCCWDecisionDimsConv!(ccwl)

  # get factor metadata -- TODO, populate, also see #784
  # TODO consolidate with ccwl??
  # FIXME do not divert Mixture for sampling
  cf = CalcFactor(ccwl; _allowThreads=true)

  # cache the measurement dimension
  @assert ccwl.zDim == calcZDim( cf ) "refactoring in progress, cannot drop assignment ccwl.zDim:$(ccwl.zDim) = calcZDim( cf ):$(calcZDim( cf ))"
  # ccwl.zDim = calcZDim( cf )  # CalcFactor(ccwl) )

  # set the 'solvefor' variable index -- i.e. which connected variable of the factor is being computed in this convolution.
  ccwl.varidx = sfidx

  # option to disable fresh samples
  if needFreshMeasurements
    # TODO refactor
    ccwl.measurement = sampleFactor(cf, maxlen)
  end

  # set each CPT
  _updateCPTs!(ccwl, sfidx)

  # calculate new gradients perhaps
  # J = ccwl.gradients(measurement..., pts...)

  return sfidx, maxlen, mani
end

function _updateCCW!( ccwl::Union{CommonConvWrapper{F},
                                                CommonConvWrapper{Mixture{N_,F,S,T}}},
                      Xi::AbstractVector{<:DFGVariable},
                      solvefor::Symbol,
                      N::Integer;
                      kw...  ) where {N_,F<:AbstractRelative,S,T}
  #
  _updateCCW!(F, ccwl, Xi, solvefor, N; kw...)
end

#


#-------------language_test.jl-------------------------------------------------#
#
# Purpose: This program uses multiple languages in julia (C, FORTRAN)
#
#   Notes: To use this program, please compile the following:
#              multiply.f90
#              multiply.c
#          Compile instructions can be found at the start of each file
#
#------------------------------------------------------------------------------#

# This function multiplies a and b into c by using the created FORTRAN library
function call_fortran()
    a = Cdouble[1.0]
    b = Cdouble[2.0]
    c = Cdouble[0.0]
    ccall((:multiply_, "/home/james/projects/skillpill-julia/examples/language_test/fortran_multiply"),
       Void,(Ptr{Float64},Ptr{Float64},Ptr{Float64}),a,b,c)

    println(c[1])
end

# This function multiplies a and b into c by using the created C library
function call_c()
    a = Cdouble(1.0)
    b = Cdouble(3.0)
    c = ccall((:multiply, "/home/james/projects/skillpill-julia/examples/language_test/c_multiply"),
       Cdouble,(Cdouble, Cdouble),a,b)

    println(c)

end

call_fortran()
call_c()


"""
    SparseMatrixCLIL{T, Ti}

The SparseMatrixCLIL represents a sparse matrix in two distinct ways:

1. As a sparse (in both row and column) n x m matrix
2. As a row-dense, column-sparse k x m matrix

The data structure keeps a permutation between the row order of the two representations.
Swapping the rows in one does not affect the other.

On construction, the second representation is equivalent to the first with fully-sparse
rows removed, though this may cease being true as row permutations are being applied
to the matrix.

The default structure of the `SparseMatrixCLIL` type is the second structure, while
the first is available via the thin `AsSubMatrix` wrapper.
"""
struct SparseMatrixCLIL{T, Ti<:Integer} <: AbstractSparseMatrix{T, Ti}
    nparentrows::Int
    ncols::Int
    nzrows::Vector{Ti}
    row_cols::Vector{Vector{Ti}} # issorted
    row_vals::Vector{Vector{T}}
end
Base.size(S::SparseMatrixCLIL) = (length(S.nzrows), S.ncols)
Base.copy(S::SparseMatrixCLIL{T, Ti}) where {T, Ti} =
    SparseMatrixCLIL(S.nparentrows, S.ncols, copy(S.nzrows), map(copy, S.row_cols), map(copy, S.row_vals))
function swaprows!(S::SparseMatrixCLIL, i, j)
    swap!(S.nzrows, i, j)
    swap!(S.row_cols, i, j)
    swap!(S.row_vals, i, j)
end

function SparseMatrixCLIL(mm::AbstractMatrix)
    nrows, ncols = size(mm)
    row_cols = [findall(!iszero, row) for row in eachrow(mm)]
    row_vals = [row[cols] for (row, cols) in zip(eachrow(mm), row_cols)]
    SparseMatrixCLIL(nrows, ncols, Int[1:length(row_cols);], row_cols, row_vals)
end

struct CLILVector{T, Ti} <: AbstractSparseVector{T, Ti}
    vec::SparseVector{T, Ti}
end
Base.size(v::CLILVector) = Base.size(v.vec)
Base.getindex(v::CLILVector, idx::Integer...) = Base.getindex(v.vec, idx...)
Base.setindex!(vec::CLILVector, v, idx::Integer...) = Base.setindex!(vec.vec, v, idx...)
Base.view(a::SparseMatrixCLIL, i::Integer, ::Colon) =
    CLILVector(SparseVector(a.ncols, a.row_cols[i], a.row_vals[i]))
SparseArrays.nonzeroinds(a::CLILVector) = SparseArrays.nonzeroinds(a.vec)
SparseArrays.nonzeros(a::CLILVector) = SparseArrays.nonzeros(a.vec)

function Base.setindex!(S::SparseMatrixCLIL, v::CLILVector, i::Integer, c::Colon)
    if v.vec.n != S.ncols
        throw(BoundsError(v, 1:S.ncols))
    end
    S.row_cols[i] = copy(v.vec.nzind)
    S.row_vals[i] = copy(v.vec.nzval)
    return v
end

zero!(a::AbstractArray{T}) where {T} = a[:] .= zero(T)
zero!(a::SparseVector) = (empty!(a.nzind); empty!(a.nzval))
zero!(a::CLILVector) = zero!(a.vec)

struct NonZeros{T <: AbstractArray}
    v::T
end
Base.pairs(nz::NonZeros{<:CLILVector}) = NonZerosPairs(nz.v)

struct NonZerosPairs{T <: AbstractArray}
    v::T
end

# N.B.: Because of how we're using this, this must be robust to modification of
# the underlying vector. As such, we treat this as an iteration over indices
# that happens to short cut using the sparse structure and sortedness of the
# array.
function Base.iterate(nzp::NonZerosPairs{<:CLILVector}, (idx, col))
    v = nzp.v.vec
    nzind = v.nzind
    nzval = v.nzval
    if idx > length(nzind)
        idx = length(col)
    end
    oldcol = nzind[idx]
    if col !== oldcol
        # The vector was changed since the last iteration. Find our
        # place in the vector again.
        tail = col > oldcol ? (@view nzind[idx+1:end]) : (@view nzind[1:idx])
        tail_i = searchsortedfirst(tail, col + 1)
        # No remaining indices.
        tail_i > length(tail) && return nothing
        new_idx = col > oldcol ? idx + tail_i : tail_i
        new_col = nzind[new_idx]
        return (new_col=>nzval[new_idx], (new_idx, new_col))
    end
    idx == length(nzind) && return nothing
    new_col = nzind[idx+1]
    return (new_col=>nzval[idx+1], (idx+1, new_col))
end

function Base.iterate(nzp::NonZerosPairs{<:CLILVector})
    v = nzp.v.vec
    nzind = v.nzind
    nzval = v.nzval
    isempty(nzind) && return nothing
    return nzind[1]=>nzval[1], (1, nzind[1])
end

# Arguably this is how nonzeros should behave in the first place, but let's
# build something that works for us here and worry about it later.
nonzerosmap(a::CLILVector) = NonZeros(a)

function bareiss_update_virtual_colswap_mtk!(zero!, M::SparseMatrixCLIL, k, swapto, pivot, last_pivot; pivot_equal_optimization=true)
    # for ei in nzrows(>= k)
    eadj = M.row_cols
    old_cadj = M.row_vals
    vpivot = swapto[2]

    ## N.B.: Micro-optimization
    #
    # For rows that do not have an entry in the eliminated column, all this
    # update does is multiply the row in question by `pivot/last_pivot` (the
    # result of which is guaranteed to be integer by general properties of the
    # bareiss algorithm, even if `pivot/last_pivot` is not).
    #
    # Thus, when `pivot == last pivot`, we can skip the update for any rows that
    # do not have an entry in the eliminated column (because we'd simply be
    # multiplying by 1).
    #
    # As an additional MTK-specific enhancement, we further allow the case
    # when the absolute values are equal, i.e. effectively multiplying the row
    # by `-1`. To ensure this is legal, we need to show two things.
    # 1. The multiplication does not change the answer and
    # 2. The multiplication does not affect the fraction-freeness of the Bareiss
    #    algorithm.
    #
    # For point 1, remember that we're working on a system of linear equations,
    # so it is always legal for us to multiply any row by a sclar without changing
    # the underlying system of equations.
    #
    # For point 2, note that the factorization we're now computing is the same
    # as if we had multiplied the corresponding row (accounting for row swaps)
    # in the original matrix by `last_pivot/pivot`, ensuring that the matrix
    # itself has integral entries when `last_pivot/pivot` is integral (here we
    # have -1, which counts). We could use the more general integrality
    # condition, but that would in turn disturb the growth bounds on the
    # factorization matrix entries that the bareiss algorithm guarantees. To be
    # conservative, we leave it at this, as this captures the most important
    # case for MTK (where most pivots are `1` or `-1`).
    pivot_equal = pivot_equal_optimization && abs(pivot) == abs(last_pivot)

    for ei in k+1:size(M, 1)
        # elimate `v`
        coeff = 0
        ivars = eadj[ei]
        vj = findfirst(isequal(vpivot), ivars)
        if vj !== nothing
            coeff = old_cadj[ei][vj]
            deleteat!(old_cadj[ei], vj)
            deleteat!(eadj[ei], vj)
        elseif pivot_equal
            continue
        end

        # the pivot row
        kvars = eadj[k]
        kcoeffs = old_cadj[k]
        # the elimination target
        ivars = eadj[ei]
        icoeffs = old_cadj[ei]

        tmp_incidence = similar(eadj[ei], 0)
        tmp_coeffs = similar(old_cadj[ei], 0)
        # TODO: We know both ivars and kvars are sorted, we could just write
        # a quick iterator here that does this without allocation/faster.
        vars = sort(union(ivars, kvars))

        for v in vars
            v == vpivot && continue
            ck = getcoeff(kvars, kcoeffs, v)
            ci = getcoeff(ivars, icoeffs, v)
            ci = (pivot*ci - coeff*ck) ÷ last_pivot
            if ci !== 0
                push!(tmp_incidence, v)
                push!(tmp_coeffs, ci)
            end
        end

        eadj[ei] = tmp_incidence
        old_cadj[ei] = tmp_coeffs
    end
end

function bareiss_update_virtual_colswap_mtk!(zero!, M::AbstractMatrix, k, swapto, pivot, last_pivot; pivot_equal_optimization=true)
    if pivot_equal_optimization
        error("MTK pivot micro-optimization not implemented for `$(typeof(M))`.
            Turn off the optimization for debugging or use a different matrix type.")
    end
    bareiss_update_virtual_colswap!(zero!, M, k, swapto, pivot, last_pivot)
end

struct AsSubMatrix{T, Ti<:Integer} <: AbstractSparseMatrix{T, Ti}
    M::SparseMatrixCLIL{T, Ti}
end
Base.size(S::AsSubMatrix) = (S.M.nparentrows, S.M.ncols)

function Base.getindex(S::SparseMatrixCLIL{T}, i1::Integer, i2::Integer) where {T}
    checkbounds(S, i1, i2)

    col = S.row_cols[i1]
    nncol = searchsortedfirst(col, i2)
    (nncol > length(col) || col[nncol] != i2) && return zero(T)

    return S.row_vals[i1][nncol]
end

function Base.getindex(S::AsSubMatrix{T}, i1::Integer, i2::Integer) where {T}
    checkbounds(S, i1, i2)
    S = S.M

    nnrow = findfirst(==(i1), S.nzrows)
    isnothing(nnrow) && return zero(T)

    col = S.row_cols[nnrow]
    nncol = searchsortedfirst(col, i2)
    (nncol > length(col) || col[nncol] != i2) && return zero(T)

    return S.row_vals[nnrow][nncol]
end


"""
	readgpcpd(filein)
Read DAILY GPCP data

**Input**
* filein: daily (GPCP)[https://precip.gsfc.nasa.gov/gpcp_daily_comb.html] binary file

**Output**
* Array{Float63}(lon,lat,time) with unchanged units (mm/day). All negative values (flag=-99999) are set to NaN

**Example**
```
# Load data for whole month and select last day of the month
# should result in approx.: https://climatedataguide.ucar.edu/climate-data/gpcp-daily-global-precipitation-climatology-project
filein = "/test/input/gpcpd_data";
datagpcp = readgpcpd(filein);
data_june30 = datagpcp(:,:,end);
timegpcp = readgpcpd_time(filein);
lon,lat = readgpcpd_lonlat(filein);
head_string = readgpcpd_head(filein);
```
"""
function readgpcpd(filein::String)
	# get header info
	headout = readgpcpd_head(filein);
	lon,lat = readgpcpd_lonlat(headout,from_file=false);
	timeout = readgpcpd_time(headout,from_file=false);
	# declare output
	dataout = Array{Float32}(undef,length(lon)*length(lat)*length(timeout));
	open(filein,"r") do fid
		# remove header lines
		seek(fid,length(headout)-3);
		# read data
		read!(fid,dataout);
	end
	#dataout[dataout.<0.] == NaN;
	return reshape(convert(Vector{Float64},dataout),length(lon),length(lat),length(timeout))
end

"""
	readgpcpd_head(filein;headersize)
Read DAILY GPCP header line

**Input**
* filein: daily (GPCP)[https://precip.gsfc.nasa.gov/gpcp_daily_comb.html] binary file
* optional header size (1440 for daily data)

**Output**
* whole header string

**Example**
```
filein = "/test/input/gpcpd_data";
head_string = readgpcpd_head(filein);
```
"""
function readgpcpd_head(filein::String;headsize::Int=1440)
	headout = ' ';
	open(filein,"r") do fid
		for i in 1:headsize
			headout = string(headout,read(fid,Char)); # concatenate characters
		end
	end
	return headout[2:end] # first value = ' '
end

"""
	readgpcpd_lonlat(filein;from_file=true)
Read DAILY GPCP longitude and latitude

**Input**
* filein: daily (GPCP)[https://precip.gsfc.nasa.gov/gpcp_daily_comb.html] binary file
* read from file (true) or parse from string (false)

**Output**
* longitude (degree east), latitude (degree north to east)

**Example**
```
filein = "/test/input/gpcpd_data";
lon,lat = readgpcpd_lonlat(filein);
```
"""
function readgpcpd_lonlat(headout::String;from_file=true)
	if from_file==true
		headout = readgpcpd_head(headout);
	end
	string_size = match(r"[0-9]{3}x[0-9]{3}",headout);
	x,y = split(string_size.match,"x") |> x -> (Meta.parse(x[1]),Meta.parse(x[2]))
	return collect(0.5:1:(x-0.5)),collect(89.5:-1:(-y/2+0.5))
end

"""
	readgpcpd_time(filein)
Read DAILY GPCP time information

**Input**
* filein: daily (GPCP)[https://precip.gsfc.nasa.gov/gpcp_daily_comb.html] binary file
* read from file (true) or parse from string (false)

**Output**
* time vector in DateTime format

**Example**
```
filein = "/test/input/gpcpd_data";
timegpcp = readgpcpd_time(filein);
```
"""
function readgpcpd_time(headout::String;from_file=true)
	if from_file==true
		headout = readgpcpd_head(headout);
	end
	string_yyyy = match(r"year=[0-9]{4}",headout);
	string_mm = match(r"month=[0-9]{2}",headout);
	string_dd = match(r"days=.-[0-9]{2}",headout) |> x->split(x.match,"=") |> x->x[2]
	time_start = DateTime(Meta.parse(split(string_yyyy.match,"=")[2]),
						Meta.parse(split(string_mm.match,"=")[2]),
						Meta.parse(split(string_dd,"-")[1]));
	time_stop = DateTime(Meta.parse(split(string_yyyy.match,"=")[2]),
						Meta.parse(split(string_mm.match,"=")[2]),
						Meta.parse(split(string_dd,"-")[2]));
	return collect(time_start:Dates.Day(1):time_stop)
end

"""
	coorShift_lon(lon,datain,to)
Convert 0 to 360 longitude coordinate (e.g. GPCP) system to -180 to 180

**Input**
* lon: longitude vector
* datain: data matrix or 3D Array{T}. If 3D matrix => first 2 dimensions are lon*lat or lat*lon
* to: switch between systems (will be converted to this system): "-180to180" | "0to360"

**Output**
* lonout: new longitude vector
* dataout: transformed data in the same dimensions as input 'datain'

**Example**
```
# convert to 0-360 system
lon = collect(-179.5:1:179.5);
datain = repeat(-179.5:1:179.5,1,180);
lonout,dataout = coorShift_lon(lon,datain,to="0to360")

# convert to -180 to 180 system
lon = collect(0.125:0.25:359.875);
datain = Array{Float64}(720,length(lon),2);
datain[:,:,1] = transpose(repeat(0.125:0.25:359.875,1,720));
datain[:,:,2] = transpose(repeat(0.125:0.25:359.875,1,720));
lonout,dataout = coorShift_lon(lon,datain,to="-180to180");
```
"""
function coorShift_lon(lon,datain;to::String="-180to180")
	lonin = copy(lon);
	# prepare output longitude
	if to == "-180to180"
		lonout = sort(lon .- 180);
		lonin[lon.>180] = lon[lon.>180] .- 360.;
	elseif to == "0to360"
		lonout = sort(lon .+ 180);
		lonin[lon.<0] = lon[lon.<0] .+ 360.;
	end
	# find indices
	i_lon = zeros(Int,length(lon))
	for (i,v) in enumerate(lonout)
		i_lon[i] = findall(v.==lonin)[1];
	end
	# Get the correct dimension and transform the data
	if size(datain,1) == length(lon)
		if size(datain,3) == 1
			return lonout,datain[i_lon,:];
		else
			return lonout,datain[i_lon,:,:];
		end
	else
		if size(datain,3) == 1
			return lonout,datain[:,i_lon];
		else
			return lonout,datain[:,i_lon,:];
		end
	end
end


# Stalin Sort

"""
A Julia implementation of the Stalin Sort algorithm.
Inspired by the Python implementation.

Elements that are out of order are eliminated.

Args:
    arr: An AbstractVector

Returns:
    Stalin sorted array.
"""

function stalinsort(arr::AbstractVector)

    if isempty(arr)
        return []
    end

    max_val = arr[1]

    function add_val(num)
        max_val = num
        return num
    end

    return [add_val(x) for x in arr if x >= max_val]
end

# Test examples

#  unordered: [1, 2, 3, 4, 5]
# -> ordered: [1, 2, 3, 4, 5]
test1 = [1, 2, 3, 4, 5]
println("# unordered: $test1\n -> ordered: ", stalinsort(test1))

#  unordered: [10, 9, 8, 7, 2]
# -> ordered: [10]
test2 = [10, 9, 8, 7, 2]
println("# unordered: $test2\n -> ordered: ", stalinsort(test2))

# unordered: [1, 2, 3, 5, 4, 6]
#-> ordered: [1, 2, 3, 5, 6]
test3 = [1, 2, 3, 5, 4, 6]
println("# unordered: $test3\n -> ordered: ", stalinsort(test3))

#  unordered: [1.0, 2.4, 2.2, 3.0, 5.0, 4.0, 6.0]
# -> ordered: [1.0, 2.4, 3.0, 5.0, 6.0]
test4 = [1, 2.4, 2.2, 3, 5, 4, 6]
println("# unordered: $test4\n -> ordered: ", stalinsort(test4))

#  unordered: [-9, 6, 4, 10, -10, -5, 1, 8, -5, 10]
# -> ordered: [-9, 6, 10, 10]
test5 = rand(-10:10, 10)
println("# unordered: $test5\n -> ordered: ", stalinsort(test5))

#  unordered: [1, 2, 2, 5, 4, 5, 3, 7]
# -> ordered: [1, 2, 2, 5, 5, 7]
test6 = [1, 2, 2, 5, 4, 5, 3, 7]
println("# unordered: $test6\n -> ordered: ", stalinsort(test6))


struct Simulation{DT <: Number, IT <: Integrator{DT}, FPRE <: Union{Function,Nothing}, FPOST <: Union{Function,Nothing}}
    int::IT
    nt::Int

    f_pre::FPRE
    f_post::FPOST

    x::Vector{Vector{DT}}

    function Simulation(x₀::Vector{Vector{DT}}, int::IT, nt::Int, f_preproc::FPRE, f_postproc::FPOST) where {DT, IT <: Integrator{DT}, FPRE, FPOST}
        new{DT, IT, FPRE, FPOST}(int, nt, f_preproc, f_postproc, copy(x₀))
    end
end

function Simulation(equ::Equation, integrator::Type{<:Integrator}, Δt::Real, nt::Int; f_preproc=nothing, f_postproc=nothing, kwargs...)
    int = integrator(equ, Δt; kwargs...)
    Simulation(equ.x₀, int, nt, f_preproc, f_postproc)
end

ndims(sim::Simulation) = ndims(sim.int.equ)
nsamples(sim::Simulation) = nsamples(sim.int.equ)
ntimesteps(sim::Simulation) = sim.nt

eachsample(sim::Simulation) = eachindex(sim.x)
eachtimestep(sim::Simulation) = 1:ntimesteps(sim)

haspreprocessing(::Simulation{<:Any, <:Any, Nothing}) = false
haspreprocessing(::Simulation{<:Any, <:Any, <:Function}) = true

haspostprocessing(::Simulation{<:Any, <:Any, <:Any, Nothing}) = false
haspostprocessing(::Simulation{<:Any, <:Any, <:Any, <:Function}) = true

preprocessing(sim::Simulation, x, n) = haspreprocessing(sim) && sim.f_pre(x, n)
postprocessing(sim::Simulation, x, n) = haspostprocessing(sim) && sim.f_post(x, n)

function (sim::Simulation{DT})() where {DT}
    for n in eachtimestep(sim)
        preprocessing(sim, sim.x, n)
        sim.int(sim.x)
        postprocessing(sim, sim.x, n)
    end
end


module SOH
include("boundary_conditions.jl")
include("flux.jl")
include("init.jl")
include("plot_save.jl")
include("run.jl")
include("scheme.jl")
include("toolbox.jl")

export boundary_conditions_xy!, boundary_conditions_ρuv!
export flux_x
export random_init, quadratic_potential_force, flat_quadratic_potential_force
export plot_rhoUV, update_plot!, save_data!, radial_density
export run!
export scheme_iter!
export make_new_dir, coefficients_Vicsek, nice_float2string

end     #module


using Pkg
Pkg.activate(@__DIR__)
Pkg.instantiate()

using Logging
using Gadfly
using JLSO
using Revise
includet("./utils.jl")
using .PlottingUtils: get_plot_data, sr_plots,
                        box_error_plots, oe_plots

"""
Load serialized results and save the
most relevant plots
"""
function plot_main_results(results::Dict,
                           path::String,
                           methods::Vector=[:SS,:DSS])::Nothing

    first_key = collect(keys(results))[1]
    first_result = results[first_key]
    samples = collect(keys(first_result))
    noise_levels = Array{Symbol}(collect(keys(first_result[samples[1]])))
    #methods = Array{Symbol}(collect(keys(first_result[samples[1]][noise_levels[1]])))

    # cur_colors = [colorant"#e6612f",colorant"#42b3d5",colorant"seagreen"]
    # [Flame, Maximum Blue, Seagreen]
    cur_colors = [colorant"#ff595e",colorant"#ffca3a",colorant"#8ac926",colorant"#1982c4"]
    # [Sizzling Red, Sunglow, Yellow Green, Green Blue Crayola]

    method_label = Dict()
    for m in methods
        method_label[m] = String(m)
    end

    method_color = Dict()
    for (m,c) in zip([:SS,:SSR,:DS,:DSS],cur_colors)
        method_color[m] = c
    end

    for (problem,result) in results
        nrmse_log = "$(problem)\n"
        sr_log = "$(problem)\n"
        oe_log = "$(problem)\n"

        dir_path = joinpath(path,string(problem))
        mkdir(dir_path)

        for sam in samples
            nrmse_log *= "Sample size:\t$(sam)\n"
            sr_log *= "Sample size:\t$(sam)\n"
            oe_log *= "Sample size:\t$(sam)\n"

            res = result[sam]
            plot_data = get_plot_data(res,noise_levels,methods)

            sr_log *= sr_plots(plot_data,noise_levels,methods,
                    method_label,method_color,sam,dir_path)

            oe_log *= oe_plots(plot_data,noise_levels,methods,
                    method_label,method_color,sam,dir_path)

            if :DS in methods
                oe_log *= oe_plots(plot_data,
                                   noise_levels,filter(x -> x != :DS, methods),
                                   method_label,method_color,sam,dir_path)
            end

            for noise in noise_levels
                plot_data = get_plot_data(res,[noise],methods)

                nrmse_log *= box_error_plots(plot_data,noise,methods,
                                  method_label,method_color,sam,dir_path)

                #parameter_plots(plot_data,noise,methods,
                #                   method_label,method_color,sam,dir_path)

                sr_log *= sr_plots(plot_data,noise,methods,
                        method_label,method_color,sam,dir_path)

                oe_log *= oe_plots(plot_data,[noise],methods,
                        method_label,method_color,sam,dir_path)

            end
        end
        @info nrmse_log
        @info sr_log
        @info oe_log
    end
    nothing
end

function main(args::Array{<:String})::Nothing
    jlso_file_or_path = string(args[1])
    save_path = string(args[2])

    println(jlso_file_or_path)
    io = open(joinpath(save_path, "plots.log"), "a+")
    logger = SimpleLogger(io)
    global_logger(logger)

    if isdir(jlso_file_or_path)
        filter_jlso = files -> filter(f -> occursin(".jlso", f), files)

        prepend_full_path = file -> joinpath(jlso_file_or_path, file)
        generate_jlso_paths = files -> map(prepend_full_path, files)

        get_jlso_files(path) =
            path |> readdir |> filter_jlso |> generate_jlso_paths

        jlso_files = get_jlso_files(jlso_file_or_path)
    else
        jlso_files = [jlso_file_or_path]
    end

    # base_dir = "/home/andrew/git/DEParamEst/"
    # save_path = joinpath(base_dir,"results/remote/results_low/")
    # jlso_file = joinpath(save_path,"experiment_results.jlso")
    results = map(JLSO.load, jlso_files)

    for r in results
        plot_main_results(r, save_path, [:SS, :SSR, :DS, :DSS])
    end
    close(io)
    nothing
end

main(ARGS)


mutable struct UnfoldGenerateState{T,U}
    score::Float64
    noise::Float64
    weight::Float64
    subtraces::Vector{U}
    retval::Vector{T}
    num_nonempty::Int
    state::T
end

function process!(gen_fn::Unfold{T,U}, params::Tuple, choices::ChoiceMap,
                  key::Int, state::UnfoldGenerateState{T,U}) where {T,U}
    local subtrace::U
    local new_state::T
    kernel_args = (key, state.state, params...)
    submap = get_submap(choices, key)
    (subtrace, weight) = generate(gen_fn.kernel, kernel_args, submap)
    state.weight += weight
    state.noise += project(subtrace, EmptySelection())
    state.num_nonempty += (isempty(get_choices(subtrace)) ? 0 : 1)
    state.score += get_score(subtrace)
    state.subtraces[key] = subtrace
    new_state = get_retval(subtrace)
    state.state = new_state
    state.retval[key] = new_state
end

function generate(gen_fn::Unfold{T,U}, args::Tuple, choices::ChoiceMap) where {T,U}
    len = args[1]
    init_state = args[2]
    params = args[3:end]
    state = UnfoldGenerateState{T,U}(0., 0., 0.,
        Vector{U}(undef,len), Vector{T}(undef,len), 0, init_state)
    for key=1:len
        process!(gen_fn, params, choices, key, state)
    end
    trace = VectorTrace{UnfoldType,T,U}(gen_fn,
        PersistentVector{U}(state.subtraces), PersistentVector{T}(state.retval),
        args, state.score, state.noise, len, state.num_nonempty)
    (trace, state.weight)
end


@testset "size_pyramid" begin
    @test SinGAN.size_pyramid(4 / 3, (32, 32), (400, 300)) ==
        [(32, 32), (42, 42), (56, 56), (75, 75), (101, 101),
         (134, 134), (179, 179), (239, 239), (319, 300), (400, 300)]
    @test SinGAN.size_pyramid(4 / 3, (32, 32), (128, 128)) ==
        [(32, 32), (42, 42), (56, 56), (75, 75), (101, 101), (128, 128)]
    @code_warntype SinGAN.size_pyramid(4 / 3, (32, 32), (128, 128))
end

@testset "channel_pyramid" begin
    @test SinGAN.channel_pyramid(5) == [32, 32, 32, 64, 64]
    @code_warntype SinGAN.channel_pyramid(5)
end

@testset "resize_and_padding" begin
    x = randn(Float32, 2, 2, 3, 1)
    @info x
    @info SinGAN.resize_and_padding(x, (3, 3), (5, 5))
    # @code_warntype SinGAN.resize_and_padding(x, (3, 3), (5, 5))
end

@testset "build_zero_pyramid" begin
    xs = Float32[1 2; 3 4] |> gpu
    ts = [(1, 1), (2, 2)]
    @test SinGAN.build_zero_pyramid(xs, ts) == [zeros(Float32, 1, 1, 3, 1), zeros(Float32, 2, 2, 3, 1)] |> gpu
    # @code_warntype SinGAN.build_zero_pyramid(xs, ts)
end

@testset "build_noise_pyramid" begin
    xs = Float32[1 2; 3 4] |> gpu
    ts = [(1, 1), (2, 2)]
    as = [1f0, 0.5f0]
    @info SinGAN.build_noise_pyramid(xs, ts, as)
    # @code_warntype SinGAN.build_noise_pyramid(xs, ts, as)
end

@testset "rec_vector_generator" begin
    xs = Float32[1 2; 3 4] |> gpu
    ts = [(1, 1), (2, 2)]
    a = 1f0
    @info SinGAN.build_rec_pyramid(xs, ts, a)
    # @code_warntype SinGAN.build_rec_pyramid(xs, ts, a)
end


#!/usr/bin/env julia
#=
   Author: Tim Sterne-Weiler, 3/16/2015
   e-mail: tim.sterne.weiler@utoronto.ca

=#
using GZip

# slow performance of HDF5 has promted [savedict, loaddict]:
function savedict{K,V}( file::AbstractString, dict::Dict{K,V} )
  GZip.open(file, "w") do fh
    println(fh, "@Dict{" * string(K) * "," * string(V) * "}:" * string(length(dict)))
    for k in keys(dict)
      println(fh, string(k) * "˧" * string(dict[k]))
    end
  end
end #--> nothing

function loaddict(file::AbstractString)
   myconvert(x::Type{Char}, y) = string(y)[1]
   myconvert{T <: AbstractString}(x::Type{T}, y) = convert(T, string(y))
   myconvert{T <: Number, S <: AbstractString}(x::Type{T}, y::S) = parse(T, y)
   myconvert(x, y) = convert(x, y)
   #hint = parse(chomp(readall(pipe(`zcat $file`, `wc -l`)))) # --deprecated 6/10/15 tsw
   dict = nothing
   GZip.open(file, "r") do fh
    heads = split(chomp( readline(fh) ), r"@Dict{|,|}:")
    @assert( length(heads) == 4 )
    @assert( ismatch(r"^[A-Z|a-z|0-9]+$", heads[2]) ) # must look like a type
    @assert( ismatch(r"^[A-Z|a-z|0-9]+$", heads[3]) ) # before we allow parse()
    ktype = eval(parse(heads[2]))
    vtype = eval(parse(heads[3]))
    dict = Dict{ktype,vtype}()
    hint  = parse(Int, heads[4])
    sizehint!(dict, hint)
    for l in eachline(fh)
      sub = split(chomp(l), '˧')
      key = myconvert(ktype, sub[1])
      val = myconvert(vtype, sub[2])
      dict[key] = val
    end
  end
  dict
end #--> Dict{K,V}

# dictionary value increment
# this should be standard imo.
function dinc!{K,V}(dict::Dict{K,V}, key::K, val::V=one(V))
  indx = Base.ht_keyindex(dict, key)
  if indx <= 0
    dict[key] = val
  else
    @assert( isa(dict.vals[indx], Number) )
    dict.vals[indx] += val
  end
end #--> nothing

# dictionary [] create or push if exists
# this also should be standard imo
function dush!{K,V}(dict::Dict{K,V}, key::K, val; arrType = Any)
  indx = Base.ht_keyindex(dict, key)
  @assert( isa(val, arrType) )
  if indx <= 0
    dict[key] = arrType[ val ]
  else
    push!(dict.vals[indx], val)
  end
end #--> nothing

# normalize numeric dictionary values
function dnorm!{K, V<:Number}(dict::Dict{K,V}, n = sum(collect(values(dict))) )
  @assert( n > 0 )
  for i in keys(dict)
    dict[i] /= n
  end #normalize
end #--> nothing

# normalize numeric dictionary values
function dnorm!(dict::Dict, n = sum(collect(values(dict))) )
  @assert( n > 0 )
  for i in keys(dict)
    @assert( isa(dict[i], Number) )
    dict[i] /= n
  end #normalize
end #--> nothing


struct Vocab
    w2i::Dict{String,Int}
    i2w::Vector{String}
    unk::Int
    eos::Int
    tokenizer
end

function Vocab(file::String; tokenizer=split, vocabsize=Inf, mincount=1, unk="<unk>", eos="<s>")
    cnt = Dict{String,Int}()
    for line in eachline(file)
        for word in tokenizer(line)
            cnt[word] = 1 + get(cnt, word, 0)
        end
    end
    delete!(cnt, unk); delete!(cnt, eos)
    words = sort!(collect(keys(cnt)); by=(w->cnt[w]), rev=true)
    if mincount > 1; i=findfirst(w -> cnt[w]<mincount, words); i!==nothing && (words=words[1:i-1]); end
    if unk != nothing; pushfirst!(words, unk); end
    if eos != nothing; pushfirst!(words, eos); end
    if length(words) > vocabsize; words = words[1:vocabsize]; end
    vocab = Dict(words[i] => i for i in 1:length(words))
    Vocab(vocab, words, get(vocab,unk,0), get(vocab,eos,0), tokenizer)
end


# --------------------------------------------------------------------------
# ACE.jl and SHIPs.jl: Julia implementation of the Atomic Cluster Expansion
# Copyright (c) 2019 Christoph Ortner <christophortner0@gmail.com>
# All rights reserved.
# --------------------------------------------------------------------------

using Documenter, ACE

makedocs(sitename="ACE.jl Documentation",
         pages = [
        "Home" => "index.md",
        "Introduction" => "intro.md",
        "Getting Started" => "gettingstarted.md",
        "Developer Docs" => "devel.md",
        "ED-Bonds" => "envpairbasis.md"
        # "Subsection" => [
        #     ...
        # ]
        ])

# deploydocs(
#     repo = "github.com/JuliaMolSim/ACE.jl.git",
# )


using BenchmarkTools
using ABM

@btime sin(1)


#!/bin/env julia

# adapted from https://github.com/JuliaEditorSupport/julia-vim/blob/master/autoload/generate_latex_symbols_table.jl

# needs postprocessing due to composing chars (removing whitespace around symbols)

using REPL

open("julia.json","w") do f
    println(f, "[")
    println(f, join([string("  [ \" ", unicode, " \", \"\\", latex, "\"],\n") for (latex,unicode) in sort!(vcat(collect(REPL.REPLCompletions.latex_symbols),collect(REPL.REPLCompletions.emoji_symbols)), by=x->x[2])]))
    println(f, "]")
end


using Pkg
Pkg.activate("/home/dabajabaza/jianguoyun/Workspace/SmartParser/")
using SmartParser

function count_frequent_tokens(fns)
    stat = Dict{String,Int}()
    good_key(x) = !any(a->occursin(a,x), ["£",".", "\"", "*****","-----","%%%%%","+++++"])
    for f in fns
        if isfile(f)
            S0 = read(f,String) |> preprocess_raw_input
            MS0 = mask(S0)
            patts, code = tokenize0(MS0)
            codeinv = revert(code)
            for pt in patts
                for p in pt
                    ky = codeinv[p]
                    if p!=0 && good_key(ky)
                        increaseindex!(stat, ky)
                    end
                end
            end
        end
    end
    return stat
end

ALL_FILES = []

for RT ∈ ["/data/ReO3_phonon_configs","/data/ScF3_phonon_configs",]
    fns=vcat([[ "$RT/$fd/$ff/$fout" for ff in readdir("$RT/$fd") if isdir("$RT/$fd/$ff")
                                        for fout in readdir("$RT/$fd/$ff") if endswith(fout,".out") ]
                                            for fd ∈ readdir(RT) if isdir("$RT/$fd") ]...)
    ALL_FILES = [ALL_FILES; fns]
end

##

STATS = count_frequent_tokens(ALL_FILES)
STATS_SORTV = sort([k=>v for (k,v) ∈ STATS if v>1], by=last) ;
L = length(STATS_SORTV)

##

CODE_ALL = Dict(k=>L-i+1 for (i,(k,v)) ∈ enumerate(STATS_SORTV)) ;
CODE_ALL |> print

##

for f in ALL_FILES
    try rm("$(f).replaced.1.txt") catch ; end
    try rm("$(f).replaced.2.txt") catch ; end
    try rm("$(f).replaced.3.txt") catch ; end
    try rm("$(f).replaced.4.txt") catch ; end
end

##


# parameters for the case study

# model and data generation files
const modelFile = "resource_20_30.jl"
const dataFile = "demand_model_iid.jl"

# parameters for estimating the true optimal objective value using SAA
const numMCScenarios = 1000

# degree of nonlinearity in demand model
const degree = Float64[1]

# scaling factor for the additive demand errors
const demand_errors_scaling = 20.0

# starting seed
const startingSeed = 1280


# This file is a part of Julia. License is MIT: https://julialang.org/license

isdefined(Main, :OffsetArrays) || @eval Main include("testhelpers/OffsetArrays.jl")
using .Main.OffsetArrays

@testset "MissingException" begin
    @test sprint(showerror, MissingException("test")) == "MissingException: test"
end

@testset "nonmissingtype" begin
    @test nonmissingtype(Union{Int, Missing}) == Int
    @test nonmissingtype(Union{Rational, Missing}) == Rational
    @test nonmissingtype(Any) == Any
    @test nonmissingtype(Missing) == Union{}
end

@testset "convert" begin
    @test convert(Union{Int, Missing}, 1) === 1
    @test convert(Union{Int, Missing}, 1.0) === 1
    @test convert(Union{Nothing, Missing}, missing) === missing
    @test convert(Union{Nothing, Missing}, nothing) === nothing
    @test convert(Union{Missing, Nothing, Float64}, 1) === 1.0

    @test_throws MethodError convert(Missing, 1)
    @test_throws MethodError convert(Union{Nothing, Missing}, 1)
    @test_throws MethodError convert(Union{Int, Missing}, "a")
end

@testset "promote rules" begin
    @test promote_type(Missing, Missing) == Missing
    @test promote_type(Missing, Int) == Union{Missing, Int}
    @test promote_type(Int, Missing) == Union{Missing, Int}
    @test promote_type(Int, Any) == Any
    @test promote_type(Any, Any) == Any
    @test promote_type(Missing, Any) == Any
    @test promote_type(Any, Missing) == Any
    @test promote_type(Union{Int, Missing}, Missing) == Union{Int, Missing}
    @test promote_type(Missing, Union{Int, Missing}) == Union{Int, Missing}
    @test promote_type(Union{Int, Missing}, Int) == Union{Int, Missing}
    @test promote_type(Int, Union{Int, Missing}) == Union{Int, Missing}
    @test promote_type(Any, Union{Int, Missing}) == Any
    @test promote_type(Union{Nothing, Missing}, Any) == Any
    @test promote_type(Union{Int, Missing}, Union{Int, Missing}) == Union{Int, Missing}
    @test promote_type(Union{Float64, Missing}, Union{String, Missing}) == Any
    @test promote_type(Union{Float64, Missing}, Union{Int, Missing}) == Union{Float64, Missing}
    @test_broken promote_type(Union{Nothing, Missing, Int}, Float64) == Any
end

@testset "promotion in various contexts" for T in (Nothing, Missing)
    @test collect(v for v in (1, T())) isa Vector{Union{Int,T}}
    @test map(identity, Any[1, T()]) isa Vector{Union{Int,T}}
    @test broadcast(identity, Any[1, T()]) isa Vector{Union{Int,T}}
    @test unique((1, T())) isa Vector{Union{Int,T}}

    @test map(ismissing, Any[1, missing]) isa Vector{Bool}
    @test broadcast(ismissing, Any[1, missing]) isa BitVector
end

@testset "comparison operators" begin
    @test (missing == missing) === missing
    @test (1 == missing) === missing
    @test (missing == 1) === missing
    @test (missing != missing) === missing
    @test (1 != missing) === missing
    @test (missing != 1) === missing
    @test isequal(missing, missing)
    @test !isequal(1, missing)
    @test !isequal(missing, 1)
    @test (missing < missing) === missing
    @test (missing < 1) === missing
    @test (1 < missing) === missing
    @test (missing <= missing) === missing
    @test (missing <= 1) === missing
    @test (1 <= missing) === missing
    @test !isless(missing, missing)
    @test !isless(missing, 1)
    @test isless(1, missing)
    @test (missing ≈ missing) === missing
    @test isapprox(missing, 1.0, atol=1e-6) === missing
    @test isapprox(1.0, missing, rtol=1e-6) === missing

    @test !any(T -> T === Union{Missing,Bool}, Base.return_types(isequal, Tuple{Any,Any}))
end

@testset "arithmetic operators" begin
    arithmetic_operators = [+, -, *, /, ^, Base.div, Base.mod, Base.fld, Base.rem]

    # All unary operators return missing when evaluating missing
    for f in [!, ~, +, -, *, &, |, xor, nand, nor]
        @test ismissing(f(missing))
    end

    # All arithmetic operators return missing when operating on two missing's
    # All arithmetic operators return missing when operating on a scalar and an missing
    # All arithmetic operators return missing when operating on an missing and a scalar
    for f in arithmetic_operators
        @test ismissing(f(missing, missing))
        @test ismissing(f(1, missing))
        @test ismissing(f(missing, 1))
    end

    @test ismissing(min(missing, missing))
    @test ismissing(max(missing, missing))
    for f in [min, max]
        for arg in ["", "a", 1, -1.0, [2]]
            @test ismissing(f(missing, arg))
            @test ismissing(f(arg, missing))
        end
    end
end

@testset "two-argument functions" begin
    two_argument_functions = [atan, hypot, log]

    # All two-argument functions return missing when operating on two missing's
    # All two-argument functions return missing when operating on a scalar and an missing
    # All two-argument functions return missing when operating on an missing and a scalar
    for f in two_argument_functions
        @test ismissing(f(missing, missing))
        @test ismissing(f(1, missing))
        @test ismissing(f(missing, 1))
    end
end

@testset "bit operators" begin
    bit_operators = [&, |, ⊻]

    # All bit operators return missing when operating on two missing's
    for f in bit_operators
        @test ismissing(f(missing, missing))
    end
end

@testset "boolean operators" begin
    @test ismissing(missing & true)
    @test ismissing(true & missing)
    @test !(missing & false)
    @test !(false & missing)
    @test ismissing(missing | false)
    @test ismissing(false | missing)
    @test missing | true
    @test true | missing
    @test ismissing(xor(missing, true))
    @test ismissing(xor(true, missing))
    @test ismissing(xor(missing, false))
    @test ismissing(xor(false, missing))
    @test ismissing(nand(missing, true))
    @test ismissing(nand(true, missing))
    @test nand(missing, false) == true
    @test nand(false, missing) == true
    @test ismissing(⊼(missing, true))
    @test ismissing(⊼(true, missing))
    @test ⊼(missing, false) == true
    @test ⊼(false, missing) == true
    @test nor(missing, true) == false
    @test nor(true, missing) == false
    @test ismissing(nor(missing, false))
    @test ismissing(nor(false, missing))
    @test ⊽(missing, true) == false
    @test ⊽(true, missing) == false
    @test ismissing(⊽(missing, false))
    @test ismissing(⊽(false, missing))

    @test ismissing(missing & 1)
    @test ismissing(1 & missing)
    @test ismissing(missing | 1)
    @test ismissing(1 | missing)
    @test ismissing(xor(missing, 1))
    @test ismissing(xor(1, missing))
    @test ismissing(nand(missing, 1))
    @test ismissing(nand(1, missing))
    @test ismissing(⊼(missing, 1))
    @test ismissing(⊼(1, missing))
    @test ismissing(nor(missing, 1))
    @test ismissing(nor(1, missing))
    @test ismissing(⊽(missing, 1))
    @test ismissing(⊽(1, missing))
end

@testset "* string/char concatenation" begin
    @test ismissing("a" * missing)
    @test ismissing('a' * missing)
    @test ismissing(missing * "a")
    @test ismissing(missing * 'a')
end

# Emulate a unitful type such as Dates.Minute
struct Unit
    value::Int
end
Base.zero(::Type{Unit}) = Unit(0)
Base.one(::Type{Unit}) = 1

@testset "elementary functions" begin
    elementary_functions = [abs, abs2, sign, real, imag,
                            acos, acosh, asin, asinh, atan, atanh, sin, sinh,
                            conj, cos, cosh, tan, tanh,
                            exp, exp2, expm1, log, log10, log1p, log2,
                            exponent, sqrt,
                            identity, zero, one, oneunit,
                            iseven, isodd, ispow2,
                            isfinite, isinf, isnan, iszero,
                            isinteger, isreal, transpose, adjoint, float, complex, inv]

    # All elementary functions return missing when evaluating missing
    for f in elementary_functions
        @test ismissing(f(missing))
    end

    @test ismissing(clamp(missing, 1, 2))

    for T in (Int, Float64)
        @test zero(Union{T, Missing}) === T(0)
        @test one(Union{T, Missing}) === T(1)
        @test oneunit(Union{T, Missing}) === T(1)
        @test float(Union{T, Missing}) === Union{float(T), Missing}
        @test complex(Union{T, Missing}) === Union{complex(T), Missing}
    end

    @test_throws MethodError zero(Union{Symbol, Missing})
    @test_throws MethodError one(Union{Symbol, Missing})
    @test_throws MethodError oneunit(Union{Symbol, Missing})
    @test_throws MethodError float(Union{Symbol, Missing})
    @test_throws MethodError complex(Union{Symbol, Missing})

    for T in (Unit,)
        @test zero(Union{T, Missing}) === T(0)
        @test one(Union{T, Missing}) === 1
        @test oneunit(Union{T, Missing}) === T(1)
    end

    @test zero(Missing) === missing
    @test one(Missing) === missing
    @test oneunit(Missing) === missing
    @test float(Missing) === Missing
    @test complex(Missing) === Missing

    @test_throws MethodError zero(Any)
    @test_throws MethodError one(Any)
    @test_throws MethodError oneunit(Any)
    @test_throws MethodError float(Any)
    @test_throws MethodError complex(Any)

    @test_throws MethodError zero(String)
    @test_throws MethodError zero(Union{String, Missing})
end

@testset "rounding functions" begin
    # All rounding functions return missing when evaluating missing as first argument

    # Check that the RoundingMode argument is passed on correctly
    @test round(Union{Int, Missing}, 0.9) === round(Int, 0.9)
    @test round(Union{Int, Missing}, 0.9, RoundToZero) === round(Int, 0.9, RoundToZero)

    # Test elementwise on mixed arrays to ensure signature of Missing methods matches that of Float methods
    test_array = [1.0, missing]

    @test isequal(round.(test_array, RoundNearest), test_array)
    @test isequal(round.(Union{Int, Missing}, test_array, RoundNearest), test_array)

    rounding_functions = [ceil, floor, round, trunc]
    for f in rounding_functions
        @test_throws MissingException f(Int, missing)
        @test isequal(f.(test_array), test_array)
        @test isequal(f.(test_array, digits=0, base=10), test_array)
        @test isequal(f.(test_array, sigdigits=1, base=10), test_array)
        @test isequal(f.(Union{Int, Missing}, test_array), test_array)
    end
end

@testset "printing" begin
    @test sprint(show, missing) == "missing"
    @test sprint(show, missing, context=:compact => true) == "missing"
    @test sprint(show, [missing]) == "[missing]"
    @test sprint(show, [1 missing]) == "$(Union{Int, Missing})[1 missing]"
    b = IOBuffer()
    display(TextDisplay(b), [missing])
    @test String(take!(b)) == "1-element Vector{$Missing}:\n missing\n"
    b = IOBuffer()
    display(TextDisplay(b), [1 missing])
    @test String(take!(b)) == "1×2 Matrix{$(Union{Int, Missing})}:\n 1  missing\n"
end

@testset "arrays with missing values" begin
    x = convert(Vector{Union{Int, Missing}}, [1.0, missing])
    @test isa(x, Vector{Union{Int, Missing}})
    @test isequal(x, [1, missing])
    x = convert(Vector{Union{Int, Missing}}, [1.0])
    @test isa(x, Vector{Union{Int, Missing}})
    @test x == [1]
    x = convert(Vector{Union{Int, Missing}}, [missing])
    @test isa(x, Vector{Union{Int, Missing}})
    @test isequal(x, [missing])
    @test eltype(adjoint([1, missing])) == Union{Int, Missing}
    # issue #32777
    let a = [0, nothing, 0.0, missing]
        @test a[1] === 0.0
        @test a[2] === nothing
        @test a[3] === 0.0
        @test a[4] === missing
        @test a isa Vector{Union{Missing, Nothing, Float64}}
    end
end

@testset "== and != on arrays" begin
    @test ismissing([1, missing] == [1, missing])
    @test ismissing(["a", missing] == ["a", missing])
    @test ismissing(Any[1, missing] == Any[1, missing])
    @test ismissing(Any[missing] == Any[missing])
    @test ismissing([missing] == [missing])
    @test ismissing(Any[missing, 2] == Any[1, missing])
    @test ismissing([missing, false] == BitArray([true, false]))
    @test ismissing(Any[missing, false] == BitArray([true, false]))
    @test Union{Int, Missing}[1] == Union{Float64, Missing}[1.0]
    @test Union{Int, Missing}[1] == [1.0]
    @test Union{Bool, Missing}[true] == BitArray([true])
    @test !([missing, 1] == [missing, 2])
    @test !(Union{Int, Missing}[1] == [2])
    @test !([1] == Union{Int, Missing}[2])
    @test !(Union{Int, Missing}[1] == Union{Int, Missing}[2])

    @test ismissing([1, missing] != [1, missing])
    @test ismissing(["a", missing] != ["a", missing])
    @test ismissing(Any[1, missing] != Any[1, missing])
    @test ismissing(Any[missing] != Any[missing])
    @test ismissing([missing] != [missing])
    @test ismissing(Any[missing, 2] != Any[1, missing])
    @test ismissing([missing, false] != BitArray([true, false]))
    @test ismissing(Any[missing, false] != BitArray([true, false]))
    @test !(Union{Int, Missing}[1] != Union{Float64, Missing}[1.0])
    @test !(Union{Int, Missing}[1] != [1.0])
    @test !(Union{Bool, Missing}[true] != BitArray([true]))
    @test [missing, 1] != [missing, 2]
    @test Union{Int, Missing}[1] != [2]
    @test [1] != Union{Int, Missing}[2]
    @test Union{Int, Missing}[1] != Union{Int, Missing}[2]
end

@testset "== and != on tuples" begin
    @test ismissing((1, missing) == (1, missing))
    @test ismissing(("a", missing) == ("a", missing))
    @test ismissing((missing,) == (missing,))
    @test ismissing((missing, 2) == (1, missing))
    @test !((missing, 1) == (missing, 2))

    longtuple = (1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16)
    @test ismissing((longtuple...,17,missing) == (longtuple...,17,18))
    @test ismissing((longtuple...,missing,18) == (longtuple...,17,18))
    @test !((longtuple...,17,missing) == (longtuple...,-17,18))
    @test !((longtuple...,missing,18) == (longtuple...,17,-18))

    @test ismissing((1, missing) != (1, missing))
    @test ismissing(("a", missing) != ("a", missing))
    @test ismissing((missing,) != (missing,))
    @test ismissing((missing, 2) != (1, missing))
    @test (missing, 1) != (missing, 2)

    @test ismissing((longtuple...,17,missing) != (longtuple...,17,18))
    @test ismissing((longtuple...,missing,18) != (longtuple...,17,18))
    @test (longtuple...,17,missing) != (longtuple...,-17,18)
    @test (longtuple...,missing,18) != (longtuple...,17,-18)
end

@testset "< and isless on tuples" begin
    @test ismissing((1, missing) < (1, 3))
    @test ismissing((1, missing) < (1, missing))
    @test ismissing((missing, 1) < (missing, 2))
    @test ismissing((1, 2) < (1, missing))
    @test ismissing((1, missing) < (1, 2))
    @test ismissing((missing,) < (missing,))
    @test ismissing((1,) < (missing,))
    @test () < (missing,)
    @test (1,) < (2, missing)
    @test (1, missing,) < (2, missing)

    @test !isless((1, missing), (1, 3))
    @test !isless((1, missing), (1, missing))
    @test isless((missing, 1), (missing, 2))
    @test isless((1, 2), (1, missing))
    @test !isless((1, missing), (1, 2))
    @test !isless((missing,), (missing,))
    @test isless((1,), (missing,))
    @test isless((), (missing,))
    @test isless((1,), (2, missing))
    @test isless((1, missing,), (2, missing))
end

@testset "any & all" begin
    @test any([true, missing])
    @test any(x -> x == 1, [1, missing])
    @test ismissing(any([false, missing]))
    @test ismissing(any(x -> x == 1, [2, missing]))
    @test ismissing(all([true, missing]))
    @test ismissing(all(x -> x == 1, [1, missing]))
    @test !all([false, missing])
    @test !all(x -> x == 1, [2, missing])
    @test 1 in [1, missing]
    @test ismissing(2 in [1, missing])
    @test ismissing(missing in [1, missing])
end

@testset "float" begin
    @test isequal(float([1, missing]), [1, missing])
    @test float([1, missing]) isa Vector{Union{Float64, Missing}}
    @test isequal(float(Union{Int, Missing}[missing]), [missing])
    @test float(Union{Int, Missing}[missing]) isa Vector{Union{Float64, Missing}}
    @test float(Union{Int, Missing}[1]) == [1]
    @test float(Union{Int, Missing}[1]) isa Vector{Union{Float64, Missing}}
    @test isequal(float([missing]), [missing])
    @test float([missing]) isa Vector{Missing}
end

@testset "skipmissing" begin
    x = skipmissing([1, 2, missing, 4])
    @test eltype(x) === Int
    @test collect(x) == [1, 2, 4]
    @test collect(x) isa Vector{Int}

    x = skipmissing([1  2; missing 4])
    @test eltype(x) === Int
    @test collect(x) == [1, 2, 4]
    @test collect(x) isa Vector{Int}

    x = collect(skipmissing([missing]))
    @test eltype(x) === Union{}
    @test isempty(collect(x))
    @test collect(x) isa Vector{Union{}}

    x = collect(skipmissing(Union{Int, Missing}[]))
    @test eltype(x) === Int
    @test isempty(collect(x))
    @test collect(x) isa Vector{Int}

    x = skipmissing([missing, missing, 1, 2, missing, 4, missing, missing])
    @test eltype(x) === Int
    @test collect(x) == [1, 2, 4]
    @test collect(x) isa Vector{Int}

    x = skipmissing(v for v in [missing, 1, missing, 2, 4])
    @test eltype(x) === Any
    @test collect(x) == [1, 2, 4]
    @test collect(x) isa Vector{Int}

    @testset "indexing" begin
        x = skipmissing([1, missing, 2, missing, missing])
        @test collect(eachindex(x)) == collect(keys(x)) == [1, 3]
        @test x[1] === 1
        @test x[3] === 2
        @test_throws MissingException x[2]
        @test_throws BoundsError x[6]
        @test findfirst(==(2), x) == 3
        @test findall(==(2), x) == [3]
        @test argmin(x) == 1
        @test findmin(x) == (1, 1)
        @test argmax(x) == 3
        @test findmax(x) == (2, 3)

        x = skipmissing([missing 2; 1 missing])
        @test collect(eachindex(x)) == [2, 3]
        @test collect(keys(x)) == [CartesianIndex(2, 1), CartesianIndex(1, 2)]
        @test x[2] === x[2, 1] === 1
        @test x[3] === x[1, 2] === 2
        @test_throws MissingException x[1]
        @test_throws MissingException x[1, 1]
        @test_throws BoundsError x[5]
        @test_throws BoundsError x[3, 1]
        @test findfirst(==(2), x) == CartesianIndex(1, 2)
        @test findall(==(2), x) == [CartesianIndex(1, 2)]
        @test argmin(x) == CartesianIndex(2, 1)
        @test findmin(x) == (1, CartesianIndex(2, 1))
        @test argmax(x) == CartesianIndex(1, 2)
        @test findmax(x) == (2, CartesianIndex(1, 2))

        for x in (skipmissing([]), skipmissing([missing, missing]))
            @test isempty(collect(eachindex(x)))
            @test isempty(collect(keys(x)))
            @test_throws BoundsError x[3]
            @test_throws BoundsError x[3, 1]
            @test findfirst(==(2), x) === nothing
            @test isempty(findall(==(2), x))
            @test_throws "reducing over an empty collection is not allowed" argmin(x)
            @test_throws "reducing over an empty collection is not allowed" findmin(x)
            @test_throws "reducing over an empty collection is not allowed" argmax(x)
            @test_throws "reducing over an empty collection is not allowed" findmax(x)
        end
    end

    @testset "mapreduce" begin
        # Vary size to test splitting blocks with several configurations of missing values
        for T in (Int, Float64),
            A in (rand(T, 10), rand(T, 1000), rand(T, 10000))
            if T === Int
                @test sum(A) === @inferred(sum(skipmissing(A))) ===
                    @inferred(reduce(+, skipmissing(A))) ===
                    @inferred(mapreduce(identity, +, skipmissing(A)))
            else
                @test sum(A) ≈ @inferred(sum(skipmissing(A))) ===
                    @inferred(reduce(+, skipmissing(A))) ===
                    @inferred(mapreduce(identity, +, skipmissing(A)))
            end
            @test mapreduce(cos, *, A) ≈
                @inferred(mapreduce(cos, *, skipmissing(A)))

            B = Vector{Union{T,Missing}}(A)
            replace!(x -> rand(Bool) ? x : missing, B)
            if T === Int
                @test sum(collect(skipmissing(B))) ===
                    @inferred(sum(skipmissing(B))) ===
                    @inferred(reduce(+, skipmissing(B))) ===
                    @inferred(mapreduce(identity, +, skipmissing(B)))
            else
                @test sum(collect(skipmissing(B))) ≈ @inferred(sum(skipmissing(B))) ===
                    @inferred(reduce(+, skipmissing(B))) ===
                    @inferred(mapreduce(identity, +, skipmissing(B)))
            end
            @test mapreduce(cos, *, collect(skipmissing(A))) ≈
                @inferred(mapreduce(cos, *, skipmissing(A)))

            # Test block full of missing values
            B[1:length(B)÷2] .= missing
            if T === Int
                @test sum(collect(skipmissing(B))) == sum(skipmissing(B)) ==
                    reduce(+, skipmissing(B)) == mapreduce(identity, +, skipmissing(B))
            else
                @test sum(collect(skipmissing(B))) ≈ sum(skipmissing(B)) ==
                    reduce(+, skipmissing(B)) == mapreduce(identity, +, skipmissing(B))
            end

            @test mapreduce(cos, *, collect(skipmissing(A))) ≈ mapreduce(cos, *, skipmissing(A))
        end

        # Patterns that exercize code paths for inputs with 1 or 2 non-missing values
        @test sum(skipmissing([1, missing, missing, missing])) === 1
        @test sum(skipmissing([missing, missing, missing, 1])) === 1
        @test sum(skipmissing([1, missing, missing, missing, 2])) === 3
        @test sum(skipmissing([missing, missing, missing, 1, 2])) === 3

        for n in 0:3
            itr = skipmissing(Vector{Union{Int,Missing}}(fill(missing, n)))
            @test sum(itr) == reduce(+, itr) == mapreduce(identity, +, itr) === 0
            @test_throws "reducing over an empty collection is not allowed" reduce(x -> x/2, itr)
            @test_throws "reducing over an empty collection is not allowed" mapreduce(x -> x/2, +, itr)
        end

        # issue #35504
        nt = NamedTuple{(:x, :y),Tuple{Union{Missing, Int},Union{Missing, Float64}}}(
            (missing, missing))
        @test sum(skipmissing(nt)) === 0

        # issues #38627 and #124
        @testset for len in [1, 2, 15, 16, 1024, 1025]
            v = repeat(Union{Int,Missing}[1], len)
            oa = OffsetArray(v, typemax(Int)-length(v))
            sm = skipmissing(oa)
            @test sum(sm) == len

            v = repeat(Union{Int,Missing}[missing], len)
            oa = OffsetArray(v, typemax(Int)-length(v))
            sm = skipmissing(oa)
            @test sum(sm) == 0
        end
    end

    @testset "filter" begin
        allmiss = Vector{Union{Int,Missing}}(missing, 10)
        @test isempty(filter(isodd, skipmissing(allmiss))::Vector{Int})
        twod1 = [1.0f0 missing; 3.0f0 missing]
        @test filter(x->x > 0, skipmissing(twod1))::Vector{Float32} == [1, 3]
        twod2 = [1.0f0 2.0f0; 3.0f0 4.0f0]
        @test filter(x->x > 0, skipmissing(twod2)) == reshape(twod2, (4,))
    end
end

@testset "coalesce" begin
    @test coalesce() === missing
    @test coalesce(1) === 1
    @test coalesce(nothing) === nothing
    @test coalesce(missing) === missing
    @test coalesce(missing, 1) === 1
    @test coalesce(1, missing) === 1
    @test coalesce(missing, missing) === missing
    @test coalesce(missing, 1, 2) === 1
    @test coalesce(1, missing, 2) === 1
    @test coalesce(missing, missing, 2) === 2
    @test coalesce(missing, missing, missing) === missing

    @test coalesce(nothing, missing) === nothing
    @test coalesce(missing, nothing) === nothing
end

@testset "@coalesce" begin
    @test @coalesce() === missing
    @test @coalesce(1) === 1
    @test @coalesce(nothing) === nothing
    @test @coalesce(missing) === missing

    @test @coalesce(1, error("failed")) === 1
    @test_throws ErrorException @coalesce(missing, error("failed"))
end

mutable struct Obj; x; end
@testset "weak references" begin
    @noinline function mk_wr(r, wr)
        x = Obj(1)
        push!(r, x)
        push!(wr, WeakRef(x))
        nothing
    end
    ref = []
    wref = []
    mk_wr(ref, wref)
    @test ismissing(wref[1] == missing)
    @test ismissing(missing == wref[1])
end

@testset "showerror missing function" begin
    me = try missing(1) catch e e end
    @test sprint(showerror, me) == "MethodError: objects of type Missing are not callable"
end

@testset "sort and sortperm with $(eltype(X))" for (X, P, RP) in
    (([2, missing, -2, 5, missing], [3, 1, 4, 2, 5], [2, 5, 4, 1, 3]),
     ([NaN, missing, 5, -0.0, NaN, missing, Inf, 0.0, -Inf],
      [9, 4, 8, 3, 7, 1, 5, 2, 6], [2, 6, 1, 5, 7, 3, 8, 4, 9]),
     ([missing, "a", "c", missing, "b"], [2, 5, 3, 1, 4], [1, 4, 3, 5, 2]))
    @test sortperm(X) == P
    @test sortperm(X, alg=QuickSort) == P
    @test sortperm(X, alg=MergeSort) == P

    XP = X[P]
    @test isequal(sort(X), XP)
    @test isequal(sort(X, alg=QuickSort), XP)
    @test isequal(sort(X, alg=MergeSort), XP)

    @test sortperm(X, rev=true) == RP
    @test sortperm(X, alg=QuickSort, rev=true) == RP
    @test sortperm(X, alg=MergeSort, rev=true) == RP

    XRP = X[RP]
    @test isequal(sort(X, rev=true), XRP)
    @test isequal(sort(X, alg=QuickSort, rev=true), XRP)
    @test isequal(sort(X, alg=MergeSort, rev=true), XRP)
end

sortperm(reverse([NaN, missing, NaN, missing]))


import LinearAlgebra

mutable struct CLArray{T, N} <: CLObject
    ctx::Context
    queue::CmdQueue
    buffer::Buffer{T}
    size::NTuple{N, Int}
end

const CLMatrix{T} = CLArray{T,2}
const CLVector{T} = CLArray{T,1}

## constructors

function CLArray(buf::Buffer{T}, queue::CmdQueue, sz::Tuple{Vararg{Int}}) where T
    ctx = context(buf)
    CLArray(ctx, queue, buf, sz)
end

function CLArray(queue::CmdQueue,
                 flags::Tuple{Vararg{Symbol}},
                 hostarray::AbstractArray{T,N}) where {T, N}
    ctx = context(queue)
    buf = Buffer(T, ctx, flags, hostbuf=hostarray)
    sz = size(hostarray)
    CLArray(ctx, queue, buf, sz)
end

CLArray(queue::CmdQueue, hostarray::AbstractArray{T,N};
        flags=(:rw, :copy)) where {T, N} = CLArray(queue, (:rw, :copy), hostarray)

Base.copy(A::CLArray; ctx=A.ctx, queue=A.queue,
          buffer=A.buffer, size=A.size) = CLArray(ctx, queue, buffer, size)

function Base.deepcopy(A::CLArray{T,N}) where {T, N}
    new_buf = Buffer(T, A.ctx, prod(A.size))
    copy!(A.queue, new_buf, A.buffer)
    return CLArray(A.ctx, A.queue, new_buf, A.size)
end

"""
Create in device memory array of type `t` and size `dims` filled by value `x`.
"""
function Base.fill(::Type{T}, q::CmdQueue, x::T, dims...) where T
    ctx = info(q, :context)
    v = opencl_version(ctx)
    if v.major == 1 && v.minor >= 2
        buf = Buffer(T, ctx, prod(dims))
        fill!(q, buf, x)
    else
        buf = Buffer(T, ctx, (:rw, :copy), prod(dims), hostbuf=fill(x, dims))
    end
    return CLArray(buf, q, dims)
end

Base.zeros(::Type{T}, q::CmdQueue, dims...) where {T} = fill(T, q, T(0), dims...)
Base.zeros(q::CmdQueue, dims...) = fill(Float64, q, Float64(0), dims...)
Base.ones(::Type{T}, q::CmdQueue, dims...) where {T} = fill(T, q, T(1), dims...)
Base.ones(q::CmdQueue, dims...) = fill(Float64, q, Float64(1), dims...)

## core functions

buffer(A::CLArray) = A.buffer
Base.pointer(A::CLArray) = A.buffer.id
context(A::CLArray) = context(A.buffer)
queue(A::CLArray) = A.queue
Base.eltype(A::CLArray{T, N}) where {T, N} = T
Base.size(A::CLArray) = A.size
Base.size(A::CLArray, dim::Integer) = A.size[dim]
Base.ndims(A::CLArray) = length(size(A))
Base.length(A::CLArray) = prod(size(A))
Base.:(==)(A:: CLArray, B:: CLArray) =
    buffer(A) == buffer(B) && size(A) == size(B)
Base.reshape(A::CLArray, dims...) = begin
    @assert prod(dims) == prod(size(A))
    return copy(A, size=dims)
end

## show

Base.show(io::IO, A::CLArray{T,N}) where {T, N} =
    print(io, "CLArray{$T,$N}($(buffer(A)),$(size(A)))")

## to_host

function to_host(A::CLArray{T,N}; queue=A.queue) where {T, N}
    hA = Array{T}(undef, size(A)...)
    copy!(queue, hA, buffer(A))
    return hA
end

## other array operations

const TRANSPOSE_PROGRAM_PATH = joinpath(dirname(@__FILE__), "kernels/transpose.cl")

function max_block_size(queue::CmdQueue, h::Int, w::Int)
    dev = info(queue, :device)
    dim1, dim2 = info(dev, :max_work_item_size)[1:2]
    wgsize = info(dev, :max_work_group_size)
    wglimit = floor(Int, sqrt(wgsize))
    return gcd(dim1, dim2, h, w, wglimit)
end

"""
Transpose CLMatrix A, write result to a preallicated CLMatrix B
"""
function LinearAlgebra.transpose!(B::CLMatrix{Float32}, A::CLMatrix{Float32};
                                  queue=A.queue)
    block_size = max_block_size(queue, size(A, 1), size(A, 2))
    ctx = context(A)
    kernel = get_kernel(ctx, TRANSPOSE_PROGRAM_PATH, "transpose",
                          block_size=block_size)
    h, w = size(A)
    lmem = LocalMem(Float32, block_size * (block_size + 1))
    set_args!(kernel, buffer(B), buffer(A), UInt32(h), UInt32(w), lmem)
    return enqueue_kernel(queue, kernel, (h, w), (block_size, block_size))
end

"""Transpose CLMatrix A"""
function LinearAlgebra.transpose(A::CLMatrix{Float32};
                        queue=A.queue)
    B = zeros(Float32, queue, reverse(size(A))...)
    ev = LinearAlgebra.transpose!(B, A, queue=queue)
    wait(ev)
    return B
end

"""Transpose CLMatrix A, write result to a preallicated CLMatrix B"""
function LinearAlgebra.transpose!(B::CLMatrix{Float64}, A::CLMatrix{Float64};
                         queue=A.queue)
    block_size = max_block_size(queue, size(A, 1), size(A, 2))
    ctx = context(A)
    kernel = get_kernel(ctx, TRANSPOSE_PROGRAM_PATH, "transpose_double",
                          block_size=block_size)
    h, w = size(A)
    # lmem = LocalMem(Float64, block_size * (block_size + 1))
    lmem = LocalMem(Float64, block_size * block_size)
    set_args!(kernel, buffer(B), buffer(A), UInt32(h), UInt32(w), lmem)
    return enqueue_kernel(queue, kernel, (h, w), (block_size, block_size))
end

"""Transpose CLMatrix A"""
function LinearAlgebra.transpose(A::CLMatrix{Float64};
                        queue=A.queue)
    B = zeros(Float64, queue, reverse(size(A))...)
    ev = LinearAlgebra.transpose!(B, A, queue=queue)
    wait(ev)
    return B
end


using ArgParse, DelimitedFiles, JLD

function readScenarioNum()
    s = ArgParseSettings(description = "arg parse here for parallel experimentation")
    @add_arg_table! s begin
        "--topology", "-t"  # topology name
        "--topoindex", "-i"  # index of IP topology
        "--cutoff", "-c"  # failure scenario cutoff
        "--scenarioID", "-o"  # generate new failure scenarios from weibull
    end
    parsed_args = parse_args(s) # the result is a Dict{String,Any}
    topology = parsed_args["topology"]
    topology_index = parsed_args["topoindex"]
    scenario_cutoff = parse(Float64, parsed_args["cutoff"])
    scenario_id = parsed_args["scenarioID"]

    failure_num = 0
    failureFileName =  "./data/topology/$(topology)/IP_topo_$(topology_index)/$(scenario_cutoff)_ip_scenarios_$(scenario_id).jld"
    if isfile(failureFileName)
        data = load(failureFileName)
        failure_num = length(data["IPScenarios"]["prob"])
    else
        failure_num = 0
    end
    println(failure_num)
end

readScenarioNum()


# TODO: seed, __getitem__, copy

"""
A tuple (i.e., product) of simpler spaces

Example usage:
tuple_obj.observation_space = spaces.Tuple((spaces.Discrete(2), spaces.Discrete(3)))
"""
mutable struct TupleSpace <: AbstractSpace
    spaces::NTuple{N, AbstractSpace} where N
    shape::Int
    TupleSpace(space_array::NTuple{N, AbstractSpace}) where N = new(space_array, length(space_array))
    TupleSpace(space_array::Array{<:AbstractSpace, 1}) = new(Tuple(space_array), length(space_array))
end

sample(tuple_obj::TupleSpace) = Tuple(sample(space) for space in tuple_obj.spaces)

function contains(x, tuple_obj::TupleSpace)
    if isa(x, Array)
        x = Tuple(x)
    end
    return isa(x, Tuple) && Base.length(x) == Base.length(tuple_obj.spaces) &&
        all(part ∈ space for (space, part) in zip(tuple_obj.spaces, x))
end

Base.length(tuple_obj::TupleSpace) = length(tuple_obj.spaces)

Base.:(==)(tuple_obj::TupleSpace, other::TupleSpace) = tuple_obj.spaces == other.spaces
# Base.getindex(::Box, index...)


module Configurations

import JLD2

using ..Trajectories
using ..Metrics
using ..Force
using ..UpdateMethods

export Configuration

struct Configuration
    traj :: Trajectory
    workheat :: WorkHeat
end

function Configuration(x0::Real, f::AbstractForce, h::Heun, τ::Real)
    t0 = 0.0
    traj = heun_fill(x0, h, (p, t) -> total_force(f, p, t), (t0, τ))
    workheat = get_workheat(traj, f, h.step)
    Configuration(traj, workheat)
end

function save_configuration(conf::Configuration, filename)
    @JLD2.save filename conf
end

function load_configuration(filename)::Configuration
    @JLD2.load filename conf
    return conf
end

end


# These tests use BenchmarkTools, and are unreliable in an
# uncontrolled environment (like Travis)

using LazyArrays, BenchmarkTools

@testset "concat" begin
    A = Vcat(Vector(1:10), Vector(1:20))
    b = Array{Int}(undef, 30)
    @test @belapsed(copyto!($b,$A)) < @belapsed(vcat($A.arrays...))

    A = Vcat(1:10, 1:20)
    b = Array{Int}(undef, 30)
    @test @belapsed(copyto!($b,$A)) < @belapsed(vcat($A.arrays...))

    A = Vcat(randn(2,10), randn(4,10))
    b = Array{Float64}(undef, 6,10)
    @test @belapsed(copyto!($b,$A)) < @belapsed(vcat($A.arrays...))

    A = Hcat(1:10, 2:11)
    b = Array{Int}(undef, 10, 2)
    @test @belapsed(copyto!($b,$A)) < @belapsed(hcat($A.arrays...))
end


mutable struct Model
    body::Expr
    params::Expr
    arguments::Tuple
    conditionings::Dict{Symbol,Any}
    observed::Set{Symbol}

    Model(body::Expr) = new(body, :(), (), Dict(), Set())
    Model(body::Expr, params::Expr) = new(body, params, (), Dict(), Set())
    Model(body::Expr, params::Expr, arguments::Tuple, conditionings::Dict{Symbol,Any}, observed::Set{Symbol}) = new(body, params, arguments, conditionings, observed)
end

macro model(body::Expr)
    Model(prettify(body))
end

macro model(param::Symbol, body::Expr)
    Model(prettify(body), :(($param,)))
end

macro model(params::Expr, body::Expr)
    Model(prettify(body), params)
end

function copy(model::Model)
    Model(model.body, model.params, model.arguments, Dict(model.conditionings), Set(model.observed))
end

function condition{T}(conds::Vararg{Pair{Symbol,T}})
    (model::Model) -> let
        newmodel = copy(model)
        for cond in conds
            newmodel.conditionings[cond.first] = cond.second
        end
        newmodel
    end
end

function observe(obs::Vararg{Symbol})
    (model::Model) -> let
        newmodel = copy(model)
        union!(newmodel.observed, Set(obs))
        newmodel
    end
end

function (model::Model)(args :: Vararg)
    newmodel = copy(model)
    newmodel.arguments = args
    newmodel
end