# This file is a part of Julia. License is MIT: https://julialang.org/license

module TestAddmul

using Base: rtoldefault
using Test
using LinearAlgebra
using LinearAlgebra: AbstractTriangular
using SparseArrays
using Random

_rand(::Type{T}) where {T <: AbstractFloat} = T(randn())
_rand(::Type{T}) where {F, T <: Complex{F}} = T(_rand(F), _rand(F))
_rand(::Type{T}) where {T <: Integer} =
    T(rand(max(typemin(T), -10):min(typemax(T), 10)))
_rand(::Type{BigInt}) = BigInt(_rand(Int))

function _rand(A::Type{<:Array}, shape)
    T = eltype(A)
    data = T[_rand(T) for _ in 1:prod(shape)]
    return copy(reshape(data, shape))
end

constructor_of(::Type{T}) where T = getfield(parentmodule(T), nameof(T))

function _rand(A::Type{<: AbstractArray}, shape)
    data = _rand(Array{eltype(A)}, shape)
    T = constructor_of(A)
    if A <: Union{Bidiagonal, Hermitian, Symmetric}
        return T(data, rand([:U, :L]))
        # Maybe test with both :U and :L?
    end
    return T(data)
end

_rand(A::Type{<: SymTridiagonal{T}}, shape) where {T} =
    SymTridiagonal(_rand(Symmetric{T}, shape))

const FloatOrC = Union{AbstractFloat, Complex{<: AbstractFloat}}
const IntegerOrC = Union{Integer, Complex{<: Integer}}
const LTri = Union{LowerTriangular, UnitLowerTriangular, Diagonal}
const UTri = Union{UpperTriangular, UnitUpperTriangular, Diagonal}

needsquare(::Type{<:Matrix}) = false
needsquare(::Type) = true

testdata = []

sizecandidates = 1:4
floattypes = [
    Float64, Float32, ComplexF64, ComplexF32,  # BlasFloat
    BigFloat,
]
inttypes = [
    Int,
    BigInt,
]
# `Bool` can be added to `inttypes` but it's hard to handle
# `InexactError` bug that is mentioned in:
# https://github.com/JuliaLang/julia/issues/30094#issuecomment-440175887
alleltypes = [floattypes; inttypes]
celtypes = [Float64, ComplexF64, BigFloat, Int]

mattypes = [
    Matrix,
    Bidiagonal,
    Diagonal,
    Hermitian,
    LowerTriangular,
    SymTridiagonal,
    Symmetric,
    Tridiagonal,
    UnitLowerTriangular,
    UnitUpperTriangular,
    UpperTriangular,
]

isnanfillable(::AbstractArray) = false
isnanfillable(::Array{<:AbstractFloat}) = true
isnanfillable(A::AbstractArray{<:AbstractFloat}) = parent(A) isa Array

"""
Sample `n` elements from `S` on average but make sure at least one
element is sampled.
"""
function sample(S, n::Real)
    length(S) <= n && return S
    xs = randsubseq(S, n / length(S))
    return length(xs) > 0 ? xs : rand(S, 1)  # sample at least one
end

function inputeltypes(celt, alleltypes = alleltypes)
    # Skip if destination type is "too small"
    celt <: Bool && return []
    filter(alleltypes) do aelt
        celt <: Real && aelt <: Complex && return false
        !(celt <: BigFloat) && aelt <: BigFloat && return false
        !(celt <: BigInt) && aelt <: BigInt && return false
        celt <: IntegerOrC && aelt <: FloatOrC && return false
        if celt <: IntegerOrC && !(celt <: BigInt)
            typemin(celt) > typemin(aelt) && return false
            typemax(celt) < typemax(aelt) && return false
        end
        return true
    end
end
# Note: using `randsubseq` instead of `rand` to avoid repetition.

function inputmattypes(cmat, mattypes = mattypes)
    # Skip if destination type is "too small"
    cmat <: Union{Bidiagonal, Tridiagonal, SymTridiagonal,
                  UnitLowerTriangular, UnitUpperTriangular,
                  Hermitian, Symmetric} && return []
    filter(mattypes) do amat
        cmat <: Diagonal && (amat <: Diagonal || return false)
        cmat <: LowerTriangular && (amat <: LTri || return false)
        cmat <: UpperTriangular && (amat <: UTri || return false)
        return true
    end
end

n_samples = 1.5
# n_samples = Inf  # to try all combinations
for cmat in mattypes,
    amat in sample(inputmattypes(cmat), n_samples),
    bmat in sample(inputmattypes(cmat), n_samples),
    celt in celtypes,
    aelt in sample(inputeltypes(celt), n_samples),
    belt in sample(inputeltypes(celt), n_samples)

    push!(testdata, (cmat{celt}, amat{aelt}, bmat{belt}))
end

@testset "Alternative multiplication operators" begin
    for T in (Int, Float32, Float64, BigFloat)
        a = [T[1, 2], T[-3, 7]]
        b = [T[5, 11], T[-13, 17]]
        @test map(⋅, a, b) == map(dot, a, b) == [27, 158]
        @test map(⊙, a, b) == map(hadamard, a, b) == [a[1].*b[1], a[2].*b[2]]
        @test map(⊗, a, b) == map(tensor, a, b) == [a[1]*transpose(b[1]), a[2]*transpose(b[2])]
        @test hadamard!(fill(typemax(Int), 2), T[1, 2], T[-3, 7]) == [-3, 14]
        @test tensor!(fill(typemax(Int), 2, 2), T[1, 2], T[-3, 7]) == [-3 7; -6 14]
    end

    @test_throws DimensionMismatch [1,2] ⊙ [3]
    @test_throws DimensionMismatch hadamard!([0, 0, 0], [1,2], [-3,7])
    @test_throws DimensionMismatch hadamard!([0, 0], [1,2], [-3])
    @test_throws DimensionMismatch hadamard!([0, 0], [1], [-3,7])
    @test_throws DimensionMismatch tensor!(Matrix{Int}(undef, 2, 2), [1], [-3,7])
    @test_throws DimensionMismatch tensor!(Matrix{Int}(undef, 2, 2), [1,2], [-3])

    u, v = [2+2im, 3+5im], [1-3im, 7+3im]
    @test u ⋅ v == conj(u[1])*v[1] + conj(u[2])*v[2]
    @test u ⊙ v == [u[1]*v[1], u[2]*v[2]]
    @test u ⊗ v == [u[1]*v[1] u[1]*v[2]; u[2]*v[1] u[2]*v[2]]
    @test hadamard(u, v) == u ⊙ v
    @test tensor(u, v)   == u ⊗ v
    dest = similar(u)
    @test hadamard!(dest, u, v) == u ⊙ v
    dest = Matrix{Complex{Int}}(undef, 2, 2)
    @test tensor!(dest, u, v) == u ⊗ v

    for (A, B, b) in (([1 2; 3 4], [5 6; 7 8], [5,6]),
                      ([1+0.8im 2+0.7im; 3+0.6im 4+0.5im],
                       [5+0.4im 6+0.3im; 7+0.2im 8+0.1im],
                       [5+0.6im,6+0.3im]))
        @test A ⊗ b == cat(A*b[1], A*b[2]; dims=3)
        @test A ⊗ B == cat(cat(A*B[1,1], A*B[2,1]; dims=3),
                           cat(A*B[1,2], A*B[2,2]; dims=3); dims=4)
    end

    A, B = reshape(1:27, 3, 3, 3), reshape(1:4, 2, 2)
    @test A ⊗ B == [a*b for a in A, b in B]

    # Adjoint/transpose is a dual vector, not an AbstractMatrix
    v = [1,2]
    @test_throws ErrorException v ⊗ v'
    @test_throws ErrorException v ⊗ transpose(v)
    @test_throws ErrorException v' ⊗ v
    @test_throws ErrorException transpose(v) ⊗ v
    @test_throws ErrorException v' ⊗ v'
    @test_throws ErrorException transpose(v) ⊗ transpose(v)
    @test_throws ErrorException v' ⊗ transpose(v)
    @test_throws ErrorException transpose(v) ⊗ v'
    @test_throws ErrorException A ⊗ v'
    @test_throws ErrorException A ⊗ transpose(v)

    # Docs comparison to `kron`
    v, w = [1,2,3], [5,7]
    @test kron(v,w) == vec(w ⊗ v)
    @test w ⊗ v == reshape(kron(v,w), (length(w), length(v)))
end

@testset "mul!(::$TC, ::$TA, ::$TB, α, β)" for (TC, TA, TB) in testdata
    if needsquare(TA)
        na1 = na2 = rand(sizecandidates)
    else
        na1, na2 = rand(sizecandidates, 2)
    end
    if needsquare(TB)
        nb2 = na2
    elseif needsquare(TC)
        nb2 = na1
    else
        nb2 = rand(sizecandidates)
    end
    asize = (na1, na2)
    bsize = (na2, nb2)
    csize = (na1, nb2)

    @testset for α in Any[true, eltype(TC)(1), _rand(eltype(TC))],
                 β in Any[false, eltype(TC)(0), _rand(eltype(TC))]

        C = _rand(TC, csize)
        A = _rand(TA, asize)
        B = _rand(TB, bsize)

        # This is similar to how `isapprox` choose `rtol` (when
        # `atol=0`) but consider all number types involved:
        rtol = max(rtoldefault.(real.(eltype.((C, A, B))))...,
                   rtoldefault.(real.(typeof.((α, β))))...)

        Cc = copy(C)
        Ac = Matrix(A)
        Bc = Matrix(B)
        returned_mat = mul!(C, A, B, α, β)
        @test returned_mat === C
        @test collect(returned_mat) ≈ α * Ac * Bc + β * Cc  rtol=rtol

        y = C[:, 1]
        x = B[:, 1]
        yc = Vector(y)
        xc = Vector(x)
        returned_vec = mul!(y, A, x, α, β)
        @test returned_vec === y
        @test collect(returned_vec) ≈ α * Ac * xc + β * yc  rtol=rtol

        if TC <: Matrix
            @testset "adjoint and transpose" begin
                @testset for fa in [identity, adjoint, transpose],
                             fb in [identity, adjoint, transpose]
                    fa === fb === identity && continue

                    Af = fa === identity ? A : fa(_rand(TA, reverse(asize)))
                    Bf = fb === identity ? B : fb(_rand(TB, reverse(bsize)))

                    Ac = collect(Af)
                    Bc = collect(Bf)
                    Cc = collect(C)

                    returned_mat = mul!(C, Af, Bf, α, β)
                    @test returned_mat === C
                    @test collect(returned_mat) ≈ α * Ac * Bc + β * Cc  rtol=rtol
                end
            end
        end

        if isnanfillable(C)
            @testset "β = 0 ignores C .= NaN" begin
                parent(C) .= NaN
                Ac = Matrix(A)
                Bc = Matrix(B)
                returned_mat = mul!(C, A, B, α, zero(eltype(C)))
                @test returned_mat === C
                @test collect(returned_mat) ≈ α * Ac * Bc  rtol=rtol
            end
        end

        if isnanfillable(A)
            @testset "α = 0 ignores A .= NaN" begin
                parent(A) .= NaN
                Cc = copy(C)
                returned_mat = mul!(C, A, B, zero(eltype(A)), β)
                @test returned_mat === C
                @test collect(returned_mat) ≈ β * Cc  rtol=rtol
            end
        end
    end
end

end  # module


reshapeVec2Mat(vec::Vector, rows::Int) = reshape(vec, rows, round(Int,length(vec)/rows))
# function reshapeVec2Mat(vec::Vector, rows::Int)
#   M = reshape(vec, rows, round(Int,length(vec)/rows))
#   return ndims(M) < 2 ? (M')' : M
# end

# get vertex from factor graph according to label symbol "x1"
getVert(fgl::FactorGraph, lbl::Symbol; api::DataLayerAPI=dlapi, nt::Symbol=:var) = api.getvertex(fgl, lbl, nt=nt)
getVert(fgl::FactorGraph, id::Int; api::DataLayerAPI=dlapi) = api.getvertex(fgl, id)

# TODO -- upgrade to dedicated memory location in Graphs.jl
# see JuliaArchive/Graphs.jl#233
getData(v::Graphs.ExVertex) = v.attributes["data"]
# Convenience functions
getData(fgl::FactorGraph, lbl::Symbol; api::DataLayerAPI=dlapi) = getData(getVert(fgl, lbl, api=api))
getData(fgl::FactorGraph, id::Int; api::DataLayerAPI=dlapi) = getData(getVert(fgl, id, api=api))

function setData!(v::Graphs.ExVertex, data)
  v.attributes["data"] = data
  nothing
end

function getVal(v::Graphs.ExVertex)
  return getData(v).val
end
function getVal(v::Graphs.ExVertex, idx::Int)
  return getData(v).val[:,idx]
end

# Convenience function to get values for given variable label
function getVal(fgl::FactorGraph, lbl::Symbol; api::DataLayerAPI=dlapi)
  #getVal(dlapi.getvertex(fgl, lbl))
  getVal(getVert(fgl, lbl, api=api))
end
function getVal(fgl::FactorGraph, exvertid::Int; api::DataLayerAPI=dlapi)
  # getVal(dlapi.getvertex(fgl, exvertid))
  getVal(getVert(fgl, exvertid, api=api))
end

function getNumPts(v::Graphs.ExVertex)
  return size(getData(v).val,2)
end

function getfnctype(data)
  if typeof(data).name.name == :VariableNodeData
    return VariableNodeData
  end
  data.fnc.usrfnc!
end

function getfnctype(vertl::Graphs.ExVertex)
  data = getData(vertl)
  getfnctype(data)
end

function getfnctype(fgl::FactorGraph, exvertid::Int; api::DataLayerAPI=dlapi)
  #
  # data = getData(fgl, exvertid, api=api)
  # data.fnc.usrfnc!
  getfnctype(getVert(fgl, exvertid, api=api))
end

# setVal! assumes you will update values to database separate, this used for local graph mods only
function setVal!(v::Graphs.ExVertex, val::Array{Float64,2})
  getData(v).val = val
  # v.attributes["data"].val = val
  nothing
end
function getBWVal(v::Graphs.ExVertex)
  return getData(v).bw
end
function setBW!(v::Graphs.ExVertex, bw::Array{Float64,2})
  getData(v).bw = bw
  # v.attributes["data"].bw = bw
  nothing
end
function setVal!(v::Graphs.ExVertex, val::Array{Float64,2}, bw::Array{Float64,2})
  setVal!(v,val)
  setBW!(v,bw)
  nothing
end
function setVal!(v::Graphs.ExVertex, val::Array{Float64,2}, bw::Vector{Float64})
  setVal!(v,val,reshape(bw,length(bw),1)) #(bw')')
  nothing
end
function setVal!(fg::FactorGraph, sym::Symbol, val::Array{Float64,2}; api::DataLayerAPI=localapi)
  setVal!(api.getvertex(fg, sym), val)
end
function setValKDE!(v::Graphs.ExVertex, val::Array{Float64,2})
  p = kde!(val)
  setVal!(v,val,getBW(p)[:,1]) # TODO -- this can be little faster
  nothing
end
function setValKDE!(v::Graphs.ExVertex, em::EasyMessage)
  setVal!(v, em.pts, em.bws ) # getBW(p)[:,1]
  nothing
end
function setValKDE!(v::Graphs.ExVertex, p::BallTreeDensity)
  pts = getPoints(p)
  setVal!(v, pts, getBW(p)[:,1]) # BUG ...al!(., val, . ) ## TODO -- this can be little faster
  nothing
end
setVal!(v::Graphs.ExVertex, em::EasyMessage) = setValKDE!(v, em)
setVal!(v::Graphs.ExVertex, p::BallTreeDensity) = setValKDE!(v, p)

function kde!(em::EasyMessage)
  return kde!(em.pts,em.bws)
end

# TODO -- there should be a better way, without retrieving full vertex
getOutNeighbors(fgl::FactorGraph, v::ExVertex; api::DataLayerAPI=dlapi, needdata::Bool=false, ready::Int=1,backendset::Int=1 ) = api.outneighbors(fgl, v, needdata=needdata, ready=ready, backendset=backendset )
getOutNeighbors(fgl::FactorGraph, vertid::Int; api::DataLayerAPI=dlapi, needdata::Bool=false, ready::Int=1,backendset::Int=1 ) = api.outneighbors(fgl, api.getvertex(fgl,vertid), needdata=needdata, ready=ready, backendset=backendset )

function updateFullVert!(fgl::FactorGraph, exvert::ExVertex;
            api::DataLayerAPI=IncrementalInference.dlapi,
            updateMAPest::Bool=false  )
  #
  warn("use of updateFullVert! should be clarified for local or remote operations.")
  api.updatevertex!(fgl, exvert, updateMAPest=updateMAPest)
end

function setDefaultNodeData!(v::Graphs.ExVertex,
                             initval::Array{Float64,2},
                             stdev::Array{Float64,2},
                             dodims::Int,
                             N::Int,
                             dims::Int;
                             gt=nothing, initialized::Bool=true,
                             softtype=nothing)
  # TODO review and refactor this function, exists as legacy from pre-v0.3.0
  # this should be the only function allocating memory for the node points (unless number of points are changed)
  data = nothing
  if initialized
    if size(initval,2) < N && size(initval, 1) == dims
      warn("setDefaultNodeData! -- deprecated use of stdev.")
      p = kde!(initval,diag(stdev));
      pN = resample(p,N)
    elseif size(initval,2) < N && size(initval, 1) != dims
      println("Node value memory allocated but not initialized")
      pN = kde!(randn(dims, N));
    else
      pN = kde!(initval)
    end
    # dims = size(initval,1) # rows indicate dimensions
    sp = Int[0;] #round.(Int,linspace(dodims,dodims+dims-1,dims))
    gbw = getBW(pN)[:,1]
    gbw2 = Array{Float64}(length(gbw),1)
    gbw2[:,1] = gbw[:]
    pNpts = getPoints(pN)
    data = VariableNodeData(initval, stdev, pNpts,
                            gbw2, Int[], sp,
                            dims, false, 0, Int[], gt, softtype, true) #initialized
  else
      sp = round.(Int,linspace(dodims,dodims+dims-1,dims))
      data = VariableNodeData(initval, stdev, zeros(dims, N),
                              zeros(dims,1), Int[], sp,
                              dims, false, 0, Int[], gt, softtype, false) #initialized
  end
  #
  setData!(v, data)
  # v.attributes["data"] = data

  # what about dedicated user variable metadata
  # p.factormetadata

  nothing
end

"""
    $(SIGNATURES)

Initialize a new Graphs.ExVertex which will be added to some factor graph.
"""
function addNewVarVertInGraph!(fgl::FactorGraph,
            vert::Graphs.ExVertex,
            id::Int,
            lbl::Symbol,
            ready::Int,
            smalldata  )
  #
  vert.attributes = Graphs.AttributeDict() #fg.v[fg.id]
  vert.attributes["label"] = string(lbl) #fg.v[fg.id]
  fgl.IDs[lbl] = id

  # used for cloudgraph solving
  vert.attributes["ready"] = ready
  vert.attributes["backendset"] = 0

  # store user metadata
  vert.attributes["smalldata"] = smalldata
  nothing
end

"""
$(SIGNATURES)

Add a node (variable) to a graph. Use this over the other dispatches.
"""
function addNode!(fg::FactorGraph,
                  lbl::Symbol,
                  softtype::T;
                  N::Int=100,
                  autoinit::Bool=true,  # does init need to be separate from ready? TODO
                  ready::Int=1,
                  labels::Vector{<:AbstractString}=String[],
                  api::DataLayerAPI=dlapi,
                  uid::Int=-1,
                  smalldata=""  ) where {T <:InferenceVariable}
  #
  currid = fg.id+1
  if uid==-1
    fg.id=currid
  else
    currid = uid
  end
  # dims = dims != -1 ? dims : st.dims
  lblstr = string(lbl)
  vert = ExVertex(currid,lblstr)
  addNewVarVertInGraph!(fg, vert, currid, lbl, ready, smalldata)
  # dlapi.setupvertgraph!(fg, vert, currid, lbl) #fg.v[currid]
  # dodims = fg.dimID+1
  setDefaultNodeData!(vert, zeros(softtype.dims,N), zeros(0,0), 0, N, softtype.dims, initialized=!autoinit, softtype=softtype) # dodims

  vnlbls = union(string.(labels), softtype.labels, String["VARIABLE";])
  push!(vnlbls, fg.sessionname)

  api.addvertex!(fg, vert, labels=vnlbls)

  # fg.dimID+=dims # DONE -- drop this, rows indicate dimensions, move to last dimension
  push!(fg.nodeIDs, currid)

  vert
end

"""
$(SIGNATURES)

Add a node (variable) to a graph. Use this over the other dispatches.
"""
function addNode!(fg::FactorGraph,
                  lbl::Symbol,
                  softtype::Type{<:InferenceVariable};
                  N::Int=100,
                  autoinit::Bool=true,
                  ready::Int=1,
                  labels::Vector{<:AbstractString}=String[],
                  api::DataLayerAPI=dlapi,
                  uid::Int=-1,
                  # dims::Int=-1,
                  smalldata=""  )
  #
  addNode!(fg,
           lbl,
           softtype();
           N=N,
           autoinit=autoinit,
           ready=ready,
           labels=labels,
           api=api,
           uid=uid,
           smalldata=smalldata  )
end

function getVal(vA::Array{Graphs.ExVertex,1})
  warn("getVal(::Vector{ExVertex}) is obsolete, use getVal.(ExVertex) instead.")
  len = length(vA)
  vals = Array{Array{Float64,2},1}()
  cols = Array{Int,1}()
  push!(cols,0)
  rows = Array{Int,1}()
  for v in vA
      push!(vals, getVal(v))
      c = size(vals[end],2)
      r = size(vals[end],1)
      push!(cols, floor(Int,c))
      push!(rows, floor(Int,r))
  end
  cols = cumsum(cols)
  sc = cols[end]
  rw = floor(Int,rows[1])
  val = Array{Float64,2}(rw, sc)
  for i in 1:(len-1)
      val[:,(cols[i]+1):cols[i+1]] = vals[i]
  end
  val[:,(cols[len]+1):cols[len+1]] = vals[len] # and the last one
  return val
end

"""
    $(SIGNATURES)

Prepare the particle arrays `ARR` to be used for approximate convolution.  This function ensures that ARR has te same dimensions among all the parameters.  Function returns with ARR[sfidx] pointing at newly allocated deepcopy of the existing values in getVal(Xi[.index==solvefor]).  Return values `sfidx` is the element in ARR where `Xi.index==solvefor` and `maxlen` is length of all (possibly resampled) `ARR` contained particles.  Note `Xi` is order sensitive.
"""
function prepareparamsarray!(ARR::Array{Array{Float64,2},1},
            Xi::Vector{Graphs.ExVertex},
            N::Int,
            solvefor::Int  )
  #
  LEN = Int[]
  maxlen = N
  count = 0
  sfidx = 0

  for xi in Xi
    push!(ARR, getVal(xi))
    len = size(ARR[end], 2)
    push!(LEN, len)
    if len > maxlen
      maxlen = len
    end
    count += 1
    if xi.index == solvefor
      sfidx = count #xi.index
    end
  end
  SAMP=LEN.<maxlen
  for i in 1:count
    if SAMP[i]
      ARR[i] = KernelDensityEstimate.sample(getKDE(Xi[i]), maxlen)[1]
    end
  end

  # TODO --rather define reusable memory for the proposal
  # we are generating a proposal distribution, not direct replacement for existing memory and hence the deepcopy.
  if sfidx > 0 ARR[sfidx] = deepcopy(ARR[sfidx]) end
  return maxlen, sfidx
end

function parseusermultihypo(multihypo::Void)
  verts = Symbol[]
  mh = nothing
  return mh
end
function parseusermultihypo(multihypo::Union{Tuple,Vector{Float64}})
  mh = nothing
  if multihypo != nothing
    multihypo2 = Float64[multihypo...]
    # verts = Symbol.(multihypo[1,:])
    for i in 1:length(multihypo)
      if multihypo[i] > 0.999999
        multihypo2[i] = 0.0
      end
    end
    mh = Categorical(Float64[multihypo2...] )
  end
  return mh
end

function prepgenericconvolution(
            Xi::Vector{Graphs.ExVertex},
            usrfnc::T;
            multihypo::Union{Void, Distributions.Categorical}=nothing,
            threadmodel=MultiThreaded  ) where {T <: FunctorInferenceType}
      # multiverts::Vector{Symbol}=Symbol[]
  #
  ARR = Array{Array{Float64,2},1}()
  maxlen, sfidx = prepareparamsarray!(ARR, Xi, 0, 0)
  fldnms = fieldnames(usrfnc)
  zdim = typeof(usrfnc) != GenericMarginal ? size(getSample(usrfnc, 2)[1],1) : 0
  ccw = CommonConvWrapper(
          usrfnc,
          zeros(1,0),
          zdim,
          ARR,
          specialzDim = sum(fldnms .== :zDim) >= 1,
          partial = sum(fldnms .== :partial) >= 1,
          hypotheses=multihypo,
          threadmodel=threadmodel
        )
  #
  for i in 1:Threads.nthreads()
    ccw.cpt[i].factormetadata.variableuserdata = []
    ccw.cpt[i].factormetadata.solvefor = :null
    for xi in Xi
      push!(ccw.cpt[i].factormetadata.variableuserdata, getData(xi).softtype)
    end
  end
  return ccw
end

function setDefaultFactorNode!(
      fgl::FactorGraph,
      vert::Graphs.ExVertex,
      Xi::Vector{Graphs.ExVertex},
      usrfnc::T;
      multihypo::Union{Void,Tuple,Vector{Float64}}=nothing,
      threadmodel=MultiThreaded) where {T <: Union{FunctorInferenceType, InferenceType}}
  #
  ftyp = typeof(usrfnc) # maybe this can be T
  # @show "setDefaultFactorNode!", usrfnc, ftyp, T
  mhcat = parseusermultihypo(multihypo)
  # gwpf = prepgenericwrapper(Xi, usrfnc, getSample, multihypo=mhcat)
  ccw = prepgenericconvolution(Xi, usrfnc, multihypo=mhcat, threadmodel=threadmodel)

  m = Symbol(ftyp.name.module)

  # experimental wip
  data_ccw = FunctionNodeData{CommonConvWrapper{T}}(Int[], false, false, Int[], m, ccw)
  vert.attributes["data"] = data_ccw

  # existing interface
  # data = FunctionNodeData{GenericWrapParam{T}}(Int[], false, false, Int[], m, gwpf)
  # vert.attributes["data"] = data

  nothing
end

function addNewFncVertInGraph!(fgl::FactorGraph, vert::Graphs.ExVertex, id::Int, lbl::Symbol, ready::Int)
  vert.attributes = Graphs.AttributeDict() #fg.v[fg.id]
  vert.attributes["label"] = lbl #fg.v[fg.id]
  # fgl.f[id] = vert #  -- not sure if this is required, using fg.g.vertices
  fgl.fIDs[lbl] = id # fg.id,

  # used for cloudgraph solving
  vert.attributes["ready"] = ready
  vert.attributes["backendset"] = 0

  # for graphviz drawing
  vert.attributes["shape"] = "point"
  vert.attributes["width"] = 0.2
  nothing
end
addNewFncVertInGraph!{T <: AbstractString}(fgl::FactorGraph, vert::Graphs.ExVertex, id::Int, lbl::T, ready::Int) =
    addNewFncVertInGraph!(fgl,vert, id, Symbol(lbl), ready)

function isInitialized(vert::Graphs.ExVertex)::Bool
  return getData(vert).initialized
end
function isInitialized(fgl::FactorGraph, vsym::Symbol)::Bool
  # TODO, make cloudgraphs work and make faster by avoiding all the getVerts
  isInitialized(getVert(fgl, vsym))
end

"""
    $(SIGNATURES)

initialize destination variable nodes based on this factor in factor graph, fg, generally called
during addFactor!. Destination factor is first (singletons) or second (dim 2 pairwise) variable vertex in Xi.
"""
function doautoinit!(fgl::FactorGraph,
                     Xi::Vector{Graphs.ExVertex};
                     api::DataLayerAPI=dlapi,
                     singles::Bool=true,
                     N::Int=100)
  # Mighty inefficient function, since we only need very select fields nearby from a few neighboring nodes
  # do double depth search for variable nodes
  # TODO this should maybe stay localapi only...
  for xi in Xi
    if !isInitialized(xi)
      vsym = Symbol(xi.label)
      neinodes = ls(fgl, vsym)
      if (length(neinodes) > 1 || singles) # && !isInitialized(xi)
        # Which of the factors can be used for initialization
        useinitfct = Symbol[]
        # println("Consider all pairwise factors connected to $vsym...")
        for xifct in neinodes #potntlfcts
          xfneivarnodes = lsf(fgl, xifct)
          for vsym2 in xfneivarnodes
            # println("find all variables that are initialized for $vsym2")
            vert2 = getVert(fgl, vsym2)
            if (isInitialized(vert2) && sum(useinitfct .== xifct) == 0 ) || length(xfneivarnodes) == 1
              # OR singleton  TODO get faster version of isInitialized for database version
              # println("adding $xifct to init factors list")
              push!(useinitfct, xifct)
            end
          end
        end
        # println("Consider all singleton (unary) factors to $vsym...")

        # calculate the predicted belief over $vsym
        pts = predictbelief(fgl, vsym, useinitfct, api=api)
        setValKDE!(xi, pts)
        getData(xi).initialized = true
        api.updatevertex!(fgl, xi, updateMAPest=false)
      end
    end
  end

  nothing
end

"""
    $(SIGNATURES)

initialize destination variable nodes based on this factor in factor graph, fg, generally called
during addFactor!.  Destination factor is first (singletons) or second (dim 2 pairwise) variable vertex in Xi.
"""
function doautoinit!(fgl::FactorGraph,
                     xsyms::Vector{Symbol};
                     api::DataLayerAPI=dlapi,
                     singles::Bool=true,
                     N::Int=100)
  #
  verts = getVert.(fgl, xsyms, api=api)
  doautoinit!(fgl, verts, api=api, singles=singles, N=N)
end
function doautoinit!(fgl::FactorGraph,
                     xsym::Symbol;
                     api::DataLayerAPI=dlapi,
                     singles::Bool=true,
                     N::Int=100)
  #
  doautoinit!(fgl, [getVert(fgl, xsym, api=api);], api=api, singles=singles, N=N)
end

function ensureAllInitialized!(fgl::FactorGraph; api::DataLayerAPI=dlapi)
  xx, xl = ls(fgl)
  allvarnodes = union(xx, xl)
  for vsym in allvarnodes
    if !isInitialized(fgl, vsym)
      println("$vsym is not initialized, and will do so now...")
      doautoinit!(fgl, Graphs.ExVertex[getVert(fgl, vsym, api=api);], api=api, singles=true)
    end
  end
  nothing
end

function assembleFactorName(fgl::FactorGraph, Xi::Vector{Graphs.ExVertex}; maxparallel::Int=50)
  namestring = ""
  for vert in Xi #f.Xi
    namestring = string(namestring,vert.attributes["label"])
  end
  for i in 1:maxparallel
    tempnm = string(namestring, "f$i")
    if !haskey(fgl.fIDs, Symbol(tempnm))
      namestring = tempnm
      break
    end
    i != maxparallel ? nothing : error("Cannot currently add more than $(maxparallel) factors in parallel, please open an issue if this is too restrictive.")
  end
  return namestring
end

"""
    $(SIGNATURES)

Add factor with user defined type <: FunctorInferenceType to the factor graph object.  Define whether the automatic initialization of variables should be performed.  Use order sensitive `multihypo` keyword argument to define if any variables are related to data association uncertainty.
"""
function addFactor!(fgl::FactorGraph,
      Xi::Vector{Graphs.ExVertex},
      usrfnc::R;
      multihypo::Union{Void,Tuple,Vector{Float64}}=nothing,
      ready::Int=1,
      api::DataLayerAPI=dlapi,
      labels::Vector{T}=String[],
      uid::Int=-1,
      autoinit::Bool=true,
      threadmodel=MultiThreaded  ) where
        {R <: Union{FunctorInferenceType, InferenceType},
         T <: AbstractString}
  #
  currid = fgl.id+1
  if uid==-1
    fgl.id=currid
  else
    currid = uid
  end
  namestring = assembleFactorName(fgl, Xi)
  # fgl.id+=1
  newvert = ExVertex(currid,namestring)
  addNewFncVertInGraph!(fgl, newvert, currid, namestring, ready)
  setDefaultFactorNode!(fgl, newvert, Xi, deepcopy(usrfnc), multihypo=multihypo, threadmodel=threadmodel)
  push!(fgl.factorIDs,currid)

  for vert in Xi
    push!(getData(newvert).fncargvID, vert.index)
    # push!(newvert.attributes["data"].fncargvID, vert.index)
  end

  fnlbls = deepcopy(labels)
  fnlbls = union(fnlbls, String["FACTOR";])
  push!(fnlbls, fgl.sessionname)
  # TODO -- multiple accesses to DB with this method, must refactor!
  newvert = api.addvertex!(fgl, newvert, labels=fnlbls)  # used to be two be three lines up ##fgl.g
  for vert in Xi
    api.makeaddedge!(fgl, vert, newvert)
  end

  # TODO change this operation to update a conditioning variable
  if autoinit
    doautoinit!(fgl, Xi, api=api, singles=false)
  end

  return newvert
end

"""
    $(SIGNATURES)

Add factor with user defined type <: FunctorInferenceType to the factor graph object.  Define whether the automatic initialization of variables should be performed.  Use order sensitive `multihypo` keyword argument to define if any variables are related to data association uncertainty.
"""
function addFactor!(
      fgl::FactorGraph,
      xisyms::Vector{Symbol},
      usrfnc::R;
      multihypo::Union{Void,Tuple,Vector{Float64}}=nothing,
      ready::Int=1,
      api::DataLayerAPI=dlapi,
      labels::Vector{T}=String[],
      uid::Int=-1,
      autoinit::Bool=true,
      threadmodel=MultiThreaded  ) where
        {R <: Union{FunctorInferenceType, InferenceType},
         T <: AbstractString}
  #
  verts = Vector{Graphs.ExVertex}()
  for xi in xisyms
      push!( verts, api.getvertex(fgl,xi) )
  end
  addFactor!(fgl, verts, usrfnc, multihypo=multihypo, ready=ready, api=api, labels=labels, uid=uid, autoinit=autoinit, threadmodel=threadmodel )
end

function prtslperr(s)
  println(s)
  sleep(0.1)
  error(s)
end

# for computing the Bayes Net-----------------------------------------------------
function getEliminationOrder(fg::FactorGraph; ordering::Symbol=:qr)
    s = fg.nodeIDs
    lens = length(s)
    sf = fg.factorIDs
    lensf = length(sf)
    adjm, dictpermu = adjacency_matrix(fg.g,returnpermutation=true)
    permuteds = Vector{Int}(lens)
    permutedsf = Vector{Int}(lensf)
    for j in 1:length(dictpermu)
      semap = 0
      for i in 1:lens
        if dictpermu[j] == s[i]
          permuteds[i] = j#dictpermu[j]
          semap += 1
          if semap >= 2  break; end
        end
      end
      for i in 1:lensf
        if dictpermu[j] == sf[i]
          permutedsf[i] = j#dictpermu[j]
          semap += 1
          if semap >= 2  break; end
        end
      end
    end

      A=convert(Array{Int},adjm[permutedsf,permuteds]) # TODO -- order seems brittle
      p = Int[]
      if ordering==:chol
        p = cholfact(A'A,:U,Val{true})[:p] #,pivot=true
      elseif ordering==:qr
        q,r,p = qr(A,Val{true})
      else
        prtslperr("getEliminationOrder -- cannot do the requested ordering $(ordering)")
      end

    # we need the IDs associated with the Graphs.jl and our Type fg
    return  dictpermu[permuteds[p]] # fg.nodeIDs[p]
end

# lets create all the vertices first and then deal with the elimination variables thereafter
function addBayesNetVerts!(fg::FactorGraph, elimOrder::Array{Int,1})
  for p in elimOrder
    vert = getVert(fg, p, api=localapi)
    @show vert.label, getData(vert).BayesNetVertID
    if getData(vert).BayesNetVertID == 0
      fg.bnid+=1
      vert.attributes["data"].BayesNetVertID = p
      localapi.updatevertex!(fg, vert)
    else
      println("addBayesNetVerts -- something is very wrong, should not have a Bayes net vertex")
    end
  end
end

function addConditional!(fg::FactorGraph, vertID::Int, lbl, Si)
  bnv = getVert(fg, vertID, api=localapi) #fg.v[vertID]
  bnvd = getData(bnv) # bnv.attributes["data"]
  bnvd.separator = Si
  for s in Si
    push!(bnvd.BayesNetOutVertIDs, s)
  end
  localapi.updatevertex!(fg, bnv)
  nothing
end

function addChainRuleMarginal!(fg::FactorGraph, Si)
  lbls = String[]
  genmarg = GenericMarginal()
  Xi = Graphs.ExVertex[]
  for s in Si
    push!(Xi, getVert(fg, s, api=localapi))
  end
  println("adding marginal to")
  for x in Xi @show x.index end
  addFactor!(fg, Xi, genmarg, api=localapi, autoinit=false)
  nothing
end

# TODO -- Cannot have any CloudGraph calls at this level, must refactor
function rmVarFromMarg(fgl::FactorGraph, fromvert::Graphs.ExVertex, gm::Array{Graphs.ExVertex,1})
  for m in gm
    # get all out edges
    # get neighbors
    for n in localapi.outneighbors(fgl, m)
      if n.index == fromvert.index
        alleids = m.attributes["data"].edgeIDs
        i = 0
        for id in alleids
          i+=1
          edge = localapi.getedge(fgl, id)
          if edge != nothing # hack to avoid dictionary use case
            if edge.SourceVertex.exVertexId == m.index || edge.DestVertex.exVertexId == m.index
              warn("removing edge $(edge.neo4jEdgeId), between $(m.index) and $(n.index)")
              localapi.deleteedge!(fgl, edge)
              m.attributes["data"].edgeIDs = alleids[[collect(1:(i-1));collect((i+1):length(alleids))]]
              localapi.updatevertex!(fgl, m)
            end
          end
        end
      end
    end
    # if 0 edges, delete the marginal
    if length(localapi.outneighbors(fgl, m)) <= 1
      warn("removing vertex id=$(m.index)")
      localapi.deletevertex!(fgl,m)
    end
  end
  nothing
end

function buildBayesNet!(fg::FactorGraph, p::Array{Int,1})
    addBayesNetVerts!(fg, p)
    for v in p
      println()
      println("Eliminating $(v)")
      println("===============")
      println()
      # which variable are we eliminating

      # all factors adjacent to this variable
      fi = Int[]
      Si = Int[]
      gm = ExVertex[]
      # TODO -- optimize outneighbor calls like this
      vert = localapi.getvertex(fg,v)
      for fct in localapi.outneighbors(fg, vert)
        if (getData(fct).eliminated != true)
          push!(fi, fct.index)
          for sepNode in localapi.outneighbors(fg, fct)
            if sepNode.index != v && length(findin(sepNode.index,Si)) == 0
              push!(Si,sepNode.index)
            end
          end
          getData(fct).eliminated = true #fct.attributes["data"].eliminated = true
          localapi.updatevertex!(fg, fct) # TODO -- this might be a premature statement
        end

        if typeof(getData(fct).fnc) == GenericMarginal
          push!(gm, fct)
        end
      end

      if v != p[end]
        addConditional!(fg, v, "", Si)
        # not yet inserting the new prior p(Si) back into the factor graph
      end

      tuv = localapi.getvertex(fg, v) # TODO -- This may well through away valuable data
      tuv.attributes["data"].eliminated = true # fg.v[v].
      localapi.updatevertex!(fg, tuv)

      # TODO -- remove links from current vertex to any marginals
      rmVarFromMarg(fg, vert, gm)

      #add marginal on remaining variables... ? f(xyz) = f(x | yz) f(yz)
      # new function between all Si
      addChainRuleMarginal!(fg, Si)

    end
    nothing
end

# some plotting functions on the factor graph
function stackVertXY(fg::FactorGraph, lbl::String)
    v = dlapi.getvertex(fg,lbl)
    vals = getVal(v)
    X=vec(vals[1,:])
    Y=vec(vals[2,:])
    return X,Y
end

function getKDE(v::Graphs.ExVertex)
  return kde!(getVal(v), getBWVal(v)[:,1])
end

function getVertKDE(v::Graphs.ExVertex)
  return getKDE(v)
end

function getVertKDE(fgl::FactorGraph, id::Int; api::DataLayerAPI=dlapi)
  v = api.getvertex(fgl,id)
  return getKDE(v)
end
function getVertKDE(fgl::FactorGraph, lbl::Symbol; api::DataLayerAPI=dlapi)
  v = api.getvertex(fgl,lbl)
  return getKDE(v)
end

function drawCopyFG(fgl::FactorGraph)
  fgd = deepcopy(fgl)
  for (sym,vid) in fgd.IDs
    delete!(fgd.g.vertices[vid].attributes,"data")
    !haskey(fgd.g.vertices[vid].attributes,"frtend") ? nothing : delete!(fgd.g.vertices[vid].attributes,"frtend")
  end
  for (sym,vid) in fgd.fIDs
    delete!(fgd.g.vertices[vid].attributes,"data")
    !haskey(fgd.g.vertices[vid].attributes,"frtend") ? nothing : delete!(fgd.g.vertices[vid].attributes,"frtend")
  end
  return fgd
end

function writeGraphPdf(fgl::FactorGraph;
                       pdfreader::Union{Void, String}="evince",
                       filename::AS="/tmp/fg.pdf"  ) where {AS <: AbstractString}
  #
  fgd = drawCopyFG(fgl)
  println("Writing factor graph file")
  dotfile = split(filename, ".pdf")[1]*".dot"
  fid = open(dotfile,"w")
  write(fid,Graphs.to_dot(fgd.g))
  close(fid)
  run(`dot $(dotfile) -Tpdf -o $(filename)`)
  try
    pdfreader != nothing ? (@async run(`$(pdfreader) $(filename)`)) : nothing
  catch e
    warn("not able to show $(filename) with pdfreader=$(pdfreader). Exception e=$(e)")
  end
  nothing
end

function expandEdgeListNeigh!(fgl::FactorGraph,
                              vertdict::Dict{Int,Graphs.ExVertex},
                              edgedict::Dict{Int,Graphs.Edge{Graphs.ExVertex}})
  #asfd
  for vert in vertdict
    for newedge in out_edges(vert[2],fgl.g)
      if !haskey(edgedict, newedge.index)
        edgedict[newedge.index] = newedge
      end
    end
  end

  nothing
end

# dictionary of unique vertices from edgelist
function expandVertexList!(fgl::FactorGraph,
  edgedict::Dict{Int,Graphs.Edge{Graphs.ExVertex}},
  vertdict::Dict{Int,Graphs.ExVertex})

  # go through all source and target nodes
  for edge in edgedict
    if !haskey(vertdict, edge[2].source.index)
      vertdict[edge[2].source.index] = edge[2].source
    end
    if !haskey(vertdict, edge[2].target.index)
      vertdict[edge[2].target.index] = edge[2].target
    end
  end
  nothing
end

function edgelist2edgedict(edgelist::Array{Graphs.Edge{Graphs.ExVertex},1})
  edgedict = Dict{Int,Graphs.Edge{Graphs.ExVertex}}()
  for edge in edgelist
    edgedict[edge.index] = edge
  end
  return edgedict
end

# TODO -- convert to use add_vertex! instead, since edges type must be made also
function addVerticesSubgraph(fgl::FactorGraph,
    fgseg::FactorGraph,
    vertdict::Dict{Int,Graphs.ExVertex})

    for vert in vertdict
      fgseg.g.vertices[vert[1]] = vert[2]
      if haskey(fgl.v,vert[1])
        fgseg.g.vertices[vert[1]] = vert[2]
        fgseg.IDs[Symbol(vert[2].label)] = vert[1]

        # add edges going in opposite direction
        elr = Graphs.out_edges(vert[2], fgl.g)
        len = length(elr)
        keeprm = trues(len)
        j = 0
        for i in 1:len
          if !haskey(vertdict, elr[i].target.index) # a function node in set, so keep ref
            keeprm[i] = false
            j+=1
          end
        end
        if j < len
          elridx = elr[1].source.index
          fgseg.g.inclist[elridx] = elr[keeprm]
        end
      elseif haskey(fgl.f, vert[1])
        fgseg.f[vert[1]] = vert[2] # adding element to subgraph
        fgseg.fIDs[Symbol(vert[2].label)] = vert[1]
        # get edges associated with function nodes and push edges onto incidence list
        el = Graphs.out_edges(vert[2], fgl.g)
        elidx = el[1].source.index
        fgseg.g.inclist[elidx] = el # okay because treating function nodes only
        fgseg.g.nedges += length(el)
      else
        error("Unknown type factor graph vertex type, something is wrong")
      end
    end
    nothing
end

# NOTICE, nodeIDs and factorIDs are not brough over by this method yet
# must sort out for incremental updates
function genSubgraph(fgl::FactorGraph,
    vertdict::Dict{Int,Graphs.ExVertex})
    # edgedict::Dict{Int,Graphs.Edge{Graphs.ExVertex}},

  fgseg = FactorGraph() # new handle for just a segment of the graph
  fgseg.g = Graphs.inclist(Graphs.ExVertex,is_directed=false)

  fgseg.v = Dict{Int,Graphs.ExVertex}()
  fgseg.f = Dict{Int,Graphs.ExVertex}()
  fgseg.IDs = Dict{AbstractString,Int}()
  fgseg.fIDs = Dict{AbstractString,Int}()

  # TODO -- convert to use empty constructor since Graphs.incdict now works
  fgseg.g.vertices = Array{Graphs.ExVertex,1}(length(fgl.g.vertices))
  fgseg.g.inclist = Array{Array{Graphs.Edge{Graphs.ExVertex},1},1}(length(fgl.g.inclist))

  addVerticesSubgraph(fgl, fgseg, vertdict)

  fgseg.id = fgl.id
  fgseg.bnid = fgl.bnid
  fgseg.dimID = fgl.dimID

  return fgseg
end

function getShortestPathNeighbors(fgl::FactorGraph;
    from::Graphs.ExVertex=nothing,
    to::Graphs.ExVertex=nothing,
    neighbors::Int=0 )

  edgelist = shortest_path(fgl.g, ones(num_edges(fgl.g)), from, to)
  vertdict = Dict{Int,Graphs.ExVertex}()
  edgedict = edgelist2edgedict(edgelist)
  expandVertexList!(fgl, edgedict, vertdict) # grow verts
  for i in 1:neighbors
    expandEdgeListNeigh!(fgl, vertdict, edgedict) # grow edges
    expandVertexList!(fgl, edgedict, vertdict) # grow verts
  end
  return vertdict
end

function subgraphShortestPath(fgl::FactorGraph;
    from::Graphs.ExVertex=nothing,
    to::Graphs.ExVertex=nothing,
    neighbors::Int=0  )

  vertdict = getShortestPathNeighbors(fgl, from=from, to=to, neighbors=neighbors)
  return genSubgraph(fgl, vertdict)
end

# explore all shortest paths combinations in verts, add neighbors and reference subgraph
function subgraphFromVerts(fgl::FactorGraph,
    verts::Dict{Int,Graphs.ExVertex};
    neighbors::Int=0  )

    allverts = Dict{Int,Graphs.ExVertex}()
    allkeys = collect(keys(verts))
    len = length(allkeys)
    # union all shortest path combinations in a vertdict
    for i in 1:len, j in (i+1):len
      from = verts[allkeys[i]]
      to = verts[allkeys[j]]
      vertdict = getShortestPathNeighbors(fgl, from=from, to=to, neighbors=neighbors)
      for vert in vertdict
        if !haskey(allverts, vert[1])
          allverts[vert[1]] = vert[2]
        end
      end
    end

  return genSubgraph(fgl, allverts)
end

# explore all shortest paths combinations in verts, add neighbors and reference subgraph
# Using unique index into graph data structure
function subgraphFromVerts(fgl::FactorGraph,
    verts::Array{Int,1};
    neighbors::Int=0  )

  vertdict = Dict{Int,Graphs.ExVertex}()
  for vert in verts
    vertdict[vert] = fgl.g.vertices[vert]
  end

  return subgraphFromVerts(fgl,vertdict,neighbors=neighbors)
end

# explore all shortest paths combinations in verts, add neighbors and reference subgraph
# Using unique index into graph data structure
function subgraphFromVerts(fgl::FactorGraph,
    verts::Array{String,1};
    neighbors::Int=0  )

  vertdict = Dict{Int,Graphs.ExVertex}()
  for vert in verts
    id = -1
    if haskey(fgl.IDs, vert)
      id = fgl.IDs[Symbol(vert)]
    else
      error("FactorGraph01 only supports variable node subgraph search at this point")
    end
    vertdict[id] = fgl.g.vertices[id]
  end

  return subgraphFromVerts(fgl,vertdict,neighbors=neighbors)
end


# script to run all SLICOT models in several benchmark settings

# Plots is required here, unless never used, to avoid an issue with "world age"
using Reachability, Plots

# model list
models = String[
    "motor",
    "building",
    "pde",
    "heat",
    "iss",
    "beam",
    "mna1",
    "fom",
    "mna5"
   ]

# load models and benchmark scripts
model_library = include(@relpath "model_library.jl")
for model in models
    path = (@relpath "../") * model_library[model]
    include(path)
end
include(@relpath "reach_1D_single.jl")
include(@relpath "reach_1D_all.jl")
include(@relpath "reach_2D_box_two.jl")
include(@relpath "reach_2D_eps_two.jl")
include(@relpath "reach_kD_all.jl")
include(@relpath "check_1D_discrete.jl")
include(@relpath "check_kD_discrete.jl")
include(@relpath "check_1D_dense.jl")
include(@relpath "check_kD_dense.jl")

# create plots (option)?
create_plots = true
if create_plots
    include(@relpath "plot_reach.jl")
end

# run benchmarks

# -- reach_1D_single --
# no modification, optionally create plots
reach_1D_single(models, create_plots)

# -- reach_1D_all --
# remove mna5
models_filtered = filter(e -> e != "mna5", models)
reach_1D_all(models_filtered)

# -- reach_2D_box_two --
# no modification, optionally create plots
reach_2D_box_two(models, create_plots)

# -- reach_2D_eps_two --
# no modification, optionally create plots
reach_2D_eps_two(models, create_plots)

# -- reach_kD_all --
# only use pde, optionally create plots
models_filtered = filter(e -> e ∈ ["fom"], models)
reach_kD_all(models_filtered, create_plots)

# -- check_1D_discrete --
# no modification
check_1D_discrete(models)

# -- check_kD_discrete --
# only use iss and fom; use default partition
models_filtered = filter(e -> e ∈ ["iss", "fom"], models)
check_kD_discrete(models_filtered)

# -- check_1D_dense --
# remove iss and fom; define deltas
models_filtered = filter(e -> e ∉ ["iss", "fom"], models)
delta_map = Dict(
    "motor" => 1e-3,
    "building" => 2e-3,
    "pde" => 3e-4,
    "heat" => 1e-3,
    "beam" => 5e-5,
    "mna1" => 4e-4,
    "mna5" => 3e-1
   )
deltas = Vector{Float64}(length(models_filtered))
for (i, model) in enumerate(models_filtered)
    deltas[i] = delta_map[model]
end
check_1D_dense(models_filtered, deltas)

# -- check_kD_dense --
# only use iss; use default partition
models_filtered = filter(e -> e ∈ ["iss"], models)
delta_map = Dict(
    "iss" => 6e-4,
   )
deltas = Vector{Float64}(length(models_filtered))
for (i, model) in enumerate(models_filtered)
    deltas[i] = delta_map[model]
end
check_kD_dense(models_filtered, deltas)

return nothing


#=
Probe at randomly selected corpus points.
A fixed count of probes, with short circuit for perfect match.

Should be kept very similar to heuristicPatchMatch
TODO refactor so common code is in one place
=#
function matchPatchesAtRandomCorpusPoints(
    synthPatch,  # const IN
    targetImage, # mutable, IN and OUT
    corpusImage, # const IN
    bestKnownPatchDiff, # IN
    synthResult,    # mutable, OUT
    searchResult    # mutable, IN/OUT to be mutated by the search
    )

    # OLD setStartingSearchResult(searchResult, bestKnownPatchDiff)
    #@debug  "Random search diff to beat %f\n" result.bestProbeResult.patchDifference

    for i = 1:parameters.maxProbeCount
        corpusPatchCenterPoint = generateRandomMaskedPoint(corpusImage)
        probeResult = patchMatch(
            targetImage, corpusImage,
            corpusPatchCenterPoint,
            synthPatch,
            searchResult.bestProbeResult.patchDifference)

        if probeResult.betterment != NotBetter
            # Better or equal to any point probed in this pass

            # Call method of SearchResult to lift the probeResult into the searchResult
            setBetterSearchResult(searchResult, probeResult, corpusPatchCenterPoint)

            @debug  "    Better random probe, at" probeResult corpusPatchCenterPoint

            # Is better, might be perfect
            if probeResult.betterment == PerfectMatch
                @debug "Perfect match"
                break
            end
        end
    end

    @debug  "  >>Random search result " searchResult
    return searchResult
end


struct Sample{T}
    q::JointSegmentedVector{T}
    v::JointSegmentedVector{T}
    v̇::JointSegmentedVector{T}
    τ::JointSegmentedVector{T}
end


# Multivariate optimization
function check_kwargs(kwargs, fallback_method)
    kws = Dict{Symbol, Any}()
    method = nothing
    for kwarg in kwargs
        if kwarg[1] != :method
            kws[kwarg[1]] = kwarg[2]
        else
            method = kwarg[2]
        end
    end

    if method == nothing
        method = fallback_method
    end
    kws, method
end

default_options(method::AbstractOptimizer) = Dict{Symbol, Any}()

function add_default_opts!(opts::Dict{Symbol, Any}, method::AbstractOptimizer)
    for newopt in default_options(method)
        if !haskey(opts, newopt[1])
            opts[newopt[1]] = newopt[2]
        end
    end
end

fallback_method(f) = NelderMead()
fallback_method(f, g!) = LBFGS()
fallback_method(f, g!, h!) = Newton()

fallback_method(d::OnceDifferentiable) = LBFGS()
fallback_method(d::TwiceDifferentiable) = Newton()

# promote the objective (tuple of callables or an AbstractObjective) according to method requirement
promote_objtype(method, initial_x, autodiff::Symbol, inplace::Bool, args...) = error("No default objective type for $method and $args.")
# actual promotions, notice that (args...) captures FirstOrderOptimizer and NonDifferentiable, etc
promote_objtype(method::ZerothOrderOptimizer, x, autodiff::Symbol, inplace::Bool, args...) = NonDifferentiable(args..., x, real(zero(eltype(x))))
promote_objtype(method::FirstOrderOptimizer,  x, autodiff::Symbol, inplace::Bool, f) = OnceDifferentiable(f, x, real(zero(eltype(x))); autodiff = autodiff)
promote_objtype(method::FirstOrderOptimizer,  x, autodiff::Symbol, inplace::Bool, args...) = OnceDifferentiable(args..., x, real(zero(eltype(x))); inplace = inplace)
promote_objtype(method::FirstOrderOptimizer,  x, autodiff::Symbol, inplace::Bool, f, g, h) = OnceDifferentiable(f, g, x, real(zero(eltype(x))); inplace = inplace)
promote_objtype(method::SecondOrderOptimizer, x, autodiff::Symbol, inplace::Bool, f) = TwiceDifferentiable(f, x, real(zero(eltype(x))); autodiff = autodiff)
promote_objtype(method::SecondOrderOptimizer, x, autodiff::Symbol, inplace::Bool, f::NotInplaceObjective) = TwiceDifferentiable(f, x, real(zero(eltype(x))))
promote_objtype(method::SecondOrderOptimizer, x, autodiff::Symbol, inplace::Bool, f::InplaceObjective) = TwiceDifferentiable(f, x, real(zero(eltype(x))))
promote_objtype(method::SecondOrderOptimizer, x, autodiff::Symbol, inplace::Bool, f, g) = TwiceDifferentiable(f, g, x, real(zero(eltype(x))); inplace = inplace, autodiff = autodiff)
promote_objtype(method::SecondOrderOptimizer, x, autodiff::Symbol, inplace::Bool, f, g, h) = TwiceDifferentiable(f, g, h, x, real(zero(eltype(x))); inplace = inplace)
# no-op
promote_objtype(method::ZerothOrderOptimizer, x, autodiff::Symbol, inplace::Bool, nd::NonDifferentiable)  = nd
promote_objtype(method::ZerothOrderOptimizer, x, autodiff::Symbol, inplace::Bool, od::OnceDifferentiable) = od
promote_objtype(method::FirstOrderOptimizer,  x, autodiff::Symbol, inplace::Bool, od::OnceDifferentiable) = od
promote_objtype(method::ZerothOrderOptimizer, x, autodiff::Symbol, inplace::Bool, td::TwiceDifferentiable) = td
promote_objtype(method::FirstOrderOptimizer,  x, autodiff::Symbol, inplace::Bool, td::TwiceDifferentiable) = td
promote_objtype(method::SecondOrderOptimizer, x, autodiff::Symbol, inplace::Bool, td::TwiceDifferentiable) = td

# if no method or options are present
function optimize(f,         initial_x::AbstractArray; inplace = true, autodiff = :finite, kwargs...)
    method = fallback_method(f)
    checked_kwargs, method = check_kwargs(kwargs, method)
    d = promote_objtype(method, initial_x, autodiff, inplace, f)
    add_default_opts!(checked_kwargs, method)

    options = Options(; checked_kwargs...)
    optimize(d, initial_x, method, options)
end
function optimize(f, g, initial_x::AbstractArray; inplace = true, autodiff = :finite, kwargs...)

    method = fallback_method(f, g)
    checked_kwargs, method = check_kwargs(kwargs, method)
    d = promote_objtype(method, initial_x, autodiff, inplace, f, g)
    add_default_opts!(checked_kwargs, method)

    options = Options(; checked_kwargs...)
    optimize(d, initial_x, method, options)
end
function optimize(f, g, h, initial_x::AbstractArray; inplace = true, autodiff = :finite, kwargs...)

    method = fallback_method(f, g, h)
    checked_kwargs, method = check_kwargs(kwargs, method)
    d = promote_objtype(method, initial_x, autodiff, inplace, f, g, h)
    add_default_opts!(checked_kwargs, method)

    options = Options(; checked_kwargs...)
    optimize(d, initial_x, method, options)
end

# no method supplied with objective
function optimize(d::T, initial_x::AbstractArray, options::Options) where T<:AbstractObjective
    optimize(d, initial_x, fallback_method(T), options)
end
# no method supplied with inplace and autodiff keywords becauase objective is not supplied
function optimize(f, initial_x::AbstractArray, options::Options; inplace = true, autodiff = :finite)
    method = fallback_method(f)
    d = promote_objtype(method, initial_x, autodiff, inplace, f)
    optimize(d, initial_x, method, options)
end
function optimize(f, g, initial_x::AbstractArray, options::Options; inplace = true, autodiff = :finite)

    method = fallback_method(f, g!)
    d = promote_objtype(method, initial_x, autodiff, inplace, f, g)
    optimize(d, initial_x, method, options)
end
function optimize(f, g, h, initial_x::AbstractArray, options::Options; inplace = true, autodiff = :finite)

    method = fallback_method(f, g!, h!)
    d = promote_objtype(method, initial_x, autodiff, inplace, f, g, h)

    optimize(d, initial_x, method, options)
end

# potentially everything is supplied (besides caches)
function optimize(f, initial_x::AbstractArray, method::AbstractOptimizer,
                     options::Options = Options(;default_options(method)...); inplace = true, autodiff = :finite)

    d = promote_objtype(method, initial_x, autodiff, inplace, f)
    optimize(d, initial_x, method, options)
end
function optimize(f, g, initial_x::AbstractArray, method::AbstractOptimizer,
         options::Options = Options(;default_options(method)...); inplace = true, autodiff = :finite)

    d = promote_objtype(method, initial_x, autodiff, inplace, f, g)

    optimize(d, initial_x, method, options)
end
function optimize(f, g, h, initial_x::AbstractArray{T}, method::AbstractOptimizer,
         options::Options = Options(;default_options(method)...); inplace = true, autodiff = :finite) where T

    d = promote_objtype(method, initial_x, autodiff, inplace, f, g, h)

    optimize(d, initial_x, method, options)
end

function optimize(d::D, initial_x::AbstractArray, method::SecondOrderOptimizer,
                  options::Options = Options(;default_options(method)...); autodiff = :finite, inplace = true) where {D <: Union{NonDifferentiable, OnceDifferentiable}}
    d = promote_objtype(method, initial_x, autodiff, inplace, d)
    optimize(d, initial_x, method, options)
end


#------------------------------------------------------------------------------------------------------------------------------#

abstract type AbstractRobot end

import HorizonSideRobots: move!, isborder, putmarker!, ismarker, temperature

#get(robot::AbstractRobot) = robot::Robot

move!(robot::AbstractRobot, side) = move!(get(robot), side)
isborder(robot::AbstractRobot, side) = isborder(get(robot), side)
putmarker!(robot::AbstractRobot) = putmarker!(get(robot))
ismarker(robot::AbstractRobot) = ismarker(get(robot))
temperature(robot::AbstractRobot) = temperature(get(robot))

#------------------------------------------------------------------------------------------------------------------------------#

mutable struct CountmarkersRobot <: AbstractRobot
    CmR::Robot
    counter::Integer
end

CmR = CountmarkersRobot(originRobot,0)

get(robot::CountmarkersRobot) = robot.CmR

function move!(robot::CountmarkersRobot, side::HorizonSide)
    move!(robot.CmR, side) # move!(get(CmR), side)
    if (ismarker(robot.CmR))
        robot.counter += 1
    end
end

#------------------------------------------------------------------------------------------------------------------------------#

mutable struct CoordsRobot <: AbstractRobot
    coordsrobot::Robot
    x::Int
    y::Int
end

coordsrobot = CoordsRobot(originRobot, 0, 0)

get(robot::CoordsRobot) = robot.coordsrobot

function move!(robot::CoordsRobot, side::HorizonSide)
    if side == Nord
        robot.y += 1
    elseif side == Sud
        robot.y -= 1
    elseif side == Ost
        robot.x += 1
    else
        robot.x -= 1
    end
    move!(robot.coordsrobot, side)
end

get_coords(robot::CoordsRobot) = (robot.x , robot.y)

#------------------------------------------------------------------------------------------------------------------------------#

mutable struct Beta <: AbstractRobot
    beta::Robot
    path::Vector
end

beta = Beta(originRobot, [])

get(robot::Beta) = robot.beta

function move!(robot::Beta, side::HorizonSide)
    push!(robot.path, Int(side))
    move!(robot.beta, side)
end

function get_back!(robot::Beta)
    for i in 1:length(robot.path)
        move!(robot.beta, anti_side(HorizonSide(robot.path[ length(robot.path) - i + 1 ])))
    end
    empty!(robot.path)
end

function try_move!(robot::Beta, side::HorizonSide)::Bool

    # если перед роботом есть перегородка, он начнет выполнять действия, чтобы попытаться её обойти, иначе просто пройдет в заданном направлении
    if isborder(robot.beta, side)
        steps = 0
        nextside = next_side(side)
        # робот пытается обойти перегородку со строны, которая следует за заданной по часовой стрелке
        while isborder(robot.beta, side)
            if !isborder(robot.beta, nextside)
                move!(robot, nextside)
                steps += 1
            else
                # срабатывает, когда робот уперся в угол. Он возвращается в то место, с которого начал попытку обойти по часовой стороне
                for j in 1:steps
                    move!(robot, anti_side(nextside))
                end
                return false
            end
        end

        # если у робота получилось обойти перегородку с первого раза, будет выполнено слудущее:
        move!(robot, side)
        while isborder(robot.beta, anti_side(nextside))
            move!(robot, side)
        end

        for j in 1:steps
            move!(robot, anti_side(nextside))
        end
        return true

    else
        # перегородки не было, робот прошел в заданном направлении
        move!(robot, side)
        return true
    end
end

#------------------------------------------------------------------------------------------------------------------------------#

mutable struct Gamma <: AbstractRobot
    gamma::Robot
end

gamma = Gamma(originRobot)

get(robot::Gamma) = robot.gamma

function try_move_var2!(robot::Robot, side::HorizonSide)::Bool
    steps = 0
    nextside = next_side_pr(side)
    # робот пытается обойти перегодку со стороны противоположной часовой стрелке
    while isborder(robot, side)
        if !isborder(robot, nextside)
            move!(robot, nextside)
            steps += 1
        else
            # если обойти невозможно вернется на исходное место
            for j in 1:steps
                move!(robot, anti_side(nextside))
            end
            # обойти прегородку невозможно
            return false
        end
    end

    # обход перегородки в ширину
    move!(robot, side)
    while isborder(robot, anti_side(nextside))
        move!(robot, side)
    end

    moving_defsteps!(robot, anti_side(nextside), steps)
    move!(robot,side)
    return true
end

function move!(robot::Gamma, side::HorizonSide)
    # если перед роботом есть перегородка, он начнет выполнять действия, чтобы попытаться её обойти, иначе просто пройдет в заданном направлении
    if isborder(robot.gamma, side)
        flag_nextside = true
        steps = 0
        nextside = next_side(side)
        # сначала робот пытается обойти перегородку со строны, которая следует за заданной по часовой стрелке
        while isborder(robot.gamma, side)
            if !isborder(robot.gamma, nextside)
                move!(robot.gamma, nextside)
                steps += 1
            else
                # срабатывает, когда робот уперся в угол. Он возвращается в то место, с которого начал попытку обойти по часовой стороне
                for j in 1:steps
                    move!(robot.gamma, anti_side(nextside))
                end

                flag_nextside = false

                # робот попытается обойти перегородку со сторны противоположной часовой стрелке
                return try_move_var2!(robot.gamma, side)
            end
        end

        # если у робота получилось обойти перегородку с первого раза, будет выполнено слудущее:
        if (flag_nextside)
            move!(robot.gamma, side)
            while isborder(robot.gamma, anti_side(nextside))
                move!(robot.gamma, side)
            end
            moving_defsteps!(robot.gamma, anti_side(nextside), steps)

            return true
        end
    else

        # перегородки не было, робот прошел в заданном направлении
        move!(robot.gamma,side)
        return true
    end
end

# moving_in_angle! передвинет робота в Юго-Западный угол поля и вернёт в переменную back_bath ОБРАТНЫЙ путь робота
function moving_in_angle!(robot::Gamma)::Vector{Int}
    back_path = []
    while (!(isborder(robot.gamma, West)) || !(isborder(robot.gamma, Sud)))
        for side in (Sud, West)
            while (!(isborder(robot.gamma,side)))
                move!(robot,side)
                push!(back_path, Int(side))
            end
        end
    end
    return reverse!(back_path)
end

# moving_back_to_start! вернёт робота в начальное положение через обратный путь робота
function moving_back_to_start!(robot::Gamma, back_path::Vector)::Nothing
    for i in 1:length(back_path)
        move!(robot, anti_side(HorizonSide(back_path[i])))
    end
end

#------------------------------------------------------------------------------------------------------------------------------#

struct Alpha{TypeRobot}
    alpha::TypeRobot
end

alpha = Alpha(originRobot)

get(robot::Alpha) = robot.alpha

#=
function try_move!(robot::PutmarkersRobot, side)::Bool
    result = try_move!(robot.robot, side)
    if result
        putmarker!(robot.robot) # предполагается, что для каждого возможного конкретного типа TypeRobot определен метод putmarker!
    end
    return result
end
=#

#------------------------------------------------------------------------------------------------------------------------------#

#=mutable struct CountmarkersAndTemperatureRobot <: CountmarkersRobot
    robot::CountmarkersRobot
    temperature::Real
end

getTemperature(robot::CountmarkersAndTemperatureRobot) = robot.temperature
getCountOfMarkers(robot::CountmarkersAndTemperatureRobot) = robot.CmR.counter

CmR = CountmarkersRobot(originRobot,0)
markerAndTemperature = CountmarkersAndTemperatureRobot(CmR, 0)

function move!(robot::CountmarkersAndTemperatureRobot)
    move!(markerAndTemperature.robot.CmR, side)
    if (ismarker(rmarkerAndTemperature.robot.CmR))
        markerAndTemperature.robot.counter += 1
        markerAndTemperature.temperature += temperature(markerAndTemperature.robot.CmR)
    end
end=#

mutable struct MODcountmarkersRobot <: AbstractRobot
    robot::Robot
    markers::Integer
    temperature::Real
end

mod_CmR = MODcountmarkersRobot(originRobot,0,0)

getTemperature(robot::MODcountmarkersRobot) = robot.temperature
getCountOfMarkers(robot::MODcountmarkersRobot) = robot.markers

function move!(robot::MODcountmarkersRobot, side::HorizonSide)
    if (ismarker(robot.robot))
        robot.markers += 1
        robot.temperature += temperature(robot.robot)
    end
    move!(robot.robot, side)
    if (ismarker(robot.robot))
        robot.markers += 1
        robot.temperature += temperature(robot.robot)
    end
end


@testset "----- Testing IO -----" begin

  fname = "test.dat"
  q = rand(Complex128, 2, 3, 4)
  write_binary(fname, q)

  q2 = zeros(q)
  read_binary!(fname, q2)

  @test isapprox( vecnorm(q - q2), 0.0) atol=1e-14

  # test error conditions
  q = zeros(Complex128, 2, 3)
  @test_throws Exception  read_binary!(fname, q)

  q = zeros(Complex, 2, 2)
  @test_throws Exception  write_binary(fname, q)
  @test_throws Exception  read_binary!(fname, q)

  opts = Dict{Any, Any}()
  mesh = ODLCommonTools.ConcreteDGMesh{Float64}(opts)
  sbp = ODLCommonTools.ConcreteSBP()
  eqn = ODLCommonTools.ConcreteSolutionData{Float64, Float64}(mesh, sbp, opts)

  rand!(eqn.q_vec)

  writeSolutionFiles(mesh, sbp, eqn, opts, "checkpoint")
  q_vec2 = copy(eqn.q_vec)
  fill!(eqn.q_vec, 0.0)
  readSolutionFiles(mesh, sbp, eqn, opts, "checkpoint")
  @test isapprox( norm(q_vec2 - eqn.q_vec), 0.0) atol=1e-14

  # check writing into a directory
  if !isdir("tmpdir")
    mkdir("tmpdir")
  end
  fname = "tmpdir/checkpoint"
  writeSolutionFiles(mesh, sbp, eqn, opts, fname)
  q_vec2 = copy(eqn.q_vec)
  fill!(eqn.q_vec, 0.0)
  readSolutionFiles(mesh, sbp, eqn, opts, fname)
  @test isapprox( norm(q_vec2 - eqn.q_vec), 0.0) atol=1e-14

end


using Random
import Future
import Base.Threads: @threads
#macro threads(ex) :($(esc(ex))) end

"""
   function to create a population for GAState or ga
"""
function initializepop(model::GAModel,npop::Integer,
                       nelites::Integer, baserng,sort=true)
    # each thread gets its own auxiliary scratch space
    nthreads = Threads.nthreads()
    rngs = let m = MersenneTwister(1)
                [m; accumulate(Future.randjump, fill(big(10)^20, nthreads-1), init=m)]
            end;
    aux = map(i -> genauxga(model), 1:nthreads)
    if npop > 0
        # initialize population
        pop1 = randcreature(model,aux[1],rngs[1])
        pop = Vector{typeof(pop1)}(undef, npop)
        pop[1] = pop1
        @threads for i = 2:npop
            threadid = Threads.threadid()
            pop[i] = randcreature(model,aux[threadid],rngs[threadid])
        end
        if sort
            sort!(pop,by=fitness,rev=true,
                  alg=PartialQuickSort(max(1,nelites)))
        end
    else
        pop = nothing
    end
    return pop,aux,rngs
end

mutable struct GAState
    model::GAModel
    pop::Vector
    ngen::Integer
    curgen::Integer
    npop::Integer
    elite_fraction::Real
    crossover_params
    mutation_params
    print_fitness_iter::Real
    save_creature_iter::Real
    save_state_iter::Integer
    file_name_prefix::AbstractString
    baserng::AbstractRNG
end
function GAState(model::GAModel;
                 ngen=10,
                 npop=100,
                 elite_fraction=0.01,
                 crossover_params=nothing,
                 mutation_params=Dict(:rate=>0.1),
                 print_fitness_iter=1,
                 save_creature_iter=0,
                 save_state_iter=0,
                 file_name_prefix="gamodel",
                 baserng=Random.GLOBAL_RNG)
    0 <= elite_fraction <= 1 || error("elite_fraction bounds")
    nelites = Int(floor(elite_fraction*npop))
    pop,aux,rngs = initializepop(model, npop, nelites, baserng)
    return GAState(model,pop,ngen,0,npop,
                   elite_fraction,
                   crossover_params,
                   mutation_params,
                   print_fitness_iter,
                   save_creature_iter,
                   save_state_iter,
                   file_name_prefix,
                   baserng)
end

"""
       Saves ga state to file
       Doesn't support GAModels or GACreatures containing functions (e.g. CoordinateModel)
       since JLD doesn't support saving functions
"""
function savegastate(file_name_prefix::AbstractString, curgen::Integer, state::GAState)
    filename = "$(file_name_prefix)_state_$(curgen).jld"
    println("Saving state to file $filename")
    save(filename,"state",state)
end

function loadgastate(filename::AbstractString)
    println("Load state from file $filename")
    load(filename, "state")
end

"""
    ga function
    x in each generation, the following is done
        - select parents from all creatures in population
        - create children using crossover
        - replace non-elites in population with children
        - mutate all creatures (both elites and children) in population
        - logging
            x saves state every save_state_iter iterations to file
                - restart using state = loadgastate(filename) & ga(state)
            x outputs creature every save_creature_iter iterations to file
            x prints fitness value every print_fitness_iter iterations to screen
        - go to next generation
            """
function ga(state::GAState)
    # load from state
    model = state.model
    pop = state.pop
    ngen = state.ngen
    curgen = state.curgen
    npop = state.npop
    elite_fraction = state.elite_fraction
    crossover_params = state.crossover_params == nothing ? "Nothing" : state.crossover_params
    mutation_params = state.mutation_params
    print_fitness_iter = state.print_fitness_iter
    save_creature_iter = state.save_creature_iter
    save_state_iter = state.save_state_iter
    file_name_prefix = state.file_name_prefix
    baserng = state.baserng

    nelites = Int(floor(elite_fraction*npop))
    nchildren = npop-nelites
    children = deepcopy(pop[nelites+1:end])
    # initialize auxiliary space
    _,aux,rngs = initializepop(model, 0, 0, baserng)

    println("Running genetic algorithm with
            population size $npop,
            generation number $ngen,
            elite fraction $elite_fraction,
            children created $nchildren,
            crossover_params $crossover_params,
            mutation_params $mutation_params,
            printing fitness every $print_fitness_iter iteration(s),
            saving creature to file every $save_state_iter iteration(s),
            saving state every $save_state_iter iteration(s),
            with file name prefix $file_name_prefix.")

    # main loop:
    # 1. select parents
    # 2. crossover parents & create children
    # 4. replace non-elites in current generation with children
    # 3. mutate population
    # 5. sort population
    for outer curgen = curgen+1:ngen
        # crossover. uses multi-threading when available
        parents = selection(pop, nchildren, rngs[1])
        @threads for i = 1:nchildren
            threadid = Threads.threadid()
            p1,p2 = parents[i]
            children[i] = crossover(children[i], pop[p1], pop[p2],
                                    model, crossover_params, curgen,
                                    aux[threadid], rngs[threadid])
        end
        # moves children and elites to current pop
        for i = 1:nchildren
            ip = nelites+i
            # swapping instead of deepcopy-ing
            pop[ip], children[i] = children[i], pop[ip]
        end
        # mutate pop (including elites; except for the most elite creature
        # in order to preserve monotonocity wrt best fitness)
        @threads for i = 2:npop
            threadid = Threads.threadid()
            pop[i] = mutate(pop[i], model, mutation_params, curgen,
                            aux[threadid], rngs[threadid])
        end
        sort!(pop,by=fitness,rev=true,alg=PartialQuickSort(max(1,nelites)))

        if print_fitness_iter>0 && mod(curgen,print_fitness_iter)==0
            printfitness(curgen, pop[1])
        end

        if save_creature_iter>0 && mod(curgen,save_creature_iter)==0
            savecreature(file_name_prefix, curgen, pop[1], model)
        end

        if save_state_iter>0 && mod(curgen,save_state_iter)==0
            state = GAState(model,pop,ngen,curgen,npop,
                            elite_fraction,
                            crossover_params,
                            mutation_params,
                            print_fitness_iter,
                            save_creature_iter,
                            save_state_iter,
                            file_name_prefix,
                            baserng)
            savegastate(file_name_prefix, curgen, state)
        end
    end
    state.curgen = curgen
    pop[1]
end


module Simulate_Pendulum

using  Modia3D
import Modia3D.ModiaMath

# visualization material
visuMaterial1 = Modia3D.Material(color="LightBlue", transparency=0.5)
visuMaterial2 = Modia3D.Material(color="Red")

@assembly Pendulum(;Lx = 1.0, Ly=0.2*Lx, Lz=0.2*Lx) begin
   # world is the reference point and parent Object3D of all other Object3D
   world  = Modia3D.Object3D(Modia3D.CoordinateSystem(0.4*Lx))

   # Pendulum
   # the beam is made of Aluminium and is visualized with blue color
   beam     = Modia3D.Object3D(Modia3D.Solid(Modia3D.SolidBeam(Lx,Ly,Lz), "Aluminium", visuMaterial1))
   # the bearing is a helping Object3D, its parent is the beam
   bearing  = Modia3D.Object3D(beam; r=[-Lx/2, 0.0, 0.0])
   # for visualizing the revolute joint
   cylinder = Modia3D.Object3D(bearing, Modia3D.Cylinder(Ly/2,1.2*Ly; material=visuMaterial2))

   # the revolute joint connects the world with the bearing, which belongs to the beam
   revolute    = Modia3D.Revolute(world, bearing)
end
# a Pendulum object is created, where SceneOptions are set (in this case visualization properties of frames)
pendulum = Pendulum(Lx=1.6, sceneOptions=Modia3D.SceneOptions(visualizeFrames=true, defaultFrameLength=0.3))

# a simulation model is created, simulated and potted
model  = Modia3D.SimulationModel(pendulum)
result = ModiaMath.simulate!(model, stopTime=5.0, tolerance=1e-6,log=true)
ModiaMath.plot(result, ["revolute.phi", "revolute.w", "revolute.a"])

println("... success of Simulate_Pendulum.jl!")
end


using CrystallographyBase
using Test
using Unitful: @u_str
using UnitfulAtomic

@testset "CrystallographyBase.jl" begin
    include("lattice.jl")
    include("miller.jl")
    include("metric.jl")
    include("transform.jl")
    include("reciprocal.jl")
    # include("wignerseitz.jl")
end


export ishom, ishet, _ishom, _ishet, isbiallelic, isbinary

"""
    isbiallelic(data::GenoArray)
Returns `true` if the `GenoArray` is biallelic, `false` if not.
"""
@inline function isbiallelic(data::T) where T<:GenoArray
    length(unique(Base.Iterators.flatten(skipmissing(data)))) == 2
end

# method for using just a genodata input
function isbiallelic(data::DataFrame)
    tmp = sort(data, [:locus, :name])
    prt = Base.Iterators.partition(tmp.genotype, length(data.name.pool))
    bi = findfirst(!isbiallelic, collect(prt))
    isnothing(bi) ? true : false
end

"""
    isbiallelic(data::PopData)
Returns `true` all the loci in the `PopData` are biallelic, `false` if not.
"""
isbiallelic(data::PopData) = data.metadata.biallelic

#TODO how to treat haploids?
"""
```
ishom(locus::T) where T <: GenoArray
ishom(locus::T) where T<:Base.SkipMissing
ishom(locus::Genotype)
ishom(locus::Missing)
```
A series of methods to test if a locus or loci are homozygous and return `true` if
it is, `false` if it isn't (or missing). For calculations, we recommend using `_ishom()`,
which returns `missing` if the genotype is `missing`. The vector version
simply maps the function over the elements.
"""
ishom(locus::Genotype) = all(@inbounds first(locus) .== locus)

# public facing method
ishom(locus::Missing) = false
ishom(locus::T) where T<:GenoArray = @inbounds map(ishom, locus)
ishom(locus::T) where T<:Base.SkipMissing = @inbounds map(ishom, locus)

# API computational methods
_ishom(locus::Genotype) = all(@inbounds first(locus) .== locus)
_ishom(locus::T) where T<:GenoArray = @inbounds map(_ishom, locus)
_ishom(locus::T) where T<:Base.SkipMissing = @inbounds map(_ishom, locus)
_ishom(locus::Missing) = missing

#= scales for size, which isn't super necessary yet
function ishom(geno::Genotype)
    x = true
    y = 2
    while x
        x = geno[1] == geno[y]
        y += 1
    end
    x
end
=#2
"""
    ishom(locus::Genotype, allele::Signed)
    ishom(loci::GenoArray, allele::Signed)
Returns `true` if the `locus`/`loci` is/are homozygous for the specified `allele`.
"""
function ishom(geno::T, allele::U) where T<:Genotype where U<:Integer
    ∈(allele, geno) & ishom(geno) ? true : false
end

ishom(geno::T, allele::U) where T<:GenoArray where U<:Integer = map(i -> ishom(i, allele), geno)

# public facing method
ishom(geno::Missing, allele::U) where U<:Integer = false

# API computational method
_ishom(geno::T, allele::U) where T<:Genotype where U<:Integer = ∈(allele, geno) & _ishom(geno) ? true : false
_ishom(geno::T, allele::U) where T<:GenoArray where U<:Integer = map(i -> _ishom(i, allele), geno)
_ishom(geno::Missing, allele::U) where U<:Integer = missing

# public facing method
"""
```
ishet(locus::T) where T <: GenoArray
ishet(locus::Genotype)
ishet(locus::Missing)
```
A series of methods to test if a locus or loci are heterozygous and return `true` if
it is, `false` if it isn't (or missing). For calculations, we recommend using `_ishet()`,
which returns `missing` if the genotype is `missing`. The vector version
simply maps the function over the elements.
"""
ishet(locus::Genotype) = !ishom(locus)
ishet(locus::Missing) = false
ishet(locus::T) where T<:GenoArray = @inbounds map(ishet, locus)
ishet(locus::T) where T<:Base.SkipMissing = @inbounds map(ishet, locus)

_ishet(locus::Genotype) = !_ishom(locus)
_ishet(locus::Missing) = missing
_ishet(locus::T) where T<:GenoArray = @inbounds map(_ishet, locus)
_ishet(locus::T) where T<:Base.SkipMissing = @inbounds map(_ishet, locus)

# public facing methods
"""
    ishet(locus::Genotype, allele::Signed)
    ishet(loci::GenoArray, allele::Signed)
Returns `true` if the `locus`/`loci` is/are heterozygous for the specified `allele`.
"""
function ishet(geno::T, allele::U) where T<:Genotype where U<:Integer
    ∈(allele, geno) & !ishom(geno) ? true : false
end

ishet(geno::T, allele::U) where T<:GenoArray where U<:Integer = map(i -> ishet(i, allele), geno)
ishet(geno::Missing, allele::U) where U<:Integer = false

_ishet(geno::T, allele::U) where T<:Genotype where U<:Integer = ∈(allele, geno) & !ishom(geno) ? true : false
_ishet(geno::T, allele::U) where T<:GenoArray where U<:Integer = map(i -> ishet(i, allele), geno)
_ishet(geno::Missing, allele::U) where U<:Integer = missing


if !("." in LOAD_PATH) # for easier local testing
    push!(LOAD_PATH, ".")
end
using Test
import TurbulenceConvection
const TC = TurbulenceConvection
const tc_dir = dirname(dirname(pathof(TC)))

include(joinpath(tc_dir, "driver", "main.jl"))
include(joinpath(tc_dir, "driver", "generate_namelist.jl"))
include(joinpath(tc_dir, "post_processing", "compute_mse.jl"))
include(joinpath(tc_dir, "post_processing", "mse_tables.jl"))
import .NameList
best_mse = all_best_mse["Bomex"]

@testset "Unit tests" begin
    # TODO: add unit tests.
end

@testset "Bomex" begin

    case_name = "Bomex"
    println("Running $case_name...")
    namelist = NameList.default_namelist(case_name)
    namelist["meta"]["uuid"] = "01"
    ds_tc_filename, return_code = main(namelist)

    computed_mse = compute_mse_wrapper(
        case_name,
        best_mse,
        ds_tc_filename;
        plot_comparison = true,
        t_start = 4 * 3600,
        t_stop = 6 * 3600,
    )
    nothing
end

@testset "ClimaCore extensions" begin
    include("clima_core_extensions.jl")
end


module Questionnaires

using Codex
using CSV
using DataFrames
using Dates

struct Items
    normal::Array{Int,1}
    reversed::Array{Int,1}
end

const NATO_NAMES = [
    "alfa",
    "bravo",
    "charlie",
    "delta",
    "echo",
    "foxtrot",
    "golf",
    "hotel",
    "india",
    "julliet",
    "kilo",
    "lima",
    "mike"
]

reverse(x::Int) = 6 - x
reverse(x::Missing) = missing

function get_scores(df::DataFrame, items::Items; average=true)::Array
    if length(items.normal) != 0
        v_normal = [Symbol("v$i") for i in items.normal]
        normal_scores = [sum(row[v_normal]) for row in eachrow(df[:, v_normal])]
    else
        normal_scores = repeat([0], nrow(df))
    end

    if length(items.reversed) != 0
        v_reversed = [Symbol("v$i") for i in items.reversed]
        reversed_scores = [sum(map(reverse, row)) for row in eachrow(df[:, v_reversed])]
    else
        reversed_scores = repeat([0], nrow(df))
    end

    totals = normal_scores .+ reversed_scores
    n_items = length(items.normal) + length(items.reversed)

    return average ? totals ./ n_items : totals
end

include("demographics.jl")
include("resilience.jl")
include("commitment.jl")
include("self-efficacy.jl")
include("personality.jl")
include("intelligence.jl")
include("toughness.jl")
include("optimism.jl")
include("coping.jl")
include("inspire.jl")

"""
    get_hnd(path::AbstractString)::DataFrame

Get HowNutsAreTheDutch data and select big five, age and more.
"""
function get_hnd(path::AbstractString)::DataFrame
    path = string(path)::String
    # Don't need the added complexity of weakrefstrings here.
    df = try
        CSV.read(path, DataFrame; stringtype=String)
    catch
        # CSV version 0.8.
        CSV.read(path, DataFrame)
    end
    # Assumes that all neo scores are missing if neo_neurot is 999.
    filter!(:neo_neurot => !=(999), df)
    df[!, :group] .= "civilians"
    domains = (:neo_neurot => :N, :neo_extraversion => :E, :neo_openness => :O,
        :neo_agreeable => :A, :neo_conscient => :C)
    select!(df, :id, :age, :start_educ => :education, domains...)
    @assert nrow(df) == 4_984
    df
end

transformation_map = Dict{String,Function}(
    "bravo" => unify_demographics,
    "charlie" => resilience2scores,
    "delta" => Commitment.delta2scores,
    "echo" => self_efficacy2scores,
    "foxtrot" => Intelligence.foxtrot2scores,
    "golf" => Intelligence.golf2scores,
    "india" => Toughness.india2scores,
    "julliet" => Coping.julliet2scores,
    "kilo" => Optimism.kilo2scores,
    "lima" => personality2scores,
    "mike" => Inspire.mike2scores
)

"""
    responses(data_dir::String, nato_name::String)::DataFrame

Responses for questionnaire `nato_name` as contained in directory `data_dir`.
Returns a DataFrame with rows `{ id, r...}` where `id` is a long identifier and not the one from the backend.
"""
function responses(data_dir::String, nato_name::String)::DataFrame
    responses_dir = joinpath(data_dir, "responses")
    responses_file = joinpath(responses_dir, "$nato_name.csv")
    responses_data = try
        CSV.read(responses_file, DataFrame; delim=';', stringtype=String)
    catch
        # CSV.jl 0.8.
        CSV.read(responses_file, DataFrame; delim=';')
    end
    # To speed up further processing.
    Codex.TransformExport.rm_timing!(responses_data)
    Codex.TransformExport.rm_boring_timestamps!(responses_data)
    Codex.TransformExport.rm_boring_foreign_keys!(responses_data)

    people_file = joinpath(data_dir, "people.csv")
    people_data = Codex.TransformExport.read_csv(people_file, delim=';')

    if nato_name != "mike" || !contains(data_dir, "2018")
        select!(people_data, :person_id => :backend_id, :first_name => :id)
        rename!(responses_data, Dict(:filled_out_by_id => "backend_id"))
        # If people_data is free from missings, then matchmissing equal should not introduce
        # missings, I think.
        dropmissing!(people_data)
        joined = nothing
        joined = innerjoin(people_data, responses_data, on=:backend_id, matchmissing=:equal)
        select!(joined, Not(:backend_id))
    else
        joined = responses_data
    end

    if "locale" in names(joined)
        select!(joined, Not(:locale))
    end

    if nato_name in keys(transformation_map)
        f = transformation_map[nato_name]
        joined = f(joined)
    end
    return joined
end

remove_auth_prefix(s) = ismissing(s) ? missing : s[7:end]

"""
    replace_usernames(df::DataFrame, id_col::Symbol, id_username::DataFrame)

For every username in `df[!, id_col]`, replace the name by the long identifier.
The long identifier is necessary, because usernames where not anonymized in 2018.
"""
function replace_usernames(df::DataFrame, id_col::Symbol, id_username::DataFrame)
    matchmissing = :notequal
    on = id_col => :username
    makeunique = true
    df = leftjoin(df, id_username; on, matchmissing, makeunique)
    new_col = Symbol("$(id_col)_1")
    df[!, new_col] = remove_auth_prefix.(df[!, new_col])
    df[!, :id] = [ismissing(new_id) ? id : new_id for (id, new_id) in zip(df[!, :id], df[!, new_col])]
    select!(df, Not(new_col))
    df
end

"""
    dropouts(raw_dir::String)

Returns dropout data where all IDs are in the long identifier format.
On 2021-07-19, returned a 197x7 dataset.
"""
function dropouts(raw_dir::String)
    dropouts_file = joinpath(raw_dir, "dropouts.csv")
    dropouts = Codex.TransformExport.read_csv(dropouts_file; delim=';')

    id_username_file = joinpath(raw_dir, "id-username.csv")
    id_username = Codex.TransformExport.read_csv(id_username_file; delim=',')

    dropouts = replace_usernames(dropouts, :id, id_username)
    transform!(dropouts, :dropout => ByRow(Bool) => :dropout)
    msg = "Unable to convert all usernames. Got:\n$dropouts"
    @assert all([length(id) == 24 for id in dropouts.id]) msg
    dropouts
end

function clean_graduates_dropouts(responses::DataFrame, dropouts::DataFrame, group, cohort)
    @assert group != "operators"

    responses = transform(responses, :id => ByRow(remove_auth_prefix) => :id)
    df = leftjoin(responses, dropouts; on=:id)

    function filter_group(dropout::Bool, dropout_reason::Union{Missing,String})
        if group == "graduates"
            !dropout
        else
            # Read as: if the query asks for "dropouts-medical", then get data with:
            medical_reason_match() = group == "dropouts-medical" ?
                dropout_reason == "B" || dropout_reason == "medisch" :
                dropout_reason == "A" || dropout_reason == "C" || dropout_reason == "ontheffing"
            dropout && medical_reason_match()
        end
    end
    function filter_group(dropout::Missing, dropout_reason)
        false
    end

    df = subset!(df, [:dropout, :dropout_reason] => ByRow(filter_group))
    df[!, :group] .= group
    df = select!(df, :group, names(responses)...)
end

"""
    validate_dropouts(df::DataFrame)

Validate "dropouts.csv".
"""
function validate_dropouts(df::DataFrame)
    disallowmissing!(df, :dropout)
    valid(dropout_reason::Missing, dropout::Bool) = !dropout
    valid(dropout_reason::String, dropout::Bool) = dropout
    invs = subset(df, [:dropout_reason, :dropout] => ByRow(!valid))
    @assert nrow(invs) == 0 "Found invalid rows:\n$invs"
end

"""
    responses(data_dir::String, nato_name::String, group::String; measurement=999)::DataFrame

Responses for group `group` and measurement `measurement`, where `group` is one of `graduates`, `operators`, `dropouts-medical` or `dropouts-non-medical`.
`measurement` is only used to split the 2018 data, for the later datasets it is ignored.

## Example in 2021-07:

```
julia> dir = joinpath(ysf_raw, "2020-first");

julia> df = Codex.Questionnaires.responses(dir, "kilo", "dropouts");

julia> nrow(df)
25

julia> select(first(df, 5), Not([:id, :completed_at]))
5×3 DataFrame
 Row │ group     optimism  pessimism
     │ String    Int64     Int64
─────┼───────────────────────────────
   1 │ dropouts        10         10
   2 │ dropouts        13          6
   3 │ dropouts        10          9
   4 │ dropouts        12          6
   5 │ dropouts        12          5
```
"""
function responses(data_dir::String, nato_name::String, group::String; measurement=999)
    responses_data = responses(data_dir, nato_name)

    cohort = parse(Int, match(r"[0-9]{4}", data_dir).match)
    dropouts_data = dropouts(dirname(data_dir))
    validate_dropouts(dropouts_data)

    if group == "operators"
        df = responses_data
        df[:, :group] .= "operators"
    else # Graduates and dropouts.
        df = clean_graduates_dropouts(responses_data, dropouts_data, group, cohort)
    end

    if cohort == 2018
        if !(measurement in [1, 2])
            throw(AssertionError("Measurement has to be specified for the 2018 data"))
        end
        threshold = Date("2019-01-01")
        function filter_date(x)::Bool
            date = Date(first(split(x, ' ')), DateFormat("dd-mm-yyyy"))
            measurement == 1 ? date < threshold : threshold < date
        end
        return nato_name != "mike" ? filter([:completed_at] => filter_date, df) : df
    else # After 2018, the datasets are already split on before and after selection.
        df
    end
end

function first_measurement(raw_dir::AbstractString, nato_name::AbstractString)
    parameters = [
        (raw_dir, "2018-first", "graduates"),
        (raw_dir, "2018-first", "dropouts-medical"),
        (raw_dir, "2018-first", "dropouts-non-medical"),
        (raw_dir, "2019-first", "graduates"),
        (raw_dir, "2019-first", "dropouts-medical"),
        (raw_dir, "2019-first", "dropouts-non-medical"),
        (raw_dir, "2020-operators", "operators"),
        (raw_dir, "2020-first", "graduates"),
        (raw_dir, "2020-first", "dropouts-medical"),
        (raw_dir, "2020-first", "dropouts-non-medical"),
        (raw_dir, "2021-march-first", "graduates"),
        (raw_dir, "2021-march-first", "dropouts-medical"),
        (raw_dir, "2021-march-first", "dropouts-non-medical"),
    ]
    measurement = 1
    function helper(dir, cohort_dir, group)
        data_dir = joinpath(dir, cohort_dir)
        responses(data_dir, nato_name, group; measurement)
    end
    vcat([helper(p...) for p in parameters]...)
end

function second_measurement(raw_dir::AbstractString, nato_name::AbstractString)
    parameters = [
        (raw_dir, "2018-second", "graduates", 2),
        (raw_dir, "2018-second", "dropouts-medical", 2),
        (raw_dir, "2018-second", "dropouts-non-medical", 2),
        (raw_dir, "2019-second", "graduates", 2),
        (raw_dir, "2019-second", "dropouts-medical", 2),
        (raw_dir, "2019-second", "dropouts-non-medical", 2),

#        (raw_dir, "2020-first", "dropouts-medical", 1),
#        (raw_dir, "2020-first", "dropouts-non-medical", 1),
    ]
    if nato_name == "mike"
        filter!(p -> p[2] != "2018-second", parameters)
    end
    helper(dir, cohort_dir, group, measurement::Int) =
        responses(joinpath(dir, cohort_dir), nato_name, group; measurement)
    vcat([helper(p...) for p in parameters]...)

end

"""
    join_questionnaires(raw_dir::String, questionnaires::Array{String,1}, groups::Array{String,1})::DataFrame

Joines multiple `questionnaires` for the members of `groups`, where `groups` ensures that the joins do not remove rows which shouldn't be removed.
For instance, given questionnaires `"A"` and `"B"` containing respectively columns `:a1`, `:a2` and `:b1`, this method will return `{ group, id, A_a1, A_a2, B_b1 }`.
"""
function join_questionnaires(raw_dir::String, questionnaires::Array{String,1}, groups::Array{String,1})::DataFrame
    function prepare_responses(q::String)::DataFrame
        df = first_measurement(raw_dir, q)
        filter!(:group => x -> x in groups, df)
        select!(df, Not(:completed_at))
    end

    function ysf_join(A::DataFrame, b::String)
        B = prepare_responses(b)
        # Group information is already provided by `A`.
        select!(B, Not(:group))
        rename!(s -> s == "id" ? s : "$(b)_$s", B)
        innerjoin(A, B, on = :id)
    end

    function ysf_join(a::String, b::String)
        A = prepare_responses(a)
        rename!(s -> s == "id" || s == "group" ? s : "$(a)_$s", A)
        ysf_join(A, b)
    end

    reduce(ysf_join, questionnaires)
end

"""
    join_vo_questionnaires(raw_dir::String)::DataFrame

Combine information from multiple questionnaires to allow model fitting.
"""
function join_vo_questionnaires(raw_dir::String)::DataFrame
    questionnaires = sort(collect(keys(transformation_map)))
    # Ignoring delta since only two operators participated in it.
    questionnaires = filter(!in(["bravo", "delta"]), questionnaires)
    df = Codex.Questionnaires.join_questionnaires(
        raw_dir,
        questionnaires,
        ["graduates", "operators", "dropouts-non-medical"]
    )
    df[:, :binary_group] = [x == "graduates" || x == "operators" ? 1 : 0 for x in df[:, :group]]
    df
end

function csv_files(dir::String)
    return filter(endswith(".csv"), readdir(dir))
end

"""
    unfinished_questionnaires(responses_dir::String, id::String)

Return questionnaire names in `responses_dir` for which `id` is not in `filled_out_by_id` column.
This assumes that unfinished questionnaires have a `missing` in the `filled_out_by_id` column.
"""
function unfinished_questionnaires(responses_dir::String, id::String)
    csvs = csv_files(responses_dir)
    unfinished_csv_files = filter(csvs) do csv_file
        csv_path = joinpath(responses_dir, csv_file)
        df = CSV.read(csv_path, DataFrame)
        if "filled_out_by_id" in names(df)
            return !(id in skipmissing(df.filled_out_by_id))
        else
            # We can ignore the dataset since it is not a valid u-can-act output.
            return true
        end
    end
    unfinished = first.(splitext.(unfinished_csv_files))
    return unfinished
end

"""
    all_ids(responses_dir)::Vector{String}

"""
function all_ids(responses_dir)::Vector{String}
    csvs = csv_files(responses_dir)
    ids = map(csvs) do csv_file
        csv_path = joinpath(responses_dir, csv_file)
        df = CSV.read(csv_path, DataFrame)
        if "filled_out_by_id" in names(df)
            return collect(skipmissing(df.filled_out_by_id))
        else
            return String[]
        end
    end
    return sort(unique(Iterators.flatten(ids)))
end

struct Unfinished
    id::String
    unfinished_questionnaires::Vector{String}
end

"""
    unfinished_info(responses_dir; required::Union{Nothing,Vector{String}}=missing)::Vector{Unfinished}

Return the ids for which not all questionnaires in `required` have been filled in.
When `ismissing(required)`, take all the questionnaires in `responses_dir`.

# Example
```
julia> responses_dir = joinpath(homedir(), "git", "ysf-raw", "2021-08", "responses");

julia> required = ["alfa", "bravo", "charlie", "delta", "echo", "foxtrot", "golf", "hotel", "india", "julliet", "kilo", "lima", "mike"];

julia> Codex.Questionnaires.unfinished_info(responses_dir; required)

"""
function unfinished_info(responses_dir; required::Union{Missing,Vector{String}}=missing)::Vector{Unfinished}
    if ismissing(required)
        required = first.(splitext.(csv_files(responses_dir)))
    end
    ids = all_ids(responses_dir)
    U = [unfinished_questionnaires(responses_dir, id) for id in ids]
    U = [filter(in(required), unfinished) for unfinished in U]
    Z = collect(zip(ids, U))
    Z = filter(t -> !isempty(last(t)), Z)
    return [Unfinished(id, unfinished) for (id, unfinished) in Z]
end

end # module


using DSGE, JLD2
using Test

path = dirname(@__FILE__)

# Initialize models
function make_test_model(year::Int)
    m = Model990()
    vint = Dates.format(Dates.Date(year, 4, 10), "yymmdd")
    m <= Setting(:data_vintage, vint)
    m <= Setting(:cond_vintage, vint)
    m <= Setting(:date_forecast_start, DSGE.quartertodate("$year-Q1"))
    m <= Setting(:date_conditional_end, DSGE.quartertodate("$year-Q1"))
    m <= Setting(:forecast_horizons, 16)
    return m
end

m_new = make_test_model(2016)
m_old = make_test_model(2014)

# Read in data and parameters
@load "$path/../reference/decompose_forecast_args.jld2" df_new df_old params_new params_old

# Read in expected outputs
@load "$path/../reference/decompose_forecast_out.jld2" exp_decomps

cond_types = [(:none, :none), (:none, :full), (:full, :none), (:full, :full)]
for (cond_new, cond_old) in cond_types
    # Test inner method
    @time decomps = decompose_forecast(m_new, m_old,
                                       cond_new == :full ? df_new : df_new[1:end-1, :],
                                       cond_old == :full ? df_old : df_old[1:end-1, :],
                                       params_new, params_old, cond_new, cond_old,
                                       [:obs, :pseudo]; check = true)

    for k in keys(decomps)
        @test decomps[k] ≈ exp_decomps[(cond_new, cond_old)][k]
    end
end

# Test outer method
m_new = AnSchorfheide(testing = true)
m_new <= Setting(:saveroot, tempdir())
m_new <= Setting(:date_forecast_start, quartertodate("2015-Q4"))
m_new <= Setting(:date_conditional_end, quartertodate("2015-Q4"))
m_new <= Setting(:use_population_forecast, true)
m_new <= Setting(:forecast_horizons, 12)

estroot = normpath(joinpath(dirname(@__FILE__), "..", "reference"))
overrides = forecast_input_file_overrides(m_new)
overrides[:mode] = joinpath(estroot, "optimize.h5")

m_old = deepcopy(m_new)
m_old <= Setting(:date_forecast_start, quartertodate("2014-Q4"))
m_old <= Setting(:date_conditional_end, quartertodate("2014-Q4"))

df_new = load_data(m_new)
df_old = df_new[1:end-4, :]

@time decompose_forecast(m_new, m_old, df_new, df_old, :mode, :none, :none, [:obs, :pseudo];
                         verbose = :none)
@time decomposition_means(m_new, m_old, :mode, :none, :none, [:obs, :pseudo], verbose = :none)

nothing