using Serialization

"""
Supertype for types encoding the family of perturbations allowed.
"""
abstract type PerturbationFamily end

struct UnrestrictedPerturbationFamily <: PerturbationFamily end
Base.show(io::IO, pp::UnrestrictedPerturbationFamily) = print(io, "unrestricted")

abstract type RestrictedPerturbationFamily <: PerturbationFamily end

"""
For blurring perturbations, we currently allow colors to "bleed" across color channels -
that is, the value of the output of channel 1 can depend on the input to all channels.
(This is something that is worth reconsidering if we are working on color input).
"""
struct BlurringPerturbationFamily <: RestrictedPerturbationFamily
    blur_kernel_size::NTuple{2}
end
Base.show(io::IO, pp::BlurringPerturbationFamily) =
    print(io, filter(x -> !isspace(x), "blur-$(pp.blur_kernel_size)"))

struct LInfNormBoundedPerturbationFamily <: RestrictedPerturbationFamily
    norm_bound::Real

    function LInfNormBoundedPerturbationFamily(norm_bound::Real)
        @assert(norm_bound > 0, "Norm bound $(norm_bound) should be positive")
        return new(norm_bound)
    end
end
Base.show(io::IO, pp::LInfNormBoundedPerturbationFamily) =
    print(io, "linf-norm-bounded-$(pp.norm_bound)")

struct BrightnessPerturbationFamily <: RestrictedPerturbationFamily
    bright_bound::Real

    function BrightnessPerturbationFamily(bright_bound::Real)
        @assert(bright_bound > 0, "Norm bound $(bright_bound) should be positive")
        return new(bright_bound)
    end
end
Base.show(io::IO, pp::BrightnessPerturbationFamily) =
    print(io, "brightness-bounded-$(pp.bright_bound)")

function get_model(
    nn::NeuralNet,
    input::Array{<:Real},
    pp::PerturbationFamily,
    optimizer,
    tightening_options::Dict,
    tightening_algorithm::TighteningAlgorithm,
)::Dict{Symbol,Any}
    notice(
        MIPVerify.LOGGER,
        "Determining upper and lower bounds for the input to each non-linear unit.",
    )
    m = Model(optimizer_with_attributes(optimizer, tightening_options...))
    m.ext[:MIPVerify] = MIPVerifyExt(tightening_algorithm)

    d_common = Dict(
        :Model => m,
        :PerturbationFamily => pp,
        :TighteningApproach => string(tightening_algorithm),
    )

    return merge(d_common, get_perturbation_specific_keys(nn, input, pp, m))
end

function get_model(
    nn::NeuralNet,
    input_size,
    global_predicted,
    global_confidence,
    target_indexes,
    pp::PerturbationFamily,
    optimizer,
    tightening_options::Dict,
    tightening_algorithm::TighteningAlgorithm,
)::Dict{Symbol,Any}
    notice(
        MIPVerify.LOGGER,
        "Determining upper and lower bounds for the REGULAR input to each non-linear unit.",
    )
    m = Model(optimizer_with_attributes(optimizer, tightening_options...))
    m.ext[:MIPVerify] = MIPVerifyExt(tightening_algorithm)

    d_common = Dict(
        :Model => m,
        :PerturbationFamily => pp,
        :TighteningApproach => string(tightening_algorithm),
    )

    input_range = CartesianIndices(input_size)
    # input is the input without perturbation added
    input = map(_ -> @variable(m, lower_bound = 0, upper_bound = 1), input_range)
    d_common[:Input] = input

    v_output = input |> nn

    target_vars = v_output[Bool[i ∈ target_indexes for i in 1:length(v_output)]]
    @constraint(m, target_vars .<= v_output[global_predicted] - global_confidence)

    notice(
        MIPVerify.LOGGER,
        "Determining upper and lower bounds for the PERTUBATED input to each non-linear unit.",
    )

    m.ext[:MIPVerify] = MIPVerifyExt(tightening_algorithm)

    return merge(d_common, get_perturbation_specific_keys(nn, input, pp, m))
end

function get_perturbation_specific_keys(
    nn::NeuralNet,
    input::Array{<:Real},
    pp::UnrestrictedPerturbationFamily,
    m::Model,
)::Dict{Symbol,Any}
    input_range = CartesianIndices(size(input))

    # v_x0 is the input with the perturbation added
    v_x0 = map(_ -> @variable(m, lower_bound = 0, upper_bound = 1), input_range)

    v_output = v_x0 |> nn

    return Dict(:PerturbedInput => v_x0, :Perturbation => v_x0 - input, :Output => v_output)
end

function get_perturbation_specific_keys(
    nn::NeuralNet,
    input::Array{<:Real},
    pp::BlurringPerturbationFamily,
    m::Model,
)::Dict{Symbol,Any}

    input_size = size(input)
    num_channels = size(input)[4]
    filter_size = (pp.blur_kernel_size..., num_channels, num_channels)

    v_f = map(_ -> @variable(m, lower_bound = 0, upper_bound = 1), CartesianIndices(filter_size))
    @constraint(m, sum(v_f) == num_channels)
    v_x0 = map(_ -> @variable(m, lower_bound = 0, upper_bound = 1), CartesianIndices(input_size))
    @constraint(m, v_x0 .== input |> Conv2d(v_f))

    v_output = v_x0 |> nn

    return Dict(
        :PerturbedInput => v_x0,
        :Perturbation => v_x0 - input,
        :Output => v_output,
        :BlurKernel => v_f,
    )
end

function get_perturbation_specific_keys(
    nn::NeuralNet,
    input::Array{<:Real},
    pp::LInfNormBoundedPerturbationFamily,
    m::Model,
)::Dict{Symbol,Any}

    input_range = CartesianIndices(size(input))
    # v_e is the perturbation added
    v_e = map(
        _ -> @variable(m, lower_bound = -pp.norm_bound, upper_bound = pp.norm_bound),
        input_range,
    )
    # v_x0 is the input with the perturbation added
    v_x0 = map(
        i -> @variable(
            m,
            lower_bound = max(0, input[i] - pp.norm_bound),
            upper_bound = min(1, input[i] + pp.norm_bound)
        ),
        input_range,
    )
    @constraint(m, v_x0 .== input + v_e)

    v_output = v_x0 |> nn

    return Dict(:PerturbedInput => v_x0, :Perturbation => v_e, :Output => v_output)
end

function get_perturbation_specific_keys(
    nn::NeuralNet,
    input::Array{<:Real},
    pp::BrightnessPerturbationFamily,
    m::Model,
)::Dict{Symbol,Any}

    input_range = CartesianIndices(size(input))
    # v_e is the perturbation added
    v_tmp = @variable(m, lower_bound = -pp.bright_bound, upper_bound = pp.bright_bound)
    v_e = map(
        _ -> v_tmp,
        input_range,
    )
    # v_x0 is the input with the perturbation added
    v_x0 = map(
        i -> @variable(
            m,
            lower_bound = max(0, input[i] - pp.bright_bound),
            upper_bound = min(1, input[i] + pp.bright_bound)
        ),
        input_range,
    )
    @constraint(m, v_x0 .== input + v_tmp)

    v_output = v_x0 |> nn

    return Dict(:PerturbedInput => v_x0, :Perturbation => v_e, :Output => v_output)
end

function get_perturbation_specific_keys(
    nn::NeuralNet,
    input::AbstractArray{<:JuMPLinearType},
    pp::LInfNormBoundedPerturbationFamily,
    m::Model,
)::Dict{Symbol,Any}

    input_range = CartesianIndices(size(input))
    # v_e is the perturbation added
    v_e = map(
        _ -> @variable(m, lower_bound = -pp.norm_bound, upper_bound = pp.norm_bound),
        input_range,
    )
    # v_x0 is the input with the perturbation added
    v_x0 = map(
        i -> @variable(
            m,
            lower_bound = 0,
            upper_bound = 1
        ),
        input_range,
    )
    @constraint(m, v_x0 .== input + v_e)

    v_output = v_x0 |> nn

    return Dict(:PerturbedInput => v_x0, :Perturbation => v_e, :Output => v_output)
end

function get_perturbation_specific_keys(
    nn::NeuralNet,
    input::AbstractArray{<:JuMPLinearType},
    pp::BrightnessPerturbationFamily,
    m::Model,
)::Dict{Symbol,Any}

    input_range = CartesianIndices(size(input))
    # v_e is the perturbation added
    v_tmp = @variable(m, lower_bound = -pp.bright_bound, upper_bound = pp.bright_bound)
    v_e = [v_tmp for i in input_range]
    # v_x0 is the input with the perturbation added
    v_x0 = map(
        i -> @variable(
            m,
            lower_bound = 0,
            upper_bound = 1,
        ),
        input_range,
    )
    @constraint(m, v_x0 .== input + v_e)

    v_output = v_x0 |> nn

    return Dict(:PerturbedInput => v_x0, :Perturbation => v_e, :Output => v_output)
end

struct MIPVerifyExt
    tightening_algorithm::TighteningAlgorithm
end


# This file is part of Kpax3. License is MIT.

function writeresults(x::KData,
                      state::State,
                      file::String;
                      what::Int=1,
                      verbose::Bool=false)
  dirpath = dirname(file)
  if !isdir(dirpath)
    mkpath(dirpath)
  end

  fp = open("$(file)_partition.csv", "w")
  for i in 1:size(x.data, 2)
    write(fp, "\"$(x.id[i])\",$(state.R[i])\n")
  end
  close(fp)

  write("$(file)_logposterior_value.txt",
        string(state.logpR + state.logpC[1] + state.loglik, "\n"))

  if verbose
    println("Partition has been written to file: $(file)_partition.csv")
    println("Log posterior probability (minus a constant) has been written ",
            "to file: $(file)_logposterior.txt")
  end

  if what >= 2
    writeattributes(x, state, file, verbose)

    if what >= 3
      V = writecvalues(x, state, file, verbose)

      if what >= 4
        writedataset(x, state, file, verbose, V)
      end
    end
  end

  nothing
end

function writeattributes(x::KData,
                         state::State,
                         file::String,
                         verbose::Bool)
  (m, n) = size(x.data)
  M = length(x.ref)

  isuint8 = eltype(x.ref) == UInt8

  polval = if isuint8
              UInt8('.')
            else
              "."
            end

  # attributes
  C = ones(UInt8, state.k, M)

  # column index in x.key
  s = 0

  # column indices in state.C
  t = 1
  b = 1

  # temporary value
  v = 0x01

  for j in 1:M
    if x.ref[j] == polval
      s += 1
      b = t
      v = 0x01

      while (b <= m) && (x.key[b] == s)
        if state.C[state.cl[1], b] > v
          v = state.C[state.cl[1], b]
        end
        b += 1
      end

      if v > 0x02
        for u in t:(b - 1), l in 1:state.k
          if state.C[state.cl[l], u] > C[l, j]
            C[l, j] = state.C[state.cl[l], u]
          end
        end
      elseif v == 0x02
        for l in 1:state.k
          C[l, j] = 0x02
        end
      end

      t = b
    end
  end

  fp = open("$(file)_attributes.csv", "w")
  for l in 1:(state.k - 1)
    write(fp, string("\"Cluster ", l, "\","))
  end
  write(fp, string("\"Cluster ", state.k, "\"\n"))

  for j in 1:M
    for l in 1:(state.k - 1)
      write(fp, string(Int(C[l, j]), ","))
    end
    write(fp, string(Int(C[state.k, j]), "\n"))
  end
  close(fp)

  if verbose
    println("Attributes have been written to file: $(file)_attributes.csv")
  end

  nothing
end

function writecvalues(x::KData,
                      state::State,
                      file::String,
                      verbose::Bool)
  (m, n) = size(x.data)
  M = length(x.ref)

  isuint8 = eltype(x.ref) == UInt8

  polval = if isuint8
              UInt8('.')
            else
              "."
            end

  # characteristic values
  V = fill(" ", (state.k, M))

  # column index in x.key
  s = 0

  # column indices in state.C
  t = 1
  b = 1

  # temporary values
  v = 0x01
  z = 0
  flag = false

  fp = open("$(file)_characteristic.csv", "w")
  write(fp, "\"Site\",")
  for l in 1:(state.k - 1)
    write(fp, string("\"Cluster ", l, "\","))
  end
  write(fp, string("\"Cluster ", state.k, "\"\n"))

  for j in 1:M
    if x.ref[j] == polval
      s += 1
      b = t
      v = 0x01

      while (b <= m) && (x.key[b] == s)
        if state.C[state.cl[1], b] > v
          v = state.C[state.cl[1], b]
        end
        b += 1
      end

      if v > 0x02
        write(fp, string(j, ","))

        for l in 1:(state.k - 1)
          z = t
          flag = false

          while !flag && z < b
            flag = state.C[state.cl[l], z] == 0x04
            z += 1
          end

          if flag
            V[l, j] = uppercase(isuint8 ?
                                string(Char(x.val[z - 1])) :
                                x.val[z - 1])
            write(fp, string("\"", V[l, j], "\","))
          else
            write(fp, ",")
          end
        end

        z = t
        flag = false

        while !flag && z < b
          flag = state.C[state.cl[state.k], z] == 0x04
          z += 1
        end

        if flag
          V[state.k, j] = uppercase(isuint8 ?
                                    string(Char(x.val[z - 1])) :
                                    x.val[z - 1])
          write(fp, string("\"", V[state.k, j], "\"\n"))
        else
          write(fp, "\n")
        end
      end

      t = b
    end
  end
  close(fp)

  if verbose
    println("Characteristic values have been written to file: ",
            "$(file)_characteristic.csv")
  end

  V
end

function writedataset(x::KData,
                      state::State,
                      file::String,
                      verbose::Bool,
                      V::Matrix{String})
  (m, n) = size(x.data)
  M = length(x.ref)

  isuint8 = eltype(x.ref) == UInt8

  polval = if isuint8
              UInt8('.')
            else
              "."
            end

  # reconstruct original dataset
  D = fill(" ", (M, n))

  # column index in x.key
  s = 0

  # column indices in state.C
  t = 1
  b = 1

  # temporary values
  g = 0

  for a in 1:n
    s = 0
    t = 1
    g = state.R[a]

    for j in 1:M
      if x.ref[j] == polval
        s += 1
        b = t

        while (b <= m) && (x.key[b] == s)
          if x.data[b, a] == 0x01
            D[j, a] = (state.C[g, b] == 0x04) ?
                      uppercase(isuint8 ? string(Char(x.val[b])) : x.val[b]) :
                      lowercase(isuint8 ? string(Char(x.val[b])) : x.val[b])
          end
          b += 1
        end

        t = b
      end
    end
  end

  fp = open("$(file)_dataset.txt", "w")
  for l in 1:(state.k - 1)
    for j in 1:M
      write(fp, V[l, j])
    end
    write(fp, "\n")

    for i in 1:state.v[state.cl[l]]
      a = state.unit[state.cl[l]][i]
      for j in 1:M
        write(fp, D[j, a])
      end
      write(fp, string(" ; ", x.id[a], "\n"))
    end

    write(fp, "\n")
  end

  for j in 1:M
    write(fp, V[state.k, j])
  end
  write(fp, "\n")

  for i in 1:state.v[state.cl[state.k]]
    a = state.unit[state.cl[state.k]][i]
    for j in 1:M
      write(fp, D[j, a])
    end
    write(fp, string(" ; ", x.id[a], "\n"))
  end
  close(fp)

  nothing
end


export setpolymodule!

setdefault!(m::JuMP.Model, S::Type, T::Type) = setdefault!(getpolydata(m), S, T)
setdefault!(data::Data, ::Type{NonNegPoly}, S::Type) = data.nonnegpoly_default = S
setdefault!(data::Data, ::Type{PosDefPolyMatrix}, S::Type) = data.posdefpolymatrix_default = S

getdefault(m::JuMP.Model, s) = getdefault(getpolydata(m), s)

# includes Poly but also custom types defined by other modules
getdefault(data::Data, p) = p
getdefault(data::Data, s::Union{NonNegPoly, PosDefPolyMatrix}) = getdefault(data, typeof(s))()
getdefault(data::Data, ::Type{NonNegPoly}) = get_default(data.nonnegpoly_default)
getdefault(data::Data, ::Type{PosDefPolyMatrix}) = get_default(data.posdefpolymatrix_default)

setpolymodule!(m::JuMP.Model, pm::Module) = setpolymodule!(getpolydata(m), pm)
setpolymodule!(data::Data, pm::Module) = pm.setdefaults!(data)

get_default(d) = d
function get_default(::Nothing)
    error("PolyJuMP is just a JuMP extension for modelling Polynomial Optimization: it does not implement any reformulation. To use automatic sums of squares (SOS) reformulations, install the SumOfSquares Julia package and try \`using SumOfSquares\` and \`setpolymodule!(SumOfSquares)\` or use \`SOSModel\` instead of \`Model\`.")
end


module miraccess1
using StaticArrays
using MeshCore: VecAttrib
using MeshCore: P1, L2, Q4, ShapeColl, manifdim, nfacets, facetdesc, nshapes
using MeshCore: Q4ShapeDesc, shapedesc, n1storderv, nridges, nshifts, nvertices
using MeshCore: IncRel
using Test
function test()
    a = VecAttrib([1, 2, 4]);
    # @test show(a) != ""
    c = [(1, 2, 6, 5), (5, 6, 10, 9), (2, 3, 7, 6), (6, 7, 11, 10), (3, 4, 8, 7), (7, 8, 12, 11)]
    cc = [SVector{nvertices(Q4)}(c[idx]) for idx in 1:length(c)]
    q4s = ShapeColl(Q4, 6)
    vrts = ShapeColl(P1, 12)
    ir = IncRel(q4s, vrts, cc)
    # @show ir[1]
    # @show ir[1][3]
    q4s = ShapeColl(Q4, 6, "q4s")
    vrts = ShapeColl(P1, 12, "vrts")
    ir = IncRel(q4s, vrts, cc)
    @test summary(ir.left) == "q4s = 6 x Q4"
    @test summary(ir.right) == "vrts = 12 x P1"
    @test summary(ir) == "(q4s, vrts): q4s = 6 x Q4, vrts = 12 x P1"
    # @show Q4
    # @show q4s
    true
end
end
using .miraccess1
miraccess1.test()

module mtattr10
using StaticArrays
using MeshCore: VecAttrib
using MeshCore: P1, L2, Q4, ShapeColl, manifdim, nfacets, facetdesc, nshapes
using MeshCore: Q4ShapeDesc, shapedesc, n1storderv, nridges, nshifts, nvertices
using MeshCore: IncRel
using Test
function test()
    a = VecAttrib([1, 2, 4]);
    # @test show(a) != ""
    c = [(1, 2, 6, 5), (5, 6, 10, 9), (2, 3, 7, 6), (6, 7, 11, 10), (3, 4, 8, 7), (7, 8, 12, 11)]
    cc = [SVector{nvertices(Q4)}(c[idx]) for idx in 1:length(c)]
    q4s = ShapeColl(Q4, 6)
    vrts = ShapeColl(P1, 12)
    ir = IncRel(q4s, vrts, cc)
    q4s = ShapeColl(Q4, 6, "q4s")
    vrts = ShapeColl(P1, 12, "vrts")
    ir = IncRel(q4s, vrts, cc)
    @test summary(ir.left) == "q4s = 6 x Q4"
    @test summary(ir.right) == "vrts = 12 x P1"
    @test summary(ir) == "(q4s, vrts): q4s = 6 x Q4, vrts = 12 x P1"
    # @show Q4
    # @show q4s
    true
end
end
using .mtattr10
mtattr10.test()

module mod1ame
using StaticArrays
using MeshCore: P1, L2, Q4, ShapeColl, manifdim, nfacets, facetdesc, nshapes
using MeshCore: Q4ShapeDesc, shapedesc, n1storderv, nridges, nshifts, nvertices
using MeshCore: IncRel
using Test
function test()
    c = [(1, 2, 6, 5), (5, 6, 10, 9), (2, 3, 7, 6), (6, 7, 11, 10), (3, 4, 8, 7), (7, 8, 12, 11)]
    cc = [SVector{nvertices(Q4)}(c[idx]) for idx in 1:length(c)]
    q4s = ShapeColl(Q4, 6)
    vrts = ShapeColl(P1, 12)
    ir = IncRel(q4s, vrts, cc)
    q4s = ShapeColl(Q4, 6, "q4s")
    vrts = ShapeColl(P1, 12, "vrts")
    ir = IncRel(q4s, vrts, cc)
    @test summary(ir.left) == "q4s = 6 x Q4"
    @test summary(ir.right) == "vrts = 12 x P1"
    @test summary(ir) == "(q4s, vrts): q4s = 6 x Q4, vrts = 12 x P1"

    true
end
end
using .mod1ame
mod1ame.test()

module mtest1
using MeshCore: NoSuchShape, P1, L2, L3, SHAPE_DESC
using MeshCore: NoSuchShape, ShapeColl, manifdim, nfacets
using MeshCore: shapedesc, nshapes, manifdim, nvertices, facetdesc, nfacets, ridgedesc, nridges
using Test
function test()
    @test manifdim(P1) == 0
    @test nfacets(L2) == 2
    vrts = ShapeColl(P1, 12)
    lines = ShapeColl(L2, 32)
    @test shapedesc(vrts) == P1
    @test nshapes(vrts) == 12
    @test manifdim(vrts) == manifdim(P1)
    @test nvertices(lines) == nvertices(L2)
    @test facetdesc(lines) == P1
    @test nfacets(lines) == nfacets(L2)
    @test ridgedesc(lines) == NoSuchShape
    @test nridges(lines) == 0

    lines3 = ShapeColl(L3, 13)
    @test nvertices(lines3) == nvertices(L3)
    @test nvertices(lines3) == 3
    @test facetdesc(lines3) == P1
    @test nfacets(lines3) == nfacets(L3)
    @test ridgedesc(lines3) == NoSuchShape
    @test nridges(lines3) == 0
    @test manifdim(lines3) == 1

    @test SHAPE_DESC["P1"].name == "P1"
    @test SHAPE_DESC["L2"].name == "L2"
    @test SHAPE_DESC["T3"].name == "T3"
    @test SHAPE_DESC["T4"].name == "T4"
    @test SHAPE_DESC["Q4"].name == "Q4"
    @test SHAPE_DESC["H8"].name == "H8"
    @test SHAPE_DESC["L3"].name == "L3"
    true
end
end
using .mtest1
mtest1.test()

module mtattr11
using MeshCore: VecAttrib
using Test
function test()
    a = VecAttrib([1, 2, 4]);
    @test getindex(a, 3)     == 4
    setindex!(a, 6, 3)
    @test  getindex(a, 3) == 6
    @test a[3] == 6
    a[3] = -3
    @test a[3] == -3
    # @show a
    true
end
end
using .mtattr11
mtattr11.test()

module mtest1a1
using MeshCore: _sense
using Test
function test()
    oc = [4, 1, 5, 13]
    fc = [4, 1, 5, 13]
    @test _sense(fc, oc, 4) == +1
    fc = [13, 4, 1, 5]
    @test _sense(fc, oc, 4) == +1
    fc = [1, 5, 13, 4]
    @test _sense(fc, oc, 4) == +1
    fc = [1, 4, 13, 5]
    @test _sense(fc, oc, 4) == -1
    fc = [5, 1, 4, 13]
    @test _sense(fc, oc, 4) == -1
    fc = [13, 5, 1, 4]
    @test _sense(fc, oc, 4) == -1
    fc = [4, 13, 5, 1]
    @test _sense(fc, oc, 4) == -1
    true
end
end
using .mtest1a1
mtest1a1.test()

module mtest2
using StaticArrays
using MeshCore: VecAttrib
using Test
function test()
    xyz = [0.0 0.0; 633.3333333333334 0.0; 1266.6666666666667 0.0; 1900.0 0.0; 0.0 400.0; 633.3333333333334 400.0; 1266.6666666666667 400.0; 1900.0 400.0; 0.0 800.0; 633.3333333333334 800.0; 1266.6666666666667 800.0; 1900.0 800.0]
    N, T = size(xyz, 2), eltype(xyz)
    a =  VecAttrib([SVector{N, T}(xyz[i, :]) for i in 1:size(xyz, 1)])

    @test length(a) == 12
    x = a[SVector{2}([2, 4])]
    @test x[1] == SVector{2}([633.3333333333334 0.0])
    @test length(a[1]) == 2
    @test eltype(a[1]) == Float64

    true
end
end
using .mtest2
mtest2.test()

module mtest3
using StaticArrays
using MeshCore: P1, L2, Q4, ShapeColl, manifdim, nfacets, facetdesc, nshapes
using MeshCore: Q4ShapeDesc, shapedesc, n1storderv, nridges, nshifts, nvertices
using MeshCore: IncRel
using Test
function test()
    c = [(1, 2, 6, 5), (5, 6, 10, 9), (2, 3, 7, 6), (6, 7, 11, 10), (3, 4, 8, 7), (7, 8, 12, 11)]
    cc = [SVector{nvertices(Q4)}(c[idx]) for idx in 1:length(c)]
    q4s = ShapeColl(Q4, 6)
    vrts = ShapeColl(P1, 12)
    ir = IncRel(q4s, vrts, cc)
    q4s = ShapeColl(Q4, 6, "q4s")
    vrts = ShapeColl(P1, 12, "vrts")
    ir = IncRel(q4s, vrts, cc)
    @test ir.name == "(q4s, vrts)"
    @test manifdim(q4s) == 2
    @test nvertices(q4s) == 4
    @test facetdesc(q4s) == L2
    @test nfacets(q4s) == 4
    @test nshapes(q4s) == 6
    @test manifdim(shapedesc(q4s)) == 2
    @test nvertices(shapedesc(q4s)) == 4
    @test nfacets(shapedesc(q4s)) == 4
    @test nridges(shapedesc(q4s)) == 4
    @test n1storderv(shapedesc(q4s)) == 4
    @test nshifts(shapedesc(q4s)) == 4
    @test ir[3, 3] == 7
    @test ir[1, 4] == 5
    @test ir[6, 1] == 7

    true
end
end
using .mtest3
mtest3.test()

module mtest5
using StaticArrays
using MeshCore: P1, L2, Q4, ShapeColl, manifdim, nvertices, nfacets, facetdesc, nrelations, facetconnectivity
using MeshCore: IncRel
using MeshCore: ir_skeleton
using Test
function test()
    c = [(1, 2, 6, 5), (5, 6, 10, 9), (2, 3, 7, 6), (6, 7, 11, 10), (3, 4, 8, 7), (7, 8, 12, 11)]
    cc = [SVector{nvertices(Q4)}(c[idx]) for idx in 1:length(c)]
    q4s = ShapeColl(Q4, 6)
    vrts = ShapeColl(P1, 12)
    mesh = IncRel(q4s, vrts, cc)

    facemesh = ir_skeleton(mesh)
    @test nrelations(facemesh) == 17
    true
end
end
using .mtest5
mtest5.test()

module mtest6
using StaticArrays
using MeshCore: P1, L2, Q4, ShapeColl, manifdim, nvertices, nfacets, facetdesc, nrelations, facetconnectivity
using MeshCore: VecAttrib, IncRel
using MeshCore: ir_skeleton
using Test
function test()
    xyz = [0.0 0.0; 633.3333333333334 0.0; 1266.6666666666667 0.0; 1900.0 0.0; 0.0 400.0; 633.3333333333334 400.0; 1266.6666666666667 400.0; 1900.0 400.0; 0.0 800.0; 633.3333333333334 800.0; 1266.6666666666667 800.0; 1900.0 800.0]
    N, T = size(xyz, 2), eltype(xyz)
    locs =  VecAttrib([SVector{N, T}(xyz[i, :]) for i in 1:size(xyz, 1)])
    c = [(1, 2, 6, 5), (5, 6, 10, 9), (2, 3, 7, 6), (6, 7, 11, 10), (3, 4, 8, 7), (7, 8, 12, 11)]
    cc = [SVector{nvertices(Q4)}(c[idx]) for idx in 1:length(c)]
    q4s = ShapeColl(Q4, 6)
    vrts = ShapeColl(P1, 12)
    mesh = IncRel(q4s, vrts, cc)

    facemesh = ir_skeleton(mesh)
    @test nrelations(facemesh) == 17
    # @show facemeshx
    for i in 1:nrelations(facemesh)
        x = locs[facemesh[i]]
    end #
    @test locs[facemesh[17]] == StaticArrays.SArray{Tuple{2},Float64,1,2}[[1266.6666666666667, 800.0], [1900.0, 800.0], ]
    @test locs[facemesh[17][1]] == [1266.6666666666667, 800.0]
    true
end
end
using .mtest6
mtest6.test()

module mtest7
using StaticArrays
using MeshCore: P1, L2, Q4, ShapeColl, manifdim, nvertices, nfacets, facetdesc, nrelations, facetconnectivity
using MeshCore: VecAttrib, IncRel
using MeshCore: ir_skeleton, ir_boundary, nshapes
using Test
function test()
    xyz = [0.0 0.0; 633.3333333333334 0.0; 1266.6666666666667 0.0; 1900.0 0.0; 0.0 400.0; 633.3333333333334 400.0; 1266.6666666666667 400.0; 1900.0 400.0; 0.0 800.0; 633.3333333333334 800.0; 1266.6666666666667 800.0; 1900.0 800.0]
    N, T = size(xyz, 2), eltype(xyz)
    locs =  VecAttrib([SVector{N, T}(xyz[i, :]) for i in 1:size(xyz, 1)])
    c = [(1, 2, 6, 5), (5, 6, 10, 9), (2, 3, 7, 6), (6, 7, 11, 10), (3, 4, 8, 7), (7, 8, 12, 11)]
    cc = [SVector{nvertices(Q4)}(c[idx]) for idx in 1:length(c)]
    q4s = ShapeColl(Q4, 6)
    vrts = ShapeColl(P1, 12)
    mesh = IncRel(q4s, vrts, cc)

    bdrymesh = ir_boundary(mesh)
    @test nrelations(bdrymesh) == 10
    # @show bdrymesh.left, bdrymesh.right
    @test nshapes(bdrymesh.left) == 10
    @test nshapes(bdrymesh.right) == 12
    true
end
end
using .mtest7
mtest7.test()

module mtopoop1
using StaticArrays
using MeshCore: P1, L2, Q4, ShapeColl, manifdim, nvertices, nfacets, facetdesc, nshapes
using MeshCore: IncRel, ir_transpose, nshapes, VecAttrib
using Test
function test()
    xyz = [0.0 0.0; 633.3333333333334 0.0; 1266.6666666666667 0.0; 1900.0 0.0; 0.0 400.0; 633.3333333333334 400.0; 1266.6666666666667 400.0; 1900.0 400.0; 0.0 800.0; 633.3333333333334 800.0; 1266.6666666666667 800.0; 1900.0 800.0]
    N, T = size(xyz, 2), eltype(xyz)
    locs =  VecAttrib([SVector{N, T}(xyz[i, :]) for i in 1:size(xyz, 1)])
    c = [(1, 2, 6, 5), (5, 6, 10, 9), (2, 3, 7, 6), (6, 7, 11, 10), (3, 4, 8, 7), (7, 8, 12, 11)]
    cc = [SVector{nvertices(Q4)}(c[idx]) for idx in 1:length(c)]
    q4s = ShapeColl(Q4, 6)
    vrts = ShapeColl(P1, 12)
    mesh = IncRel(q4s, vrts, cc)
    # Test the transpose incidence relation
    tmesh = ir_transpose(mesh)
    shouldget = Array{Int64,1}[[1], [1, 3], [3, 5], [5], [1, 2], [1, 2, 3, 4], [3, 4, 5, 6], [5, 6], [2], [2, 4], [4, 6], [6]]
    allmatch = true
    for j in 1:length(shouldget)
        for k in 1:length(shouldget[j])
            allmatch = allmatch && (tmesh[j, k] == shouldget[j][k])
        end
    end
    @test allmatch
    true
end
end
using .mtopoop1
mtopoop1.test()

module mtopoop2
using StaticArrays
using MeshCore: P1, L2, Q4, ShapeColl, manifdim, nvertices, nfacets, facetdesc, nshapes
using MeshCore: ir_bbyfacets, ir_skeleton, ir_transpose
using MeshCore: IncRel, VecAttrib
using Test
function test()
    xyz = [0.0 0.0; 633.3333333333334 0.0; 1266.6666666666667 0.0; 1900.0 0.0; 0.0 400.0; 633.3333333333334 400.0; 1266.6666666666667 400.0; 1900.0 400.0; 0.0 800.0; 633.3333333333334 800.0; 1266.6666666666667 800.0; 1900.0 800.0]
    N, T = size(xyz, 2), eltype(xyz)
    locs =  VecAttrib([SVector{N, T}(xyz[i, :]) for i in 1:size(xyz, 1)])
    c = [(1, 2, 6, 5), (5, 6, 10, 9), (2, 3, 7, 6), (6, 7, 11, 10), (3, 4, 8, 7), (7, 8, 12, 11)]
    cc = [SVector{nvertices(Q4)}(c[idx]) for idx in 1:length(c)]
    q4s = ShapeColl(Q4, 6)
    vrts = ShapeColl(P1, 12)
    mesh = IncRel(q4s, vrts, cc)
    fmesh = ir_skeleton(mesh)
    # Test the transpose incidence relation
    tmesh = ir_bbyfacets(mesh, fmesh)
    shouldget = StaticArrays.SArray{Tuple{4},Int64,1,4}[
    [2, 4, 1, 8],
    [9, 11, 8, 15],
    [4, 6, 3, 10],
    [11, 13, 10, 16],
    [6, 7, 5, 12],
    [13, 14, 12, 17]   ]
    allmatch = true
    for j in 1:length(shouldget)
        for k in 1:length(shouldget[j])
            allmatch = allmatch && (tmesh[j, k] == shouldget[j][k])
        end
    end
    @test allmatch
    true
end
end
using .mtopoop2
mtopoop2.test()

module mtopoop3
using StaticArrays
using MeshCore: P1, L2, Q4, ShapeColl, manifdim, nvertices, nfacets, facetdesc, nshapes
using MeshCore: ir_bbyfacets, ir_skeleton, ir_transpose
using MeshCore: IncRel, VecAttrib
using Test
function test()
    xyz = [0.0 0.0; 633.3333333333334 0.0; 1266.6666666666667 0.0; 1900.0 0.0; 0.0 400.0; 633.3333333333334 400.0; 1266.6666666666667 400.0; 1900.0 400.0; 0.0 800.0; 633.3333333333334 800.0; 1266.6666666666667 800.0; 1900.0 800.0]
    N, T = size(xyz, 2), eltype(xyz)
        locs =  VecAttrib([SVector{N, T}(xyz[i, :]) for i in 1:size(xyz, 1)])
    c = [(1, 2, 6, 5), (5, 6, 10, 9), (2, 3, 7, 6), (6, 7, 11, 10), (3, 4, 8, 7), (7, 8, 12, 11)]
    cc = [SVector{nvertices(Q4)}(c[idx]) for idx in 1:length(c)]
    q4s = ShapeColl(Q4, 6)
    vrts = ShapeColl(P1, 12)
    mesh = IncRel(q4s, vrts, cc)
    fmesh = ir_skeleton(mesh)
    # Test the bounded-by incidence relation
    bbmesh = ir_bbyfacets(mesh, fmesh)
    shouldget = StaticArrays.SArray{Tuple{4},Int64,1,4}[
    [2, 4, 1, 8],
    [9, 11, 8, 15],
    [4, 6, 3, 10],
    [11, 13, 10, 16],
    [6, 7, 5, 12],
    [13, 14, 12, 17]    ]
    allmatch = true
    for j in 1:length(shouldget)
        for k in 1:length(shouldget[j])
            allmatch = allmatch && (bbmesh[j, k] == shouldget[j][k])
        end
    end
    @test allmatch
    # Now test the transposed incidence relation
    tbbmesh = ir_transpose(bbmesh)
    shouldget = Array{Int64,1}[[1], [1], [3], [1, 3], [5], [3, 5], [5], [1, 2], [2], [3, 4], [2, 4], [5, 6],
    [4, 6], [6], [2], [4], [6]]
    allmatch = true
    for j in 1:length(shouldget)
        for k in 1:length(shouldget[j])
            allmatch = allmatch && (tbbmesh[j, k] == shouldget[j][k])
        end
    end
    @test allmatch
    true
end
end
using .mtopoop3
mtopoop3.test()

module mtopoop4
using StaticArrays
using MeshCore: P1, L2, Q4, ShapeColl, manifdim, nvertices, nfacets, facetdesc, nshapes
using MeshCore: ir_bbyridges, ir_skeleton, ir_transpose
using MeshCore: IncRel, VecAttrib
using Test
function test()
    xyz = [0.0 0.0; 633.3333333333334 0.0; 1266.6666666666667 0.0; 1900.0 0.0; 0.0 400.0; 633.3333333333334 400.0; 1266.6666666666667 400.0; 1900.0 400.0; 0.0 800.0; 633.3333333333334 800.0; 1266.6666666666667 800.0; 1900.0 800.0]
    N, T = size(xyz, 2), eltype(xyz)
        locs =  VecAttrib([SVector{N, T}(xyz[i, :]) for i in 1:size(xyz, 1)])
    c = [(1, 2, 6, 5), (5, 6, 10, 9), (2, 3, 7, 6), (6, 7, 11, 10), (3, 4, 8, 7), (7, 8, 12, 11)]
    cc = [SVector{nvertices(Q4)}(c[idx]) for idx in 1:length(c)]
    q4s = ShapeColl(Q4, 6)
    vrts = ShapeColl(P1, 12)
    mesh = IncRel(q4s, vrts, cc)
    # fmesh = skeleton(mesh)
    vmesh = IncRel(ShapeColl(P1, 12), vrts, [[i] for i in 1:nshapes(vrts)] )
    # Test the bounded-by incidence relation
    bbmesh = ir_bbyridges(mesh, vmesh)
    shouldget = Array{Int64,1}[[1, 2, 6, 5], [5, 6, 10, 9], [2, 3, 7, 6], [6, 7, 11, 10], [3, 4, 8, 7], [7, 8, 12, 11]]
    allmatch = true
    for j in 1:length(shouldget)
        for k in 1:length(shouldget[j])
            allmatch = allmatch && (bbmesh[j, k] == shouldget[j][k])
        end
    end
    @test allmatch
    # Now test the transposed incidence relation
    tbbmesh = ir_transpose(bbmesh)
    shouldget = Array{Int64,1}[[1], [1, 3], [3, 5], [5], [1, 2], [1, 2, 3, 4], [3, 4, 5, 6], [5, 6], [2], [2, 4], [4, 6], [6]]
    allmatch = true
    for j in 1:length(shouldget)
        for k in 1:length(shouldget[j])
            allmatch = allmatch && (tbbmesh[j, k] == shouldget[j][k])
        end
    end
    @test allmatch
    true
end
end
using .mtopoop4
mtopoop4.test()

include("samplet4.jl")

module mt4topo1
using StaticArrays
using MeshCore: P1, T4, ShapeColl,  manifdim, nvertices, nridges, nshapes
using MeshCore: ir_bbyridges, ir_skeleton, ir_bbyfacets, nshifts, _sense
using MeshCore: IncRel, ir_transpose, nrelations, nentities, VecAttrib
using ..samplet4: samplet4mesh
using Test
function test()
    xyz, cc = samplet4mesh()
    # Construct the initial incidence relation
    N, T = size(xyz, 2), eltype(xyz)
    locs =  VecAttrib([SVector{N, T}(xyz[i, :]) for i in 1:size(xyz, 1)])
    vrts = ShapeColl(P1, length(locs))
    tets = ShapeColl(T4, size(cc, 1))
    ir30 = IncRel(tets, vrts, cc)
    # Test the incidence relations 3 -> 0 & 0 -> 3
    ir03 = ir_transpose(ir30)
    allmatch = true
    for j in 1:nrelations(ir03)
        for k in 1:nentities(ir03, j)
            f = ir03[j, k]
            allmatch = allmatch && (j in ir30[f])
        end # k
    end # j
    @test allmatch

    # Test the incidence relations 2 -> 0 & 0 -> 2
    ir20 = ir_skeleton(ir30)
    ir02 = ir_transpose(ir20)
    allmatch = true
    for j in 1:nrelations(ir02)
        for k in 1:nentities(ir02, j)
            f = ir02[j, k]
            allmatch = allmatch && (j in ir20[f])
        end # k
    end # j
    @test allmatch

    # Test the incidence relations 1 -> 0 & 0 -> 1
    ir10 = ir_skeleton(ir20)
    ir01 = ir_transpose(ir10)
    allmatch = true
    for j in 1:nrelations(ir01)
        for k in 1:nentities(ir01, j)
            f = ir01[j, k]
            allmatch = allmatch && (j in ir10[f])
        end # k
    end # j
    @test allmatch

    # Test the incidence relations 3 -> 2 & 2 -> 3
    ir20 = ir_skeleton(ir30)
    ir32 = ir_bbyfacets(ir30, ir20)
    # Check that the tetrahedra connectivity refers to all three vertices of the faces
    allmatch = true
    @test nrelations(ir30) == nrelations(ir32)
    for j in 1:nrelations(ir32)
        for k in 1:nentities(ir32, j)
            f = ir32[j, k]
            for m in 1:3
                allmatch = allmatch && (ir20[abs(f), m] in ir30[j])
            end # m
        end # k
    end # j
    @test allmatch
    # Check that the orientations of the faces is correct
    allmatch = true
    @test nrelations(ir30) == nrelations(ir32)
    for j in 1:nrelations(ir32)
        for k in 1:nentities(ir32, j)
            for f in ir32[j]
                orientation = sign(f)
                f = abs(f)
                fc = ir20[f]
                matchone = false
                for m in 1:4
                    oc = ir30[j][ir30.left.shapedesc.facets[m, :]]
                    if length(intersect(fc, oc)) == 3
                        s = _sense(fc, oc, nshifts(ir20.left.shapedesc))
                        matchone = (s == orientation)
                    end
                    if matchone
                        break
                    end
                end # m
                allmatch = allmatch && matchone
            end
        end # k
    end # j
    @test allmatch
    # Check the transpose incidence relation
    ir23 = ir_transpose(ir32)
    allmatch = true
    for j in 1:nrelations(ir23)
        for k in 1:nentities(ir23, j)
            e = ir23[j, k]
            allmatch = allmatch && (j in abs.(ir32[e]))
        end # k
    end # j
    @test allmatch
    true
end
end
using .mt4topo1
mt4topo1.test()

module mtestattr1
using StaticArrays
using MeshCore: VecAttrib, P1, ShapeColl, attribute
using Test
# using BenchmarkTools

function test()
    xyz = [0.0 0.0; 633.3333333333334 0.0; 1266.6666666666667 0.0; 1900.0 0.0; 0.0 400.0; 633.3333333333334 400.0; 1266.6666666666667 400.0; 1900.0 400.0; 0.0 800.0; 633.3333333333334 800.0; 1266.6666666666667 800.0; 1900.0 800.0]
    N, T = size(xyz, 2), eltype(xyz)
    locs =  VecAttrib([SVector{N, T}(xyz[i, :]) for i in 1:size(xyz, 1)])

    @test locs[10] == [633.3333333333334, 800.0]

    # @show typeof(Dict(a.name=>a))
    # @show typeof(Dict("geom"=>locs))
    vertices = ShapeColl(P1, size(xyz, 1))
    vertices.attributes["geom"] = locs
    locs = vertices.attributes["geom"]
    @test locs[10] == [633.3333333333334, 800.0]
    @test length(locs) == 12

    locs = attribute(vertices, "geom")
    @test locs[10] == [633.3333333333334, 800.0]

    true
end
end
using .mtestattr1
mtestattr1.test()

module m113651
using Test
using StaticArrays
using MeshCore: VecAttrib, ShapeColl, P1, attribute
function test()
	xyz = [0.0 0.0; 633.3333333333334 0.0; 1266.6666666666667 0.0]
	N, T = size(xyz, 2), eltype(xyz)
	locs =  VecAttrib([SVector{N, T}(xyz[i, :]) for i in 1:size(xyz, 1)])
	vertices = ShapeColl(P1, size(xyz, 1))
    vertices.attributes["geom"] = locs
	a = attribute(vertices, "geom")
	@test a[2] == [633.3333333333334, 0.0]

    dofn =  VecAttrib([i for i in 1:size(xyz, 1)])
    vertices.attributes["dofn"] = dofn
    # @show keys(vertices.attributes)
    @test "geom" in keys(vertices.attributes)
    @test "dofn" in keys(vertices.attributes)
end
end
using .m113651
m113651.test()

# module m1136193
# using Test
# using StaticArrays
# using MeshCore: VecAttrib, ShapeColl, P1, attribute
# function test()
# 	xyz = [0.0 0.0; 633.3333333333334 0.0; 1266.6666666666667 0.0]
# 	a = Attrib(i -> 1)
# 	vertices = ShapeColl(P1, size(xyz, 1), Dict("label"=>a))
# 	a = attribute(vertices, "label")
# 	@test a.co(3) == 1
# end
# end
# using .m1136193
# m1136193.test()

module mtest3a1
using StaticArrays
using MeshCore: P1, L2, Q4, ShapeColl, manifdim, nfacets, facetdesc, nshapes
using MeshCore: Q4ShapeDesc, shapedesc, n1storderv, nridges, nshifts, nvertices
using MeshCore: IncRel
using Test
function test()
    c = [(1, 2, 6, 5), (5, 6, 10, 9), (2, 3, 7, 6), (6, 7, 11, 10), (3, 4, 8, 7), (7, 8, 12, 11)]
    cc = [SVector{nvertices(Q4)}(c[idx]) for idx in 1:length(c)]
    q4s = ShapeColl(Q4, 6)
    vrts = ShapeColl(P1, 12)
    ir = IncRel(q4s, vrts, cc)
    q4s = ShapeColl(Q4, 6, "q4s")
    vrts = ShapeColl(P1, 12, "vrts")
    ir = IncRel(q4s, vrts, cc)
    @test ir.name == "(q4s, vrts)"
    @test manifdim(q4s) == 2
    @test nvertices(q4s) == 4
    @test facetdesc(q4s) == L2
    @test nfacets(q4s) == 4
    @test nshapes(q4s) == 6
    @test manifdim(shapedesc(q4s)) == 2
    @test nvertices(shapedesc(q4s)) == 4
    @test nfacets(shapedesc(q4s)) == 4
    @test nridges(shapedesc(q4s)) == 4
    @test n1storderv(shapedesc(q4s)) == 4
    @test nshifts(shapedesc(q4s)) == 4
    @test ir[3, 3] == 7
    @test ir[1, 4] == 5
    @test ir[6, 1] == 7

    ir = IncRel(q4s, vrts, cc, "vertices")
    @test ir.name == "vertices"

    true
end
end
using .mtest3a1
mtest3a1.test()

module mtest3a2
using StaticArrays
using MeshCore: P1, L2, Q4, ShapeColl, manifdim, nfacets, facetdesc, nshapes
using MeshCore: Q4ShapeDesc, shapedesc, n1storderv, nridges, nshifts, nvertices
using MeshCore: IncRel, ir_subset, nrelations
using Test
function test()
    c = [(1, 2, 6, 5), (5, 6, 10, 9), (2, 3, 7, 6), (6, 7, 11, 10), (3, 4, 8, 7), (7, 8, 12, 11)]
    cc = [SVector{nvertices(Q4)}(c[idx]) for idx in 1:length(c)]
    q4s = ShapeColl(Q4, 6)
    vrts = ShapeColl(P1, 12)
    ir = IncRel(q4s, vrts, cc)
    sir = ir_subset(ir, [1, 2])
    @test nrelations(sir) == 2
    @test isapprox(sir[1], [1, 2, 6, 5])

    true
end
end
using .mtest3a2
mtest3a2.test()

module mfeat1
using StaticArrays
using MeshCore: P1, L2, Q4, H8, T3, T4
using MeshCore: nfeatofdim
# using BenchmarkTools
using Test
function test()
    @test nfeatofdim(P1, 0) == 1
    @test nfeatofdim(P1, 1) == 0
    @test nfeatofdim(P1, 2) == 0
    @test nfeatofdim(P1, 3) == 0

    @test nfeatofdim(L2, 0) == 2
    @test nfeatofdim(L2, 1) == 1
    @test nfeatofdim(L2, 2) == 0
    @test nfeatofdim(L2, 3) == 0

    @test nfeatofdim(T3, 0) == 3
    @test nfeatofdim(T3, 1) == 3
    @test nfeatofdim(T3, 2) == 1
    @test nfeatofdim(T3, 3) == 0

    @test nfeatofdim(T4, 0) == 4
    @test nfeatofdim(T4, 1) == 6
    @test nfeatofdim(T4, 2) == 4
    @test nfeatofdim(T4, 3) == 1

    @test nfeatofdim(Q4, 0) == 4
    @test nfeatofdim(Q4, 1) == 4
    @test nfeatofdim(Q4, 2) == 1
    @test nfeatofdim(Q4, 3) == 0

    @test nfeatofdim(H8, 0) == 8
    @test nfeatofdim(H8, 1) == 12
    @test nfeatofdim(H8, 2) == 6
    @test nfeatofdim(H8, 3) == 1

    # @btime nfeatofdim($H8, 0)
    # @btime nfeatofdim($H8, 1)
    # @btime nfeatofdim($H8, 2)
    # @btime nfeatofdim($H8, 3)
    true
end
end
using .mfeat1
mfeat1.test()

module mtest4a5
using StaticArrays
using MeshCore: P1, L2, Q4, ShapeColl, manifdim, nfacets, facetdesc, nshapes
using MeshCore: Q4ShapeDesc, shapedesc, n1storderv, nridges, nshifts, nvertices
using MeshCore: IncRel, ir_subset, nrelations, ir_identity
using Test
function test()
    c = [(1, 2, 6, 5), (5, 6, 10, 9), (2, 3, 7, 6), (6, 7, 11, 10), (3, 4, 8, 7), (7, 8, 12, 11)]
    cc = [SVector{nvertices(Q4)}(c[idx]) for idx in 1:length(c)]
    q4s = ShapeColl(Q4, 6)
    vrts = ShapeColl(P1, 12)
    ir = IncRel(q4s, vrts, cc)
    sir = ir_subset(ir, [1, 2])
    @test nrelations(sir) == 2
    @test isapprox(sir[1], [1, 2, 6, 5])
    ir1 = ir_identity(ir, :left)
    @test nrelations(ir1) == 6
    @test nshapes(ir1.left) == 6
    @test nshapes(ir1.right) == 6
    true
end
end
using .mtest4a5
mtest4a5.test()

module mtest1q
using MeshCore: NoSuchShape, P1, L3, T6, SHAPE_DESC
using MeshCore: NoSuchShape, ShapeColl, manifdim, nfacets
using MeshCore: shapedesc, nshapes, manifdim, nvertices, facetdesc, nfacets, ridgedesc, nridges, n1storderv
using Test
function test()
    @test manifdim(P1) == 0
    @test nfacets(L3) == 2
    @test nvertices(L3) == 3
    @test n1storderv(L3) == 2
    @test nfacets(T6) == 3

    vrts = ShapeColl(P1, 12)
    lines = ShapeColl(L3, 32)
    @test shapedesc(vrts) == P1
    @test nshapes(vrts) == 12
    @test manifdim(vrts) == manifdim(P1)
    @test nvertices(lines) == nvertices(L3)
    @test facetdesc(lines) == P1
    @test nfacets(lines) == nfacets(L3)
    @test ridgedesc(lines) == NoSuchShape
    @test nridges(lines) == 0

    tris6 = ShapeColl(T6, 13)
    @test nvertices(tris6) == nvertices(T6)
    @test nvertices(tris6) == 6
    @test n1storderv(T6) == 3
    @test facetdesc(tris6) == L3
    @test nfacets(tris6) == nfacets(T6)
    @test ridgedesc(tris6) == P1
    @test nridges(tris6) == 3
    @test manifdim(tris6) == 2

    @test SHAPE_DESC["P1"].name == "P1"
    @test SHAPE_DESC["L2"].name == "L2"
    @test SHAPE_DESC["T3"].name == "T3"
    @test SHAPE_DESC["T6"].name == "T6"
    @test SHAPE_DESC["T4"].name == "T4"
    @test SHAPE_DESC["Q4"].name == "Q4"
    @test SHAPE_DESC["H8"].name == "H8"
    @test SHAPE_DESC["L3"].name == "L3"
    true
end
end
using .mtest1q
mtest1q.test()

module mtest2q
using MeshCore: NoSuchShape, P1, L3, Q8, SHAPE_DESC
using MeshCore: NoSuchShape, ShapeColl, manifdim, nfacets
using MeshCore: shapedesc, nshapes, manifdim, nvertices, facetdesc, nfacets, ridgedesc, nridges, n1storderv
using Test
function test()
    @test manifdim(P1) == 0
    @test nfacets(L3) == 2
    @test nvertices(L3) == 3
    @test n1storderv(L3) == 2
    @test nfacets(Q8) == 4
    @test nridges(Q8) == 4

    vrts = ShapeColl(P1, 12)
    lines = ShapeColl(L3, 32)
    @test shapedesc(vrts) == P1
    @test nshapes(vrts) == 12
    @test manifdim(vrts) == manifdim(P1)
    @test nvertices(lines) == nvertices(L3)
    @test facetdesc(lines) == P1
    @test nfacets(lines) == nfacets(L3)
    @test ridgedesc(lines) == NoSuchShape
    @test nridges(lines) == 0

    q8s = ShapeColl(Q8, 13)
    @test nvertices(q8s) == nvertices(Q8)
    @test nvertices(q8s) == 8
    @test n1storderv(Q8) == 4
    @test facetdesc(q8s) == L3
    @test nfacets(q8s) == nfacets(Q8)
    @test ridgedesc(q8s) == P1
    @test nridges(q8s) == 4
    @test manifdim(q8s) == 2

    @test SHAPE_DESC["P1"].name == "P1"
    @test SHAPE_DESC["L2"].name == "L2"
    @test SHAPE_DESC["T3"].name == "T3"
    @test SHAPE_DESC["T6"].name == "T6"
    @test SHAPE_DESC["T4"].name == "T4"
    @test SHAPE_DESC["Q4"].name == "Q4"
    @test SHAPE_DESC["Q8"].name == "Q8"
    @test SHAPE_DESC["H8"].name == "H8"
    @test SHAPE_DESC["L3"].name == "L3"
    true
end
end
using .mtest2q
mtest2q.test()

module mtestbb1
using MeshCore: @_check
using Test
function test()
    @_check 1 == 1 "Really?"
    @_check 1 == 1 "Why " * "not?"
    a = 13
    b = 13
    @_check a == b "Why is $a " * "not equal to $b?"
    a = 61
    @_check a > b "Why is $a " * "not greater than $b?"
    Test.@test_throws AssertionError @_check(a < b, "Why is $a " * "not greater than $b?")
    true
end
end
using .mtestbb1
mtestbb1.test()

module mtestattr6
using StaticArrays
using MeshCore: VecAttrib, P1, ShapeColl, attribute
using Test
# using BenchmarkTools

function test()
    xyz = [0.0 0.0; 633.3333333333334 0.0; 1266.6666666666667 0.0; 1900.0 0.0; 0.0 400.0; 633.3333333333334 400.0; 1266.6666666666667 400.0; 1900.0 400.0; 0.0 800.0; 633.3333333333334 800.0; 1266.6666666666667 800.0; 1900.0 800.0]
    N, T = size(xyz, 2), eltype(xyz)
    locs =  VecAttrib([SVector{N, T}(xyz[i, :]) for i in 1:size(xyz, 1)])

    @test locs[10] == [633.3333333333334, 800.0]
    locs[10] = -locs[10]
    @test locs[10] == [-633.3333333333334, -800.0]
    @test isapprox(locs[8:10], StaticArrays.SArray{Tuple{2},Float64,1,2}[[1900.0, 400.0], [0.0, 800.0], [-633.3333333333334, -800.0]])
    locs[8:10] .= StaticArrays.SArray{Tuple{2},Float64,1,2}[[1.0, 1.0], [1.0, 1.0], [1.0, 1.0]]
    @test isapprox(locs[8:10], StaticArrays.SArray{Tuple{2},Float64,1,2}[[1.0, 1.0], [1.0, 1.0], [1.0, 1.0]])

    true
end
end
using .mtestattr6
mtestattr6.test()

module mtesat4a5
using StaticArrays
using MeshCore: P1, L2, Q4, ShapeColl, manifdim, nfacets, facetdesc, nshapes
using MeshCore: Q4ShapeDesc, shapedesc, n1storderv, nridges, nshifts, nvertices
using MeshCore: IncRel, ir_subset, nrelations, ir_identity, indextype, ir_code
using Test
function test()
    c = [(1, 2, 6, 5), (5, 6, 10, 9), (2, 3, 7, 6), (6, 7, 11, 10), (3, 4, 8, 7), (7, 8, 12, 11)]
    cc = [SVector{nvertices(Q4)}(c[idx]) for idx in 1:length(c)]
    q4s = ShapeColl(Q4, 6)
    vrts = ShapeColl(P1, 12)
    ir = IncRel(q4s, vrts, cc)
    @test indextype(ir) == Int64
    @test ir_code(ir) == (2, 0)
    sir = ir_subset(ir, [1, 2])
    @test nrelations(sir) == 2
    @test isapprox(sir[1], [1, 2, 6, 5])
    ir1 = ir_identity(ir, :left)
    @test nrelations(ir1) == 6
    @test nshapes(ir1.left) == 6
    @test nshapes(ir1.right) == 6
    true
end
end
using .mtesat4a5
mtesat4a5.test()

module mfeatx1
using StaticArrays
using MeshCore: P1, L2, Q4, H8, T3, T4, T6, L3
using MeshCore: nfeatofdim
# using BenchmarkTools
using Test
function test()

    @test nfeatofdim(L3, 0) == 3
    @test nfeatofdim(L3, 1) == 1
    @test nfeatofdim(L3, 2) == 0
    @test nfeatofdim(L3, 3) == 0

    @test nfeatofdim(T6, 0) == 6
    @test nfeatofdim(T6, 1) == 3
    @test nfeatofdim(T6, 2) == 1
    @test nfeatofdim(T6, 3) == 0

    true
end
end
using .mfeatx1
mfeatx1.test()

module mtest3y1
using StaticArrays
using MeshCore.Exports
using Test
function test()
    c = [(1, 2, 6, 5), (5, 6, 10, 9), (2, 3, 7, 6), (6, 7, 11, 10), (3, 4, 8, 7), (7, 8, 12, 11)]
    cc = [SVector{nvertices(Q4)}(c[idx]) for idx in 1:length(c)]
    q4s = ShapeColl(Q4, 6)
    vrts = ShapeColl(P1, 12)
    ir = IncRel(q4s, vrts, cc)
    q4s = ShapeColl(Q4, 6, "q4s")
    vrts = ShapeColl(P1, 12, "vrts")
    ir = IncRel(q4s, vrts, cc)
    @test ir.name == "(q4s, vrts)"
    @test manifdim(q4s) == 2
    @test nvertices(q4s) == 4
    @test facetdesc(q4s) == L2
    @test nfacets(q4s) == 4
    @test nshapes(q4s) == 6
    @test manifdim(shapedesc(q4s)) == 2
    @test nvertices(shapedesc(q4s)) == 4
    @test nfacets(shapedesc(q4s)) == 4
    @test nridges(shapedesc(q4s)) == 4
    @test n1storderv(shapedesc(q4s)) == 4
    @test nshifts(shapedesc(q4s)) == 4
    @test ir[3, 3] == 7
    @test ir[1, 4] == 5
    @test ir[6, 1] == 7

    true
end
end
using .mtest3y1
mtest3y1.test()

module mtest1y2
using MeshCore.Exports
using Test
function test()
    @test manifdim(P1) == 0
    @test nfacets(L2) == 2
    vrts = ShapeColl(P1, 12)
    lines = ShapeColl(L2, 32)
    @test shapedesc(vrts) == P1
    @test nshapes(vrts) == 12
    @test manifdim(vrts) == manifdim(P1)
    @test nvertices(lines) == nvertices(L2)
    @test facetdesc(lines) == P1
    @test nfacets(lines) == nfacets(L2)
    @test ridgedesc(lines) == NoSuchShape
    @test nridges(lines) == 0

    lines3 = ShapeColl(L3, 13)
    @test nvertices(lines3) == nvertices(L3)
    @test nvertices(lines3) == 3
    @test facetdesc(lines3) == P1
    @test nfacets(lines3) == nfacets(L3)
    @test ridgedesc(lines3) == NoSuchShape
    @test nridges(lines3) == 0
    @test manifdim(lines3) == 1

    @test SHAPE_DESC["P1"].name == "P1"
    @test SHAPE_DESC["L2"].name == "L2"
    @test SHAPE_DESC["T3"].name == "T3"
    @test SHAPE_DESC["T4"].name == "T4"
    @test SHAPE_DESC["Q4"].name == "Q4"
    @test SHAPE_DESC["H8"].name == "H8"
    @test SHAPE_DESC["L3"].name == "L3"
    true
end
end
using .mtest1y2
mtest1y2.test()

module mtestattrbc1
using StaticArrays
using MeshCore: VecAttrib, P1, ShapeColl, attribute, FunAttrib, datavaluetype
using Test
# using BenchmarkTools

function test()
    xyz = [0.0 0.0; 633.3333333333334 0.0; 1266.6666666666667 0.0; 1900.0 0.0; 0.0 400.0; 633.3333333333334 400.0; 1266.6666666666667 400.0; 1900.0 400.0; 0.0 800.0; 633.3333333333334 800.0; 1266.6666666666667 800.0; 1900.0 800.0]
    N, T = size(xyz, 2), eltype(xyz)
    locs =  VecAttrib([SVector{N, T}(xyz[i, :]) for i in 1:size(xyz, 1)])

    @test locs[10] == [633.3333333333334, 800.0]

    # @show typeof(Dict(a.name=>a))
    # @show typeof(Dict("geom"=>locs))
    vertices = ShapeColl(P1, size(xyz, 1))
    vertices.attributes["geom"] = locs
    locs = vertices.attributes["geom"]
    @test locs[10] == [633.3333333333334, 800.0]
    @test length(locs) == 12

    locs = attribute(vertices, "geom")
    @test locs[10] == [633.3333333333334, 800.0]
    @test locs[[3, 5]] == StaticArrays.SArray{Tuple{2},Float64,1,2}[[1266.6666666666667, 0.0], [0.0, 400.0]]

    @test datavaluetype(locs) == typeof(SVector{N, T}([0.0 0.0]))
    @test size(locs) == (size(xyz, 1),)

    flocs = FunAttrib(SVector{N, T}([0.0 0.0]), size(xyz, 1), i -> SVector{N, T}(xyz[i, :]))
    vertices.attributes["geom"] = flocs
    flocs = vertices.attributes["geom"]
    @test flocs[10] == [633.3333333333334, 800.0]

    @test datavaluetype(flocs) == typeof(SVector{N, T}([0.0 0.0]))
    @test size(flocs) == (size(xyz, 1),)
    @test flocs[[3, 5]] == StaticArrays.SArray{Tuple{2},Float64,1,2}[[1266.6666666666667, 0.0], [0.0, 400.0]]

    true
end
end
using .mtestattrbc1
mtestattrbc1.test()

module mtestattrbc2
using StaticArrays
using MeshCore: VecAttrib, P1, ShapeColl, attribute, FunAttrib, datavaluetype
using Test
# using BenchmarkTools

function test()
    list = [3, 7, 1, 2, 5, 4, 6]
    N, T = size(list, 2), eltype(list)
    la =  VecAttrib(list)

    @test la[4] == 2

    la = FunAttrib(0, size(list, 1), i -> list[i])
    @test la[4] == 2

    list = collect(1:7)
    N, T = size(list, 2), eltype(list)
    la =  VecAttrib(list)

    @test la[4] == 4

    la = FunAttrib(0, size(list, 1), i -> i)
    @test la[4] == 4

    true
end
end
using .mtestattrbc2
mtestattrbc2.test()

module mtestattrbc3
using StaticArrays
using MeshCore: VecAttrib, P1, ShapeColl, attribute, FunAttrib, datavaluetype
using Test
# using BenchmarkTools

function test()
    list = [3, 7, 1, 2, 5, 4, 6]
    N, T = size(list, 2), eltype(list)
    la =  VecAttrib(list)

    @test la[4] == 2

    la = FunAttrib(0, size(list, 1), i -> list[i])
    @test la[4] == 2
    @test_throws ErrorException la[4] = -4
    @test la[4] == 2

    list = collect(1:7)
    N, T = size(list, 2), eltype(list)
    la =  VecAttrib(list)

    @test la[4] == 4

    la = FunAttrib(0, size(list, 1), i -> i)
    @test la[4] == 4
    @test_throws ErrorException la[4] = -4
    @test la[4] == 4

    true
end
end
using .mtestattrbc3
mtestattrbc3.test()

module mtestattrbc4
using StaticArrays
using MeshCore: VecAttrib, P1, ShapeColl, attribute, FunAttrib, datavaluetype
using Test
# using BenchmarkTools

function test()
    xyz = [0.0 0.0; 633.3333333333334 0.0; 1266.6666666666667 0.0]
    a = FunAttrib(0, size(xyz, 1), i -> 1)
    vertices = ShapeColl(P1, size(xyz, 1))
    vertices.attributes["label"] = a
    @test a[3] == 1

    true
end
end
using .mtestattrbc4
mtestattrbc4.test()

module mfunattraccess1
using StaticArrays
using MeshCore: VecAttrib
using MeshCore: P1, L2, H8, ShapeColl, manifdim, nfacets, facetdesc, nshapes
using MeshCore: Q4ShapeDesc, shapedesc, n1storderv, nridges, nshifts, nvertices
using MeshCore: IncRel
using MeshCore: FunAttrib
using Test
function test()
    L, W, H = 1.5*pi, 3.0, 1.5
    R = 4.0
    M, N, K = 13, 4, 2
    linix = LinearIndices((M+1, N+1, K+1))
    sn(m, n, k) = linix[m, n, k]

    C = SVector{8, Int}[]
    for k in 1:K, n in 1:N, m in 1:M
        c = (
        sn(m, n, k), sn(m+1, n, k), sn(m+1, n+1, k), sn(m, n+1, k),
        sn(m, n, k+1), sn(m+1, n, k+1), sn(m+1, n+1, k+1), sn(m, n+1, k+1)
        )
        push!(C, SVector{8, Int}(c))
    end
    loc = SVector{3, Float64}[]
    for k in 1:K+1, n in 1:N+1, m in 1:M+1
        a = (m-1)/M*L
        x, y, z = (R+(n-1)/N*W) * cos(a), (R+(n-1)/N*W) * sin(a), (k-1)/K*H * (1 + a)
        push!(loc, SVector{3, Float64}(x, y, z))
    end

    elements = ShapeColl(H8, length(C), "elements")
    vertices = ShapeColl(P1, length(loc), "vertices")

    vertices.attributes["geom"] = VecAttrib(loc)

    connectivities = IncRel(elements, vertices, C)

    cart = CartesianIndices((M+1, N+1, K+1))
    access_location(i) = let
        ci  = cart[i]
        m, n, k = ci[1], ci[2], ci[3]
        a = (m-1)/M*L
        x, y, z = (R+(n-1)/N*W) * cos(a), (R+(n-1)/N*W) * sin(a), (k-1)/K*H * (1 + a)
        SVector{3, Float64}(x, y, z)
    end

    vertices2 = ShapeColl(P1, length(loc), "vertices2")
    vertices2.attributes["geom"] = FunAttrib(0.0, prod((M+1, N+1, K+1)), access_location);

    @test size(vertices.attributes["geom"]) == size(vertices2.attributes["geom"])

    n = 0.0
    for i in 1:size(vertices2.attributes["geom"], 1)
        n += sum((vertices.attributes["geom"][i] - vertices2.attributes["geom"][i]).^2)
    end
    @test n ≈ 0

    true
end
end
using .mfunattraccess1
mfunattraccess1.test()


module Var

using Random: AbstractRNG
using ..Basis

"""
`recurse(f, ω)`
Recursively apply contextual execution to internals of `f`"""
function recurse end

include("variable.jl")          # Random / Parametric Variables
include("member.jl")            # Families
include("primparam.jl")         # Primitive Parameters
include("distributions.jl")     # Primitive Distributions
include("multivariate.jl")      # Multivariate Distributions
include("constant.jl")          # Constant distribution
include("pointwise.jl")         # Point wise variable application
include("dispatch.jl")          # Contextual application

end


#
# Cache routines for Business Days precalculated days
#

"""
Holds caches for Holiday Calendars.

* Key = `HolidayCalendar` instance
* Value = instance of `HolidayCalendarCache`
"""
global _CACHE_DICT = Dict{HolidayCalendar, HolidayCalendarCache}()

function _getcachestate(hc::HolidayCalendar)
    return haskey(_CACHE_DICT, hc)
end

function _getholidaycalendarcache(hc::HolidayCalendar)
    return _CACHE_DICT[hc]
end

function checkbounds(hcc::HolidayCalendarCache, dt::Date)
    if !((hcc.dtmin <= dt) && (dt <= hcc.dtmax))
        error("Date out of cache bounds. Use initcache function with a wider time spread. Provided date: $(dt).")
    end
end

function _linenumber(hcc::HolidayCalendarCache, dt::Date)
    return Dates.days(dt) - Dates.days(hcc.dtmin) + 1
end

function isbday(hcc::HolidayCalendarCache, dt::Date)
    checkbounds(hcc, dt)
    return hcc.isbday_array[ _linenumber(hcc, dt) ]
end

function bdays(hcc::HolidayCalendarCache, dt0::Date, dt1::Date)
    # Computation is always based on next Business Days if given dates are not Business Days, inspired by Banking Account convention.
    dt0 = tobday(hcc.hc, dt0) # cache bounds are checked inside tobday -> isbday
    dt1 = tobday(hcc.hc, dt1) # cache bounds are checked inside tobday -> isbday

    return Day(convert(Int, hcc.bdayscounter_array[_linenumber(hcc, dt1)]) - convert(Int, hcc.bdayscounter_array[_linenumber(hcc, dt0)]))
end

# Be sure to use this function on a syncronized code (not multithreaded).
"""
    initcache(calendar, [d0], [d1])

Creates cache for a given Holiday Calendar. After calling this function, any call to `isbday`
function, or any function that uses `isbday`, will be optimized to use this cache.

You can pass `calendar` as an instance of `HolidayCalendar`, `Symbol` or `AbstractString`.
You can also pass `calendar` as an `AbstractArray` of those types.
"""
function initcache(hc::HolidayCalendar, d0::Date, d1::Date)
    isbday_array , bdayscounter_array = _createbdayscache(hc, d0, d1)
    _CACHE_DICT[hc] = HolidayCalendarCache(hc, isbday_array, bdayscounter_array, min(d0, d1), max(d0, d1))
end

initcache(calendar, d0::Date, d1::Date) = initcache(convert(HolidayCalendar, calendar), d0, d1)

# Defaults to d0 = 1st Jan 1980 , d1 = 20th dec 2150
function initcache(hc::HolidayCalendar)
    initcache(hc, Date(1980, 01, 01), Date(2150, 12, 20))
end

initcache(calendar) = initcache(convert(HolidayCalendar,calendar))
initcache{A<:AbstractArray}(calendars::A) = initcache(convert(Vector{HolidayCalendar}, calendars))

@vectorize_1arg HolidayCalendar initcache

# remove all elements from cache
function cleancache()
    for k in keys(_CACHE_DICT)
        delete!(_CACHE_DICT, k)
    end
end

"""
    cleancache([calendar])

Cleans cache for a given instance or list of `HolidayCalendar`, `Symbol` or `AbstractString`.
"""
function cleancache(hc::HolidayCalendar)
    if haskey(_CACHE_DICT, hc)
        delete!(_CACHE_DICT, hc)
    end
end

function cleancache(hc_vec::Vector{HolidayCalendar})
    for k in hc_vec
        if k in keys(_CACHE_DICT)
            delete!(_CACHE_DICT, k)
        end
    end
end

cleancache(calendar) = cleancache(convert(HolidayCalendar, calendar))
cleancache{A<:AbstractArray}(calendars::A) = cleancache(convert(Vector{HolidayCalendar}, calendars))

# Returns tuple
# tuple[1] = Array of Bool (isBday) , tuple[2] = Array of UInt32 (bdaycounter)
function _createbdayscache(hc::HolidayCalendar, d0::Date, d1::Date)

    const d0_ = min(d0, d1)
    const d1_ = max(d0, d1)

    const d0_rata = Dates.days(d0_)
    const d1_rata = Dates.days(d1_)

    # length of the cache arrays
    const len::Int = d1_rata - d0_rata + 1

    # This function uses UInt32 to store bdayscounter array
    # We need to check if we'll exceed typemax(UInt32)
    if len > convert(Int, typemax(UInt32))
        error("Maximum size allowed for bdays array is $(typemax(UInt32)). Required lenght of $(len).")
    end

    isbday_array = zeros(Bool, len)
    bdayscounter_array = zeros(UInt32, len)

    isbday_array[1] = isbday(hc, d0_)
    bdayscounter_array[1] = 0

    for i in 2:len
        isbday_array[i] = isbday(hc, d0_ + Dates.Day(i-1))
        bdayscounter_array[i] = bdayscounter_array[i-1] + isbday_array[i]
    end

    return isbday_array, bdayscounter_array
end


# Molecular Dynamics Result Abstract Interface

"""
    MolecularDynamicsResult
Abstract type to be extended by all concrete structs representing the result of a molecular dynamics simulation.
"""
abstract type MolecularDynamicsResult end

"""
    get_system(result::MolecularDynamicsResult, t::Integer = 0)::AbstractSystem

Extract the underlying system at a particular timestep from the simulation result.

An implementer of this API should implement a method of this function for their custom result type.
"""
function get_system(result::MolecularDynamicsResult, t::Integer = 0)::AbstractSystem
    throw(UnimplementedError(:get_system, result))
end
"""
    get_time_range(result::MolecularDynamicsResult)::Vector{<:Real}

Extract the time range from the simulation result in atomic units.

An implementer of this API should implement a method of this function for their custom result type.
"""
function get_time_range(result::MolecularDynamicsResult)::Vector{<:Real}
    throw(UnimplementedError(:get_time_range, result))
end

"""
    temperature(result::MolecularDynamicsResult, t::Integer = 0)::Real

Extract the temperature of the simulation at a particular timestep from the simulation result in atomic units.

An implementer of this API should implement a method of this function for their custom result type.
"""
function temperature(result::MolecularDynamicsResult, t::Integer = 0)::Real
    throw(UnimplementedError(:temperature, result))
end
"""
    reference_temperature(result::MolecularDynamicsResult)::Union{Real,Missing}

Extract the reference temperature of the simulation from the simulation result.
If there is no thermostat with a reference temperature in this simulation, return missing.

An implementer of this API should implement a method of this function for their custom result type if it supports thermostats.
If not implmented, the default implemenation just returns missing.
"""
function reference_temperature(result::MolecularDynamicsResult)::Union{Real,Missing}
    missing
end

"""
    kinetic_energy(result::MolecularDynamicsResult, t::Integer = 0)::Real

Extract the kinetic energy of the simulation at a particular timestep from the simulation result in atomic units.

An implementer of this API should implement a method of this function for their custom result type.
"""
function kinetic_energy(result::MolecularDynamicsResult, t::Integer = 0)::Real
    throw(UnimplementedError(:kinetic_energy, result))
end
"""
    potential_energy(result::MolecularDynamicsResult, t::Integer = 0)::Real

Extract the potential energy of the simulation at a particular timestep from the simulation result in atomic units.

An implementer of this API should implement a method of this function for their custom result type.
"""
function potential_energy(result::MolecularDynamicsResult, t::Integer = 0)::Real
    throw(UnimplementedError(:potential_energy, result))
end
"""
    total_energy(result::MolecularDynamicsResult, t::Integer = 0)::Real

Extract the total energy of the simulation at a particular timestep from the simulation result in atomic units.

The default implementation simply sums the kinetic and potential energy at the timestep.
An implementer of this API could implement a method of this function for their custom result type if it supports a more efficient way to calculate this quantity.
"""
function total_energy(result::MolecularDynamicsResult, t::Integer = 0)::Real
    kinetic_energy(result, t) + potential_energy(result, t)
end

"""
    rdf(result::MolecularDynamicsResult, sample_fraction::Float64 = 1.0)::Tuple{Vector{<:Real},Vector{<:Real}}

Calculate the radial distribution function from the simulation result.

To include only a trailing portion of the timesteps for reduced noise and faster computation, set sample_fraction to be less than 1.0.
The result is a tuple of vectors which represent the radial distances (in atomic units) and the value of the rdf at each distance respectively.
"""
function rdf(result::MolecularDynamicsResult, sample_fraction::Float64 = 1.0)::Tuple{Vector{<:Real},Vector{<:Real}}
    throw(UnimplementedError(:rdf, result))
end


##################################################################
# Abstract types

"""
$(TYPEDEF)
Apex type for grid components.
"""
abstract type AbstractGridComponent <: AbstractExtendableGridApexType end

"""
$(TYPEDEF)
1D Array of floating point data
"""
abstract type AbstractGridFloatArray1D <: AbstractGridComponent end

"""
$(TYPEDEF)
2D Array of floating point data
"""
abstract type AbstractGridFloatArray2D <: AbstractGridComponent end

"""
$(TYPEDEF)
1D Array of interger data
"""
abstract type AbstractGridIntegerArray1D <: AbstractGridComponent end

"""
$(TYPEDEF)
2D Array of integer data
"""
abstract type AbstractGridIntegerArray2D <: AbstractGridComponent end

"""
$(TYPEDEF)
Integer number
"""
abstract type AbstractGridIntegerConstant <: AbstractGridComponent end

"""
$(TYPEDEF)
Floating point number
"""
abstract type AbstractGridFloatConstant <: AbstractGridComponent end

"""
$(TYPEDEF)

Any kind of adjacency between grid components
"""
abstract type AbstractGridAdjacency <: AbstractGridComponent end

"""
$(TYPEDEF)

Array of element geometry information.
"""
abstract type AbstractElementGeometries <: AbstractGridComponent end

"""
$(TYPEDEF)

Array of element region number information.
"""
abstract type AbstractElementRegions <: AbstractGridComponent end

##################################################################
# Component key types

"""
$(TYPEDEF)

Node coordinates
"""
abstract type Coordinates <: AbstractGridFloatArray2D end

"""
$(TYPEDEF)

Adjacency describing nodes per grid cell
"""
abstract type CellNodes <: AbstractGridAdjacency end

"""
$(TYPEDEF)

Adjacency describing nodes per grid boundary face
"""
abstract type BFaceNodes <: AbstractGridAdjacency end

"""
$(TYPEDEF)

Description of cell geometries
"""
abstract type CellGeometries <: AbstractElementGeometries end

"""
Description of boundary face geometries

$(TYPEDEF)
"""
abstract type BFaceGeometries <: AbstractElementGeometries end

"""
$(TYPEDEF)

Cell region number per cell
"""
abstract type CellRegions <: AbstractElementRegions end

"""
$(TYPEDEF)

Boundary region number per boundary face
"""
abstract type BFaceRegions <: AbstractElementRegions end

"""
$(TYPEDEF)

Number of cell regions
"""
abstract type NumCellRegions <: AbstractGridIntegerConstant end

"""
$(TYPEDEF)

Number of boundary face regions
"""
abstract type NumBFaceRegions <: AbstractGridIntegerConstant end

"""
$(TYPEDEF)

Coordinate system
"""
abstract type CoordinateSystem <: AbstractGridComponent end

"""
$(TYPEDEF)

Boundary edge region number per boundary edge
"""
abstract type BEdgeRegions <: AbstractElementRegions end

"""
$(TYPEDEF)

Number of boundary edge regions
"""
abstract type NumBEdgeRegions <: AbstractGridIntegerConstant end

############################################################
# Grid type

"""
$(TYPEDEF)
Grid type wrapping Dict
"""
mutable struct ExtendableGrid{Tc,Ti}
    components::Dict{Type{<:AbstractGridComponent},Any}
    ExtendableGrid{Tc,Ti}() where{Tc,Ti} =new(Dict{Type{<:AbstractGridComponent},Any}())
end

#############################################################
#
"""
````
const ElementInfo{T}=Union{Vector{T},VectorOfConstants{T}}`
````

Union type for element information arrays. If all elements have
the same information, it can be stored in an economical form
as a [`VectorOfConstants`](@ref).
"""
const ElementInfo{T}=Union{Vector{T},VectorOfConstants{T}}

############################################################
# Generic component methods

"""
$(TYPEDSIGNATURES)
Default veryform method.

"veryform"  means "verify and/or transform"  and is called to check
and possibly transform components to be added to the grid via `setindex!`.

The default method just passes data through.
"""
veryform(grid::ExtendableGrid,v,::Type{<:AbstractGridComponent})=v

"""
$(TYPEDSIGNATURES)

Set new grid component
"""
Base.setindex!(grid::ExtendableGrid,v,T::Type{<:AbstractGridComponent})= grid.components[T]=veryform(grid,v,T)

"""
$(TYPEDSIGNATURES)

To be called by getindex. This triggers lazy creation of
non-existing gridcomponents
"""
Base.get!(grid::ExtendableGrid,T::Type{<:AbstractGridComponent})= get!( ()->veryform(grid,instantiate(grid,T),T),  grid.components,T)

"""
$(TYPEDSIGNATURES)

"Hook" for methods instantiating lazy components.
"""
function instantiate end

"""
````
Base.getindex(grid::ExtendableGrid,T::Type{<:AbstractGridComponent})
````

Generic method for obtaining grid component.

This method is mutating in the sense that non-existing grid components
are created on demand.

Due to the fact that components are stored as Any the return
value triggers type instability.
"""
Base.getindex(grid::ExtendableGrid,T::Type{<:AbstractGridComponent})=get!(grid,T)

############################################################
# Verfication of inserted data

"""
````
veryform(grid::ExtendableGrid{Tc,Ti},v,T::Type{<:AbstractGridAdjacency}) where{Tc,Ti}
````

Check proper type of adjacencies upon insertion
"""
veryform(grid::ExtendableGrid{Tc,Ti},v,T::Type{<:AbstractGridAdjacency}) where{Tc,Ti}= typeof(v)<:Adjacency{Ti} ? v : throw("Type mismatch")

############################################################
# Type stable component access

"""
````
Base.getindex(grid::ExtendableGrid{Tc,Ti},T::Type{<:AbstractGridAdjacency})::Adjacency{Ti} where{Tc,Ti}
````

Type stable return of adjacency component
"""
function Base.getindex(grid::ExtendableGrid{Tc,Ti},T::Type{<:AbstractGridAdjacency})::Adjacency{Ti} where{Tc,Ti}
    get!(grid,T)
end

"""
````
Base.getindex(grid::ExtendableGrid{Tc,Ti},T::Type{<:AbstractGridFloatArray2D})::Array{Tc,2}
````

Type stable c method to obtain 2D array from grid
"""
function Base.getindex(grid::ExtendableGrid{Tc,Ti},T::Type{<:AbstractGridFloatArray2D})::Array{Tc,2} where{Tc,Ti}
    get!(grid,T)
end

"""
````
Base.getindex(grid::ExtendableGrid{Tc,Ti},T::Type{<:AbstractGridFloatArray1D})::Array{Tc,1} where{Tc,Ti}
````

Type stable method to obtain 1D array from grid
"""
function Base.getindex(grid::ExtendableGrid{Tc,Ti},T::Type{<:AbstractGridFloatArray1D})::Array{Tc,1} where{Tc,Ti}
    get!(grid,T)
end

"""
````
Base.getindex(grid::ExtendableGrid{Tc,Ti},T::Type{<:AbstractElementGeometries})::ElementInfo{DataType}
````

Type stable method to obtain  element type
"""
function Base.getindex(grid::ExtendableGrid{Tc,Ti},T::Type{<:AbstractElementGeometries})::ElementInfo{DataType} where{Tc,Ti}
    get!(grid,T)
end

"""
````
Base.getindex(grid::ExtendableGrid{Tc,Ti},T::Type{<:AbstractElementRegions})::ElementInfo{Ti} where{Tc,Ti}
````

Type stable method to obtain element region number
"""
function Base.getindex(grid::ExtendableGrid{Tc,Ti},T::Type{<:AbstractElementRegions})::ElementInfo{Ti} where{Tc,Ti}
    get!(grid,T)
end

"""
````
Base.getindex(grid::ExtendableGrid{Tc,Ti},T::Type{<:AbstractGridIntegerConstant})::Ti where{Tc,Ti}
````

Type stable method to obtain  integer constant stored on elements
"""
function Base.getindex(grid::ExtendableGrid{Tc,Ti},T::Type{<:AbstractGridIntegerConstant})::Ti where{Tc,Ti}
    get!(grid,T)
end

"""
````
Base.getindex(grid::ExtendableGrid{Tc,Ti},T::Type{<:AbstractGridFloatConstant})::Tc where{Tc,Ti}
````

Type stable  method to obtain float constant stored on elements
"""
function Base.getindex(grid::ExtendableGrid{Tc,Ti},T::Type{<:AbstractGridFloatConstant})::Tc where{Tc,Ti}
    get!(grid,T)
end

############################################################
# Instantiation methods

"""
$(TYPEDSIGNATURES)

Instantiate number of cell regions
"""
instantiate(grid, ::Type{NumCellRegions})=maximum(grid[CellRegions])

"""
$(TYPEDSIGNATURES)

Instantiate number of bface regions
"""
instantiate(grid, ::Type{NumBFaceRegions})=maximum(grid[BFaceRegions])

"""
$(TYPEDSIGNATURES)

Instantiate number of boundary edge regions
"""
instantiate(grid, ::Type{NumBEdgeRegions})=maximum(grid[BEdgeRegions])

function prepare_bedgeregions!(grid::ExtendableGrid)
    bedges             = grid[BEdgeNodes]
    bedgeregions       = zeros(Int32, num_bedges(grid))
    grid[BEdgeRegions] = bedgeregions
end

instantiate(grid, ::Type{BEdgeRegions})=prepare_bedgeregions!(grid)

#############################################################
# General methods
"""
$(TYPEDSIGNATURES)

Print the hierarchy of grid component key types (subtypes of [`AbstractGridComponent`](@ref).
This includes additionally user defined subptypes.
"""
gridcomponents()=AbstractTrees.print_tree(AbstractGridComponent,5,indicate_truncation=false)

"""
$(TYPEDSIGNATURES)

Keys in grid
"""
Base.keys(g::ExtendableGrid)=Base.keys(g.components)

"""
$(TYPEDSIGNATURES)

Check if key is in grid
"""
Base.haskey(g::ExtendableGrid,k) = Base.haskey(g.components,k)

"""
$(SIGNATURES)

Space dimension of grid
"""
dim_space(grid::ExtendableGrid)= size(grid[Coordinates],1)

"""
$(SIGNATURES)

Grid dimension dimension of grid (larges element dimension)
"""
dim_grid(grid::ExtendableGrid)=  dim_element(grid[CellGeometries][1])

"""
$(SIGNATURES)

Number of nodes in grid
"""
num_nodes(grid::ExtendableGrid)::Int= size(grid[Coordinates],2)

"""
$(TYPEDSIGNATURES)

Number of cells in grid
"""
num_cells(grid::ExtendableGrid)= num_sources(grid[CellNodes])

"""
$(TYPEDSIGNATURES)

Number of boundary faces in grid.
"""
num_bfaces(grid::ExtendableGrid)= haskey(grid,BFaceNodes) ? num_sources(grid[BFaceNodes]) : 0

"""
$(TYPEDSIGNATURES)

Number of boundary edges in grid.
"""
num_bedges(grid::ExtendableGrid)= haskey(grid,BEdgeNodes) ? num_sources(grid[BEdgeNodes]) : 0

"""
$(TYPEDSIGNATURES)

Maximum  cell  region number
"""
num_cellregions(grid::ExtendableGrid)=grid[NumCellRegions]

"""
$(TYPEDSIGNATURES)

Maximum  boundary face region numbers
"""
num_bfaceregions(grid::ExtendableGrid)=grid[NumBFaceRegions]

"""
$(TYPEDSIGNATURES)

Maximum boundary edge region numbers
"""
num_bedgeregions(grid::ExtendableGrid)=grid[NumBEdgeRegions]

"""
$(SIGNATURES)
Type of coordinates in grid
"""
coord_type(grid::ExtendableGrid{Tc, Ti}) where {Tc,Ti}=Tc

"""
$(SIGNATURES)

Type of indices
"""
index_type(grid::ExtendableGrid{Tc, Ti}) where {Tc,Ti}=Ti

"""
$(TYPEDSIGNATURES)

Map a function onto node coordinates of grid
"""
function Base.map(f::Function, grid::ExtendableGrid)
    coord=grid[Coordinates]
    dim=dim_space(grid)
    if dim==1
        @views Base.map(f,coord[1,:])
    elseif dim==2
        @views Base.map(f,coord[1,:], coord[2,:])
    else
        @views Base.map(f,coord[1,:], coord[2,:], coord[3,:])
    end
end

function Base.show(io::IO,grid::ExtendableGrid)
    if num_edges(grid)>0
        str=@sprintf("%s;\ndim: %d nodes: %d cells: %d bfaces: %d, edges: %d\n",
                     typeof(grid),dim_space(grid),num_nodes(grid), num_cells(grid), num_bfaces(grid), num_edges(grid))
    else
        str=@sprintf("%s;\ndim: %d nodes: %d cells: %d bfaces: %d\n",
                     typeof(grid),dim_space(grid),num_nodes(grid), num_cells(grid), num_bfaces(grid))
    end
    println(io,str)
end

"""
$(SIGNATURES)

Recursively check seeming equality of two grids. Seemingly means
that long arrays are only compared via random samples
"""
function seemingly_equal(grid1::ExtendableGrid, grid2::ExtendableGrid)
    for key in keys(grid1)
        if !haskey(grid2,key)
            return false
        end
        if !seemingly_equal(grid1[key],grid2[key])
            return false
        end
    end
    return true
end

"""
$(SIGNATURES)

Check for seeming equality of two arrays by random sample.
"""
function seemingly_equal(array1::AbstractArray, array2::AbstractArray)
    if size(array1)!=size(array2)
        return false
    end
    l=length(array1)
    ntests=Float64(min(l,50))
    p=min(0.5,ntests/l)
    for i in randsubseq( 1:l,p)
        if !seemingly_equal(array1[i],array2[i])
            return false
        end
    end
    true
end

seemingly_equal(a1::VariableTargetAdjacency, a2::VariableTargetAdjacency)=seemingly_equal(a1.colentries,a2.colentries)&& seemingly_equal(a1.colstart, a2.colstart)
seemingly_equal(x1::Type, x2::Type)=(x1==x2)
seemingly_equal(x1::Number, x2::Number)=(x1≈x2)
seemingly_equal(x1::Any, x2::Any)=(x1==x2)

Base.extrema(grid::ExtendableGrid)=Base.extrema(grid[Coordinates],dims=2)

function bbox(grid)
    e=extrema(grid)
    map(a->a[1],e),map(a->a[2],e)
end


# sorting in data.frame is slow
if false
    N = 100_000_000
    K = 100
    using DataFrames
    srand(1)
    @time df = DataFrame(idstr = rand([@sprintf "id%03d" k for k in 1:(N/K)], N)
        , id = rand(1:K, N)
        , val = rand(1:5,N))

    @time sort!(df,cols=[:id, :val])

    using uCSV
    @time uCSV.write("df.csv", df);

    using CSV
    @time CSV.write("df.csv", df);
end


module PyPSANetwork

using AxisArrays

using PyCall
import Pandas
const PD = Pandas
# const PD = Base.require(Base.PkgId(Base.UUID("eadc2687-ae89-51f9-a5d9-86b5a6373a9c"), "Pandas"))

using DataFrames
using Dates

using ..EnergyModels
const EM = EnergyModels

using ..EnergyModels:
    AbstractData, Data, pushaxes!, resolve, @consense, Component, Device,
    DeviceFormulation, attributes, components, naming, astype, typenames

import ..EnergyModels: store_results!

using JuMP: value, dual, ConstraintRef, VariableRef, GenericAffExpr, LowerBoundRef, UpperBoundRef

export PypsaNetworkData

function EM.load(network::PyObject)::AbstractData
    pypsa = pyimport("pypsa")
    if pyisinstance(network, pypsa.Network)
        Data(fallback=PypsaNetworkData(network))
    else
        error("Python object is no PyPSA Network")
    end
end

abstract type AbstractAttrDesc end

struct AttrDesc{T,S} <: AbstractAttrDesc
    data::S
    indices::Union{Colon,UnitRange{Int64},StepRange{Int64,Int64},Vector{Int64}}
    default::Union{T,Missing}

    AttrDesc{T}(data::S, indices, default) where {T,S} = new{T,S}(data, indices, default)
end

struct SingleAttrDesc{T} <: AbstractAttrDesc
    value::T
end

struct ComponentDesc
    name::Symbol
    axis::Axis
    componenttype
    attributes::Dict{Symbol,AbstractAttrDesc}
    typeparams::Union{Nothing,Dict{Symbol,Any}}
end

struct PypsaNetworkData <: AbstractData
    network::PyObject
    components::Dict{Symbol}{ComponentDesc}
    axes::Dict{Symbol}{Axis}

    function PypsaNetworkData(network, components)
        snapshots = PD.values(PD.Index(network.snapshots))
        if pyisinstance(network.snapshots, pyimport("pandas").DatetimeIndex)
            snapshots = DateTime.(snapshots)
        end
        axes = Dict(:snapshots => Axis{:snapshots}(snapshots),
                    (name => cd.axis for (name, cd) in components)...)
        new(network, components, axes)
    end
end

splitgroup(df, group) = [group]
function splitgroup(df, group, attr, attrs...)
    gdf = groupby(DataFrame(attr=>df[group.idx,attr], :idx=>group.idx, copycols=false), attr);

    if length(gdf) == 1
        ng = attr == :carrier ? (idx=group.idx, name=(group.name..., first(parent(gdf)[!,attr]))) : group
        groups = splitgroup(df, ng, attrs...)
    else
        groups = Any[]
        for sdf in gdf
            ng = (idx=collect(sdf.idx),
                  name=(group.name..., first(sdf[!,attr])))
            append!(groups, splitgroup(df, ng, attrs...))
        end
    end

    groups
end

function AttrDesc(::Type{Val{:single}}, attr, name, df, pnl, idx, axis)
    val = @consense(PD.values(df[attr[:PyPSA]])[idx],
                    "$(attr[:PyPSA]) for group $name must be a single value")
    SingleAttrDesc(astype(attr[:dtype], val))
end

AttrDesc(::Type{Val{:static}}, attr, name, df, pnl, idx, axis) =
    AttrDesc{typenames[attr[:dtype]]}(df[attr[:PyPSA]], idx, attr[:default])

function AttrDesc(::Type{Val{:series}}, attr, name, df, pnl, idx, axis)
    df_t = get(pnl, attr[:PyPSA], nothing)
    indices_t = !isnothing(df_t) ? findall(in(Symbol.(PD.values(PD.columns(df_t)))), axis) : []
    if length(indices_t) == 0
        AttrDesc(Val{:static}, attr, name, df, pnl, idx, axis)
    else
        @assert(length(indices_t) == length(axis),
                "$name has static and time-dependent $(attr[:PyPSA])")
        AttrDesc{typenames[attr[:dtype]]}(df_t, indices_t, attr[:default])
    end
end

attributedescriptions(name, df, pnl, idx, axis; attributes=nothing) =
    Dict(attr[:attribute] => AttrDesc(Val{attr[:dimensions]}, attr, name, df, pnl, idx, axis)
         for attr in eachrow(attributes)
         if attr[:PyPSA] in PD.values(PD.columns(df)))

function typeparamdescriptions(name, df, indices; types=nothing)
    isnothing(types) && return nothing
    typename = @consense(PD.values(df.type)[indices], "$name may not have more than one type")
    typename == "" && return nothing
    typ = types[types.name .== typename, :]
    Dict(c => typ[1, c] for c in names(typ) if c != :name)
end

"""
    PypsaNetworkData

"""
function PypsaNetworkData(net)
    # Determine defined components and split them into classes
    components = vcat(
        getcomponents(net, :buses),
        getcomponents(net, :generators, withname=false, exp=:p_nom_extendable, uc=:committable),
        getcomponents(net, :storageunits, withname=false, pypsaname=:storage_units, exp=:p_nom_extendable),
        getcomponents(net, :stores, carrier=:bus, exp=:e_nom_extendable),
        getcomponents(net, :loads, carrier=:bus),
        getcomponents(net, :links, carrier=(:bus0, :bus1), exp=:p_nom_extendable),
        getcomponents(net, :lines, carrier=:bus0, exp=:s_nom_extendable, types=:line_types),
        getcomponents(net, :transformers, carrier=:bus0, exp=:s_nom_extendable, types=:transformer_types)
    )

    compdict = Dict(cd.name => cd for cd in components)

    PypsaNetworkData(net, compdict)
end

function _maybe_as_range(x::AbstractArray{T,1}) where T<:Number
    if length(x) < 3 return Vector(x) end
    d = diff(x)
    s = d[1]
    if all(d .== s)
        s == 1 ? (x[1]:x[end]) : (x[1]:s:x[end])
    else
        Vector(x)
    end
end

_join(a...; delim="_") = join(a, delim)

function getcomponents(net, name; pypsaname=false, withname=true, carrier=true, exp=false, uc=false, types=nothing)
    if pypsaname === false
        pypsaname = name
    end

    df = PD.DataFrame(getproperty(net, pypsaname))
    pnl = Dict(k=>PD.DataFrame(v) for (k,v) in getproperty(net, Symbol(pypsaname, :_t)))

    components = ComponentDesc[]

    if isempty(df)
        return components
    end

    initialname = withname ? (name,) : ()

    attrs = DataFrame()
    if carrier === true
        attrs[!, :carrier] = PD.values(df.carrier)
    elseif carrier !== false
        buses_carrier = PD.Series(net.buses.carrier)
        attrs[!, :carrier] =
            if carrier isa Tuple
                _join.((map(buses_carrier, df[field]) for field in carrier)...)
            else
                PD.values(map(buses_carrier, df[carrier]))
            end
    end

    if exp !== false
        attrs[!, exp] = recode(Bool.(PD.values(df[exp])), false=>:fix, true=>:ext)
    end
    if uc !== false
        attrs[!, uc] = recode(Bool.(PD.values(df[uc])), false=>:lin, true=>:uc)
    end

    if !isnothing(types)
        types = DataFrame(PD.reset_index(PD.DataFrame(getproperty(net, types))))
    end

    componenttypeunion = resolve(Component, name)
    componentattributes = copy(attributes(componenttypeunion))
    componentattributes = componentattributes[componentattributes.quantitytype .!= "Variable", :]
    componentattributes[!, :PyPSA] = string.(componentattributes.attribute)
    componentattributes[!, :dimensions] = recode(length.(componentattributes.dimensions),
                                                 0=>:single, 1=>:static, 2=>:series,
                                                 3:20=>missing)

    names = PD.values(PD.index(df))
    for (timeattr, df) in pnl
        if !isempty(df)
            label = Symbol(:t_, timeattr)
            attrs[!, label] = recode(in.(names, Ref(PD.values(PD.columns(df)))),
                                     false=>missing, true=>label)
        end
    end

    groups =
        if isempty(attrs)
            [(idx=1:length(names),
              name=Symbol(join(skipmissing(initialname), "_")),
              attrs=Dict{Symbol}{Any}())]
        else
            [(idx=_maybe_as_range(group.idx),
              name=Symbol(join(skipmissing(group.name), "_")),
              attrs=Dict(pairs(attrs[first(group.idx),:])...))
             for group in splitgroup(attrs, (idx=1:size(attrs, 1), name=initialname), Base.names(attrs)...)]
        end

    for g in groups
        axis = Axis{g.name}(Symbol.(names[g.idx]))

        formulation = join(skipmissing([
            exp !== false ? ((coalesce(g.attrs[exp], :fix) == :ext) ? "linexp" : missing) : missing,
            uc !== false ?  ((coalesce(g.attrs[uc], :lin) == :uc)   ? "ucdisp" : "lindisp") : "lindisp"
        ]), "_")

        componenttype =
            if componenttypeunion <: Device && !isempty(formulation)
                componenttypeunion{resolve(DeviceFormulation, Symbol(formulation))}
            else
                componenttypeunion
            end

        attributes = attributedescriptions(g.name, df, pnl, g.idx, axis; attributes=componentattributes)
        typeparams = typeparamdescriptions(g.name, df, g.idx; types=types)

        push!(components, ComponentDesc(g.name, axis, componenttype, attributes, typeparams))
    end

    components
end

_value(a::AxisArray) = a
_value(a::AxisArray{<:ConstraintRef}) = AxisArray(dual.(a), AxisArrays.axes(a)...)
_value(a::AxisArray{<:VariableRef}) = AxisArray(value.(a), AxisArrays.axes(a)...)
_value(a::AxisArray{<:GenericAffExpr}) = AxisArray(value.(a), AxisArrays.axes(a)...)

function _set!(df, args...)
    @assert(length(args) % 2 == 0, "Needs an even number of arguments")
    arrays = map(_value, args[2:2:end])
    axes = AxisArrays.axes(first(arrays))
    factors = [EM.WrappedArray(f, axes...) for f in args[1:2:end]]
    agg = AxisArray(Array{Float64}(undef, size(first(arrays))), axes)
    for I in eachindex(IndexCartesian(), agg)
        agg[I] = sum(f[Tuple(I)...] * v[I] for (f,v) in zip(factors, arrays))
    end
    _set!(df, agg)
end

function _set!(ser::PD.Series, data)
    ax, = AxisArrays.axes(data)
    PD.loc(ser)[ax.val] = _value(data)
end

function _set!(df::PD.DataFrame, data)
    B, T = AxisArrays.axes(data)
    for (idx, b) in zip(PD.columns(df).pyo.get_indexer(B), B)
        if idx == -1
            df[b] = PD.Series(_value(data[b, :]), index=T.val)
        else
            PD.loc(df)[T.val, idx] = _value(data[b, :])
        end
    end
end

function _set!(df::PyObject, data)
    pd = pyimport("pandas")
    if pyisinstance(df, pd.DataFrame)
        _set!(PD.DataFrame(df), data)
    elseif pyisinstance(df, pd.Series)
        _set!(PD.Series(df), data)
    else
        error("Needs pandas series or dataframe")
    end
end

function store_results!(data::PypsaNetworkData, g::EM.Generator)
    _set!(data.network.generators_t["p"], g[:p])
    _set!(data.network.generators.p_nom_opt, g[:p_nom])
end

function store_results!(data::PypsaNetworkData, s::EM.StorageUnit)
    _set!(data.network.storage_units_t["p"], 1, s[:p_dispatch], -1, s[:p_store])
    _set!(data.network.storage_units_t["state_of_charge"], s[:state_of_charge])
    _set!(data.network.storage_units_t["spill"], s[:spill])
    _set!(data.network.storage_units.p_nom_opt, s[:p_nom])
end

function store_results!(data::PypsaNetworkData, s::EM.Store)
    _set!(data.network.stores_t["p"], s[:p])
    _set!(data.network.stores_t["e"], s[:e])
    _set!(data.network.stores.e_nom_opt, s[:e_nom])
end

function store_results!(data::PypsaNetworkData, l::EM.Load)
    m = EM.model(l)
    T = axis(m, :snapshots)
    L = axis(m, l)
    # TODO: FIX
    # _set!(data.network.loads_t["p"], EM.get(l, :p_set, L, T))
end

function _pypsaname(::Type{T}) where T<:EM.Component
    S = T
    while !(S isa Symbol)
        S = S.name
    end
    string(S)
end

_mu_lower(b::EM.Branch{DF}) where DF <: EM.ExpansionForm = b[:p_lower]
_mu_lower(b::EM.Branch{DF}) where DF <: EM.DispatchForm = AxisArray(LowerBoundRef.(b[:p]), AxisArrays.axes(b[:p])...)
_mu_upper(b::EM.Branch{DF}) where DF <: EM.ExpansionForm = l[:p_upper]
_mu_upper(b::EM.Branch{DF}) where DF <: EM.DispatchForm = AxisArray(UpperBoundRef.(b[:p]), AxisArrays.axes(b[:p])...)

function store_results!(data::PypsaNetworkData, b::T) where T <: EM.PassiveBranch
    pnl = data.network.pnl(_pypsaname(T))
    _set!(pnl["p0"], b[:p])
    _set!(pnl["p1"], -1, b[:p])
    _set!(pnl["mu_lower"], _mu_lower(b))
    _set!(pnl["mu_upper"], -1, _mu_upper(b))
    _set!(data.network.df(_pypsaname(T)).s_nom_opt, b[:s_nom])
end

function store_results!(data::PypsaNetworkData, l::EM.Link)
    pnl = data.network.links_t
    _set!(pnl["p0"], l[:p])
    _set!(pnl["p1"], AxisArray(- l[:efficiency], AxisArrays.axes(l[:efficiency])...), l[:p])
    _set!(pnl["mu_lower"], _mu_lower(l))
    _set!(pnl["mu_upper"], -1, _mu_upper(l))
end

function store_results!(data::PypsaNetworkData, b::EM.Bus)
    pnl = data.network.buses_t
    _set!(pnl["marginal_price"], b[:p_balance])
    _set!(pnl["p"], EM.expression(EM.NodalActivePower(), EM.model(b), b))
    # TODO: v_ang and v_mag_pu needs to be calculated from B
end

function store_results!(data::PypsaNetworkData, c)
    @warn "Storing nothing for $(naming(c))"
end

function store_results!(data::PypsaNetworkData, model::EM.EnergyModel)
    map(b->store_results!(data, b), values(model.buses))
    for d in devices(model)
        store_results!(data, d)
    end
end

Base.get(::PypsaNetworkData, attrdesc::SingleAttrDesc, _) = attrdesc.value

Base.get(data::PypsaNetworkData, attrdesc::AttrDesc{Symbol,S}, ax) where S<:PD.Series =
    AxisArray(Symbol.(PD.values(attrdesc.data)[attrdesc.indices]), ax)

Base.get(data::PypsaNetworkData, attrdesc::AttrDesc{T,S}, ax) where {T, S<:PD.Series} =
    AxisArray(PD.values(attrdesc.data)[attrdesc.indices], ax)

Base.get(data::PypsaNetworkData, attrdesc::AttrDesc{T,S}, ax) where {T, S<:PD.DataFrame} =
    AxisArray(attrdesc.data.pyo.values[:,attrdesc.indices], data.axes[:snapshots], ax)

function Base.get(data::PypsaNetworkData, component::Component, param::Symbol)
    componentdesc = data.components[naming(component)]
    attrdesc = get(componentdesc.attributes, param, nothing)
    !isnothing(attrdesc) ? get(data, componentdesc.attributes[param], componentdesc.axis) : nothing
end

EM.gettypeparams(data::PypsaNetworkData, device::Device) =
    data.components[naming(device)].typeparams

EM.axis(data::PypsaNetworkData, c::Component) = data.axes[naming(c)]
EM.axis(data::PypsaNetworkData, name::Symbol) = data.axes[name]

EM.components(data::PypsaNetworkData) = EM._components(data)

end


"""
GenX: An Configurable Capacity Expansion Model
Copyright (C) 2021,  Massachusetts Institute of Technology
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.
A complete copy of the GNU General Public License v2 (GPLv2) is available
in LICENSE.txt.  Users uncompressing this from an archive may not have
received this license file.  If not, see <http://www.gnu.org/licenses/>.
"""

@doc raw"""
	configure_cplex(solver_settings_path::String)

Reads user-specified solver settings from cplex\_settings.yml in the directory specified by the string solver\_settings\_path.

Returns a MathOptInterface OptimizerWithAttributes CPLEX optimizer instance to be used in the GenX.generate_model() method.

The CPLEX optimizer instance is configured with the following default parameters if a user-specified parameter for each respective field is not provided:

 - CPX\_PARAM\_EPRHS = 1e-6 (Constraint (primal) feasibility tolerances. See https://www.ibm.com/support/knowledgecenter/en/SSSA5P_12.5.1/ilog.odms.cplex.help/CPLEX/Parameters/topics/EpRHS.html)
 - CPX\_PARAM\_EPOPT = 1e-6 (Dual feasibility tolerances. See https://www.ibm.com/support/knowledgecenter/en/SSSA5P_12.5.1/ilog.odms.cplex.help/CPLEX/Parameters/topics/EpOpt.html)
 - CPX\_PARAM\_AGGFILL = 10 (Allowed fill during presolve aggregation. See https://www.ibm.com/support/knowledgecenter/en/SSSA5P_12.5.1/ilog.odms.cplex.help/CPLEX/Parameters/topics/AggFill.html)
 - CPX\_PARAM\_PREDUAL = 0 (Decides whether presolve should pass the primal or dual linear programming problem to the LP optimization algorithm. See https://www.ibm.com/support/knowledgecenter/en/SSSA5P_12.5.1/ilog.odms.cplex.help/CPLEX/Parameters/topics/PreDual.html)
 - CPX\_PARAM\_TILIM = 1e+75	(Limits total time solver. See https://www.ibm.com/support/knowledgecenter/en/SSSA5P_12.5.1/ilog.odms.cplex.help/CPLEX/Parameters/topics/TiLim.html)
 - CPX\_PARAM\_EPGAP = 1e-4	(Relative (p.u. of optimal) mixed integer optimality tolerance for MIP problems (ignored otherwise). See https://www.ibm.com/support/knowledgecenter/en/SSSA5P_12.5.1/ilog.odms.cplex.help/CPLEX/Parameters/topics/EpGap.html)
 - CPX\_PARAM\_LPMETHOD = 0 (Algorithm used to solve continuous models (including MIP root relaxation). See https://www.ibm.com/support/knowledgecenter/de/SSSA5P_12.7.0/ilog.odms.cplex.help/CPLEX/Parameters/topics/LPMETHOD.html)
 - CPX\_PARAM\_BAREPCOMP = 1e-8 (Barrier convergence tolerance (determines when barrier terminates). See https://www.ibm.com/support/knowledgecenter/en/SSSA5P_12.5.1/ilog.odms.cplex.help/CPLEX/Parameters/topics/BarEpComp.html)
 - CPX\_PARAM\_NUMERICALEMPHASIS = 0 (Numerical precision emphasis. See https://www.ibm.com/support/knowledgecenter/en/SSSA5P_12.5.1/ilog.odms.cplex.help/CPLEX/Parameters/topics/NumericalEmphasis.html)
 - CPX\_PARAM\_BAROBJRNG = 1e+75 (Sets the maximum absolute value of the objective function. See https://www.ibm.com/support/knowledgecenter/en/SSSA5P_12.5.1/ilog.odms.cplex.help/CPLEX/Parameters/topics/BarObjRng.html)
 - CPX\_PARAM\_SOLUTIONTYPE = 2 (Solution type for LP or QP. See https://www.ibm.com/support/knowledgecenter/hr/SSSA5P_12.8.0/ilog.odms.cplex.help/CPLEX/Parameters/topics/SolutionType.html)

"""
function configure_cplex(solver_settings_path::String)

    solver_settings = YAML.load(open(solver_settings_path))

    # Set solve to use CPLEX for MIP or LP problems
    # Optional setup parameters ############################################
    # Constraint (primal) feasibility tolerances. See https://www.ibm.com/support/knowledgecenter/en/SSSA5P_12.5.1/ilog.odms.cplex.help/CPLEX/Parameters/topics/EpRHS.html
    FeasibilityTol = 1e-6
    if (haskey(solver_settings, "Feasib_Tol"))
        FeasibilityTol = solver_settings["Feasib_Tol"]
    end

    # Dual feasibility tolerances. See https://www.ibm.com/support/knowledgecenter/en/SSSA5P_12.5.1/ilog.odms.cplex.help/CPLEX/Parameters/topics/EpOpt.html
    OptimalityTol = 1e-6
    if (haskey(solver_settings, "Optimal_Tol"))
        OptimalityTol = solver_settings["Optimal_Tol"]
    end

    # Allowed fill during presolve aggregation. See https://www.ibm.com/support/knowledgecenter/en/SSSA5P_12.5.1/ilog.odms.cplex.help/CPLEX/Parameters/topics/AggFill.html
    AggFill = 10
    if (haskey(solver_settings, "AggFill"))
        AggFill = solver_settings["AggFill"]
    end

    # Decides whether presolve should pass the primal or dual linear programming problem to the LP optimization algorithm. See https://www.ibm.com/support/knowledgecenter/en/SSSA5P_12.5.1/ilog.odms.cplex.help/CPLEX/Parameters/topics/PreDual.html
    PreDual = 0
    if (haskey(solver_settings, "PreDual"))
        PreDual = solver_settings["PreDual"]
    end

    # Limits total time solver. See https://www.ibm.com/support/knowledgecenter/en/SSSA5P_12.5.1/ilog.odms.cplex.help/CPLEX/Parameters/topics/TiLim.html
    TimeLimit = 1e+75
    if (haskey(solver_settings, "TimeLimit"))
        TimeLimit = solver_settings["TimeLimit"]
    end

    # Relative (p.u. of optimal) mixed integer optimality tolerance for MIP problems (ignored otherwise). See https://www.ibm.com/support/knowledgecenter/en/SSSA5P_12.5.1/ilog.odms.cplex.help/CPLEX/Parameters/topics/EpGap.html
    MIPGap = 1e-4
    if (haskey(solver_settings, "MIPGap"))
        MIPGap = solver_settings["MIPGap"]
    end

    # Algorithm used to solve continuous models (including MIP root relaxation). See https://www.ibm.com/support/knowledgecenter/de/SSSA5P_12.7.0/ilog.odms.cplex.help/CPLEX/Parameters/topics/LPMETHOD.html
    Method = 0
    if (haskey(solver_settings, "Method"))
        Method = solver_settings["Method"]
    end

    # Barrier convergence tolerance (determines when barrier terminates). See https://www.ibm.com/support/knowledgecenter/en/SSSA5P_12.5.1/ilog.odms.cplex.help/CPLEX/Parameters/topics/BarEpComp.html
    BarConvTol = 1e-8
    if (haskey(solver_settings, "BarConvTol"))
        BarConvTol = solver_settings["BarConvTol"]
    end

    # Numerical precision emphasis. See https://www.ibm.com/support/knowledgecenter/en/SSSA5P_12.5.1/ilog.odms.cplex.help/CPLEX/Parameters/topics/NumericalEmphasis.html
    NumericFocus = 0
    if (haskey(solver_settings, "NumericFocus"))
        NumericFocus = solver_settings["NumericFocus"]
    end

    # Sets the maximum absolute value of the objective function. See https://www.ibm.com/support/knowledgecenter/en/SSSA5P_12.5.1/ilog.odms.cplex.help/CPLEX/Parameters/topics/BarObjRng.html
    BarObjRng = 1e+75
    if (haskey(solver_settings, "BarObjRng"))
        BarObjRng = solver_settings["BarObjRng"]
    end

    # Solution type for LP or QP. See https://www.ibm.com/support/knowledgecenter/hr/SSSA5P_12.8.0/ilog.odms.cplex.help/CPLEX/Parameters/topics/SolutionType.html
    SolutionType = 2
    if (haskey(solver_settings, "SolutionType"))
        SolutionType = solver_settings["SolutionType"]
    end
    ########################################################################

    OPTIMIZER = optimizer_with_attributes(
        CPLEX.Optimizer,
        "CPX_PARAM_EPRHS" => FeasibilityTol,
        "CPX_PARAM_EPOPT" => OptimalityTol,
        "CPX_PARAM_AGGFILL" => AggFill,
        "CPX_PARAM_PREDUAL" => PreDual,
        "CPX_PARAM_TILIM" => TimeLimit,
        "CPX_PARAM_EPGAP" => MIPGap,
        "CPX_PARAM_LPMETHOD" => Method,
        "CPX_PARAM_BAREPCOMP" => BarConvTol,
        "CPX_PARAM_NUMERICALEMPHASIS" => NumericFocus,
        "CPX_PARAM_BAROBJRNG" => BarObjRng,
        "CPX_PARAM_SOLUTIONTYPE" => SolutionType,
    )
    return OPTIMIZER
end


"""
# module Potentials

# Examples

```jldoctest
julia>
```
"""
module Potentials

include("InfinitePotentialWells.jl")

end


# XXX Should this permute the dimensions?
dims(H::Hermitian{<:Number,<:Tensor}) = dims(parent(H))

# XXX Should this permute the dimensions?
dim(H::Hermitian{<:Number,<:Tensor}, i::Int) = dim(parent(H), i)

matrix(H::Hermitian{<:Number,<:Tensor}) = Hermitian(matrix(parent(H)))

# XXX Should this permute the indices?
inds(H::Hermitian{<:Number,<:Tensor}) = inds(parent(H))

# XXX Should this permute the indices?
ind(H::Hermitian{<:Number,<:Tensor}, i::Int) = ind(parent(H), i)

# XXX Should this tranpose the block locations?
nnzblocks(H::Hermitian{<:Number,<:Tensor}) = nnzblocks(parent(H))

# XXX Should this tranpose the block locations?
nzblocks(H::Hermitian{<:Number,<:Tensor}) = nzblocks(parent(H))

# XXX Should this tranpose the block locations?
eachnzblock(H::Hermitian{<:Number,<:Tensor}) = eachnzblock(parent(H))

nblocks(H::Hermitian{<:Number,<:Tensor}) = nblocks(parent(H))

blockview(H::Hermitian{<:Number,<:Tensor}, block) = Hermitian(blockview(parent(H), block))


module FedDCD

using LinearAlgebra
using Printf
using SparseArrays
using Random
using StatsBase
using Distributed
using LinearMaps
using IterativeSolvers
using DataFrames
using CSV
using Zygote
using Flux

export FedProxClient, ScaffoldClient, FedDCDClient, AccFedDCDClient, getObjValue
export FedProxServer, ScaffoldServer, FedDCDServer, AccFedDCDServer, getObjValue
export FedDCDClientNN, FedDCDServerNN, AccFedDCDClientNN, AccFedDCDServerNN
export FedProxClientNN, FedProxServerNN, ScaffoldClientNN, ScaffoldServerNN
export softmax, neg_log_loss
export split_data, splitDataByRow, labelTransform, writeToFile, splitDataByClass
export read_libsvm
export fedAvgAndProx, Scaffold, fedDCD, accfedDCD
export obj, accuracy
export getStochasticGrad, getGradient
export ComputeNewtonDirection, SoftmaxNewtonMethod, lineSearch, lineSearch2
export dot_product
export sgd!, newton!, adam!

include("./utils.jl")
include("./LogReg/logReg.jl")
include("./oracle.jl")
include("./Client/client.jl")
include("./Server/server.jl")
include("./training.jl")

end # module