function hasquadratic(m::LinQuadOptimizer)
    (m.obj_type == QuadraticObjective) || (length(cmap(m).q_less_than) + length(cmap(m).q_greater_than) + length(cmap(m).q_equal_to) > 0)
end

#=
    Optimize the model
=#

function MOI.optimize!(m::LinQuadOptimizer)
    # reset storage
    fill!(m.variable_primal_solution, NaN)
    fill!(m.variable_dual_solution, NaN)
    fill!(m.constraint_primal_solution, NaN)
    fill!(m.constraint_dual_solution, NaN)
    m.primal_status = MOI.UnknownResultStatus
    m.dual_status   = MOI.UnknownResultStatus
    m.primal_result_count = 0
    m.dual_result_count = 0

    t = time()
    if hasinteger(m)
        solve_mip_problem!(m)
    elseif hasquadratic(m)
        solve_quadratic_problem!(m)
    else
        solve_linear_problem!(m)
    end
    m.solvetime = time() - t

    # termination_status
    m.termination_status = get_termination_status(m)
    m.primal_status = get_primal_status(m)
    m.dual_status = get_dual_status(m)

    if m.primal_status in [MOI.FeasiblePoint, MOI.InfeasiblePoint]
        # primal solution exists
        get_variable_primal_solution!(m, m.variable_primal_solution)
        get_linear_primal_solution!(m, m.constraint_primal_solution)
        if hasquadratic(m)
            get_quadratic_primal_solution!(m, m.qconstraint_primal_solution)
        end
        m.primal_result_count = 1
        # CPLEX can return infeasible points
    elseif m.primal_status == MOI.InfeasibilityCertificate
        get_unbounded_ray!(m, m.variable_primal_solution)
        m.primal_result_count = 1
    end
    if m.dual_status in [MOI.FeasiblePoint, MOI.InfeasiblePoint]
        # dual solution exists
        get_variable_dual_solution!(m, m.variable_dual_solution)
        get_linear_dual_solution!(m, m.constraint_dual_solution)
        if hasquadratic(m)
            get_quadratic_dual_solution!(m, m.qconstraint_dual_solution)
        end
        m.dual_result_count = 1
        # dual solution may not be feasible
    elseif m.dual_status == MOI.InfeasibilityCertificate
        get_farkas_dual!(m, m.constraint_dual_solution)
        m.dual_result_count = 1
    end

    #=
        CPLEX has the dual convention that the sign of the dual depends on the
        optimization sense. This isn't the same as the MOI convention so we need
        to correct that.
    =#
    # TODO
    if MOI.get(m, MOI.ObjectiveSense()) == MOI.MaxSense
        m.constraint_dual_solution *= -1
        m.variable_dual_solution *= -1
    end
end

#=
    Result Count
=#
function MOI.get(m::LinQuadOptimizer, ::MOI.ResultCount)
    max(m.primal_result_count, m.dual_result_count)
end
MOI.canget(m::LinQuadOptimizer, ::MOI.ResultCount) = true

#=
    Termination status
=#

function MOI.get(m::LinQuadOptimizer, ::MOI.TerminationStatus)
    m.termination_status
end
MOI.canget(m::LinQuadOptimizer, ::MOI.TerminationStatus) = true

#=
    Primal status
=#

function MOI.get(m::LinQuadOptimizer, p::MOI.PrimalStatus)
    m.primal_status
end
function MOI.canget(m::LinQuadOptimizer, p::MOI.PrimalStatus)
    m.primal_result_count >= p.N
end

#=
    Dual status
=#

function MOI.get(m::LinQuadOptimizer, d::MOI.DualStatus)
    m.dual_status
end
function MOI.canget(m::LinQuadOptimizer, d::MOI.DualStatus)
    m.dual_result_count >= d.N
end

#=
    Objective Value
=#

function MOI.get(m::LinQuadOptimizer, attr::MOI.ObjectiveValue)
    if attr.resultindex == 1
        get_objective_value(m) + m.objective_constant
    else
        error("Unable to access multiple objective values")
    end
end
function MOI.canget(m::LinQuadOptimizer, attr::MOI.ObjectiveValue)
    if attr.resultindex == 1
        return true
    else
        return false
    end
end

#=
    Variable Primal solution
=#

function MOI.get(m::LinQuadOptimizer, ::MOI.VariablePrimal, v::VarInd)
    col = m.variable_mapping[v]
    return m.variable_primal_solution[col]
end
MOI.canget(m::LinQuadOptimizer, ::MOI.VariablePrimal, ::Type{VarInd}) = true

function MOI.get(m::LinQuadOptimizer, ::MOI.VariablePrimal, v::Vector{VarInd})
    MOI.get.(m, MOI.VariablePrimal(), v)
end
MOI.canget(m::LinQuadOptimizer, ::MOI.VariablePrimal, ::Type{Vector{VarInd}}) = true

#=
    Variable Dual solution
=#
isbinding(set::LE, value::Float64) = isapprox(set.upper, value)
isbinding(set::GE, value::Float64) = isapprox(set.lower, value)
isbinding(set::EQ, value::Float64) = isapprox(set.value, value)
isbinding(set::IV, value::Float64) = isapprox(set.lower, value) || isapprox(set.upper, value)
function MOI.get(m::LinQuadOptimizer, ::MOI.ConstraintDual, c::SVCI{<: LinSets})
    vref = m[c]
    col = m.variable_mapping[vref]
    # the variable reduced cost is only the constriant dual if the bound is active.
    set = MOI.get(m, MOI.ConstraintSet(), c)
    solval = m.variable_primal_solution[col]
    if isbinding(set, solval)
        return m.variable_dual_solution[col]
    else
        return 0.0
    end
end
MOI.canget(m::LinQuadOptimizer, ::MOI.ConstraintDual, ::Type{SVCI{S}}) where S <: LinSets = true

function MOI.get(m::LinQuadOptimizer, ::MOI.ConstraintDual, c::VVCI{<: Union{MOI.Zeros, MOI.Nonnegatives, MOI.Nonpositives}})
    rowrefs = m[c]
    out = Float64[]
    sizehint!(out, length(rowrefs))
    for ref in rowrefs
        push!(out, m.constraint_dual_solution[ref])
    end
    return out
end
MOI.canget(m::LinQuadOptimizer, ::MOI.ConstraintDual, ::Type{VVCI{S}}) where S <: Union{MOI.Zeros, MOI.Nonnegatives, MOI.Nonpositives} = true

#=
    Variable Bound Primal solution
=#

function MOI.get(m::LinQuadOptimizer, ::MOI.ConstraintPrimal, c::SVCI{<: LinSets})
    vref = m[c]
    col = m.variable_mapping[vref]
    return m.variable_primal_solution[col]
end
MOI.canget(m::LinQuadOptimizer, ::MOI.ConstraintPrimal, ::Type{SVCI{S}}) where S <: LinSets = true

function MOI.get(m::LinQuadOptimizer, ::MOI.ConstraintPrimal, c::VVCI{<: Union{MOI.Zeros, MOI.Nonnegatives, MOI.Nonpositives}})
    rowrefs = m[c]
    out = Float64[]
    sizehint!(out, length(rowrefs))
    for ref in rowrefs
        push!(out, m.constraint_primal_solution[ref])
    end
    return out
end
MOI.canget(m::LinQuadOptimizer, ::MOI.ConstraintPrimal, ::Type{VVCI{S}}) where S <: Union{MOI.Zeros, MOI.Nonnegatives, MOI.Nonpositives} = true

#=
    Constraint Primal solution
=#

function MOI.get(m::LinQuadOptimizer, ::MOI.ConstraintPrimal, c::LCI{<: LinSets})
    row = m[c]
    return m.constraint_primal_solution[row]+m.constraint_constant[row]
end
MOI.canget(m::LinQuadOptimizer, ::MOI.ConstraintPrimal, ::Type{LCI{S}}) where S <: LinSets = true

function MOI.get(m::LinQuadOptimizer, ::MOI.ConstraintPrimal, c::QCI{<: LinSets})
    row = m[c]
    return m.qconstraint_primal_solution[row]#+m.qconstraint_constant[row]
end
MOI.canget(m::LinQuadOptimizer, ::MOI.ConstraintPrimal, ::Type{QCI{S}}) where S <: LinSets = true

# vector valued constraint duals
function MOI.get(m::LinQuadOptimizer, ::MOI.ConstraintPrimal, c::VLCI{<: Union{MOI.Zeros, MOI.Nonnegatives, MOI.Nonpositives}})
    row = m[c]
    return m.constraint_primal_solution[row]+m.constraint_constant[row]
end
MOI.canget(m::LinQuadOptimizer, ::MOI.ConstraintPrimal, ::Type{VLCI{S}}) where S <: Union{MOI.Zeros, MOI.Nonnegatives, MOI.Nonpositives} = true

#=
    Constraint Dual solution
=#

_checkdualsense(::LCI{LE}, dual) = dual <= 0.0
_checkdualsense(::LCI{GE}, dual) = dual >= 0.0
_checkdualsense(::LCI{IV}, dual) = true
_checkdualsense(::LCI{EQ}, dual) = true

function MOI.get(m::LinQuadOptimizer, ::MOI.ConstraintDual, c::LCI{<: LinSets})
    row = m[c]
    dual = m.constraint_dual_solution[row]
    @assert _checkdualsense(c, dual)
    return dual
end
MOI.canget(m::LinQuadOptimizer, ::MOI.ConstraintDual, ::Type{LCI{S}}) where S <: LinSets = true

function MOI.get(m::LinQuadOptimizer, ::MOI.ConstraintDual, c::QCI{<: LinSets})
    row = m[c]
    return m.qconstraint_dual_solution[row]#+m.qconstraint_constant[row]
end
MOI.canget(m::LinQuadOptimizer, ::MOI.ConstraintDual, ::Type{QCI{S}}) where S <: LinSets = true

# vector valued constraint duals
MOI.get(m::LinQuadOptimizer, ::MOI.ConstraintDual, c::VLCI{<: Union{MOI.Zeros, MOI.Nonnegatives, MOI.Nonpositives}}) = m.constraint_dual_solution[m[c]]
MOI.canget(m::LinQuadOptimizer, ::MOI.ConstraintDual, ::Type{VLCI{S}}) where S <: Union{MOI.Zeros, MOI.Nonnegatives, MOI.Nonpositives} = true

#=
    Solution Attributes
=#

# struct ObjectiveBound <: MOI.AbstractOptimizerAttribute end
MOI.get(m::LinQuadOptimizer, ::MOI.ObjectiveBound) = get_objective_bound(m)
MOI.canget(m::LinQuadOptimizer, ::MOI.ObjectiveBound) = true

# struct RelativeGap <: MOI.AbstractOptimizerAttribute  end
MOI.get(m::LinQuadOptimizer, ::MOI.RelativeGap) = get_relative_mip_gap(m)
MOI.canget(m::LinQuadOptimizer, ::MOI.RelativeGap) = true

# struct SolveTime <: MOI.AbstractOptimizerAttribute end
MOI.get(m::LinQuadOptimizer, ::MOI.SolveTime) = m.solvetime
MOI.canget(m::LinQuadOptimizer, ::MOI.SolveTime) = true

# struct SimplexIterations <: MOI.AbstractOptimizerAttribute end
MOI.get(m::LinQuadOptimizer, ::MOI.SimplexIterations) = get_iteration_count(m)
MOI.canget(m::LinQuadOptimizer, ::MOI.SimplexIterations) = true

# struct BarrierIterations <: MOI.AbstractOptimizerAttribute end
MOI.get(m::LinQuadOptimizer, ::MOI.BarrierIterations) = get_barrier_iterations(m)
MOI.canget(m::LinQuadOptimizer, ::MOI.BarrierIterations) = true

# struct NodeCount <: MOI.AbstractOptimizerAttribute end
MOI.get(m::LinQuadOptimizer, ::MOI.NodeCount) = get_node_count(m)
MOI.canget(m::LinQuadOptimizer, ::MOI.NodeCount) = true

# struct RawSolver <: MOI.AbstractOptimizerAttribute end
MOI.get(m::LinQuadOptimizer, ::MOI.RawSolver) = m
MOI.canget(m::LinQuadOptimizer, ::MOI.RawSolver) = true


module GradientBoost

# Load sources files
include("util.jl")
include("loss.jl")
include("gb.jl")
include("gb_bl.jl")
include("gb_dt.jl")
include("ml.jl")

end # module


println("##### methods #####")

using Test
using TreeTools

## Testing equality operator
root_1 = TreeTools.read_newick("$(dirname(pathof(TreeTools)))/../test/methods/tree1.nwk")
root_2 = TreeTools.read_newick("$(dirname(pathof(TreeTools)))/../test/methods/tree1_reordered.nwk")
@testset "Equality `==`" begin
	@test root_1 == root_2
end

@testset "node2tree" begin
	@test typeof(node2tree(root_1)) <: Tree
	@test typeof(node2tree(root_2)) <: Tree
end

# Testing ancestors
@testset "Ancestors" begin
    root_1 = TreeTools.read_newick("$(dirname(pathof(TreeTools)))/../test/methods/tree1.nwk")
    @test TreeTools.isancestor(root_1, root_1.child[1])
    @test TreeTools.isancestor(root_1, root_1.child[1].child[1])
    @test !TreeTools.isancestor(root_1.child[1],root_1.child[2])
    root_2 = TreeTools.read_newick("$(dirname(pathof(TreeTools)))/../test/methods/tree2.nwk")
    @test lca((root_2.child[1].child[1], root_2.child[1].child[2])).label == "ABC"
    @test lca((root_2.child[1].child[1].child[1], root_2.child[1].child[1].child[2], root_2.child[1].child[2])).label == "ABC"
    @test lca((root_2.child[1].child[1], root_2.child[2])).label == "ABCD"
end

@testset "Count" begin
	t1 = node2tree(root_1)
	@test count(isleaf, t1) == 4
	@test count(n -> n.label[1] == 'A', t1) == 3
	@test count(isleaf, t1.lnodes["AB"]) == 2
	@test count(n -> n.label[1] == 'A', t1.lnodes["AB"]) == 2
end

@testset "Copy type" begin
	t1 = Tree(TreeNode(MiscData(Dict(1=>2))))
	tc = copy(t1)
	@test tc.root.data[1] == 2
	t1.root.data[1] = 3
	@test tc.root.data[1] == 2
end

@testset "Copy" begin
	t1 = node2tree(root_1)
	t2 = copy(t1)
	@test typeof(t1) == typeof(t2)
	prunesubtree!(t2, ["A"])
	@test haskey(t1.lnodes, "A")
	@test !haskey(t2.lnodes, "A")
end

@testset "Convert" begin
	t1 = Tree(TreeNode(MiscData(Dict(1=>2))))
	# No op
	@test convert(Tree{MiscData}, t1) === t1
	@test convert(Tree{MiscData}, t1).root.data === t1.root.data

	# Converting to EmptyData and back
	t2 = convert(Tree{TreeTools.EmptyData}, t1)
	t3 = convert(Tree{MiscData}, t2)
	@test typeof(t2) == Tree{TreeTools.EmptyData}
	@test t2.root.data == TreeTools.EmptyData()

	@test typeof(t3) == Tree{TreeTools.MiscData}
	@test !haskey(t3.root.data, 1)

	###

	t1 = Tree(TreeNode(TreeTools.EmptyData()))
	# No op
	@test convert(Tree{TreeTools.EmptyData}, t1) === t1

	# Converting to MiscData and back
	t2 = convert(Tree{TreeTools.MiscData}, t1)
	@test typeof(t2) == Tree{TreeTools.MiscData}
	@test isempty(t2.root.data)

	@test typeof(convert(Tree{TreeTools.EmptyData}, t2)) == Tree{TreeTools.EmptyData}
end

nwk = "(A:3,(B:1,C:1):2)"
@testset "Distance" begin
	t = parse_newick_string(nwk)
	# Branch length
	@test distance(t, "A", "B") == 6
	@test distance(t.lnodes["A"], t.lnodes["B"]) == 6
	@test distance(t, "A", "B") == distance(t, "A", "C")
	@test distance(t.root, t.lnodes["A"]) == 3
	# Topological
	@test distance(t.root, t.lnodes["A"]; topological=true) == 1
	@test distance(t, "A", "B"; topological=true) == distance(t, "A", "C"; topological=true)
	@test distance(t, "A", "B"; topological=true) == 3
	for n in nodes(t)
		@test distance(t[n.label], t[n.label]; topological=true) == 0
		@test distance(t[n.label], t[n.label]; topological=false) == 0
	end
	# tests below can be removed when `divtime` is removed
	@test divtime(t.lnodes["A"], t.lnodes["B"]) == 6
	@test divtime(t.root, t.lnodes["A"]) == 3
end


export rule

@rule GammaShapeRate(:out, Marginalisation) (m_α::PointMass, m_β::PointMass) = GammaShapeRate(mean(m_α), mean(m_β))

@rule GammaShapeRate(:out, Marginalisation) (q_α::PointMass, q_β::PointMass) = GammaShapeRate(mean(q_α), mean(q_β))


function cuITensor(::Type{T}, inds::IndexSet) where {T<:Number}
  return ITensor(Dense{float(T)}(CUDA.zeros(float(T), dim(inds))), inds)
end
cuITensor(::Type{T}, inds::Index...) where {T<:Number} = cuITensor(T, IndexSet(inds...))

cuITensor(is::IndexSet) = cuITensor(Float64, is)
cuITensor(inds::Index...) = cuITensor(IndexSet(inds...))

cuITensor() = ITensor()
function cuITensor(x::S, inds::IndexSet{N}) where {S<:Number,N}
  dat = CuVector{float(S)}(undef, dim(inds))
  fill!(dat, float(x))
  return ITensor(Dense{S}(dat), inds)
end
cuITensor(x::S, inds::Index...) where {S<:Number} = cuITensor(x, IndexSet(inds...))

#TODO: check that the size of the Array matches the Index dimensions
function cuITensor(A::Array{S}, inds) where {S<:Number}
  return ITensor(Dense(CuArray{S}(A)), inds)
end
function cuITensor(A::CuArray{S}, inds::IndexSet) where {S<:Number}
  return ITensor(Dense(A), inds)
end
cuITensor(A::Array{S}, inds::Index...) where {S<:Number} = cuITensor(A, IndexSet(inds...))
cuITensor(A::CuArray{S}, inds::Index...) where {S<:Number} = cuITensor(A, IndexSet(inds...))
function cuITensor(A::ITensor)
  return if storage(tensor(A)) isa ITensors.EmptyStorage
    cuITensor(zero(eltype(storage(tensor(A)))), inds(A)...)
  else
    cuITensor(data(tensor(A)), inds(A)...)
  end
end

cu(A::ITensor) = cuITensor(A)

function cpu(A::ITensor)
  typeof(data(storage(A))) <: CuArray && return ITensor(cpu(storage(A)), inds(A))
  return A
end

function randomCuITensor(::Type{S}, inds::Indices) where {S<:Real}
  T = cuITensor(S, inds)
  randn!(T)
  return T
end
function randomCuITensor(::Type{S}, inds::Indices) where {S<:Complex}
  Tr = cuITensor(real(S), inds)
  Ti = cuITensor(real(S), inds)
  randn!(Tr)
  randn!(Ti)
  return complex(Tr) + im * Ti
end
function randomCuITensor(::Type{S}, inds::Index...) where {S<:Number}
  return randomCuITensor(S, IndexSet(inds...))
end
randomCuITensor(inds::IndexSet) = randomCuITensor(Float64, inds)
randomCuITensor(inds::Index...) = randomCuITensor(Float64, IndexSet(inds...))

CuArray(T::ITensor) = CuArray(tensor(T))

function CuArray{ElT,N}(T::ITensor, is::Vararg{Index,N}) where {ElT,N}
  ndims(T) != N && throw(
    DimensionMismatch(
      "cannot convert an $(ndims(T)) dimensional ITensor to an $N-dimensional CuArray."
    ),
  )
  TT = tensor(permute(T, is...; allow_alias=true))
  return CuArray{ElT,N}(TT)::CuArray{ElT,N}
end

function CuArray{ElT}(T::ITensor, is::Vararg{Index,N}) where {ElT,N}
  return CuArray{ElT,N}(T, is...)
end

function CuArray(T::ITensor, is::Vararg{Index,N}) where {N}
  return CuArray{eltype(T),N}(T, is...)::CuArray{<:Number,N}
end

CUDA.CuMatrix(A::ITensor) = CuArray(A)

function CuVector(A::ITensor)
  if ndims(A) != 1
    throw(DimensionMismatch("Vector() expected a 1-index ITensor"))
  end
  return CuArray(A)
end

function CuMatrix(T::ITensor, i1::Index, i2::Index)
  ndims(T) != 2 &&
    throw(DimensionMismatch("ITensor must be order 2 to convert to a Matrix"))
  return CuArray(T, i1, i2)
end


### BasicMCJob

# BasicMCJob is used for sampling a single parameter via serial Monte Carlo
# It is the most elementary and typical Markov chain Monte Carlo (MCMC) job

type BasicMCJob <: MCJob
  model::GenericModel # Model of single parameter
  pindex::Integer # Index of single parameter in model.vertices
  parameter::Parameter # Points to model.vertices[pindex] for faster access
  sampler::MCSampler
  tuner::MCTuner
  range::BasicMCRange
  vstate::VariableStateVector # Vector of variable states ordered according to variables in model.vertices
  pstate::ParameterState # Points to vstate[pindex] for faster access
  sstate::MCSamplerState # Internal state of MCSampler
  outopts::Dict # Options related to output
  output::Union{VariableNState, VariableIOStream, Void} # Output of model's single parameter
  count::Integer # Current number of post-burnin iterations
  resetpstate::Bool # If resetpstate=true then pstate is reset by reset(job), else pstate is not modified by reset(job)
  plain::Bool # If plain=false then job flow is controlled via tasks, else it is controlled without tasks
  task::Union{Task, Void}
  verbose::Bool
  resetplain!::Function
  reset!::Function
  save!::Union{Function, Void}
  iterate!::Function
  run!::Function

  function BasicMCJob(
    model::GenericModel,
    pindex::Integer,
    sampler::MCSampler,
    tuner::MCTuner,
    range::BasicMCRange,
    vstate::VariableStateVector,
    outopts::Dict,
    resetpstate::Bool,
    plain::Bool,
    verbose::Bool,
    check::Bool
  )
    instance = new()

    instance.model = model
    instance.pindex = pindex
    instance.vstate = vstate
    instance.resetpstate = resetpstate

    if check
      checkin(instance)
    end

    instance.sampler = sampler
    instance.tuner = tuner
    instance.range = range
    instance.plain = plain
    instance.verbose = verbose

    instance.parameter = instance.model.vertices[instance.pindex]
    instance.parameter.states = instance.vstate

    instance.outopts = isa(outopts, Dict{Symbol, Any}) ? outopts : convert(Dict{Symbol, Any}, outopts)
    augment_parameter_outopts!(instance.outopts)

    instance.pstate = instance.vstate[instance.pindex]
    if any(isnan, instance.pstate.value)
      if instance.parameter.pdf != nothing
        instance.pstate.value = rand(instance.parameter.pdf)
      elseif instance.parameter.prior != nothing
        instance.pstate.value = rand(instance.parameter.prior)
      else
        error("Not possible to initialize pstate with missing pstate.value and without parameter.pdf or parameter.prior")
      end
    end
    initialize!(instance.pstate, instance.parameter, sampler, instance.outopts)

    instance.sstate = sampler_state(instance.parameter, sampler, tuner, instance.pstate, instance.vstate)

    instance.output = initialize_output(instance.pstate, range.npoststeps, instance.outopts)

    instance.count = 0

    instance.resetplain! = eval(codegen(:resetplain, instance))

    instance.save! = (instance.output == nothing) ? nothing : eval(codegen(:save, instance))

    instance.iterate! = eval(codegen(:iterate, instance))

    if plain
      instance.task = nothing
      instance.reset! = instance.resetplain!
    else
      instance.task = Task(() -> initialize_task!(instance))
      instance.reset! = eval(codegen(:resettask, instance))
    end

    instance.run! = eval(codegen(:run, instance))

    instance
  end
end

BasicMCJob(
  model::GenericModel,
  sampler::MCSampler,
  range::BasicMCRange,
  v0::VariableStateVector;
  pindex::Integer=findfirst(v::Variable -> isa(v, Parameter), model.vertices),
  tuner::MCTuner=VanillaMCTuner(),
  outopts::Dict=Dict(:destination=>:nstate, :monitor=>[:value], :diagnostics=>Symbol[]),
  resetpstate::Bool=true,
  plain::Bool=true,
  verbose::Bool=false,
  check::Bool=false
) =
  BasicMCJob(model, pindex, sampler, tuner, range, v0, outopts, resetpstate, plain, verbose, check)

function BasicMCJob{S<:VariableState}(
  model::GenericModel,
  sampler::MCSampler,
  range::BasicMCRange,
  v0::Dict{Symbol, S};
  pindex::Integer=findfirst(v::Variable -> isa(v, Parameter), model.vertices),
  tuner::MCTuner=VanillaMCTuner(),
  outopts::Dict=Dict(:destination=>:nstate, :monitor=>[:value], :diagnostics=>Symbol[]),
  resetpstate::Bool=true,
  plain::Bool=true,
  verbose::Bool=false,
  check::Bool=false
)
  vstate = Array(S, length(v0))
  for (k, v) in v0
    vstate[model.ofkey[k]] = v
  end
  BasicMCJob(model, pindex, sampler, tuner, range, vstate, outopts, resetpstate, plain, verbose, check)
end

function BasicMCJob(
  model::GenericModel,
  sampler::MCSampler,
  range::BasicMCRange,
  v0::Vector;
  pindex::Integer=findfirst(v::Variable -> isa(v, Parameter), model.vertices),
  tuner::MCTuner=VanillaMCTuner(),
  outopts::Dict=Dict(:destination=>:nstate, :monitor=>[:value], :diagnostics=>Symbol[]),
  resetpstate::Bool=true,
  plain::Bool=true,
  verbose::Bool=false,
  check::Bool=false
)
  vstate = default_state(model.vertices, v0, [outopts], [pindex])
  BasicMCJob(model, pindex, sampler, tuner, range, vstate, outopts, resetpstate, plain, verbose, check)
end

function BasicMCJob(
  model::GenericModel,
  sampler::MCSampler,
  range::BasicMCRange,
  v0::Dict;
  pindex::Integer=findfirst(v::Variable -> isa(v, Parameter), model.vertices),
  tuner::MCTuner=VanillaMCTuner(),
  outopts::Dict=Dict(:destination=>:nstate, :monitor=>[:value], :diagnostics=>Symbol[]),
  resetpstate::Bool=true,
  plain::Bool=true,
  verbose::Bool=false,
  check::Bool=false
)
  vstate = Array(Any, length(v0))
  for (k, v) in v0
    vstate[model.ofkey[k]] = v
  end

  BasicMCJob(
    model,
    sampler,
    range,
    vstate,
    pindex=pindex,
    tuner=tuner,
    outopts=outopts,
    resetpstate=resetpstate,
    plain=plain,
    verbose=verbose,
    check=check
  )
end

# It is likely that MCMC inference for parameters of ODEs will require a separate ODEBasicMCJob

codegen(f::Symbol, job::BasicMCJob) = codegen(Val{f}, job)

function codegen(::Type{Val{:resetplain}}, job::BasicMCJob)
  fsignature::Vector{Union{Symbol, Expr}}
  body = []

  if job.resetpstate
    push!(body, :(reset!(_job.pstate, _x, _job.parameter, _job.sampler)))
  end

  push!(body, :(reset!(_job.sstate, _job.pstate, _job.parameter, _job.sampler, _job.tuner)))

  if isa(job.output, VariableIOStream)
    push!(body, :(reset(_job.output)))
    push!(body, :(mark(_job.output)))
  end

  push!(body, :(_job.count = 0))

  @gensym _resetplain

  if job.resetpstate
    vform = variate_form(job.pstate)
    if vform == Univariate
      fsignature = Union{Symbol, Expr}[_resetplain, :(_job::BasicMCJob), :(_x::Real)]
    elseif vform == Multivariate
      fsignature = Union{Symbol, Expr}[:($_resetplain{N<:Real}), :(_job::BasicMCJob), :(_x::Vector{N})]
    else
      error("It is not possible to define plain reset for given job")
    end
  else
    fsignature = Union{Symbol, Expr}[_resetplain, :(_job::BasicMCJob)]
  end

  Expr(:function, Expr(:call, fsignature...), Expr(:block, body...))
end

function codegen(::Type{Val{:resettask}}, job::BasicMCJob)
  fsignature::Vector{Union{Symbol, Expr}}
  fbody::Expr

  @gensym _resettask

  if job.resetpstate
    vform = variate_form(job.pstate)
    if vform == Univariate
      fsignature = Union{Symbol, Expr}[_resettask, :(_job::BasicMCJob), :(_x::Real)]
    elseif vform == Multivariate
      fsignature = Union{Symbol, Expr}[:($_resettask{N<:Real}), :(_job::BasicMCJob), :(_x::Vector{N})]
    else
      error("It is not possible to define plain reset for given job")
    end
    fbody = :(_job.task.storage[:reset](_job, _x))
  else
    fsignature = Union{Symbol, Expr}[_resettask, :(_job::BasicMCJob)]
    fbody = :(_job.task.storage[:reset](_job))
  end

  Expr(:function, Expr(:call, fsignature...), Expr(:block, fbody))
end

function codegen(::Type{Val{:save}}, job::BasicMCJob)
  body = []

  if isa(job.output, VariableNState)
    push!(body, :(copy!(_job.output, _job.pstate, _i)))
  elseif isa(job.output, VariableIOStream)
    push!(body, :(write(_job.output, _job.pstate)))
    if job.outopts[:flush]
      push!(body, :(flush(_job.output)))
    end
  else
    error("To save output, :destination must be set to :nstate or :iostream, got $(job.outopts[:destination])")
  end

  @gensym _save

  quote
    function $_save(_job::BasicMCJob, _i::Integer)
      $(body...)
    end
  end
end

function codegen(::Type{Val{:run}}, job::BasicMCJob)
  result::Expr
  ifforbody = []
  forbody = []
  body = []

  if job.verbose
    fmt_iter = format_iteration(ndigits(job.range.nsteps))
    push!(forbody, :(println("Iteration ", $(fmt_iter)(i), " of ", _job.range.nsteps)))
  end

  if job.task == nothing
    push!(forbody, :(iterate(_job)))
  else
    push!(forbody, :(consume(_job.task)))
  end

  push!(ifforbody, :(_job.count+=1))
  if job.save! != nothing
    push!(ifforbody, :(save(_job, _job.count)))
  end

  push!(forbody, Expr(:if, :(in(i, _job.range.postrange)), Expr(:block, ifforbody...)))

  push!(body, Expr(:for, :(i = 1:_job.range.nsteps), Expr(:block, forbody...)))

  if isa(job.output, VariableIOStream)
    push!(body, :(close(_job.output)))
  end

  @gensym _run

  result = quote
    function $_run(_job::BasicMCJob)
      $(body...)
    end
  end

  result
end

function checkin(job::BasicMCJob)
  pindex = find(v::Variable -> isa(v, Parameter), job.model.vertices)
  np = length(pindex)

  if np == 0
    error("The model does not have a parameter, but a BasicMCJob requires a parameter")
  elseif np == 1
    if pindex[1] != job.pindex
      error("Parameter located in model.vertices[$(pindex[1])], but pindex = $(job.pindex)")
    end
  elseif np >= 2
    warn("The model has $np parameters, but a BasicMCJob requires exactly one parameter")
    if in(job.pindex, pindex)
      error("Indices of parameters in model.vertices do not contain pindex")
    end
  end

  nv = num_vertices(job.model)
  nvstate = length(job.vstate)

  if nv != nvstate
    warn("Number of variables ($nv) not equal to number of variable states ($nvstate)")
  end

  pstate = job.vstate[job.pindex]

  if !isa(pstate, ParameterState)
    error("The parameter's state must be saved in a ParameterState subtype, got $(typeof(pstate)) state type")
  else
    check_support(job.model.vertices[job.pindex], pstate)
  end
end

output(job::BasicMCJob) = job.output

function Base.show(io::IO, job::BasicMCJob)
  isplain = job.plain ? "job flow not controlled by tasks" : "job flow controlled by tasks"

  indentation = "  "

  println(io, "BasicMCJob:")
  print(io, indentation)
  show(io, job.parameter)
  print(io, "\n"*indentation)
  show(io, job.model)
  print(io, "\n"*indentation)
  show(io, job.sampler)
  print(io, "\n"*indentation)
  show(io, job.tuner)
  print(io, "\n"*indentation)
  show(io, job.range)
  print(io, "\n"*indentation*"plain = $(job.plain) ($isplain)")
end

Base.writemime(io::IO, ::MIME"text/plain", job::BasicMCJob) = show(io, job)


"""
GenX: An Configurable Capacity Expansion Model
Copyright (C) 2021,  Massachusetts Institute of Technology
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.
A complete copy of the GNU General Public License v2 (GPLv2) is available
in LICENSE.txt.  Users uncompressing this from an archive may not have
received this license file.  If not, see <http://www.gnu.org/licenses/>.
"""

function write_esr_prices(path::AbstractString, inputs::Dict, setup::Dict, EP::Model)
	dfESR = DataFrame(ESR_Price = convert(Array{Union{Missing, Float64}}, dual.(EP[:cESRShare])))
	if setup["ParameterScale"] == 1
		dfESR[!,:ESR_Price] = dfESR[!,:ESR_Price] * ModelScalingFactor # Converting MillionUS$/GWh to US$/MWh
	end
	CSV.write(joinpath(path, "ESR_prices.csv"), dfESR)
	return dfESR
end


module DenoiseDNA
using FASTX, BioSequences,  BioAlignments # BioJulia
import BioAlignments.pairalign
using StatsFuns, Loess,LazyStack, StatsBase,LoopVectorization, CodecZlib,DataFrames
using CairoMakie,UnicodePlots, Poppler_jll # visualisation
using RCall

include.(["plotting.jl",
        "fileIO.jl",
        "kmers.jl",
        "DADA.jl"])

# Write your package code here.
export quality_plot,
        dada,
        custering,
        trim,
        dereplicate,
        get_files,
        learnErrors
end


include("../../RawFileofflineProc/dataStruct.jl")
include("WindIDen.jl")
using  Base.Threads
function CalDynmicTh(CumSumDataobj::spaceCumSumData, vPos)
    segID = getAbsPos(CumSumDataobj ,vPos)
    CalTimePeriod,calCFactor  = getSegDynParaValue( glbSegAlarmParasDictObj,segID )
    # curAlarmsinSEG[idx].DynTh    =  typemax(UInt ) # getSegStaticThPar
    vsc=1
    avg = calRectAvg(CumSumDataobj, vPos ,vsc,CalTimePeriod  )
    if calCFactor != typemax( UInt )
        curAlarmsinSEG[idx].DynTh    = avg * calCFactor
    else
        curAlarmsinSEG[idx].DynTh    = typemax( UInt )
    end
end

function postProcFeatureData(dataObject::spaceCumSumData ,data::Array{UInt16,1} )
    cumSumProc( dataObject,  data  )
    # println("          <?>                          ",length(data ))
    vsc = 4 #dataObject.defaultSpaceCnt
    vTcNum = 20 #
    # vTcNum = dataObject.defaultTimeCnt
    ######### calRectSum( CumSumDataobj::spaceCumSumData, vPos ,vsc ,vTcNum   )
    arng =getSpaceLoopRng( dataObject )
    # println( "---------------",arng )

     @threads for vpos = arng
        # println("vpos ,vsc ,vTcNum :$vpos ,$vsc ,$vTcNum   "  )
        calRectSum( dataObject, vpos ,vsc ,vTcNum   )
        # CalDynmicTh(CumSumDataobj, vPos)
    end

    AlarmCal(dataObject)
end

function AlarmCal( CumSumDataobj::spaceCumSumData,threshold = 3000)
    chnID = CumSumDataobj.startID > 1 ? 2 : 1
    # println("-CHnID:-$chnID, $(CumSumDataobj.startID)----")
    return  doesWinding( CumSumDataobj) # in windIDen.jl
    # return  AlarmCalAna( CumSumDataobj,threshold)
end


cols_1_to_30 = [
    -2.5   0.5         0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   5.26957  -2.45701   -0.812567   0.0        0.0         0.0         0.0        0.0         0.0         0.0
    1.0  -3.0         0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   5.26957  -2.45701   -0.812567   0.0        0.0         0.0         0.0        0.0         0.0         0.0
    0.0   0.0        -2.5   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  -0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   2.0       0.0        0.0        0.0        0.0         0.0         0.0        0.0         0.0         0.728864
    0.0   0.0         0.0  -2.5  -0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  -0.0  -0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0       2.0        0.0        0.0        0.0         0.0         0.0        0.0         0.0         1.05443
    0.0   0.0         0.0   0.0  -2.5   0.0   0.0   0.0   0.0   0.0   0.0   0.0  -0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0       0.0        2.0        0.0        0.0         0.0         0.0        0.0         0.0         0.564472
    0.0   0.0         0.0   0.0   0.0  -2.5   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  -0.0   0.0   0.0   0.0   0.0   0.0       0.0        0.0        2.0        0.0         0.0         0.0        0.0         0.0         0.0
    0.0   0.0         0.0   0.0   0.0   0.0  -2.5  -0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  -0.0  -0.0   0.0   0.0   0.0   0.0       0.0        0.0        0.0        2.0         0.0         0.0        0.0         0.0         0.0
    0.0   0.0         0.0   0.0   0.0   0.0   0.0  -2.5   0.0   0.0   0.0   0.0   0.0   0.0   0.0  -0.0   0.0   0.0   0.0   0.0   0.0       0.0        0.0        0.0        0.0         2.0         0.0        0.0         0.0         0.0
    0.0   0.0         0.0   0.0   0.0   0.0   0.0   0.0  -2.5   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  -0.0   0.0   0.0       0.0        0.0        0.0        0.0         0.0         2.0        0.0         0.0         0.0
    0.0   0.0         0.0   0.0   0.0   0.0   0.0   0.0   0.0  -2.5  -0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  -0.0  -0.0   0.0       0.0        0.0        0.0        0.0         0.0         0.0        2.0         0.0         0.0
    0.0   0.0         0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  -2.5   0.0   0.0   0.0   0.0   0.0   0.0   0.0  -0.0   0.0   0.0       0.0        0.0        0.0        0.0         0.0         0.0        0.0         2.0         0.0
    0.0   0.0         0.0  -0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  -2.5   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   2.91546  -1.1331     0.217645   0.0        0.0         0.0         0.0        0.0         0.0         0.5
    0.0   0.0         0.0  -0.0  -0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  -2.5  -0.0   0.0   0.0   0.0   0.0   0.0   0.0   4.21772  -2.19645   -0.021269   0.0        0.0         0.0         0.0        0.0         0.0         0.0
    0.0   0.0         0.0  -0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  -2.5   0.0   0.0   0.0   0.0   0.0   0.0   2.25789  -0.845166   0.587279   0.0        0.0         0.0         0.0        0.0         0.0         0.0
    0.0   0.0         0.0   0.0   0.0   0.0  -0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  -2.5   0.0   0.0   0.0   0.0   0.0   0.0       0.0        0.0        2.91546   -1.1331      0.217645    0.0        0.0         0.0         0.0
    0.0   0.0         0.0   0.0   0.0   0.0  -0.0  -0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  -2.5  -0.0   0.0   0.0   0.0   0.0       0.0        0.0        4.21772   -2.19645    -0.021269    0.0        0.0         0.0         0.0
    0.0   0.0         0.0   0.0   0.0   0.0  -0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  -2.5   0.0   0.0   0.0   0.0       0.0        0.0        2.25789   -0.845166    0.587279    0.0        0.0         0.0         0.0
    0.0   0.0         0.0   0.0   0.0   0.0   0.0   0.0   0.0  -0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  -2.5   0.0   0.0   0.0       0.0        0.0        0.0        0.0         0.0         2.91546   -1.1331      0.217645    0.0
    0.0   0.0         0.0   0.0   0.0   0.0   0.0   0.0   0.0  -0.0  -0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  -2.5  -0.0   0.0       0.0        0.0        0.0        0.0         0.0         4.21772   -2.19645    -0.021269    0.0
    0.0   0.0         0.0   0.0   0.0   0.0   0.0   0.0   0.0  -0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  -2.5   0.0       0.0        0.0        0.0        0.0         0.0         2.25789   -0.845166    0.587279    0.0
    0.0   0.0         1.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  -0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  -3.12854   0.0        0.0        2.97348   -1.38642    -0.458509    0.0        0.0         0.0         1.45773
    0.0   0.0         0.0   1.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  -0.0  -0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0      -3.12854   -1.16909    6.05379   -2.82266    -0.933492    0.0        0.0         0.0         2.10886
    0.0   0.0         0.0   0.0   1.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  -0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0       1.16909   -3.12854   -0.106837   0.0498141   0.0164742   0.0        0.0         0.0         1.12894
    0.0   0.0         0.0   0.0   0.0   1.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  -0.0   0.0   0.0   0.0   0.0   0.0       0.0        0.0       -3.12854    0.0         0.0         2.97348   -1.38642    -0.458509    0.0
    0.0   0.0         0.0   0.0   0.0   0.0   1.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  -0.0  -0.0   0.0   0.0   0.0   0.0       0.0        0.0        0.0       -3.12854    -1.16909     6.05379   -2.82266    -0.933492    0.0
    0.0   0.0         0.0   0.0   0.0   0.0   0.0   1.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  -0.0   0.0   0.0   0.0   0.0   0.0       0.0        0.0        0.0        1.16909    -3.12854    -0.106837   0.0498141   0.0164742   0.0
    0.0   0.0         0.0   0.0   0.0   0.0   0.0   0.0   1.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  -0.0   0.0   0.0       0.0        0.0        0.0        0.0         0.0        -3.12854    0.0         0.0         0.0
    0.0   0.0         0.0   0.0   0.0   0.0   0.0   0.0   0.0   1.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  -0.0  -0.0   0.0       0.0        0.0        0.0        0.0         0.0         0.0       -3.12854    -1.16909     0.0
    0.0   0.0         0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   1.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  -0.0   0.0   0.0       0.0        0.0        0.0        0.0         0.0         0.0        1.16909    -3.12854     0.0
    0.0   1.69282     0.0  -0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   1.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   2.91546  -1.1331     0.217645   0.0        0.0         0.0         0.0        0.0         0.0        -4.69282
    0.0   3.44646     0.0  -0.0  -0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   1.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   4.21772  -2.19645   -0.021269   0.0        0.0         0.0         0.0        0.0         0.0         0.0
    0.0  -0.0608229   0.0  -0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   1.0   0.0   0.0   0.0   0.0   0.0   0.0   2.25789  -0.845166   0.587279   0.0        0.0         0.0         0.0        0.0         0.0         0.0
    0.0   0.0         0.0   0.0   0.0   0.0  -0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   1.0   0.0   0.0   0.0   0.0   0.0   0.0       0.0        0.0        2.91546   -1.1331      0.217645    0.0        0.0         0.0         4.46021
    0.0   0.0         0.0   0.0   0.0   0.0  -0.0  -0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   1.0   0.0   0.0   0.0   0.0   0.0       0.0        0.0        4.21772   -2.19645    -0.021269    0.0        0.0         0.0         9.08068
    0.0   0.0         0.0   0.0   0.0   0.0  -0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   1.0   0.0   0.0   0.0   0.0       0.0        0.0        2.25789   -0.845166    0.587279    0.0        0.0         0.0        -0.160255
    0.0   0.0         0.0   0.0   0.0   0.0   0.0   0.0   0.0  -0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   1.0   0.0   0.0   0.0       0.0        0.0        0.0        0.0         0.0         2.91546   -1.1331      0.217645    0.0
    0.0   0.0         0.0   0.0   0.0   0.0   0.0   0.0   0.0  -0.0  -0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   1.0   0.0   0.0       0.0        0.0        0.0        0.0         0.0         4.21772   -2.19645    -0.021269    0.0
    0.0   0.0         0.0   0.0   0.0   0.0   0.0   0.0   0.0  -0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   1.0   0.0       0.0        0.0        0.0        0.0         0.0         2.25789   -0.845166    0.587279    0.0
    0.0   0.0         0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0       0.0        0.0        0.0        0.0         0.0         0.0        0.0         0.0         0.0
    0.0   0.0         0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0       0.0        0.0        0.0        0.0         0.0         0.0        0.0         0.0         0.0
]

cols_31_to_40 = [
    0.0         0.0          0.0        0.0         0.0          0.0        0.0        0.0          0.0   0.0
    0.0         0.0          0.0        0.0         0.0          0.0        0.0        0.0          0.0   0.0
   -0.283275    0.0544112    0.0        0.0         0.0          0.0        0.0        0.0          0.0   0.0
   -0.549114   -0.00531726   0.0        0.0         0.0          0.0        0.0        0.0          0.0   0.0
   -0.211292    0.14682      0.0        0.0         0.0          0.0        0.0        0.0          0.0   0.0
    0.0         0.0          0.728864  -0.283275    0.0544112    0.0        0.0        0.0          0.0   0.0
    0.0         0.0          1.05443   -0.549114   -0.00531726   0.0        0.0        0.0          0.0   0.0
    0.0         0.0          0.564472  -0.211292    0.14682      0.0        0.0        0.0          0.0   0.0
    0.0         0.0          0.0        0.0         0.0          0.728864  -0.283275   0.0544112    0.0   0.0
    0.0         0.0          0.0        0.0         0.0          1.05443   -0.549114  -0.00531726   0.0   0.0
    0.0         0.0          0.0        0.0         0.0          0.564472  -0.211292   0.14682      0.0   0.0
    0.0         0.0          0.0        0.0         0.0          0.0        0.0        0.0          0.0   0.0
    0.5         0.0          0.0        0.0         0.0          0.0        0.0        0.0          0.0   0.0
    0.0         0.5          0.0        0.0         0.0          0.0        0.0        0.0          0.0   0.0
    0.0         0.0          0.5        0.0         0.0          0.0        0.0        0.0          0.0   0.0
    0.0         0.0          0.0        0.5         0.0          0.0        0.0        0.0          0.0   0.0
    0.0         0.0          0.0        0.0         0.5          0.0        0.0        0.0          0.0   0.0
    0.0         0.0          0.0        0.0         0.0          0.5        0.0        0.0          0.0   0.0
    0.0         0.0          0.0        0.0         0.0          0.0        0.5        0.0          0.0   0.0
    0.0         0.0          0.0        0.0         0.0          0.0        0.0        0.5          0.0   0.0
   -0.56655     0.108822     0.0        0.0         0.0          0.0        0.0        0.0          0.0   0.0
   -1.09823    -0.0106345    0.0        0.0         0.0          0.0        0.0        0.0          0.0   0.0
   -0.422583    0.29364      0.0        0.0         0.0          0.0        0.0        0.0          0.0   0.0
    0.0         0.0          1.45773   -0.56655     0.108822     0.0        0.0        0.0          0.0   0.0
    0.0         0.0          2.10886   -1.09823    -0.0106345    0.0        0.0        0.0          0.0   0.0
    0.0         0.0          1.12894   -0.422583    0.29364      0.0        0.0        0.0          0.0   0.0
    0.0         0.0          0.0        0.0         0.0          1.45773   -0.56655    0.108822     0.0   1.12854
    0.0         0.0          0.0        0.0         0.0          2.10886   -1.09823   -0.0106345    0.0   2.29764
    0.0         0.0          0.0        0.0         0.0          1.12894   -0.422583   0.29364      0.0  -0.0405486
    0.0         0.0          0.0        0.0         0.0          0.0        0.0        0.0          0.0   0.0
   -4.69282    -1.75364      0.0        0.0         0.0          0.0        0.0        0.0          0.0   0.0
    1.75364    -4.69282      0.0        0.0         0.0          0.0        0.0        0.0          0.0   0.0
   -2.07963    -0.687763    -4.69282    0.0         0.0          0.0        0.0        0.0          0.0   0.0
   -4.23399    -1.40024      0.0       -4.69282    -1.75364      0.0        0.0        0.0          0.0   0.0
    0.0747212   0.0247113    0.0        1.75364    -4.69282      0.0        0.0        0.0          0.0   0.0
    0.0         0.0          4.46021   -2.07963    -0.687763    -4.69282    0.0        0.0          0.0   0.0
    0.0         0.0          9.08068   -4.23399    -1.40024      0.0       -4.69282   -1.75364      0.0   0.0
    0.0         0.0         -0.160255   0.0747212   0.0247113    0.0        1.75364   -4.69282      0.0   0.0
    0.0         0.0          0.0        0.0         0.0          1.31739   -0.614252  -0.203142    -2.5   2.0
    0.0         0.0          0.0        0.0         0.0          2.63479   -1.2285    -0.406283     1.0  -2.0
]

B_data = [cols_1_to_30 cols_31_to_40]


function mass_matrix_inner_entries!(
    mass_matrix,
    inner_control::IC,
    global_index::Base.ImmutableDict{Symbol, Int64},
) where {IC <: PSY.InnerControl}
    @debug "Using default mass matrix entries $IC"
end

function mass_matrix_inner_entries!(
    mass_matrix,
    inner_control::PSY.RECurrentControlB,
    global_index::ImmutableDict{Symbol, Int64},
)
    mass_matrix[global_index[:Vt_filt], global_index[:Vt_filt]] =
        PSY.get_T_rv(inner_control)
    if PSY.get_Q_Flag(inner_control) == 0
        mass_matrix[global_index[:I_icv], global_index[:I_icv]] =
            PSY.get_T_iq(inner_control)
    end
end

#####################################################
### Auxiliary ODE calculations via Flags dispatch ###
#####################################################

### Inner Controllers ###

#Q_Flag = 0
function _mdl_ode_RE_inner_controller_B!(
    inner_controller_ode::AbstractArray{<:ACCEPTED_REAL_TYPES},
    inner_controller_states::AbstractArray{<:ACCEPTED_REAL_TYPES},
    ::Type{Base.RefValue{0}},
    inner_control::PSY.RECurrentControlB,
    dynamic_device::DynamicWrapper{
        PSY.DynamicInverter{C, O, PSY.RECurrentControlB, DC, P, F},
    },
    inner_vars::AbstractVector,
) where {
    C <: PSY.Converter,
    O <: PSY.OuterControl,
    DC <: PSY.DCSource,
    P <: PSY.FrequencyEstimator,
    F <: PSY.Filter,
}
    #Obtain inner variables for component
    V_t = sqrt(inner_vars[Vr_inv_var]^2 + inner_vars[Vi_inv_var]^2)
    Ip_oc = inner_vars[Id_oc_var]
    Iq_oc = inner_vars[Iq_oc_var]

    #Get Current Controller parameters
    PQ_Flag = PSY.get_PQ_Flag(inner_control)
    dbd1, dbd2 = PSY.get_dbd_pnts(inner_control)
    K_qv = PSY.get_K_qv(inner_control)
    I_ql1, I_qh1 = PSY.get_Iqinj_lim(inner_control)
    V_ref0 = PSY.get_V_ref0(inner_control)

    #Read local states
    Vt_filt = inner_controller_states[1]
    I_icv = inner_controller_states[2]

    #Compute additional states
    V_err = deadband_function(V_ref0 - Vt_filt, dbd1, dbd2)
    Iq_inj = clamp(K_qv * V_err, I_ql1, I_qh1)
    Iq_cmd = I_icv + Iq_inj
    Ip_min, Ip_max, Iq_min, Iq_max =
        current_limit_logic(inner_control, Base.RefValue{PQ_Flag}, Vt_filt, Ip_oc, Iq_cmd)
    Iq_cmd = clamp(Iq_cmd, Iq_min, Iq_max)
    Ip_cmd = clamp(Ip_oc, Ip_min, Ip_max)

    #ODE update
    inner_controller_ode[1] = V_t - Vt_filt
    inner_controller_ode[2] = Iq_oc - I_icv

    #Update Inner Vars
    inner_vars[Id_ic_var] = Ip_cmd
    inner_vars[Iq_ic_var] = Iq_cmd
end

#Q_Flag = 1
function _mdl_ode_RE_inner_controller_B!(
    inner_controller_ode::AbstractArray{<:ACCEPTED_REAL_TYPES},
    inner_controller_states::AbstractArray{<:ACCEPTED_REAL_TYPES},
    ::Type{Base.RefValue{1}},
    inner_control::PSY.RECurrentControlB,
    dynamic_device::DynamicWrapper{
        PSY.DynamicInverter{C, O, PSY.RECurrentControlB, DC, P, F},
    },
    inner_vars::AbstractVector,
) where {
    C <: PSY.Converter,
    O <: PSY.OuterControl,
    DC <: PSY.DCSource,
    P <: PSY.FrequencyEstimator,
    F <: PSY.Filter,
}
    #Obtain inner variables for component
    V_t = sqrt(inner_vars[Vr_inv_var]^2 + inner_vars[Vi_inv_var]^2)
    Ip_oc = inner_vars[Id_oc_var]
    V_oc = inner_vars[V_oc_var]

    #Get Current Controller parameters
    PQ_Flag = PSY.get_PQ_Flag(inner_control)
    K_vp = PSY.get_K_vp(inner_control)
    K_vi = PSY.get_K_vi(inner_control)
    V_ref0 = PSY.get_V_ref0(inner_control)
    dbd1, dbd2 = PSY.get_dbd_pnts(inner_control)
    K_qv = PSY.get_K_qv(inner_control)
    I_ql1, I_qh1 = PSY.get_Iqinj_lim(inner_control)

    #Read local states
    Vt_filt = inner_controller_states[1]
    ξ_icv = inner_controller_states[2]

    #Compute additional states
    V_err = deadband_function(V_ref0 - Vt_filt, dbd1, dbd2)
    Iq_inj = clamp(K_qv * V_err, I_ql1, I_qh1)
    #To do: Limits on PI non-windup
    I_icv = K_vp * V_oc + K_vi * ξ_icv
    Iq_cmd = I_icv + Iq_inj
    Ip_min, Ip_max, Iq_min, Iq_max =
        current_limit_logic(inner_control, Base.RefValue{PQ_Flag}, Vt_filt, Ip_oc, Iq_cmd)
    Iq_cmd = clamp(Iq_cmd, Iq_min, Iq_max)
    Ip_cmd = clamp(Ip_oc, Ip_min, Ip_max)

    #ODE update
    inner_controller_ode[1] = V_t - Vt_filt
    inner_controller_ode[2] = V_oc

    #Update Inner Vars
    inner_vars[Id_ic_var] = Ip_cmd
    inner_vars[Iq_ic_var] = Iq_cmd
end

############################################
### ODE calculations via device dispatch ###
############################################

function mdl_inner_ode!(
    device_states::AbstractArray{<:ACCEPTED_REAL_TYPES},
    output_ode::AbstractArray{<:ACCEPTED_REAL_TYPES},
    inner_vars::AbstractArray{<:ACCEPTED_REAL_TYPES},
    dynamic_device::DynamicWrapper{
        PSY.DynamicInverter{C, O, PSY.VoltageModeControl, DC, P, F},
    },
) where {
    C <: PSY.Converter,
    O <: PSY.OuterControl,
    DC <: PSY.DCSource,
    P <: PSY.FrequencyEstimator,
    F <: PSY.Filter,
}

    #Obtain external states inputs for component
    external_ix = get_input_port_ix(dynamic_device, PSY.VoltageModeControl)
    Ir_filter = device_states[external_ix[1]]
    Ii_filter = device_states[external_ix[2]]
    Ir_cnv = device_states[external_ix[3]]
    Ii_cnv = device_states[external_ix[4]]
    Vr_filter = device_states[external_ix[5]]
    Vi_filter = device_states[external_ix[6]]

    #Obtain inner variables for component
    ω_oc = inner_vars[ω_oc_var]
    θ_oc = inner_vars[θ_oc_var]
    v_refr = inner_vars[V_oc_var]
    Vdc = inner_vars[Vdc_var]

    #Get Voltage Controller parameters
    inner_control = PSY.get_inner_control(dynamic_device)
    filter = PSY.get_filter(dynamic_device)
    kpv = PSY.get_kpv(inner_control)
    kiv = PSY.get_kiv(inner_control)
    kffi = PSY.get_kffi(inner_control)
    cf = PSY.get_cf(filter)
    rv = PSY.get_rv(inner_control)
    lv = PSY.get_lv(inner_control)

    #Get Current Controller parameters
    kpc = PSY.get_kpc(inner_control)
    kic = PSY.get_kic(inner_control)
    kffv = PSY.get_kffv(inner_control)
    lf = PSY.get_lf(filter)
    ωad = PSY.get_ωad(inner_control)
    kad = PSY.get_kad(inner_control)

    #Obtain indices for component w/r to device
    local_ix = get_local_state_ix(dynamic_device, PSY.VoltageModeControl)

    #Define internal states for frequency estimator
    internal_states = @view device_states[local_ix]
    ξ_d = internal_states[1]
    ξ_q = internal_states[2]
    γ_d = internal_states[3]
    γ_q = internal_states[4]
    ϕ_d = internal_states[5]
    ϕ_q = internal_states[6]

    #Transformations to dq frame
    I_dq_filter = ri_dq(θ_oc + pi / 2) * [Ir_filter; Ii_filter]
    I_dq_cnv = ri_dq(θ_oc + pi / 2) * [Ir_cnv; Ii_cnv]
    V_dq_filter = ri_dq(θ_oc + pi / 2) * [Vr_filter; Vi_filter]

    ### Compute 6 states ODEs (D'Arco EPSR122 Model) ###
    ## SRF Voltage Control w/ Virtual Impedance ##
    #Virtual Impedance
    Vd_filter_ref = (v_refr - rv * I_dq_filter[d] + ω_oc * lv * I_dq_filter[q])
    Vq_filter_ref = (-rv * I_dq_filter[q] - ω_oc * lv * I_dq_filter[d])

    #Voltage Control ODEs
    #PI Integrator (internal state)
    output_ode[local_ix[1]] = (Vd_filter_ref - V_dq_filter[d])
    output_ode[local_ix[2]] = (Vq_filter_ref - V_dq_filter[q])

    #Output Control Signal - Links to SRF Current Controller
    Id_cnv_ref = (
        kpv * (Vd_filter_ref - V_dq_filter[d]) + kiv * ξ_d - cf * ω_oc * V_dq_filter[q] + kffi * I_dq_filter[d]
    )

    Iq_cnv_ref = (
        kpv * (Vq_filter_ref - V_dq_filter[q]) +
        kiv * ξ_q +
        cf * ω_oc * V_dq_filter[d] +
        kffi * I_dq_filter[q]
    )

    ## SRF Current Control ##
    #Current Control ODEs
    #PI Integrator (internal state)
    output_ode[local_ix[3]] = Id_cnv_ref - I_dq_cnv[d]
    output_ode[local_ix[4]] = Iq_cnv_ref - I_dq_cnv[q]

    #References for Converter Output Voltage
    Vd_cnv_ref = (
        kpc * (Id_cnv_ref - I_dq_cnv[d]) + kic * γ_d - ω_oc * lf * I_dq_cnv[q] +
        kffv * V_dq_filter[d] - kad * (V_dq_filter[d] - ϕ_d)
    )
    Vq_cnv_ref = (
        kpc * (Iq_cnv_ref - I_dq_cnv[q]) +
        kic * γ_q +
        ω_oc * lf * I_dq_cnv[d] +
        kffv * V_dq_filter[q] - kad * (V_dq_filter[q] - ϕ_q)
    )

    #Active Damping LPF (internal state)
    output_ode[local_ix[5]] = ωad * V_dq_filter[d] - ωad * ϕ_d
    output_ode[local_ix[6]] = ωad * V_dq_filter[q] - ωad * ϕ_q

    #Update inner_vars
    #Modulation Commands to Converter
    inner_vars[md_var] = Vd_cnv_ref / Vdc
    inner_vars[mq_var] = Vq_cnv_ref / Vdc
end

function mdl_inner_ode!(
    device_states::AbstractArray{<:ACCEPTED_REAL_TYPES},
    output_ode::AbstractArray{<:ACCEPTED_REAL_TYPES},
    inner_vars::AbstractArray{<:ACCEPTED_REAL_TYPES},
    dynamic_device::DynamicWrapper{
        PSY.DynamicInverter{C, O, PSY.CurrentModeControl, DC, P, F},
    },
) where {
    C <: PSY.Converter,
    O <: PSY.OuterControl,
    DC <: PSY.DCSource,
    P <: PSY.FrequencyEstimator,
    F <: PSY.Filter,
}

    #Obtain external states inputs for component
    external_ix = get_input_port_ix(dynamic_device, PSY.CurrentModeControl)
    # Ir_filter = device_states[external_ix[1]]
    # Ii_filter = device_states[external_ix[2]]
    Ir_cnv = device_states[external_ix[3]]
    Ii_cnv = device_states[external_ix[4]]
    Vr_filter = device_states[external_ix[5]]
    Vi_filter = device_states[external_ix[6]]

    #Obtain inner variables for component
    ω_oc = inner_vars[ω_oc_var]
    θ_oc = inner_vars[θ_oc_var]
    Vdc = inner_vars[Vdc_var]

    #Get Current Controller parameters
    inner_control = PSY.get_inner_control(dynamic_device)
    filter = PSY.get_filter(dynamic_device)
    kpc = PSY.get_kpc(inner_control)
    kic = PSY.get_kic(inner_control)
    kffv = PSY.get_kffv(inner_control)
    lf = PSY.get_lf(filter)

    #Obtain indices for component w/r to device
    local_ix = get_local_state_ix(dynamic_device, PSY.CurrentModeControl)

    #Define internal states for frequency estimator
    internal_states = @view device_states[local_ix]
    γ_d = internal_states[1]
    γ_q = internal_states[2]

    #Transformations to dq frame
    I_dq_cnv = ri_dq(θ_oc + pi / 2) * [Ir_cnv; Ii_cnv]
    V_dq_filter = ri_dq(θ_oc + pi / 2) * [Vr_filter; Vi_filter]

    #Input Control Signal - Links to SRF Current Controller
    Id_cnv_ref = inner_vars[Id_oc_var]
    Iq_cnv_ref = inner_vars[Iq_oc_var]

    #Current Control ODEs
    #PI Integrator (internal state)
    output_ode[local_ix[1]] = Id_cnv_ref - I_dq_cnv[d]
    output_ode[local_ix[2]] = Iq_cnv_ref - I_dq_cnv[q]

    #References for Converter Output Voltage
    Vd_cnv_ref = (
        kpc * (Id_cnv_ref - I_dq_cnv[d]) + kic * γ_d - ω_oc * lf * I_dq_cnv[q] +
        kffv * V_dq_filter[d]
    )
    Vq_cnv_ref = (
        kpc * (Iq_cnv_ref - I_dq_cnv[q]) +
        kic * γ_q +
        ω_oc * lf * I_dq_cnv[d] +
        kffv * V_dq_filter[q]
    )

    #Update inner_vars
    #Modulation Commands to Converter
    inner_vars[md_var] = Vd_cnv_ref / Vdc
    inner_vars[mq_var] = Vq_cnv_ref / Vdc
end

function mdl_inner_ode!(
    device_states::AbstractArray{<:ACCEPTED_REAL_TYPES},
    output_ode::AbstractArray{<:ACCEPTED_REAL_TYPES},
    inner_vars::AbstractArray{<:ACCEPTED_REAL_TYPES},
    dynamic_device::DynamicWrapper{
        PSY.DynamicInverter{C, O, PSY.RECurrentControlB, DC, P, F},
    },
) where {
    C <: PSY.Converter,
    O <: PSY.OuterControl,
    DC <: PSY.DCSource,
    P <: PSY.FrequencyEstimator,
    F <: PSY.Filter,
}
    #Get Current Controller parameters
    inner_control = PSY.get_inner_control(dynamic_device)
    Q_Flag = PSY.get_Q_Flag(inner_control)

    #Obtain indices for component w/r to device
    local_ix = get_local_state_ix(dynamic_device, PSY.RECurrentControlB)
    #Define internal states for Inner Control
    internal_states = @view device_states[local_ix]
    internal_ode = @view output_ode[local_ix]

    # TODO: Voltage Dip Freeze logic

    #Dispatch inner controller ODE calculation
    _mdl_ode_RE_inner_controller_B!(
        internal_ode,
        internal_states,
        Base.RefValue{Q_Flag},
        inner_control,
        dynamic_device,
        inner_vars,
    )
end


#=
using PowerSystems
using JuMP
base_dir = dirname(dirname(pathof(PowerSystems)))
include(joinpath(base_dir, "data/data_5bus_uc.jl"))
sys5 = PSY.System(nodes5, generators5, loads5_DA, branches5, nothing, 100.0);
using PowerSimulations
const PS = PowerSimulations

simple_reserve = PSY.StaticReserve("test_reserve", vcat(sys5.generators.thermal, sys5.generators.renewable[2]), 60.0, [gen.tech for gen in sys5.generators.thermal])
#simple_reserve = PSY.StaticReserve("test_reserve", sys5.generators.thermal, 60.0, [sys5.generators.thermal[1].tech])

@test try
    Net = PSI.CopperPlatePowerModel
    m = Model();
    netinjection = PSI.instantiate_network(Net, sys5);
    PSI.construct_device!(m, netinjection, ThermalGen, PSI.ThermalDispatch, Net, sys5);
    PSI.construct_device!(m, netinjection, RenewableGen, PSI.RenewableCurtail, Net, sys5);
    PSI.construct_device!(m, netinjection, ElectricLoad, PSI.InterruptibleLoad, Net, sys5);
    PSI.construct_network!(m, [(device=Branch, formulation=PSI.PiLine)], netinjection, Net, sys5)
    PSI.construct_service!(m, simple_reserve, PSI.RampLimitedReserve, [(device = ThermalGen, formulation =PSI.ThermalDispatch),
                                                              (device = RenewableGen, formulation = PSI.RenewableCurtail)],
                                                              sys5)
    m.obj_dict
true finally end

@test try
    Net = PSI.CopperPlatePowerModel
    m = Model();
    netinjection = PSI.instantiate_network(Net, sys5);
    PSI.construct_device!(m, netinjection, ThermalGen, PSI.ThermalStandardUnitCommitment , Net, sys5);
    PSI.construct_device!(m, netinjection, RenewableGen, PSI.RenewableCurtail, Net, sys5);
    PSI.construct_device!(m, netinjection, ElectricLoad, PSI.InterruptibleLoad, Net, sys5);
    PSI.construct_network!(m, [(device=Branch, formulation=PSI.PiLine)], netinjection, Net, sys5)
    PSI.construct_service!(m, simple_reserve, PSI.RampLimitedReserve, [(device = ThermalGen, formulation =PSI.ThermalStandardUnitCommitment ),
                                                              (device = RenewableGen, formulation = PSI.RenewableCurtail)],
                                                              sys5)
    m.obj_dict
true finally end
=#


module TightBindingApproximation

using Printf: @sprintf
using TimerOutputs: @timeit
using Optim: optimize, LBFGS
using RecipesBase: RecipesBase, @recipe, @series
using LinearAlgebra: inv, dot, Hermitian, Diagonal, eigvals, cholesky, Eigen
using QuantumLattices: getcontent, iidtype, rcoord, icoord, expand, statistics, plain, creation, annihilation, atol, rtol, periods, decimaltostr
using QuantumLattices: AbstractLattice, AbstractPID, FID, NID, Index, CompositeOID, ID, Bonds, Hilbert, Metric, Operator, Operators, OIDToTuple, Table, Term, Boundary
using QuantumLattices: Internal, Fock, Phonon, Hopping, Onsite, Pairing, PhononKinetic, PhononPotential, MatrixRepresentation, BrillouinZone
using QuantumLattices: Engine, Parameters, AbstractGenerator, CompositeGenerator, Entry, Generator, Formulation, Action, Assignment, Algorithm

import LinearAlgebra: eigen, eigvals, ishermitian
import QuantumLattices: add!, contentnames, dimension, kind, matrix, update!, prepare!, run!

export TBAKind, AbstractTBA, TBAMatrix, TBAMatrixRepresentation, commutator
export TBA, EnergyBands, BerryCurvature
export SampleNode, deviation, optimize!

"""
    TBAKind{K}

The kind of a free quantum lattice system using the tight-binding approximation.
"""
struct TBAKind{K}
    TBAKind(k::Symbol) = new{k}()
end
@inline TBAKind{K}() where K = TBAKind(K)
@inline Base.promote_rule(::Type{TBAKind{K}}, ::Type{TBAKind{K}}) where K = TBAKind{K}
@inline Base.promote_rule(::Type{TBAKind{:TBA}}, ::Type{TBAKind{:BdG}}) = TBAKind{:BdG}
@inline Base.promote_rule(::Type{TBAKind{:BdG}}, ::Type{TBAKind{:TBA}}) = TBAKind{:BdG}

"""
    TBAKind(T::Type{<:Term})

Depending on the kind of a term type, get the corresponding TBA kind.
"""
@inline TBAKind(::Type{T}) where {T<:Term} = error("TBAKind error: not defined for $(kind(T)).")
@inline TBAKind(::Type{T}) where {T<:Union{Hopping, Onsite}} = TBAKind(:TBA)
@inline TBAKind(::Type{T}) where {T<:Union{Pairing, PhononKinetic, PhononPotential}} = TBAKind(:BdG)
@inline @generated function TBAKind(::Type{TS}) where {TS<:Tuple{Vararg{Term}}}
    exprs = []
    for i = 1:fieldcount(TS)
        push!(exprs, :(typeof(TBAKind(fieldtype(TS, $i)))))
    end
    return Expr(:call, Expr(:call, :reduce, :promote_type, Expr(:tuple, exprs...)))
end

"""
    Metric(::TBAKind, hilbert::Hilbert{<:Fock} -> OIDToTuple
    Metric(::TBAKind, hilbert::Hilbert{<:Phonon}) -> OIDToTuple

Get the oid-to-tuple metric for a free fermionic/bosonic system or a free phononic system.
"""
@inline @generated Metric(::TBAKind{:TBA}, hilbert::Hilbert{<:Fock}) = OIDToTuple(fieldnames(keytype(hilbert))..., :orbital, :spin)
@inline @generated Metric(::TBAKind{:BdG}, hilbert::Hilbert{<:Fock}) = OIDToTuple(:nambu, fieldnames(keytype(hilbert))..., :orbital, :spin)
@inline @generated Metric(::TBAKind, hilbert::Hilbert{<:Phonon}) = OIDToTuple(:tag, fieldnames(keytype(hilbert))..., :dir)

"""
    commutator(k::TBAKind, hilbert::Hilbert{<:Internal}) -> Union{AbstractMatrix, Nothing}

Get the commutation relation of the single-particle operators of a free quantum lattice system using the tight-binding approximation.
"""
@inline commutator(::TBAKind{:TBA}, ::Hilbert{<:Internal}) = nothing
@inline commutator(::TBAKind{:BdG}, ::Hilbert{<:Fock{:f}}) = nothing
@inline commutator(k::TBAKind{:BdG}, hilbert::Hilbert{<:Fock{:b}}) = Diagonal(kron([1, -1], ones(Int64, dimension(hilbert, k)÷2)))
@inline commutator(k::TBAKind{:BdG}, hilbert::Hilbert{<:Phonon}) = Hermitian(kron([0 -1im; 1im 0], Diagonal(ones(Int, dimension(hilbert, k)÷2))))
@inline dimension(hilbert::Hilbert, ::TBAKind{:BdG}) = sum(dimension, values(hilbert))

"""
    AbstractTBA{K, H<:AbstractGenerator, G<:Union{Nothing, AbstractMatrix}} <: Engine

Abstract type for free quantum lattice systems using the tight-binding approximation.
"""
abstract type AbstractTBA{K, H<:AbstractGenerator, G<:Union{Nothing, AbstractMatrix}} <: Engine end
@inline contentnames(::Type{<:AbstractTBA}) = (:H, :commutator)
@inline kind(tba::AbstractTBA) = kind(typeof(tba))
@inline kind(::Type{<:AbstractTBA{K}}) where K = K
@inline Base.valtype(::Type{<:AbstractTBA{K, H} where K}) where {H<:AbstractGenerator} = valtype(eltype(H))
@inline dimension(tba::AbstractTBA{K, <:CompositeGenerator}) where K = length(getcontent(getcontent(tba, :H), :table))
@inline update!(tba::AbstractTBA; k=nothing, kwargs...) = ((length(kwargs)>0 && update!(getcontent(tba, :H); kwargs...)); tba)
@inline Parameters(tba::AbstractTBA) = Parameters(getcontent(tba, :H))

"""
    TBAMatrix{T, H<:AbstractMatrix{T}, G<:Union{AbstractMatrix, Nothing}} <: AbstractMatrix{T}

Matrix representation of a free quantum lattice system using the tight-binding approximation.
"""
struct TBAMatrix{T, H<:AbstractMatrix{T}, G<:Union{AbstractMatrix, Nothing}} <: AbstractMatrix{T}
    H::H
    commutator::G
    function TBAMatrix(H::AbstractMatrix, commutator::Union{AbstractMatrix, Nothing})
        new{eltype(H), typeof(H), typeof(commutator)}(H, commutator)
    end
end
@inline Base.size(m::TBAMatrix) = size(m.H)
@inline Base.getindex(m::TBAMatrix, i::Integer, j::Integer) = m.H[i, j]
@inline ishermitian(m::TBAMatrix) = ishermitian(typeof(m))
@inline ishermitian(::Type{<:TBAMatrix}) = true

"""
    TBAMatrixRepresentation{K<:AbstractTBA, V, T} <: MatrixRepresentation

Matrix representation of the Hamiltonian of a tight-binding system.
"""
struct TBAMatrixRepresentation{K<:AbstractTBA, V, T} <: MatrixRepresentation
    k::V
    table::T
    gauge::Symbol
    function TBAMatrixRepresentation{K}(k, table, gauge::Symbol=:icoord) where {K<:AbstractTBA}
        @assert gauge∈(:rcoord, :icoord) "TBAMatrixRepresentation error: gauge must be :rcoord or :icoord."
        return new{K, typeof(k), typeof(table)}(k, table, gauge)
    end
end
@inline TBAMatrixRepresentation{K}(table, gauge::Symbol=:icoord) where {K<:AbstractTBA} = TBAMatrixRepresentation{K}(nothing, table, gauge)
@inline Base.valtype(mr::TBAMatrixRepresentation) = valtype(typeof(mr))
@inline Base.valtype(::Type{<:TBAMatrixRepresentation{K}}) where {K<:AbstractTBA} = Matrix{promote_type(valtype(K), Complex{Int})}
@inline Base.valtype(::Type{<:TBAMatrixRepresentation{K, Nothing}}) where {K<:AbstractTBA} = Matrix{valtype(K)}
@inline Base.valtype(R::Type{<:TBAMatrixRepresentation}, ::Type{<:Union{Operator, Operators}}) = valtype(R)
@inline Base.zero(mr::TBAMatrixRepresentation) = zeros(eltype(valtype(mr)), length(mr.table), length(mr.table))
@inline Base.zero(mr::TBAMatrixRepresentation, ::Union{Operator, Operators}) = zero(mr)
@inline (mr::TBAMatrixRepresentation)(m::Operator; kwargs...) = add!(zero(mr, m), mr, m; kwargs...)

"""
    add!(dest::Matrix,
        mr::TBAMatrixRepresentation{<:AbstractTBA{TBAKind(:TBA)}},
        m::Operator;
        kwargs...
        ) -> typeof(dest)
    add!(dest::Matrix,
        mr::TBAMatrixRepresentation{<:AbstractTBA{TBAKind(:BdG)}},
        m::Operator{<:Number, <:ID{CompositeOID{<:Index{<:AbstractPID, <:FID}}}};
        atol=atol/5,
        kwargs...
        ) -> typeof(dest)
    add!(dest::Matrix,
        mr::TBAMatrixRepresentation{<:AbstractTBA{TBAKind(:BdG)}},
        m::Operator{<:Number, <:ID{CompositeOID{<:Index{<:AbstractPID, <:NID}}}};
        atol=atol/5,
        kwargs...
        ) -> typeof(dest)

Get the matrix representation of an operator and add it to destination.
"""
function add!(dest::Matrix,
        mr::TBAMatrixRepresentation{<:AbstractTBA{TBAKind(:TBA)}},
        m::Operator;
        kwargs...
        )
    seq₁, seq₂ = mr.table[m[1].index'], mr.table[m[2].index]
    coord = mr.gauge==:rcoord ? rcoord(m) : icoord(m)
    phase = isnothing(mr.k) ? one(eltype(dest)) : convert(eltype(dest), exp(-1im*dot(mr.k, coord)))
    dest[seq₁, seq₂] += m.value*phase
    return dest
end
function add!(dest::Matrix,
        mr::TBAMatrixRepresentation{<:AbstractTBA{TBAKind(:BdG)}},
        m::Operator{<:Number, <:ID{CompositeOID{<:Index{<:AbstractPID, <:FID}}}};
        atol=atol/5,
        kwargs...
        )
    seq₁, seq₂ = mr.table[m[1].index'], mr.table[m[2].index]
    coord = mr.gauge==:rcoord ? rcoord(m) : icoord(m)
    phase = isnothing(mr.k) ? one(eltype(dest)) : convert(eltype(dest), exp(-1im*dot(mr.k, coord)))
    seq₁==seq₂ || (atol = 0)
    dest[seq₁, seq₂] += m.value*phase+atol
    if m[1].index.iid.nambu==creation && m[2].index.iid.nambu==annihilation
        seq₁, seq₂ = mr.table[m[1].index], mr.table[m[2].index']
        sign = statistics(eltype(m))==:f ? -1 : +1
        dest[seq₁, seq₂] += sign*m.value*phase'+atol
    end
    return dest
end
function add!(dest::Matrix,
        mr::TBAMatrixRepresentation{<:AbstractTBA{TBAKind(:BdG)}},
        m::Operator{<:Number, <:ID{CompositeOID{<:Index{<:AbstractPID, <:NID}}}};
        atol=atol/5,
        kwargs...
        )
    if m[1] == m[2]
        seq = mr.table[m[1].index]
        dest[seq, seq] += 2*m.value+atol
    else
        seq₁, seq₂ = mr.table[m[1].index], mr.table[m[2].index]
        coord = mr.gauge==:rcoord ? rcoord(m) : icoord(m)
        phase = isnothing(mr.k) ? one(eltype(dest)) : convert(eltype(dest), exp(-1im*dot(mr.k, coord)))
        dest[seq₁, seq₂] += m.value*phase
        dest[seq₂, seq₁] += m.value'*phase'
    end
    return dest
end

"""
    TBAMatrixRepresentation(tba::AbstractTBA, k=nothing, gauge::Symbol=:icoord)

Construct the matrix representation transformation of a free quantum lattice system using the tight-binding approximation.
"""
@inline function TBAMatrixRepresentation(tba::AbstractTBA, k=nothing, gauge::Symbol=:icoord)
    table = getcontent(getcontent(tba, :H), :table)
    return TBAMatrixRepresentation{typeof(tba)}(k, table, gauge)
end

"""
    matrix(tba::Union{AbstractTBA, Algorithm{<:AbstractTBA}}; k=nothing, gauge=:icoord, atol=atol/5, kwargs...) -> TBAMatrix

Get the matrix representation of a free quantum lattice system.
"""
@inline function matrix(tba::AbstractTBA; k=nothing, gauge=:icoord, atol=atol/5, kwargs...)
    H = getcontent(tba, :H)
    commutator = getcontent(tba, :commutator)
    isnothing(commutator) && (atol = 0)
    return TBAMatrix(Hermitian(TBAMatrixRepresentation(tba, k, gauge)(expand(H); atol=atol, kwargs...)), commutator)
end
@inline function matrix(tba::AbstractTBA{TBAKind(:Analytical)}; kwargs...)
    return TBAMatrix(Hermitian(getcontent(tba, :H)(; kwargs...)), getcontent(tba, :commutator))
end
@inline matrix(tba::Algorithm{<:AbstractTBA}; kwargs...) = matrix(tba.engine; kwargs...)

"""
    eigen(m::TBAMatrix) -> Eigen

Solve the eigen problem of a free quantum lattice system.
"""
@inline eigen(m::TBAMatrix{T, H, Nothing}) where {T, H<:AbstractMatrix{T}} = eigen(m.H)
function eigen(m::TBAMatrix{T, H, G}) where {T, H<:AbstractMatrix{T}, G<:AbstractMatrix}
    W = cholesky(m.H)
    K = eigen(Hermitian(W.U*m.commutator*W.L))
    @assert length(K.values)%2==0 "eigen error: wrong dimension of matrix."
    for i = 1:(length(K.values)÷2)
        K.values[i] = -K.values[i]
    end
    V = inv(W.U)*K.vectors*sqrt(Diagonal(K.values))
    return Eigen(K.values, V)
end

"""
    eigvals(m::TBAMatrix) -> Vector

Get the eigen values of a free quantum lattice system.
"""
@inline eigvals(m::TBAMatrix{T, H, Nothing}) where {T, H<:AbstractMatrix{T}} = eigvals(m.H)
function eigvals(m::TBAMatrix{T, H, G}) where {T, H<:AbstractMatrix{T}, G<:AbstractMatrix}
    values = eigen(m.H*m.commutator).values
    @assert length(values)%2==0 "eigvals error: wrong dimension of matrix."
    for i = 1:(length(values)÷2)
        values[i] = -values[i]
    end
    return values
end

"""
    TBA{K, L<:AbstractLattice, H<:AbstractGenerator, G<:Union{AbstractMatrix, Nothing}} <: AbstractTBA{K, H, G}

The usual tight binding approximation for quantum lattice systems.
"""
struct TBA{K, L<:AbstractLattice, H<:AbstractGenerator, G<:Union{AbstractMatrix, Nothing}} <: AbstractTBA{K, H, G}
    lattice::L
    H::H
    commutator::G
    function TBA{K}(lattice::AbstractLattice, H::AbstractGenerator, commutator::Union{AbstractMatrix, Nothing}) where K
        @assert isa(K, TBAKind) "TBA error: wrong kind."
        if !isnothing(commutator)
            values = eigvals(commutator)
            num₁ = count(isapprox(+1, atol=atol, rtol=rtol), values)
            num₂ = count(isapprox(-1, atol=atol, rtol=rtol), values)
            @assert num₁==num₂==length(values)//2 "TBA error: unsupported input commutator."
        end
        new{K, typeof(lattice), typeof(H), typeof(commutator)}(lattice, H, commutator)
    end
end
@inline contentnames(::Type{<:TBA}) = (:lattice, :H, :commutator)

"""
    TBA(lattice::AbstractLattice, hilbert::Hilbert, terms::Tuple{Vararg{Term}}; boundary::Boundary=plain)

Construct a tight-binding quantum lattice system.
"""
@inline function TBA(lattice::AbstractLattice, hilbert::Hilbert, terms::Tuple{Vararg{Term}}; boundary::Boundary=plain)
    tbakind = TBAKind(typeof(terms))
    table = Table(hilbert, Metric(tbakind, hilbert))
    commt = commutator(tbakind, hilbert)
    return TBA{tbakind}(lattice, Generator(terms, Bonds(lattice), hilbert; half=false, table=table, boundary=boundary), commt)
end

"""
    TBA(lattice::AbstractLattice, hamiltonian::Function, parameters::Parameters, commt::Union{AbstractMatrix, Nothing}=nothing)

Construct a tight-binding quantum lattice system by providing the analytical expressions of the Hamiltonian.
"""
@inline function TBA(lattice::AbstractLattice, hamiltonian::Function, parameters::Parameters, commt::Union{AbstractMatrix, Nothing}=nothing)
    return TBA{TBAKind(:Analytical)}(lattice, Formulation(hamiltonian, parameters), commt)
end

"""
    EnergyBands{P, L<:Union{Colon, Vector{Int}}} <: Action

Energy bands by tight-binding-approximation for quantum lattice systems.
"""
struct EnergyBands{P, L<:Union{Colon, Vector{Int}}} <: Action
    path::P
    levels::L
    gauge::Symbol
    function EnergyBands(path, levels::Union{Colon, Vector{Int}}=Colon(), gauge::Symbol=:icoord)
        new{typeof(path), typeof(levels)}(path, levels, gauge)
    end
end
@inline prepare!(eb::EnergyBands{P, Colon}, tba::AbstractTBA) where P = (zeros(Float64, length(eb.path)), zeros(Float64, length(eb.path), dimension(tba)))
@inline prepare!(eb::EnergyBands{P, Vector{Int}}, tba::AbstractTBA) where P = (zeros(Float64, length(eb.path)), zeros(Float64, length(eb.path), length(eb.levels)))
@inline Base.nameof(tba::Algorithm{<:AbstractTBA}, eb::Assignment{<:EnergyBands}) = @sprintf "%s_%s" repr(tba, ∉(keys(eb.action.path))) eb.id
function run!(tba::Algorithm{<:AbstractTBA}, eb::Assignment{<:EnergyBands})
    for (i, params) in enumerate(pairs(eb.action.path))
        eb.data[1][i] = length(params)==1 && isa(first(params), Number) ? first(params) : i-1
        update!(tba; gauge=eb.action.gauge, params...)
        @timeit tba.timer "matrix" (m = matrix(tba.engine; params...))
        @timeit tba.timer "eigen" (eb.data[2][i, :] = eigen(m).values[eb.action.levels])
    end
end

"""
    BerryCurvature{B<:BrillouinZone} <: Action

Berry curvature of energy bands with the spirit of a momentum space discretization method by [Fukui et al, JPSJ 74, 1674 (2005)](https://journals.jps.jp/doi/10.1143/JPSJ.74.1674).
"""
struct BerryCurvature{B<:BrillouinZone} <: Action
    brillouinzone::B
    levels::Vector{Int}
end
@inline function prepare!(bc::BerryCurvature, tba::AbstractTBA)
    N₁, N₂ = periods(eltype(bc.brillouinzone))
    x = collect(Float64, 0:(N₁-1))/N₁
    y = collect(Float64, 0:(N₂-1))/N₂
    z = zeros(Float64, length(y), length(x), length(bc.levels))
    n = zeros(Float64, length(bc.levels))
    return (x, y, z, n)
end
function run!(tba::Algorithm{<:AbstractTBA}, bc::Assignment{<:BerryCurvature})
    N₁, N₂ = length(bc.data[1]), length(bc.data[2])
    eigenvectors = zeros(ComplexF64, N₁, N₂, dimension(tba.engine), length(bc.action.levels))
    for momentum in bc.action.brillouinzone
        coord = expand(momentum, bc.action.brillouinzone.reciprocals)
        @timeit tba.timer "matrix" (m = matrix(tba.engine; k=coord))
        @timeit tba.timer "eigen" (eigenvectors[Int(momentum[1])+1, Int(momentum[2])+1, :, :] = eigen(m).vectors[:, bc.action.levels])
    end
    g = isnothing(tba.engine.commutator) ? Diagonal(ones(Int, dimension(tba.engine))) : inv(tba.engine.commutator)
    @timeit tba.timer "Berry curvature" for momentum in bc.action.brillouinzone
        i₁, j₁ = Int(momentum[1]), Int(momentum[2])
        i₂, j₂ = (i₁+1)%N₁, (j₁+1)%N₂
        vs₁ = eigenvectors[i₁+1, j₁+1, :, :]
        vs₂ = eigenvectors[i₂+1, j₁+1, :, :]
        vs₃ = eigenvectors[i₂+1, j₂+1, :, :]
        vs₄ = eigenvectors[i₁+1, j₂+1, :, :]
        for k = 1:length(bc.action.levels)
            p₁ = vs₁[:, k]'*g*vs₂[:, k]
            p₂ = vs₂[:, k]'*g*vs₃[:, k]
            p₃ = vs₃[:, k]'*g*vs₄[:, k]
            p₄ = vs₄[:, k]'*g*vs₁[:, k]
            bc.data[3][j₁+1, i₁+1, k] = angle(p₁*p₂*p₃*p₄)
            bc.data[4][k] += bc.data[3][j₁+1, i₁+1, k]/2pi
        end
    end
    @info (@sprintf "Chern numbers: %s" join([@sprintf "%s(%s)" cn level for (cn, level) in zip(bc.data[4], bc.action.levels)], ", "))
end
@recipe function plot(pack::Tuple{Algorithm{<:AbstractTBA}, Assignment{<:BerryCurvature}})
    levels, chernnumbers = pack[2].action.levels, pack[2].data[4]
    layout --> length(levels)
    aspect_ratio --> :equal
    framestyle --> :none
    colorbar --> :bottom
    plot_title --> nameof(pack[1], pack[2])
    plot_titlefontsize --> 10
    title := @sprintf "level %s (C = %s)" first(levels) decimaltostr(first(chernnumbers))
    titlefontsize --> 8
    for (i, (level, chn)) in enumerate(zip(levels, chernnumbers))
        @series begin
            seriestype := :heatmap
            title := @sprintf "level %s (C = %s)" level decimaltostr(chn)
            subplot := i
            pack[2].data[1], pack[2].data[2], pack[2].data[3][:, :, i]
        end
    end
    primary := false
    ()
end

"""
    SampleNode(reciprocals::AbstractVector{<:AbstractVector}, position::Vector, levels::Vector{Int}, values::Vector, ratio::Number)
    SampleNode(reciprocals::AbstractVector{<:AbstractVector}, position::Vector, levels::Vector{Int}, values::Vector, ratios::Vector=ones(length(levels)))

A sample node of a momentum-eigenvalues pair.
"""
struct SampleNode
    k::Vector{Float64}
    levels::Vector{Int64}
    values::Vector{Float64}
    ratios::Vector{Float64}
end
@inline function SampleNode(reciprocals::AbstractVector{<:AbstractVector}, position::Vector, levels::Vector{Int}, values::Vector, ratio::Number)
    return SampleNode(reciprocals, position, levels, values, ones(length(levels))*ratio)
end
function SampleNode(reciprocals::AbstractVector{<:AbstractVector}, position::Vector, levels::Vector{Int}, values::Vector, ratios::Vector=ones(length(levels)))
    @assert length(reciprocals)==length(position) "SampleNode error: mismatched reciprocals and position."
    @assert length(levels)==length(values)==length(ratios) "SampleNode error: mismatched levels, values and ratios."
    return SampleNode(mapreduce(*, +, reciprocals, position), levels, values, ratios)
end

"""
    deviation(tba::Union{AbstractTBA, Algorithm{<:AbstractTBA}}, samplenode::SampleNode) -> Float64
    deviation(tba::Union{AbstractTBA, Algorithm{<:AbstractTBA}}, samplesets::Vector{SampleNode}) -> Float64

Get the deviation of the eigenvalues between the sample points and model points.
"""
function deviation(tba::Union{AbstractTBA, Algorithm{<:AbstractTBA}}, samplenode::SampleNode)
    diff = eigvals(matrix(tba; k=samplenode.k))[samplenode.levels] .- samplenode.values
    return real(sum(conj(diff) .* diff .* samplenode.ratios))
end
function deviation(tba::Union{AbstractTBA, Algorithm{<:AbstractTBA}}, samplesets::Vector{SampleNode})
    result = 0.0
    for samplenode in samplesets
        result += deviation(tba, samplenode)
    end
    return result
end

"""
    optimize!(tba::Union{AbstractTBA, Algorithm{<:AbstractTBA}},
        samplesets::Vector{SampleNode},
        variables=keys(Parameters(tba));
        verbose=false,
        method=LBFGS()
        ) -> Tuple{typeof(tba), Optim.MultivariateOptimizationResults}

Optimize the parameters of a tight binding system whose names are specified by `variables` so that the total deviations of the eigenvalues between the model points and sample points minimize.
"""
function optimize!(tba::Union{AbstractTBA, Algorithm{<:AbstractTBA}}, samplesets::Vector{SampleNode}, variables=keys(Parameters(tba)); verbose=false, method=LBFGS())
    v₀ = collect(getfield(Parameters(tba), name) for name in variables)
    function diff(v::Vector)
        parameters = Parameters{variables}(v...)
        update!(tba; parameters...)
        verbose && println(parameters)
        return deviation(tba, samplesets)
    end
    op = optimize(diff, v₀, method)
    parameters = Parameters{variables}(op.minimizer...)
    update!(tba; parameters...)
    return tba, op
end

end # module


function play_turn(state::DynamicState, static_state::StaticState,
    decision::Tuple{UInt8, UInt8})
    next_state = state

    fm_pending = get_fast_moves_pending(state)
    active1, active2 = get_active(next_state)
    cmp = get_cmp(state)

    if !iszero(cmp)
        agent, o_agent = isodd(cmp) ? (0x01, 0x02) : (0x02, 0x01)
        next_state = evaluate_charged_move(next_state, static_state, cmp,
            decision[agent] - 0x06, 0x64, decision[o_agent] == 0x01)
        data = next_state.data - UInt32(fm_pending[agent]) *
            UInt32(agent == 0x01 ? 16 : 112)
        if !iszero(fm_pending[o_agent])
            data -= UInt32(fm_pending[o_agent] - 1) *
                UInt32(o_agent == 0x01 ? 16 : 112)
        end
        next_state = DynamicState(next_state[0x01], next_state[0x02], data)
    else
        if fm_pending[1] == 0x01 || fm_pending[2] == 0x01
            next_state = evaluate_fast_moves(next_state, static_state,
                (fm_pending[1] == 0x01 &&
                    get_hp(next_state[0x01][active1]) != 0x0000,
                fm_pending[2] == 0x01 &&
                    get_hp(next_state[0x02][active2]) != 0x0000))
        end

        next_state = step_timers(next_state,
            decision[1] == 0x03 ?
                static_state[0x01][active1].fastMove.cooldown : Int8(0),
            decision[2] == 0x03 ?
                static_state[0x02][active2].fastMove.cooldown : Int8(0))
        if decision[1] == 0x05 || decision[1] == 0x06
            next_state = evaluate_switch(next_state,
                static_state, 0x01, decision[1] - 0x04,
                iszero(get_hp(state[0x01][active1])) ?
                0x18 : 0x00)
        end
        if decision[2] == 0x05 || decision[2] == 0x06
            next_state = evaluate_switch(next_state,
                static_state, 0x02, decision[2] - 0x04,
                iszero(get_hp(state[0x02][active2])) ?
                0x18 : 0x00)
        end
        active1, active2 = get_active(next_state)

        if get_hp(next_state[0x01][active1]) != 0x0000 &&
            get_hp(next_state[0x02][active2]) != 0x0000
            if decision[1] == 0x04
                if decision[2] == 0x04
                    atk_cmp = Base.cmp(
                        static_state[0x01][active1].stats.attack,
                        static_state[0x02][active2].stats.attack
                    )
                    return DynamicState(next_state[0x01],
                        next_state[0x02], next_state.data +
                        (atk_cmp == 1 ? UInt32(2352) :
                        atk_cmp == -1 ? UInt32(3136) : UInt32(19600)))
                else
                    return DynamicState(next_state[0x01],
                        next_state[0x02], next_state.data + UInt32(784))
                end
            elseif decision[2] == 0x04
                return DynamicState(next_state[0x01],
                    next_state[0x02], next_state.data + UInt32(2*784))
            end
        end
    end

    return next_state
end

function get_chance_state_1(state::DynamicState, static_state::StaticState,
    chance::UInt8)
    if chance == 0x05
        return DynamicState(state[0x01], state[0x02], state.data - 0x4360)
    else
        active1, active2 = get_active(state)
        agent, o_agent = chance < 0x03 ? (0x01, 0x02) : (0x02, 0x01)
        move = isodd(chance) ?
            static_state[agent][active1].charged_move_1 :
            static_state[agent][active2].charged_move_2
        a_data = state[agent].data
        d_data = state[o_agent].data
        a_data, d_data = apply_buff(a_data, d_data, move)
        return update_fm_damage(DynamicState(
            DynamicTeam(state[0x01][0x01], state[0x01][0x02],
                state[0x01][0x03], state[0x01].switchCooldown,
                agent == 0x01 ? a_data : d_data),
            DynamicTeam(state[0x02][0x01], state[0x02][0x02],
                state[0x02][0x03], state[0x02].switchCooldown,
                agent == 0x02 ? a_data : d_data),
            state.data - UInt32(chance) * UInt32(3920)), static_state)
    end
end

function get_chance_state_2(state::DynamicState, static_state::StaticState,
    chance::UInt8)
    if chance == 0x05
        return DynamicState(state[0x01], state[0x02], state.data - 0x4050)
    else
        return DynamicState(state[0x01], state[0x02],
            state.data - UInt32(chance) * UInt32(3920))
    end
end

function resolve_chance(state::DynamicState, static_state::StaticState)
    chance = get_chance(state)
    if chance == 0x00
        return state
    elseif chance == 0x05
        return rand() < 0.5 ?
            # subtract chance, add cmp
            get_chance_state_1(state, static_state, chance) :
            get_chance_state_2(state, static_state, chance)
    else
        active1, active2 = get_active(state)
        agent = chance < 0x03 ? 0x01 : 0x02
        buff_chance = isodd(chance) ?
            static_state[agent][active1].charged_move_1.buffChance :
            static_state[agent][active2].charged_move_2.buffChance
        return rand(Int8(0):Int8(99)) < buff_chance ?
            get_chance_state_1(state, static_state, chance) :
            get_chance_state_2(state, static_state, chance)
    end
end

"""
    play_battle(state, static_state)

Play through one battle, starting from the inputted state with random,
equally weighted decisions.
"""
function play_battle(state::DynamicState, static_state::StaticState;
  allow_nothing::Bool = false, allow_overfarming::Bool = false)
    active1, active2 = get_active(state)
    while true
        state = resolve_chance(state, static_state)
        d1, d2 = get_possible_decisions(state, static_state,
            allow_nothing = allow_nothing,
            allow_overfarming = allow_overfarming)
        (iszero(d1) || iszero(d2)) &&
            return battle_score(state, static_state)
        state = play_turn(
            state, static_state, select_random_decision(d1, d2))
    end
end

"""
    battle_scores(state, static_state, N)

Play through N battles, starting from the inputted state with random,
equally weighted decisions.
"""
function battle_scores(starting_state::DynamicState,
  static_state::StaticState, N::Int64)
    return map(x -> play_battle(starting_state, static_state), 1:N)
end


for ff in (exp,)
    f = nameof(ff)
    m = Base.parentmodule(ff)
    for PT in (:Particles, :StaticParticles)
        @eval begin
            @inline function ($m.$f)(p::$PT{Float64,N}) where N
                res = (SLEEFPirates.$f).(p.particles)
                return $PT{Float64,N}(res)
            end
            @inline function ($m.$f)(p::$PT{Float32,N}) where N
                res = (SLEEFPirates.$f).(p.particles)
                return $PT{Float32,N}(res)
            end
        end
    end
end

for ff in (log, sin, cos, asin, acos, atan) # tan is not faster
    f = nameof(ff)
    fs = Symbol(f,"_fast")
    m = Base.parentmodule(ff)
    for PT in (:Particles, :StaticParticles)
        @eval begin
            @inline function ($m.$f)(p::$PT{Float64,N}) where N
                res = (SLEEFPirates.$fs).(p.particles)
                return $PT{Float64,N}(res)
            end
            @inline function ($m.$f)(p::$PT{Float32,N}) where N
                res = (SLEEFPirates.$fs).(p.particles)
                return $PT{Float32,N}(res)
            end
        end
    end
end


using DocOpt

doc="""Voice conversion based on differencial spectral compensation

Usage:
    diffvc.jl [options] <input_wav> <model_jld> <dst_wav>
    diffvc.jl --version
    diffvc.jl -h | --help

Options:
    -h --help        show this message
    --period=PERIOD  frame period in msec [default: 5.0]
    --order=ORDER    order of mel cepsrum [default: 40]
    --alpha=ALPHA    all-pass constant [default: 0.0]
    --trajectory     trajectory-based parameter conversion
    --T=t            maximum length for one-step trajectory conversion [default: 100]
"""

using VoiceConversion
using MelGeneralizedCepstrums
using WAV
using SynthesisFilters
using HDF5, JLD

import WORLD: synthesis
using WORLD

using Logging
@Logging.configure(level=DEBUG, output=STDOUT)

function main()
    args = docopt(doc, version=v"0.0.1")

    x, fs = wavread(args["<input_wav>"])
    @assert size(x, 2) == 1 "The input data must be monoral."
    x = map(Float64, vec(x))
    fs = Int(fs)
    @info("length of input signal is $(length(x)/fs) sec.")

    period = parse(Float64, args["--period"])
    order = parse(Int, args["--order"])
    α = parse(Float64, args["--alpha"])
    if α == 0.0
        α = mcepalpha(fs)
    end
    trajectory = args["--trajectory"]

    # Load mapping model
    gmm = load(args["<model_jld>"])
    if !gmm["diff"]
        @warn("The model doesn't seem to be trained on differencial features")
    end

    mapper = GMMMap(gmm["weights"], gmm["means"], gmm["covars"])
    if trajectory
        mapper = TrajectoryGMMMap(mapper, int(args["--T"]))
    end

    # shape (order+1, number of frames)
    elapsed_fe = @elapsed begin
        f0, timeaxis = dio(x, fs, DioOption(period=period))
        f0 = stonemask(x, fs, timeaxis, f0)
        spectrogram = cheaptrick(x, fs, timeaxis, f0)
        src = sp2mc(spectrogram, order, α)
    end
    @info("elapsed time in feature extraction is $(elapsed_fe) sec.")
    if trajectory
        # add delta feature
        src = [src[1,:]; push_delta(src[2:end,:])]
    end

    # remove power coef.
    src[1,:] = 0.0

    # Perform conversion
    elapsed_vc = @elapsed converted = vc(mapper, src)
    @info("elapsed time in conversion process is $(elapsed_vc) sec.")

    # Waveform synthesis using Mel-Log Spectrum Approximation filter
    mf = MLSADF(order, α)
    hopsize = round(Int, (fs / (1000 / period)))

    elapsed_syn = @elapsed y = synthesis!(mf, x, mc2b(converted, α), hopsize)
    @info("elapsed time in waveform moduration is $(elapsed_syn) sec.")

    wavwrite(map(Float64, y), args["<dst_wav>"], Fs=fs)
    @info("Dumped to ", args["<dst_wav>"])
end

main()


__precompile__()
module AccelerationBenchmark

using OptimTestProblems.MultivariateProblems
using NLSolversBase
using Optim, LineSearches
using DataFrames, DataArrays
using StatPlots, GroupedErrors
using CUTEst

const MVP = MultivariateProblems
const UP = MVP.UnconstrainedProblems

export OptimizationRun, TestSetup, OACCEL2017

export createruns, createmeasures,
    createmeasuredataframe, createratiodataframe,
    createviolins, createboxplots, createstatplots,
    createperfprofiles

export RandomizeProblem, RandomizeInitialx, RandomizeInitialxMat,
    randomizeproblem!

include("types.jl")
include("api.jl")
include("utils.jl")
include("randomizer.jl")

include("benchmarks/oaccel2017.jl")

end # module