# MIT License
#
# Copyright (c) 2018 Martin Biel
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

function restore!(model::MOI.ModelLike, ::AbstractIntegerAlgorithm)
    return nothing
end

function check_optimality(lshaped::AbstractLShaped, ::AbstractIntegerAlgorithm)
    return all(1:num_subproblems(lshaped)) do idx
        if has_metadata(lshaped.execution.metadata, idx, :integral_solution)
            return get_metadata(lshaped.execution.metadata, idx, :integral_solution)
        else
            error("Worker $idx has not reported if latest solution satisfies integral restrictions.")
        end
    end
end

function is_approx_integer(val::AbstractFloat)
    return ceil(val) - val <= sqrt(eps()) || val - floor(val) <= sqrt(eps())
end

function gather_integer_variables(subproblem::SubProblem)
    model = subproblem.model
    N = num_stages(model.ext[:decisions])
    all_known = mapreduce(vcat, 1:N-1) do s
        index.(all_known_decision_variables(model, s))
    end
    all_decisions = index.(all_decision_variables(model, N))
    indices = Vector{MOI.VariableIndex}()
    for var in all_variables(model)
        vi = index(var)
        if vi in all_known
            # Known decision, skip
            continue
        end
        if vi in all_decisions
            # Decision variable
            dref = DecisionRef(model, vi)
            if is_integer(dref) || is_binary(dref)
                push!(indices, vi)
            else
                continue
            end
        else
            if is_integer(var) || is_binary(var)
                push!(indices, vi)
            else
                continue
            end
        end
    end
    return indices
end

function check_integrality_restrictions(subproblem::SubProblem, fractional = Vector{MOI.VariableIndex}())
    satisfies_integer = map(integer_variables(subproblem.integer_algorithm)) do vi
        val = MOI.get(subproblem.optimizer, MOI.VariablePrimal(), vi)
        if is_approx_integer(MOI.get(subproblem.optimizer, MOI.VariablePrimal(), vi))
            return true
        else
            push!(fractional, vi)
            return false
        end
    end
    return all(satisfies_integer)
end

mutable struct SecondStageLP{T_ <: AbstractFloat}
    T::SparseMatrixCSC{T_,Int}
    W::SparseMatrixCSC{T_,Int}
    h::Vector{T_}
    lb::Vector{T_}
    ub::Vector{T_}
end

function SecondStageLP(subproblem::SubProblem{T_}; standard_form = false) where T_ <: AbstractFloat
    model = subproblem.model
    nx = StochasticPrograms.num_known_decisions(model)
    ny = num_variables(model) - nx
    # Prepare data
    Tᵢ = Vector{Int}()
    Tⱼ = Vector{Int}()
    Tᵥ = Vector{T_}()
    Wᵢ = Vector{Int}()
    Wⱼ = Vector{Int}()
    Wᵥ = Vector{T_}()
    lhs = Vector{T_}()
    rhs = Vector{T_}()
    lb = fill(-Inf, ny)
    ub = fill(Inf, ny)
    i = 0
    slack = 0
    # Loop over constraints
    for (F,S) in list_of_constraint_types(model)
        for cref in all_constraints(model, F, S)
            if S <: StochasticPrograms.SingleDecisionSet ||
                S <: StochasticPrograms.MultipleDecisionSet
                continue
            elseif F <: VariableRef || F <: DecisionRef
                f = MOI.get(model, MOI.ConstraintFunction(), cref)
                if f isa MOI.SingleVariable
                    idx = f.variable.value - nx
                elseif f isa SingleDecision
                    idx = f.decision.value - nx
                end
                set = MOI.get(model, MOI.ConstraintSet(), cref)
                if set isa MOI.LessThan
                    ub[idx] = set.upper
                    if standard_form
                        i += 1
                        slack += 1
                        push!(Wᵢ, i)
                        push!(Wⱼ, idx)
                        push!(Wᵥ, 1.0)
                        push!(Wᵢ, i)
                        push!(Wⱼ, ny + slack)
                        push!(Wᵥ, 1.0)
                        push!(lhs, -Inf)
                        push!(rhs, set.upper)
                    end
                elseif set isa MOI.GreaterThan
                    lb[idx] = set.lower
                elseif set isa MOI.EqualTo
                    lb[idx] = set.value
                    ub[idx] = set.value
                end
            elseif F <: AffExpr
                i += 1
                f = MOI.get(model, MOI.ConstraintFunction(), cref)
                for term in f.terms
                    push!(Wᵢ, i)
                    push!(Wⱼ, term.variable_index.value - nx)
                    push!(Wᵥ, term.coefficient)
                end
                set = MOI.get(model, MOI.ConstraintSet(), cref)
                if set isa MOI.LessThan
                    push!(lhs, -Inf)
                    push!(rhs, set.upper)
                    if standard_form
                        slack += 1
                        push!(Wᵢ, i)
                        push!(Wⱼ, ny + slack)
                        push!(Wᵥ, 1.0)
                    end
                elseif set isa MOI.GreaterThan
                    push!(lhs, set.lower)
                    push!(rhs, Inf)
                    if standard_form
                        slack += 1
                        push!(Wᵢ, i)
                        push!(Wⱼ, ny + slack)
                        push!(Wᵥ, -1.0)
                    end
                elseif set isa MOI.EqualTo
                    push!(lhs, set.value)
                    push!(rhs, set.value)
                    if standard_form
                        slack += 1
                        push!(Wᵢ, i)
                        push!(Wⱼ, ny + slack)
                        push!(Wᵥ, 1.0)
                    end
                end
            elseif F <: Vector{<:AffExpr}
                f = MOI.get(model, MOI.ConstraintFunction(), cref)
                set = MOI.get(model, MOI.ConstraintSet(), cref)
                for (j,fⱼ) in enumerate(MOIU.eachscalar(f))
                    i += 1
                    for term in fⱼ.terms
                        push!(Wᵢ, i)
                        push!(Wⱼ, term.variable_index.value - nx)
                        push!(Wᵥ, term.coefficient)
                    end
                    if set isa MOI.Nonpositives
                        push!(lhs, -Inf)
                        push!(rhs, -f.constants[j])
                        if standard_form
                            slack += 1
                            push!(Wᵢ, i)
                            push!(Wⱼ, ny + slack)
                            push!(Wᵥ, 1.0)
                        end
                    elseif set isa MOI.Nonnegatives
                        push!(lhs, -f.constants[j])
                        push!(rhs, Inf)
                        if standard_form
                            slack += 1
                            push!(Wᵢ, i)
                            push!(Wⱼ, ny + slack)
                            push!(Wᵥ, -1.0)
                        end
                    elseif set isa MOI.Zeros
                        push!(lhs, -f.constants[j])
                        push!(rhs, -f.constants[j])
                        if standard_form
                            slack += 1
                            push!(Wᵢ, i)
                            push!(Wⱼ, ny + slack)
                            push!(Wᵥ, 1.0)
                        end
                    end
                end
            elseif F <: DecisionAffExpr
                i += 1
                aff = jump_function(model, MOI.get(model, MOI.ConstraintFunction(), cref))
                for (coeff, var) in linear_terms(aff.variables)
                    push!(Wᵢ, i)
                    push!(Wⱼ, index(var).value - nx)
                    push!(Wᵥ, coeff)
                end
                for (coeff, dvar) in linear_terms(aff.decisions)
                    if state(dvar) == Known
                        push!(Tᵢ, i)
                        push!(Tⱼ, index(dvar).value)
                        push!(Tᵥ, coeff)
                    else
                        push!(Wᵢ, i)
                        push!(Wⱼ, index(dvar).value - nx)
                        push!(Wᵥ, coeff)
                    end
                end
                set = MOI.get(model, MOI.ConstraintSet(), cref)
                if set isa MOI.LessThan
                    push!(lhs, -Inf)
                    push!(rhs, set.upper)
                    if standard_form
                        slack += 1
                        push!(Wᵢ, i)
                        push!(Wⱼ, ny + slack)
                        push!(Wᵥ, 1.0)
                    end
                elseif set isa MOI.GreaterThan
                    push!(lhs, set.lower)
                    push!(rhs, Inf)
                    if standard_form
                        slack += 1
                        push!(Wᵢ, i)
                        push!(Wⱼ, ny + slack)
                        push!(Wᵥ, -1.0)
                    end
                elseif set isa MOI.EqualTo
                    push!(lhs, set.value)
                    push!(rhs, set.value)
                    if standard_form
                        slack += 1
                        push!(Wᵢ, i)
                        push!(Wⱼ, ny + slack)
                        push!(Wᵥ, 1.0)
                    end
                end
            elseif F <: Vector{<:DecisionAffExpr}
                f = MOI.get(model, MOI.ConstraintFunction(), cref)
                set = MOI.get(model, MOI.ConstraintSet(), cref)
                for (j,fⱼ) in enumerate(MOIU.eachscalar(f))
                    i += 1
                    aff = jump_function(model, fⱼ)
                    for (coeff, var) in linear_terms(aff.variables)
                        push!(Wᵢ, i)
                        push!(Wⱼ, index(var).value - nx)
                        push!(Wᵥ, coeff)
                    end
                    for (coeff, dvar) in linear_terms(aff.decisions)
                        if state(dvar) == Known
                            push!(Tᵢ, i)
                            push!(Tⱼ, index(dvar).value)
                            push!(Tᵥ, coeff)
                        else
                            push!(Wᵢ, i)
                            push!(Wⱼ, index(dvar).value - nx)
                            push!(Wᵥ, coeff)
                        end
                    end
                    if set isa MOI.Nonpositives
                        push!(lhs, -Inf)
                        push!(rhs, -f.variable_part.constants[j])
                        if standard_form
                            slack += 1
                            push!(Wᵢ, i)
                            push!(Wⱼ, ny + slack)
                            push!(Wᵥ, 1.0)
                        end
                    elseif set isa MOI.Nonnegatives
                        push!(lhs, -f.variable_part.constants[j])
                        push!(rhs, Inf)
                        if standard_form
                            slack += 1
                            push!(Wᵢ, i)
                            push!(Wⱼ, ny + slack)
                            push!(Wᵥ, -1.0)
                        end
                    elseif set isa MOI.Zeros
                        push!(lhs, -f.variable_part.constants[j])
                        push!(rhs, -f.variable_part.constants[j])
                        if standard_form
                            slack += 1
                            push!(Wᵢ, i)
                            push!(Wⱼ, ny + slack)
                            push!(Wᵥ, 1.0)
                        end
                    end
                end
            else
                error("Cannot extract linear representation from $F-$S constraint.")
            end
        end
    end
    T̃ = sparse(Tᵢ, Tⱼ, Tᵥ, i, nx)
    W̃ = sparse(Wᵢ, Wⱼ, Wᵥ, i, ny + slack)
    if standard_form
        h = lhs
        i = .!isfinite.(h)
        h[i] .= rhs[i]
        return SecondStageLP(T̃, W̃, h, lb, ub)
    else
        T, h = StochasticPrograms.canonical(T̃, lhs, rhs)
        W, h = StochasticPrograms.canonical(W̃, lhs, rhs)
        return SecondStageLP(-T, -W, -h, lb, ub)
    end
end

function optimal_basis(subproblem::SubProblem)
    model = subproblem.model
    nx = StochasticPrograms.num_known_decisions(model)
    ny = num_variables(model) - nx
    variable_basis = Int[]
    slack_basis = Int[]
    slack = 0
    # Loop over constraints
    for (F,S) in list_of_constraint_types(model)
        for cref in all_constraints(model, F, S)
            if S <: StochasticPrograms.SingleDecisionSet ||
                S <: StochasticPrograms.MultipleDecisionSet
                continue
            elseif F <: VariableRef || F <: DecisionRef
                f = MOI.get(model, MOI.ConstraintFunction(), cref)
                set = MOI.get(model, MOI.ConstraintSet(), cref)
                if f isa MOI.SingleVariable
                    idx = f.variable.value - nx
                elseif f isa SingleDecision
                    idx = f.decision.value - nx
                end
                if set isa MOI.LessThan || set isa MOI.EqualTo
                    slack += 1
                    if MOI.get(model, MOI.ConstraintBasisStatus(), cref) == MOI.BASIC
                        push!(slack_basis, ny + slack)
                    end
                elseif set isa MOI.GreaterThan
                    if MOI.get(model, MOI.ConstraintBasisStatus(), cref) == MOI.BASIC
                        push!(variable_basis, idx)
                    end
                end
            elseif F <: AffExpr || F <: DecisionAffExpr
                slack += 1
                if MOI.get(model, MOI.ConstraintBasisStatus(), cref) == MOI.BASIC
                    push!(slack_basis, ny + slack)
                end
            else
                error("Cannot extract basis from $F-$S constraint.")
            end
        end
    end
    return vcat(variable_basis, slack_basis)
end

function add_row!(lp::SecondStageLP, T̄::AbstractVector, W̄::AbstractVector, h̄::AbstractFloat; standard_form = false)
    # Get current size
    m = size(lp.W, 1)
    nx = size(lp.T, 2)
    ny = size(lp.W, 2)
    if standard_form
        # Prepare to add new slack variable
        ny += 1
    end
    # Get current matrix structure
    Tᵢ, Tⱼ, Tᵥ = findnz(lp.T)
    Wᵢ, Wⱼ, Wᵥ = findnz(lp.W)
    # Add T row
    for (j, coeff) in enumerate(T̄)
        if abs(coeff) >= sqrt(eps())
            push!(Tᵢ, m + 1)
            push!(Tⱼ, j)
            push!(Tᵥ, coeff)
        end
    end
    lp.T = sparse(Tᵢ, Tⱼ, Tᵥ, m + 1, nx)
    # Add W row
    for (j, coeff) in enumerate(W̄)
        if abs(coeff) >= sqrt(eps())
            push!(Wᵢ, m + 1)
            push!(Wⱼ, j)
            push!(Wᵥ, coeff)
        end
    end
    if standard_form
        # Add slack term
        push!(Wᵢ, m + 1)
        push!(Wⱼ, ny)
        push!(Wᵥ, -1.0)
    end
    lp.W = sparse(Wᵢ, Wⱼ, Wᵥ, m + 1, ny)
    # Add new rhs
    push!(lp.h, h̄)
    return nothing
end


using JuMP, Gurobi, Ipopt
using DataFrames, JLD, CSVFiles
using MatpowerCases

### Include the file with input data functions
include("NetworkDataType.jl")
include("NetworkLoad.jl")
include("P2PTradeType.jl")
include("TradeFunction.jl")
include("DLMP.jl")

testsystem = "AP15busDN"
#testsystem = "case141_OPF"

buses, lines, generators, datamat = NetworkLoad(testsystem)
lineset = 1:length(lines)
busset = 1:length(buses)
genset = 1:length(generators)
buses[1].Vmax = 1
buses[1].Vmin = 1
windset = Int[]
for g in genset
    if datamat["gen"][g,2] == "Wind"
        push!(windset, generators[g].gindex)
    end
end

root = -1
for g in genset
    if generators[g].gtype=="Root"
        root = g
    end
end
if root == -1
    display("No generator at root node")
    generators[1].gtype = "Root"
    root = 1
end

temp = Generator
for g in genset
    if generators[g].gtype == "Root"
        temp = Generator(length(genset)+1, "DG", generators[g].location, generators[g].Pg, generators[g].Qg, generators[g].Qmax,
        generators[g].Qmin, generators[g].Vg, generators[g].mBase, generators[g].status, generators[g].Pmax, generators[g].Pmin,
        generators[g].cost, generators[g].SUcost, generators[g].SDcost, generators[g].RU, generators[g].RD, generators[g].UPtime, generators[g].DNtime)
        #push!(generators, temp)
    end
end

genset = 1:length(generators)
gensetU = root
gensetP = setdiff(genset, gensetU)

B_g = []
for g in genset
   if ~any(x->x==generators[g].location, B_g)
      push!(B_g, generators[g].location)
      buses[generators[g].location].Pd = 0
   end
end
B_d = setdiff(busset,B_g)

B_gn = Array{Array{Int64}}(length(buses),1)
for ii in 1:length(buses)
    B_gn[ii] = Int64[]
end
for g in genset
    push!(B_gn[generators[g].location], g)
end
############
partlevel = 0.0
gam = partlevel*ones(busset)
############
###
trade_scale = 1e-3 # [MWh]

agents = Agent[]
for b in 1:length(buses)
    if isempty(B_gn[b])
        temp_agent = Agent(b, b, 1, 0, 0, [], gam[b]*buses[b].Pd, gam[b]*buses[b].Qd)
        push!(agents,temp_agent)
    else
        for g in B_gn[b]
            if generators[g].gtype != "Root"
                temp_agent = Agent(b, b, 1, generators[g].Pmin, generators[g].Pmax, generators[g].cost, gam[b]*buses[b].Pd, gam[b]*buses[b].Qd)
                push!(agents,temp_agent)
            end
        end
    end
end
# agents = Agent[]
# for b in 1:length(buses)
#     if isempty(B_gn[b])
#         temp_agent = Agent(b, b, 1, 0, 0, [], gam[b]*buses[b].Pd, gam[b]*buses[b].Qd)
#         push!(agents,temp_agent)
#     elseif generators[B_gn[b][1]].gtype != "Root"
#         g = B_gn[b][1]
#         temp_agent = Agent(b, b, 1, generators[g].Pmin, generators[g].Pmax, generators[g].cost, gam[b]*buses[b].Pd, gam[b]*buses[b].Qd)
#         push!(agents,temp_agent)
#     end
# end

# Trade(tradeID, kind, Pes, Peb, As, Ab, costNw)
trades = generateTrade(agents, trade_scale)
trades_old = generateTrade(agents, trade_scale)

trades_mat = Array{Any}(length(agents),length(agents))
trades_mat_buy = Array{Any}(length(agents),length(agents))
trades_mat_sell = Array{Any}(length(agents),length(agents))
for ii in 1:length(agents), jj in 1:length(agents)
    trades_mat[ii,jj] = []
    trades_mat_buy[ii,jj] = []
    trades_mat_sell[ii,jj] = []
end
trades_selected_mat = Array{Any}(length(buses),length(buses))
trades_selected_mat_buy = Array{Any}(length(buses),length(buses))
trades_selected_mat_sell = Array{Any}(length(buses),length(buses))
for ii in 1:length(buses), jj in 1:length(buses)
    trades_selected_mat[ii,jj] = []
    trades_selected_mat_buy[ii,jj] = []
    trades_selected_mat_sell[ii,jj] = []
end

for w in 1:length(trades)
    push!(trades_mat[trades[w].As,trades[w].Ab],trades[w].tradeID)
end

trades_selected = Int[]
trades_rest = Int[]
trades_selected_old = Int[]
trades_rest_old = Int[]
dispatch = zeros(length(buses))
GenCon = zeros(length(buses))
Generation = zeros(length(buses))
Consumption = zeros(length(buses))
GenCon_old = zeros(length(buses))
dispatch_old = zeros(length(buses))
status = Symbol()
DLMP = Float64[]
DLMP_stack = Any[]
dispatch_stack = Any[]
DLMPInfo = DataFrame()
NodeInfo = DataFrame()
LineInfo = DataFrame()
dispatch_peerG = 0.0*zeros(genset)
param_delta = 1.0
tic()
while true
    #for _ in 1:10
    accepted = Array{Int64}[]
    rejected = Array{Int64}[]
    pg = Float64[]
    Revenue_Sell = Float64[]
    Cost_Buy = Float64[]
    Cost_Network = Float64[]
    Cost_DG = Float64[]
    trades_old[:] = trades[:]
    trades_selected_old = trades_selected
    for w in 1:length(trades)
        trades[w].Pes = 0
        trades[w].Peb = 0
    end

    while true
        accepted = Array{Int64}[]
        rejected = Array{Int64}[]
        pg = Float64[]

        gap = 1e-3
        for agentID in 1:length(agents)
            temp_accepted, temp_rejected, temp_pg, Revenue_Sell, Cost_Buy, Cost_Network, Cost_DG = selectTrade(trades, agents, agentID, gap, trade_scale)
            push!(accepted, temp_accepted)
            push!(rejected, temp_rejected)
            push!(pg, temp_pg)
        end

        deltaP = 5*trade_scale #$5/MWh = $5e-3/kWh
        for w in 1:length(trades)
            if (w in accepted[trades[w].Ab]) && ~(w in accepted[trades[w].As])
                if trades[w].Peb > trades[w].Pes
                    trades[w].Pes += deltaP
                else
                    trades[w].Peb += deltaP
                end
            end
        end
        # diff = sum((w in accepted[trades[w].Ab]) && ~(w in accepted[trades[w].As]) for w in 1:length(trades))
        # if diff == 0
        #     break
        # end

        ###diff = sum(abs(trades[w].Pes - trades_old[w].Pes) for w in 1:length(trades)) + sum(abs(trades[w].Peb - trades_old[w].Peb) for w in 1:length(trades))
        if isempty(trades)
            diff = 0
        else
            diff = sum((w in accepted[trades[w].Ab]) && ~(w in accepted[trades[w].As]) for w in 1:length(trades))
        end
        if diff == 0
            break
        end
    end

    trades_selected = Int[]
    trades_rest = Int[]
    for w in 1:length(trades)
        if (w in accepted[trades[w].Ab]) && (w in accepted[trades[w].As])
            push!(trades_selected, w)
        else
            push!(trades_rest, w)
        end
    end

    for i in 1:length(agents)
        dispatch[i] = ceil(agents[i].Pd/trade_scale) # MW to kW conversion
    end

    for w in trades_selected
        dispatch[agents[trades[w].Ab].location] -= 1
        dispatch[agents[trades[w].As].location] += 1
        GenCon[agents[trades[w].Ab].location] -= 1
        GenCon[agents[trades[w].As].location] += 1
        Consumption[agents[trades[w].Ab].location] -= 1
        Generation[agents[trades[w].As].location] += 1
    end

    GenCon = GenCon*trade_scale
    dispatch = dispatch*trade_scale # Kw to MW conversion
    Generation = Generation*trade_scale
    Consumption = Consumption*trade_scale

    #dispatch_peerG = 0.0*zeros(genset)
    for g in gensetP
        dispatch_peerG[g] = Generation[generators[g].location]
    end

    ###### Calculate Network Charge #####
    SMP = generators[root].cost[end-1]
    status, DLMP, pgextra, NodeInfo, LineInfo, DLMPInfo = calculateDLMP(dispatch_peerG, buses, generators, lines, SMP, gensetP, gensetU);

    ###### Update Network Charge #####
    deltaNw = 1*trade_scale # $1/MWh = $1e-3/kWh
    if status == Symbol(:Optimal)
        for w in 1:length(trades)
            if w in trades_selected
                trades[w].costNw = (DLMP[agents[trades[w].Ab].location] - DLMP[agents[trades[w].As].location])/2
            end
        end
    else
        for w in trades_selected
            trades[w].costNw += param_delta*deltaNw/trade_scale
        end
    end
    push!(dispatch_stack, dispatch)
    push!(DLMP_stack, DLMP)
    if GenCon_old == GenCon && status == Symbol(:Optimal)
        break
    else
        dispatch_old[:] = dispatch[:]
        GenCon_old[:] = GenCon[:]
    end
end
simulation_time = toc()

trades_dis = zeros(length(buses),length(buses));
NWcharge_dis = zeros(length(buses),length(buses));
Echarge_dis = zeros(length(buses),length(buses));
temp = Array{Array{Int64,1}}(length(buses),1)
for i in 1:length(buses)
    temp[i] = []
end

for w in trades_selected
    trades_dis[agents[trades[w].Ab].location,agents[trades[w].As].location] -= 1*trade_scale
    trades_dis[agents[trades[w].As].location,agents[trades[w].Ab].location] += 1*trade_scale
    Echarge_dis[agents[trades[w].Ab].location,agents[trades[w].As].location] -= trades[w].Peb
    Echarge_dis[agents[trades[w].As].location,agents[trades[w].Ab].location] += trades[w].Pes
    NWcharge_dis[agents[trades[w].Ab].location,agents[trades[w].As].location] -= round(trades[w].costNw,4)
    NWcharge_dis[agents[trades[w].As].location,agents[trades[w].Ab].location] += round(trades[w].costNw,4)
    push!(temp[agents[trades[w].Ab].location], w)
    push!(temp[agents[trades[w].As].location], w)
end

dispatch_peerG = 0.0*zeros(genset)
for g in gensetP
    dispatch_peerG[g] = Generation[generators[g].location]
end

###### Calculate Network Charge #####
SMP = generators[root].cost[end-1]
status, DLMP, pg, NodeInfo, LineInfo, DLMPInfo = calculateDLMP(dispatch_peerG, buses, generators, lines, SMP, gensetP, gensetU);

#CSV.write("PeerCentric_DLMP.csv", DLMPInfo);
#CSV.write("PeerCentric_NodeInfo.csv", NodeInfo);
#CSV.write("PeerCentric_LineInfo.csv", LineInfo);

price_mat = zeros(length(buses),length(buses))

#############################################
for w in trades_selected
    push!(trades_selected_mat[agents[trades[w].As].location,agents[trades[w].Ab].location],trades[w].tradeID)
end

price_mat = zeros(length(buses),length(buses));
pricemin = zeros(length(buses),length(buses));
pricemax = zeros(length(buses),length(buses));
for ii in 1:length(buses), jj in 1:length(buses)
    temp_mat = []
    for ww in trades_selected_mat[ii,jj]
        push!(temp_mat,trades[ww].Pes)
        pricemin[ii,jj] = minimum(temp_mat)
        pricemin[jj,ii] = -minimum(temp_mat)
        pricemax[ii,jj] = maximum(temp_mat)
        pricemax[jj,ii] = -maximum(temp_mat)
        if pricemin[ii,jj] == pricemax[ii,jj]
            price_mat[ii,jj] = pricemin[ii,jj]
            price_mat[jj,ii] = -pricemin[ii,jj]
        else
            error("pricemin != pricemax")
        end
    end
end

Nprice_mat = zeros(length(buses),length(buses));
for w in trades_selected
    Nprice_mat[agents[trades[w].Ab].location,agents[trades[w].As].location] = 0.5*round(trades[w].costNw,5)
    Nprice_mat[agents[trades[w].As].location,agents[trades[w].Ab].location] = 0.5*round(trades[w].costNw,5)
end

Nprice_mat_min = zeros(length(buses),length(buses));
Nprice_mat_max = zeros(length(buses),length(buses));
for w = 1:length(trades)
    Nprice_mat_min[agents[trades[w].Ab].location,agents[trades[w].As].location] = min(round(trades[w].costNw,5),Nprice_mat_min[agents[trades[w].Ab].location,agents[trades[w].As].location])
    Nprice_mat_min[agents[trades[w].As].location,agents[trades[w].Ab].location] = min(round(trades[w].costNw,5),Nprice_mat_min[agents[trades[w].As].location,agents[trades[w].Ab].location])
    Nprice_mat_max[agents[trades[w].Ab].location,agents[trades[w].As].location] = max(round(trades[w].costNw,5),Nprice_mat_max[agents[trades[w].Ab].location,agents[trades[w].As].location])
    Nprice_mat_max[agents[trades[w].As].location,agents[trades[w].Ab].location] = max(round(trades[w].costNw,5),Nprice_mat_max[agents[trades[w].As].location,agents[trades[w].Ab].location])
end

Eprice_mat = zeros(length(buses),length(buses));
for w in trades_selected
    Eprice_mat[agents[trades[w].Ab].location,agents[trades[w].As].location] = round(trades[w].Pes,5)
    Eprice_mat[agents[trades[w].As].location,agents[trades[w].Ab].location] = round(trades[w].Peb,5)
end

Eprice_mat_min = zeros(length(buses),length(buses));
Eprice_mat_max = zeros(length(buses),length(buses));
for w = 1:length(trades)
    Eprice_mat_min[agents[trades[w].Ab].location,agents[trades[w].As].location] = min(round(trades[w].Pes,5),Eprice_mat_min[agents[trades[w].Ab].location,agents[trades[w].As].location])
    Eprice_mat_min[agents[trades[w].As].location,agents[trades[w].Ab].location] = min(round(trades[w].Peb,5),Eprice_mat_min[agents[trades[w].As].location,agents[trades[w].Ab].location])
    Eprice_mat_max[agents[trades[w].Ab].location,agents[trades[w].As].location] = max(round(trades[w].Pes,5),Eprice_mat_max[agents[trades[w].Ab].location,agents[trades[w].As].location])
    Eprice_mat_max[agents[trades[w].As].location,agents[trades[w].Ab].location] = max(round(trades[w].Peb,5),Eprice_mat_max[agents[trades[w].As].location,agents[trades[w].Ab].location])
end

Peer_revenue = sum(Echarge_dis,2);
Consumer_cost = 0
Generator_revenue = 0
for bb in B_d
    Consumer_cost -= Peer_revenue[bb]
end
for bb in B_g
    Generator_revenue += Peer_revenue[bb]
end

line_marginrate = zeros(length(lines),1)
for ll in 1:length(lines)
    line_marginrate[ll] = round.((lines[ll].u  - sqrt((LineInfo[:fp][ll])^2 + (LineInfo[:fq][ll])^2))/ lines[ll].u,3)
end
line_loading = 1-line_marginrate
bus_voltage = sqrt.(NodeInfo[:v])

v_marginrate = round.(0.1-abs.(1-sqrt.(NodeInfo[:v])),3)/0.1

Loading = DataFrame(line_loading=vec(line_loading))
Vmargin = DataFrame(v_marginrate=vec(v_marginrate))
#####
loading = zeros(length(lines))
for l in 1:length(lines)
   loading[l] = LineInfo[:flow][l] / lines[l].u
end

flow = LineInfo[:flow]
voltage = sqrt.(NodeInfo[:v])

#######
pnm = trades_dis
demand_P2P = zeros(length(buses))
for m in B_d
    demand_P2P[m] = sum(pnm[n,m] for n in B_g)
end

EP_P = zeros(length(busset))
for w in trades_selected
   EP_P[agents[trades[w].Ab].location] += trades[w].Pes
end
EP_U = zeros(length(busset))
for m in B_d
   EP_U[m] = (buses[m].Pd-demand_P2P[m])*DLMP[m]
end

NUC = zeros(length(busset))
for w in trades_selected
   NUC[agents[trades[w].Ab].location] += 0.5*trades[w].costNw
   NUC[agents[trades[w].As].location] += 0.5*trades[w].costNw
end
UR = EP_U + NUC
CP = EP_U + EP_P + NUC

GC = zeros(length(genset))
for g in genset
   GC[g] = generators[g].cost[end-1] * round(pg[g],5)
end
sum_GC_P = sum(GC[g] for g in gensetP)
sum_GC_U = sum(GC[g] for g in gensetU)
sum_DGR = sum(EP_P)
sum_DGP = sum_DGR - sum_GC_P

UP = sum(UR) - sum_GC_U
CashFlow_mat = DataFrame(EP_P=EP_P,EP_U=EP_U,NUC=NUC,UR=UR,CP=CP)
CashFlow_sum = DataFrame(sum_CP=sum(CP), sum_EP_P=sum(EP_P), sum_EP_U=sum(EP_U), sum_NUC=sum(NUC), sum_DGR=sum_DGR, sum_DGP=sum_DGP, UP=UP)

println("partlevel = ""$partlevel")
println("status = ""$status")
println("max loading = ""$(maximum(loading)*100)""%")
println("max voltage = ""$(round(maximum(sqrt.(NodeInfo[:v])),3))")
println("min voltage = ""$(round(minimum(sqrt.(NodeInfo[:v])),3))")
println("max DLMP = ""$(round(maximum(DLMP),3))")
println("min DLMP = ""$(round(minimum(DLMP),3))")
println("simulation time = ""$simulation_time")
method = "PC"
voltagefile = "$method$(length(buses))""_""$(Int(100*partlevel))""_voltage"".csv"
loadingfile = "$method$(length(buses))""_""$(Int(100*partlevel))""_loading"".csv"
DLMPfile = "$method$(length(buses))""_""$(Int(100*partlevel))""_DLMP"".csv"
cashflowmatfile = "$method$(length(buses))""_""$(Int(100*partlevel))""_cfmat"".csv"
cashflowsumfile = "$method$(length(buses))""_""$(Int(100*partlevel))""_cfsum"".csv"
# CSVFiles.save(voltagefile, DataFrame(voltage=voltage))
# CSVFiles.save(loadingfile, DataFrame(loading=100*loading))
# CSVFiles.save(DLMPfile, DataFrame(DLMP=DLMP))
# CSVFiles.save(cashflowmatfile, CashFlow_mat)
# CSVFiles.save(cashflowsumfile, CashFlow_sum)

#######
resultfile = "$method$(length(buses))""_""$(Int(round(100*partlevel)))""_""$(Int(1000*trade_scale))""kW"".jld"
#@save "$resultfile"

#######################################
trades_selected_n = Array{Array{Int64,1}}(length(buses),1)
trades_n = Array{Array{Int64,1}}(length(buses),1)
for i in 1:length(buses)
    trades_selected_n[i] = []
    trades_n[i] = []
end
for w in trades_selected
    push!(trades_selected_n[trades[w].Ab],w)
    push!(trades_selected_n[trades[w].As],w)
end
for w in 1:length(trades)
    push!(trades_n[trades[w].Ab],w)
    push!(trades_n[trades[w].As],w)
end
Erho = zeros(length(buses),1)
Ecn = zeros(length(buses),1)
Tvol = zeros(length(buses),1)
NUC_total = zeros(length(buses),1)
for i in 1:length(buses)
    if ~isempty(trades_selected_n[i])
        Tvol[i] = length(trades_selected_n[i])
        Ecn[i] = trade_scale*sum(0.5*(DLMP[trades[w].Ab]-DLMP[trades[w].As]) for w in trades_selected_n[i]) / Tvol[i]
        NUC_total[i] = trade_scale*sum(0.5*(DLMP[trades[w].Ab]-DLMP[trades[w].As]) for w in trades_selected_n[i])
        Erho[i] = sum(trades[w].Pes for w in trades_selected_n[i]) / Tvol[i]
    end
end
println("NUC total = ",sum(NUC_total))
println("Tvol = ", sum(Tvol))
#######################################


using Base.Threads
mutable struct Seg_MicroTimeParaStruct
    StaticThreshold::Int # if equal to the typeMAxInt, the Static TH  does not be in  effect
    DynamicThreshold_CalTimePeriod::UInt ## in data Numbers # calculated from Seconds.seconds ,may be any time period ,between 10seconds and the cached data period(about 10 minutes above )

    DynamicThresHold_calCFactor::Int
    minTh::Int
    maxTh::Int
    microMergeTimePeriod::Int # in 0.426 seconds # calculated from :seconds * 0.426
    microFilterTimePeriod::Int # in 0.426 seconds # same as above
    function Seg_MicroTimeParaStruct()
        StaticThreshold= UInt( 0 )
        DynamicThreshold_CalTimePeriod=UInt(2*60) # about 1Minutes
        DynamicThresHold_calCFactor = 130 # about 1.3
        # dynamic_RTThreshold = Int(1500)
        microMergeTimePeriod = UInt(1 )
        microFilterTimePeriod = UInt(1)
        minTh=Int( 1000)
        maxTh=Int( 1000)
        new(StaticThreshold,DynamicThreshold_CalTimePeriod,DynamicThresHold_calCFactor, minTh,maxTh,
            microMergeTimePeriod, microFilterTimePeriod)
    end
end

mutable struct  Seg_MicroSpaceParaStruct
    microMergeSpaceLength::UInt # in Segment or  grain. Calculated from the space length and the segPermeters
    microFilterSpaceLength::UInt # in Segment   or  grain
    function Seg_MicroSpaceParaStruct()
        microMergeSpaceLength = UInt( 1 )
        microFilterSpaceLength = UInt( 2 )
        new(microMergeSpaceLength, microFilterSpaceLength)
    end
end

mutable struct  Seg_MacroSpaceParaStruct
    macroMaxSpaceLength::UInt # in Segment or  grain. Calculated from the space length and the segPermeters
    macroMinSpaceLength::UInt # in Segment   or  grain

    function Seg_MacroSpaceParaStruct()
        macroMaxSpaceLength = UInt( 100 )
        macroMinSpaceLength = UInt(2) #
        new(macroMaxSpaceLength, macroMinSpaceLength)
    end
end

mutable struct Seg_MacroTimeParaStruct
    macroMaxTimeInterval::UInt # in dataFrames
    macroMinTimeInterval::UInt # in  dataFrames
    function Seg_MacroTimeParaStruct()
        macroMaxTimeInterval= Int( 7 )
        macroMinTimeInterval=Int(3)
        new(macroMaxTimeInterval, macroMinTimeInterval)
    end
end

function getSegAlarmPara( SegAlarmParasDict::Array, SegID::Int)::NamedTuple
    try
        len1=length(SegAlarmParasDict )

        ret = SegAlarmParasDict[ SegID ];
        # println("---",SegID ," $len1 ,$ret ")
        return ret
    catch e
        println(":::$e===\r\n")
        println("-Error: SEgidout of range returned(1:8192) --",SegID )
	    exit();
       # return SegAlarmParasDict[0]
    end
end

function getSegStaticThPara(SegAlarmParasDict::Array, SegID::Int ) ::Int
 #   lock(glbAlarmParasdictLock)
    d1 = getSegAlarmPara( SegAlarmParasDict, SegID)
  #  unlock(glbAlarmParasdictLock)
    return d1.microtime.StaticThreshold
end

function getSegDynParaValue(SegAlarmParasDict::Array, SegID::Int )::Tuple
   # lock(glbAlarmParasdictLock)
        d1 = getSegAlarmPara( SegAlarmParasDict, SegID)
        a = (d1.microtime.DynamicThreshold_CalTimePeriod,d1.microtime.DynamicThresHold_calCFactor,d1.microtime.minTh,  d1.microtime.maxTh  )
   # unlock(glbAlarmParasdictLock)
    return a
end

function getSegMicroTimeParaValue(SegAlarmParasDict::Array, SegID::Int )::Tuple
  #  lock(glbAlarmParasdictLock)
        d1 = getSegAlarmPara( SegAlarmParasDict, SegID)
        a= (d1.microtime.microMergeTimePeriod, d1.microtime.microFilterTimePeriod)
    #unlock(glbAlarmParasdictLock)
    return a

end

function getSegMicroSpaceParaValue(SegAlarmParasDict::Array, SegID::Int ) ::Tuple
   # lock(glbAlarmParasdictLock)
        d1 = getSegAlarmPara( SegAlarmParasDict, SegID)
        a= (d1.microSpace.microMergeSpaceLength, d1.microSpace.microFilterSpaceLength)
   # unlock(glbAlarmParasdictLock)
    return a
end

function getSegMacroSpaceParaValue(SegAlarmParasDict::Array, SegID::Int ) ::Tuple
    #lock(glbAlarmParasdictLock)
        d1 = getSegAlarmPara( SegAlarmParasDict, SegID)
        a= (d1.macroSpace.macroMaxSpaceLength, d1.macroSpace.macroMinSpaceLength,)
    #unlock(glbAlarmParasdictLock)
    return a
end

function getSegMacroTimeParaValue(SegAlarmParasDict::Array, SegID::Int ) ::Tuple
    #lock(glbAlarmParasdictLock)
        d1 = getSegAlarmPara( SegAlarmParasDict, SegID)
        a= (d1.macroTime.macroMaxTimeInterval, d1.macroTime.macroMinTimeInterval,)
   # unlock(glbAlarmParasdictLock)
    return a
end

include("SegAlarmParas.jl")


#
# see: https://github.com/JuliaWeb/Mux.jl
# and:
#
using Mux
import Mux.WebSockets
using JSON
using HttpCommon
using Logging, LoggingExtras

using BudgetConstraints
using PovertyAndInequalityMeasures

import ScottishTaxBenefitModel: Definitions, FRSHouseholdGetter, ExampleHouseholdGetter
   ModelHousehold, Utils, MiniTB, GeneralTaxComponents, WebModelLibs
using .Definitions
using .FRSHouseholdGetter
using .ExampleHouseholdGetter
using .ModelHousehold
using .Utils
using .MiniTB
using .GeneralTaxComponents
using .WebModelLibs

## !!! needs Julia 1.3:w
import Base.Threads.@spawn

const DEFAULT_PORT=8000
const DEFAULT_SERVER="http://localhost:$DEFAULT_PORT/"

const DEFAULT_TEST_URL="$(DEFAULT_SERVER)/bc?it_allow=300.0&it_rate_1=0.25&it_rate_2=0.5&it_band=10000&benefit1=150.0&benefit2=60.0&ben2_min_hours = 150.0&ben2_taper=0.5&ben2_u_limit = 250.0"
const ZERO_TEST_URL="$(DEFAULT_SERVER)/bc?it_allow=0&it_rate_1=0&it_rate_2=0&it_band=0&benefit1=0&benefit2=0.0&ben2_taper=0&ben2_min_hours=0&ben2_u_limit=0"

@debug "server starting up"

# gks work round some (version?) thing in the diffeq code
function get_thing( thing::AbstractArray, key :: AbstractString, default :: AbstractString )
   for i in thing
      if i[1] == key
         return i[2]
      end
   end # loop
   default
end #get

function web_get_hh( hdstr::AbstractString )
   hid = parse(Int64, hdstr)
   JSON.json(FRSHouseholdGetter.get_household(hid))
end

function addqstrdict(app, req  :: Dict )
   req[:parsed_querystring] = qstrtodict(req[:query])
   return app(req)
end

# Better error handling
function errorCatch( app, req  :: Dict )
   try
      app(req)
   catch e
      @error "Error occured!"
      io = IOBuffer()
      showerror(io, e)
      err_text = takebuf_string(io)
      @error err_text
      resp = withHeaders(JSON.json(Dict("message" => err_text, "error" => true)), req)
      resp[:status] = 500
      return resp
   end
end

function d100( v :: Number ) :: Number
   v/100.0
end

function web_map_params( req  :: Dict, defaults = MiniTB.DEFAULT_PARAMS ) :: MiniTB.TBParameters
   querydict = req[:parsed_querystring]
   tbparams = deepcopy( defaults )
   tbparams.it_allow = get_if_set("it_allow", querydict, tbparams.it_allow, operation=weeklyise )
   tbparams.it_rate[1] = get_if_set("it_rate_1", querydict, tbparams.it_rate[1], operation=d100 )
   tbparams.it_rate[2] = get_if_set("it_rate_2", querydict, tbparams.it_rate[2], operation=d100 )
   tbparams.it_band[1] = get_if_set("it_band", querydict, tbparams.it_band[1], operation=weeklyise)
   tbparams.benefit1 = get_if_set("benefit1", querydict, tbparams.benefit1)
   tbparams.benefit2 = get_if_set("benefit2", querydict, tbparams.benefit2)
   tbparams.ben2_min_hours = get_if_set("ben2_min_hours", querydict, tbparams.ben2_min_hours)
   tbparams.ben2_taper = get_if_set("ben2_taper", querydict, tbparams.ben2_taper, operation=d100)
   tbparams.ben2_u_limit = get_if_set("ben2_u_limit", querydict, tbparams.ben2_u_limit)
   tbparams.basic_income = get_if_set("basic_income", querydict, tbparams.basic_income)
   @debug "DEFAULT_PARAMS\n$DEFAULT_PARAMS"
   @debug "tbparams\n$tbparams"
   tbparams
end

const example_names, num_households, num_people = load_data( load_examples = true, load_main = true, start_year = 2015 )

const DEFAULT_BC = local_makebc(MiniTB.DEFAULT_PERSON, MiniTB.DEFAULT_PARAMS)

const BASE_RESULTS = create_base_results( num_households, num_people )

function makeztparams() :: MiniTB.TBParameters
   pars = deepcopy( MiniTB.DEFAULT_PARAMS )
   pars.it_rate = [0.0,0.0]
   pars
end

const BC_500_SETTINGS = BCSettings( maxgross=500 )

const ZERO_TAX_PARAMS = makeztparams()
const ZERO_DEFAULT_BC = local_makebc(MiniTB.DEFAULT_PERSON, MiniTB.ZERO_PARAMS)
const ZERO_TAX_BC = local_makebc(MiniTB.DEFAULT_PERSON, ZERO_TAX_PARAMS, BC_500_SETTINGS )

function main_run_to_json( tbparams :: MiniTB.TBParameters ):: String
   results = do_one_run( tbparams, num_households, num_people, NUM_REPEATS )
   summary_output = summarise_results!( results=results, base_results=BASE_RESULTS )
   JSON.json( summary_output )
end

function web_do_one_run( req :: Dict ) :: AbstractString
   @info "web_do_one_run; running on thread $(Threads.threadid())"
   tbparams = web_map_params( req )
   json = main_run_to_json( tbparams )
   # headers could include (e.g.) a timestamp, so add after caching
end # do_one_run

function web_makebc( req  :: Dict ) :: AbstractString
   @debug "web_makebc; running on thread $(Threads.threadid())"
   tbparams = web_map_params( req )
   bc =  local_makebc( DEFAULT_PERSON, tbparams )
   JSON.json((base = DEFAULT_BC, changed = bc))
end

function web_makezbc( req  :: Dict ) :: AbstractString
   @debug "web_makezbc; running on thread $(Threads.threadid())"
   tbparams = web_map_params( req, MiniTB.ZERO_PARAMS )
   bc =  local_makebc( DEFAULT_PERSON, tbparams )
   JSON.json((base = ZERO_DEFAULT_BC, changed = bc))
end

function web_makeztbc( req  :: Dict ) :: AbstractString
   @debug "web_makeztbc; running on thread $(Threads.threadid())"
   tbparams = web_map_params( req, ZERO_TAX_PARAMS )
   bc =  local_makebc( DEFAULT_PERSON, tbparams, BC_500_SETTINGS )
   JSON.json((base = ZERO_TAX_BC, changed = bc))
end

function web_doineq( req  :: Dict ) :: AbstractString
   @debug "web_doineq; running on thread $(Threads.threadid())"
   querydict = req[:parsed_querystring]
   inc=zeros(0)
   pop=zeros(0)
   for i in 1:10 # can's see how to pass a json array from js to this, so ...
      push!( inc, Float64(get_if_set("inc_$i", querydict, 0 )))
      push!( pop, Float64(get_if_set("pop_$i", querydict, 0 )))
   end
   data = hcat( pop, inc )
   ineq = PovertyAndInequalityMeasures.make_inequality( data, 1, 2 )
   JSON.json( ( data=data, ineq=ineq ))
end

# Headers -- set Access-Control-Allow-Origin for either dev or prod
# this is from https://github.com/JuliaDiffEq/DiffEqOnlineServer
#
function add_headers( json :: AbstractString ) :: Dict
    headers  = HttpCommon.headers()
    headers["Content-Type"] = "application/json; charset=utf-8"
    headers["Access-Control-Allow-Origin"] = "*"
    Dict(
       :headers => headers,
       :body=> json
    )
end

#
# This is my attempt at starting a task using the 1.3 @spawn macro
# 1st parameter is a function that returns String (probably a json string) and accepts the req Dict
#
function do_in_thread( the_func::Function, req :: Dict ) :: Dict
   response = @spawn the_func( req )
   # note that the func returns a string but response is a Future type
   # line below converts response to a string
   @debug "do_in_thread response is $response"
   json = fetch( response )
   add_headers( json )
end

# configure logger; see: https://docs.julialang.org/en/v1/stdlib/Logging/index.html
# and: https://github.com/oxinabox/LoggingExtras.jl
logger = FileLogger("/var/tmp/stb_log.txt")
global_logger(logger)
LogLevel( Logging.Info )

#
# from diffeq thingy instead of Mux.defaults
#
# ourstack = stack(Mux.todict, errorCatch, Mux.splitquery, Mux.toresponse) # from DiffEq
#
@app stbdemo = (
   Mux.defaults,
   addqstrdict,
   page( respond("
A Scottish Tax-Benefit Model

; very early version
")),
   page( "/hhld/:hid", req -> web_get_hh((req[:params][:hid]))), # note no headers
   page("/ineq", req -> do_in_thread( web_doineq, req )),
   page("/bc", req -> do_in_thread( web_makebc, req )),
   page("/zbc", req -> do_in_thread( web_makezbc, req )),
   page("/ztbc", req -> do_in_thread( web_makeztbc, req )),
   # handle main stb run a bit differently so we can
   # check & cache the results.
   # page("/stb", req -> web_do_one_run_cached( req )),
   page("/stb", req -> do_in_thread( web_do_one_run, req )),
   Mux.notfound(),
)

port = DEFAULT_PORT
if length(ARGS) > 0
   port = parse(Int64, ARGS[1])
end

serve(stbdemo, port)

while true # FIXME better way?
   @debug "main loop; server running on port $port"
   sleep( 60 )
end


using Printf
using HTTP
using JSON

#include("white_backend.jl")
include("psql_backend.jl")
using Main.Backend

url = "https://api.telegram.org/"
key=ENV["TELEGRAM_KEY"]

function send_message(params)
    req = HTTP.request("POST",string(url,key,"/sendMessage"),["Content-Type" => "application/json"],JSON.json(params))
end

function send_welcome(contact)
    id = contact["id"]
    name = contact["first_name"]
    if "language_code" in keys(contact)
        lc = contact["language_code"]
    else
        lc = "it"
    end
    if lc=="it"
        params = Dict("chat_id"=>id, "text"=>string("Ciao ", name, "! Sono Pizzabot\n
        Ti aiuterò a preparare pizze e focacce favolose!
        Digita /cuciniamo per iniziare e poi segui le istruzioni.

        Buon lavoro e buon appetito!!"))
    else
        params = Dict("chat_id"=>id, "text"=>string("Welcome ", name, " \nClick /cuciniamo to begin.."))
    end
    send_message(params)
end

function ask_program(contact)
    id = contact["id"]
    buttons = [Dict("text"=>"pizza"), Dict("text"=>"focaccia")]
    reply_markup = Dict("keyboard"=>[buttons], "one_time_keyboard"=>true)
    params = Dict("chat_id"=>id, "text"=>"Cosa vuoi cucinare? (pizza o focaccia)", "reply_markup"=>reply_markup)
    send_message(params)
end

function ask_people(message)
    id = message["from"]["id"]
    program = message["text"]

    if !(program  in ["pizza" "focaccia"])
        program = "pizza"
    end

    params = Dict("chat_id"=>id, "text"=>string("Bene allora ti darò istruzioni per fare la ", program))
    send_message(params)
    params = Dict("chat_id"=>id, "text"=>string("per quante persone vuoi fare la ", program, "?"))
    send_message(params)
    return program
end

function read_people(message, state)
    id = message["from"]["id"]
    people = message["text"]
    program = get_key(id, "program")
    try
        people = parse(Int, people)
        if people <= 0
            throw("too few people")
        end
        send_recipe(message, people)
        return state + 1
    catch
        params = Dict("chat_id"=>id, "text"=>string("Non ho capito. Per quante persone?"))
        send_message(params)
        return state
    end
end

function send_recipe(message, people)
    id = message["from"]["id"]
    program = get_key(id, "program")
    if program == "pizza"
        send_pizza(id, people)
    elseif program == "focaccia"
        send_focaccia(id, people)
    end
end

function send_pizza(id, people)
    pizza_recipe = [1000 100 500]
    recipe = pizza_recipe / 4. * people
    params = Dict("chat_id"=>id, "text"=>string("Ecco le dosi per $(people) persone:\n
    $(@sprintf("%.1f", recipe[1]/1000)) Kg di farina\n
    $(@sprintf("%.1f", recipe[2])) ml di olio\n
    $(@sprintf("%.1f", recipe[3])) ml di acqua\n
    lievito q.b.\n
    sale q.b.\n"))
    send_message(params)
end

function send_focaccia(id, people)
    focaccia_recipe = [660 50 440]
    recipe = focaccia_recipe / 4. * people

    number, rectangular_str, circular_str, rectangular, circular = get_teglia(sum(recipe))

    params = Dict("chat_id"=>id, "text"=>string("Ecco le dosi per $(people) persone:\n
    $(@sprintf("%.3f", recipe[1]/1000)) Kg di farina di Manitoba (W=280)\n
    $(@sprintf("%.1f", recipe[2])) g di olio\n
    $(@sprintf("%.1f", recipe[3])) ml di acqua\n
    lievito q.b.\n
    sale q.b.\n\n
    Puoi usare $(circular_str) di diametro: $(@sprintf("%.1f", circular))cm\n
    oppure $(rectangular_str) $(@sprintf("%.1f", rectangular[1])) cm X $(@sprintf("%.1f", rectangular[2])) cm"))
    send_message(params)
end

function get_teglia(impasto)
    area = impasto / 0.60
    rapp = 42 / 35
    area_max = 42 * 35
    number = Int(round(floor(area/area_max) + 1))

    area = area / number # area singola teglia

    h = sqrt(area / rapp)
    b = area / h
    rectangular = [b h]
    circular = sqrt(area/pi)*2

    if number > 1
        rectangular_str = "$(@sprintf("%d", number)) teglie rettangolari"
        circular_str = "$(@sprintf("%d", number)) teglie circolari"
    else
        rectangular_str = "una teglia rettangolare"
        circular_str = "una teglia circolare"
    end

    return (number, rectangular_str, circular_str, rectangular, circular)
end

function send_bye(contact)
    id = contact["id"]
    name = contact["first_name"]
    if "language_code" in keys(contact)
        lc = contact["language_code"]
    else
        lc = "it"
    end
    if lc=="it"
        params = Dict("chat_id"=>id, "text"=>string("Buon appetito ", name, "!"))
    else
        params = Dict("chat_id"=>id, "text"=>string("Enjoy your meal ", name, "!"))
    end
    send_message(params)
end

function parse_message(message)
    id = message["message"]["from"]["id"]
    update_contact_list(message)
    state = get_state(id)

    if (state == 0) | (message["message"]["text"]=="/start")
        send_welcome(message["message"]["from"])
        set_state(id, 1)
    elseif (state == 1) | (message["message"]["text"]=="/cuciniamo")
        ask_program(message["message"]["from"])
        set_state(id, 2)
    elseif state == 2
        program = ask_people(message["message"])
        set_state(id, 3)
        set_key(id,"program",program)
    elseif state == 3
        new_state = read_people(message["message"], 3)
        state = new_state
        set_state(id, state)
    end

    if state == 4
        send_bye(message["message"]["from"])
        set_state(id, 1)
    end
end

function run()
    offset = -1

    println("ready!")
    while true
    #for i in 1:100
        if offset >= 0
            params = Dict("offset"=>offset)
        else
            params = Dict()
        end
        req = HTTP.request("GET", string(url,key,"/getUpdates"), ["Content-Type" => "application/json"],JSON.json(params))
        body = String(req.body)
        result = JSON.Parser.parse(body)

        if length(result["result"]) < 1
            continue
        end
        offset = result["result"][end]["update_id"] + 1
        for message in result["result"]
           parse_message(message)
        end
        sleep(0.1)
    end
end

init_db()
conn = get_conn()

run()

@sync close(conn)


module mzXML

using LightXML, Unitful, Codecs

struct MSscan{T<:AbstractFloat,TI<:Real}
    polarity::Char
    msLevel::Int
    retentionTime::typeof(1.0u"s")
    lowMz::Float64
    highMz::Float64
    basePeakMz::Float64
    totIonCurrent::Float64
    mz::Vector{T}
    I::Vector{TI}
    children::Vector{MSscan{T,TI}}
end

const empty32 = Array{MSscan{Float32,Float32}}[]
const empty64 = Array{MSscan{Float64,Float64}}[]

function index(filename)
    scanpositions = open(filename) do file
        # Find the indexOffset element
        seekend(file)
        p = position(file)
        l = 200
        if p < l
            error("Too short to be mzXML")
        end
        skip(file, -l)
        str = loadrest(file)
        m = match(r"<indexOffset>([0-9].*)</indexOffset>", str)
        if m == nothing
            error("Cannot find indexOffset element")
        end
        length(m.captures) == 1 || error("indexOffset should contain a single number")
        indexpos = parse(Int, m.captures[1])
        length_tail = l - m.offset
        # Read the index
        seek(file, indexpos)
        str = loadrest(file)
        xindex = parse_string(str[1:end-length_tail-1])
        xroot = root(xindex)
        scanpositions = Array{Int}(0)
        for c in child_elements(xroot)
            name(c) == "offset" || error("index could not be parsed")
            push!(scanpositions, parse(Int, content(c)))
        end
        scanpositions
    end
    scanpositions
end

function load(filename; maxlevel = 1)
    xdoc = parse_file(filename)
    xroot = root(xdoc)
    if name(xroot) != "mzXML" || name(xroot) != "indexedmzML"
        error("Not an mzXML file")
    end

    # Find the msRun node
    msRun = find_element(xroot, "msRun")

    props = Dict{Symbol,Any}()
    props[:startTime] = parse_time(attribute(msRun, "startTime"))
    props[:endTime] = parse_time(attribute(msRun, "endTime"))
    el = find_element(msRun, "msInstrument")
    el = find_element(el, "msModel")
    props[:msModel] = attribute(el, "value")

    load_scans(msRun, maxlevel-1), props
end

function load_scans(elm, ndeeper=0)
    # First we discover the data type
    local T
    local TI
    for c in child_elements(elm)
        n = name(c)
        if n != "scan"
            continue
        end
        peak = find_element(c, "peaks")
        TI, T, nochildren = precisiondict[attribute(peak, "precision")]
        break
    end
    # Now load the data
    scans = Array{MSscan{T,TI}}(0)
    load_scans!(scans, elm, ndeeper)
end

function load_scans!(scans, elm, ndeeper)
    for c in child_elements(elm)
        n = name(c)
        if n != "scan"
            continue
        end
        push!(scans, load_scan(c, ndeeper)::eltype(scans))
    end
    scans
end

polaritydict = Dict("+" => '+', "-" => '-')
precisiondict = Dict("32" => (Float32, Float32, empty32), "64" => (Float64, Float64, empty64))

function load_scan(elm, ndeeper)
    polarity = polaritydict[attribute(elm, "polarity")]
    retentionTime = parse_time(attribute(elm, "retentionTime"))
    lMza, hMza = attribute(elm, "lowMz"), attribute(elm, "highMz")
    if lMza != nothing
        lowMz, highMz = parse(Float64, lMza), parse(Float64, hMza)
    else
        lowMz = highMz = NaN
    end
    msLevela = attribute(elm, "msLevel")
    msLevel = msLevela == nothing ? 1 : parse(Int, msLevela)
    basePeakMz = parse(Float64, attribute(elm, "basePeakMz"))
    totIonCurrent = parse(Float64, attribute(elm, "totIonCurrent"))
    npeaks = parse(Int, attribute(elm, "peaksCount"))
    peak = find_element(elm, "peaks")
    data = decode(Base64, content(peak))
    TI, T, nochildren = precisiondict[attribute(peak, "precision")]
    A = reinterpret(TI, data)
    bo = attribute(peak, "byteOrder")
    if bo == "network"
        ntoh!(A)
    else
        error("Don't know what to do with byteOrder $bo")
    end
    po = attribute(peak, "pairOrder")
    if po == nothing
        po = attribute(peak, "contentType")
    end
    po == "m/z-int" || error("Don't know what to do with pairOrder/contentType $po")
    I = A[2:2:end]
    mz = reinterpret(T, A[1:2:end])
    children = ndeeper > 0 ? load_scans!(Array{MSscan{T,TI}}(0), elm, ndeeper-1) : nochildren
    MSscan{T,TI}(polarity, msLevel, retentionTime, lowMz, highMz, basePeakMz, totIonCurrent, mz, I, children)
end

function loadrest(file::IOStream)
    nb = filesize(file) - position(file)
    b = read(file, nb)
    return String(b)
end

function parse_time(tstr)
    if startswith(tstr, "PT") && endswith(tstr, "S")
        return parse(Float64, tstr[3:end-1])*u"s"
    end
    error("Time string $tstr not recognized")
end

function ntoh!(A)
    for i = 1:length(A)
        A[i] = ntoh(A[i])
    end
end

end


"""
LTESOM - Long Term Energy System Optimization Model
Copyright (C) 2022, Peking University, College of Engineering (COE),
Department of Industrial Engineering and Management
"""

println("Activating the Julia virtual environment")

using Pkg
Pkg.activate("LTESOMEnv")

## Possible solvers for GenX
# Open source solvers
Pkg.add(Pkg.PackageSpec(name="Cbc", version="0.8.0"))
Pkg.add(Pkg.PackageSpec(name="Clp", version="0.8.4"))
Pkg.add(Pkg.PackageSpec(name="GLPK", version="0.14.12"))
Pkg.add(Pkg.PackageSpec(name="Ipopt", version="0.7.0"))
# Pkg.add(Pkg.PackageSpec(name="SCIP"))
# Pkg.build("SCIP")

## Commercial solvers - depending on the host machine
# Pkg.add(Pkg.PackageSpec(name="CPLEX"))
# Pkg.build("CPLEX")
Pkg.add(Pkg.PackageSpec(name="Gurobi"))
Pkg.build("Gurobi")

## Julia interface wrapper
Pkg.add(Pkg.PackageSpec(name="JuMP", version="0.21.8"))
Pkg.add("MathOptInterface")

## Data structures
Pkg.add(Pkg.PackageSpec(name="Dates"))
Pkg.add(Pkg.PackageSpec(name="YAML", version="0.4.7"))
Pkg.add(Pkg.PackageSpec(name="CSV", version="0.8.5"))
Pkg.add(Pkg.PackageSpec(name="Distances", version="0.10.3"))
Pkg.add(Pkg.PackageSpec(name="DataFrames", version="1.0.0"))
Pkg.add(Pkg.PackageSpec(name="DataStructures", version="0.18.9"))
Pkg.add(Pkg.PackageSpec(name="Clustering", version="0.14.2"))
Pkg.add(Pkg.PackageSpec(name="Combinatorics", version="1.0.2"))

## Math base packages
Pkg.add(Pkg.PackageSpec(name="Random"))
Pkg.add(Pkg.PackageSpec(name="MathProgBase"))
Pkg.add(Pkg.PackageSpec(name="StatsBase"))
Pkg.add(Pkg.PackageSpec(name="LinearAlgebra"))
Pkg.add(Pkg.PackageSpec(name="DiffEqSensitivity"))
Pkg.add(Pkg.PackageSpec(name="OrdinaryDiffEq"))
Pkg.add(Pkg.PackageSpec(name="QuasiMonteCarlo"))
Pkg.add(Pkg.PackageSpec(name="Statistics"))
Pkg.add(Pkg.PackageSpec(name="RecursiveArrayTools"))

## Documention tools
Pkg.add(Pkg.PackageSpec(name="Documenter", version="0.27.3"))
Pkg.add(Pkg.PackageSpec(name="DocumenterTools", version="0.1.13"))

## Benchmark tools
Pkg.add(Pkg.PackageSpec(name="BenchmarkTools"))

## Show current environment
Pkg.status()


using Test
using Gadgetron
import Base.Threads.@spawn
using Random

testheader = MRD.MRDHeader("""<?xml version="1.0"?>
<ismrmrdHeader xmlns="http://www.ismrm.org/ISMRMRD" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xs="http://www.w3.org/2001/XMLSchema" xsi:schemaLocation="http://www.ismrm.org/ISMRMRD ismrmrd.xsd">
  <subjectInformation>
    <patientName>phantom</patientName>
    <patientWeight_kg>70.3068</patientWeight_kg>
  </subjectInformation>
  <acquisitionSystemInformation>
    <systemVendor>SIEMENS</systemVendor>
    <systemModel>Avanto</systemModel>
    <systemFieldStrength_T>1.494</systemFieldStrength_T>
    <receiverChannels>32</receiverChannels>
    <relativeReceiverNoiseBandwidth>0.79</relativeReceiverNoiseBandwidth>
  </acquisitionSystemInformation>
  <experimentalConditions>
    <H1resonanceFrequency_Hz>63642459</H1resonanceFrequency_Hz>
  </experimentalConditions>
  <encoding>
    <trajectory>cartesian</trajectory>
    <encodedSpace>
      <matrixSize>
        <x>256</x>
        <y>140</y>
        <z>80</z>
      </matrixSize>
      <fieldOfView_mm>
        <x>600</x>
        <y>328.153125</y>
        <z>160</z>
      </fieldOfView_mm>
    </encodedSpace>
    <reconSpace>
      <matrixSize>
        <x>128</x>
        <y>116</y>
        <z>64</z>
      </matrixSize>
      <fieldOfView_mm>
        <x>300</x>
        <y>271.875</y>
        <z>128</z>
      </fieldOfView_mm>
    </reconSpace>
    <encodingLimits>
      <kspace_encoding_step_1>
        <minimum>0</minimum>
        <maximum>83</maximum>
        28
      </kspace_encoding_step_1>
      <kspace_encoding_step_2>
        <minimum>0</minimum>
        <maximum>45</maximum>
        20
      </kspace_encoding_step_2>
      <slice>
        <minimum>0</minimum>
        <maximum>0</maximum>
        0
      </slice>
      <set>
        <minimum>0</minimum>
        <maximum>0</maximum>
        0
      </set>
    </encodingLimits>
    <parallelImaging>
    <accelerationFactor>
      <kspace_encoding_step_1>1</kspace_encoding_step_1>
      <kspace_encoding_step_2>1</kspace_encoding_step_2>
    </accelerationFactor>
    <calibrationMode>other</calibrationMode>
  </parallelImaging>
  </encoding>

  <sequenceParameters>

4.6

    <TE>2.35</TE>
    <TI>300</TI>
  </sequenceParameters>
</ismrmrdHeader>""")

function parrotserver(port)
	t = @spawn begin
		connection = listen(port)
		for msg in connection
			put!(connection,msg)
		end
		close(connection)
	end
	return t

end

function send_and_receive(connection,data )
		put!(connection,data)
		return take!(connection)
end

random_waveform_header() = MRD.WaveformHeader( [rand(Random.TaskLocalRNG(),ftype) for ftype in fieldtypes(MRD.WaveformHeader)]...)

@testset "Connection" begin
	port = 9002
	server = parrotserver(port)
  config = "< _ />"
  header = testheader
  connection = connect("localhost",port,config,header )

  data = randn(Random.TaskLocalRNG(),ComplexF32,(192,32))
	acq = MRD.Acquisition(MRD.AcquisitionHeader(),data)
  @test acq == send_and_receive(connection,acq)

  wav = MRD.Waveform(random_waveform_header(),rand(Random.TaskLocalRNG(),UInt32,(64,8)))
  @test wav == send_and_receive(connection,wav)

  img = MRD.Image(MRD.ImageHeader(),randn(Random.TaskLocalRNG(),ComplexF32,(256,256,3,1)))
  @test img == send_and_receive(connection,img)

  close(connection)
	wait(server)
end


# The MIT License (MIT)
#
# Copyright (c) 2014, 2019 Alexander Samoilov
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
# the Software, and to permit persons to whom the Software is furnished to do so,
# subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

using IterativeSolvers
using LinearAlgebra
using SparseArrays
using Test
using DocOpt
using Printf

mutable struct Pos
    x::Float64
    y::Float64

    Pos(x::Float64 = -1.0, y::Float64 = -1.0) = new(x, y)
end

mutable struct Gate
    id::UInt
    pos::Pos
    nets::Vector{UInt}

    Gate(id::UInt, nets) = new(id, Pos(), nets)
end

struct Pad
    pos::Pos
    net_num::UInt

    Pad(pos::Pos, net::UInt) = new(pos, net)
end

mutable struct Net
    weight::Float64

    # convention : positive number - gate number, negative one - inverted pad number
    gates::Vector{UInt}
    pads::Vector{UInt}

    gates_set::BitSet
    pads_set::BitSet

    Net(weight::Float64 = 1.0) = new(weight, UInt[], UInt[], BitSet(), BitSet())
end

function net_is_empty(n::Net)
    length(gates) == 0 && length(pads) == 0
end

function read_input(fname::AbstractString)

    is = open(fname, "r")
    line = readline(is)
    nums = split(line)
    (num_gates::UInt, num_nets::UInt) = (parse(UInt, nums[1]), parse(UInt, nums[2]))
    println("-I- read num_gates: $num_gates num_nets: $num_nets")

    # read gates
    gates = Array{Gate}(undef, num_gates)
    for i = 1:num_gates
        line = readline(is)
        nums = split(line)
        (gate_num, num_nets_connected) = (parse(UInt, nums[1]), parse(UInt, nums[2]))
        @assert(gate_num == i, "gate_num: $gate_num i: $i")
        nets = [parse(UInt, nums[j + 2]) for j = 1:num_nets_connected]
        gates[i] = Gate(i, nets)
    end

    # read pads
    line = readline(is)
    num_pads::UInt = parse(UInt, line)
    println("-I- read num_pads: $num_pads")
    pads = Array{Pad}(undef, num_pads)
    for i = 1:num_pads
        line = readline(is)
        nums = split(line)
        (pad_num::UInt, net_num::UInt, x::Float64, y::Float64) = (parse(UInt, nums[1]), parse(UInt, nums[2]), parse(Float64, nums[3]), parse(Float64, nums[4]))
        @assert(pad_num == i, "pad_num: $pad_num i: $i")
        pads[i] = Pad(Pos(x,y), net_num)

    end

    close(is)
    return (num_gates, num_nets, num_pads, gates, pads)
end

function fill_net_sets(n::Net)
    n.gates_set = BitSet(n.gates)
    n.pads_set  = BitSet(n.pads)
end

function create_nets(num_nets::UInt, gates::Vector{Gate}, pads::Vector{Pad})
    num_gates::UInt = length(gates)
    num_pads::UInt  = length(pads)
    nets = [Net() for i = 1:num_nets]
    for i = 1:num_gates
        gate = gates[i]
        for j = 1:length(gate.nets)
            net_no = gate.nets[j]
            push!(nets[net_no].gates, i)
        end
    end

    for i = 1:num_pads
        push!(nets[pads[i].net_num].pads, i)
    end

    for i = 1:num_nets
        fill_net_sets(nets[i])
    end

    return nets
end

function qp_step(verbose::Bool, maxiter::Int, tol::Float64, nets::Vector{Net}, gates::Vector{Gate}, pads::Vector{Pad})
    num_nets   ::UInt = length(nets)
    num_gates  ::UInt = length(gates)
    num_pads   ::UInt = length(pads)

    # connectivity matrix
    C = spzeros(num_gates, num_gates)
    for i = 1:num_nets
        net = nets[i]
        gs = net.gates
        ps = net.pads
        clique_size = length(gs)
        if verbose @printf("-D- NET : %d\n",i) end
        for j = 1:clique_size
            for k = j+1:clique_size
                jc = gs[j]
                kc = gs[k]
                if verbose @printf("-D-      Conn %d <-> %d\n", jc, kc) end
                C[jc, kc] = C[kc, jc] = net.weight
            end
        end
    end

    if verbose println("-D- C : \n$C") end

    A = spzeros(num_gates, num_gates)
    # use dense vectors for rhs
    bx = spzeros(num_gates)
    by = spzeros(num_gates)

    A = -C
    for i = 1:num_gates
        # see https://github.com/JuliaLang/julia/issues/6480
        diag_val::Float64 = sum(C[i, :])[1]
        pads_contrib::Float64 = 0.0
        b_accum = (0.0, 0.0)
        adjacent_nets = gates[i].nets
        for j = 1:length(adjacent_nets)
            net_num = adjacent_nets[j]
            the_net = nets[net_num]
            ps = the_net.pads
            net_weight = the_net.weight
            for pad_idx in ps
                pad = pads[pad_idx]
                pads_contrib += net_weight
                (x, y) = (pad.pos.x, pad.pos.y)
                b_accum = (b_accum[1] + net_weight * x, b_accum[2] + net_weight * y)
            end
        end
        if verbose @printf("-D- i: %i diag_val: %f pads_contrib: %f\n",i,diag_val,pads_contrib) end
        A[i, i] = diag_val + pads_contrib
        (bx[i], by[i]) = b_accum
    end

    if verbose
        println("-D- A : \n$A")
        println("-D- bx : \n$bx")
        println("-D- by : \n$by")
    end

    x,ch = IterativeSolvers.cg(A,bx; abstol=tol, maxiter=maxiter, log=true)
    #see https://github.com/JuliaLang/julia/issues/6485#issuecomment-40063871
    #feature request https://github.com/JuliaLang/julia/issues/6485
    #https://github.com/mfasi/julia/commit/4ceb4ea9ee46ea92d406cfced308451a112d16f9
    #@test_approx_eq_eps A*x bx cond(full(A))*sqrt(tol)
    if verbose
        println("x : $x")
        @printf("-D- length(A*x) : %d length(bx) : %d\n", length(A*x), length(bx))
    end

    atol = cond(A, Inf)*sqrt(tol)

    @test norm(A * x - bx) ≤ atol
    @test ch.isconverged
    println("ch_x : $ch")

    y,ch = IterativeSolvers.cg(A,by; abstol=tol, maxiter=maxiter, log=true)
    @test norm(A * y - by) ≤ atol
    @test ch.isconverged
    if verbose println("ch_y : $ch") end

    return [(i,Pos(x[i], y[i])) for i = 1:num_gates]
end

my_isless(a::Tuple{UInt,Pos}, b::Tuple{UInt,Pos}) = a[2].x * 1e6 + a[2].y < b[2].x * 1e6 + b[2].y

const IndexedPosVec = Vector{Tuple{UInt,Pos}}

function assignment_cut(verbose::Bool, gate_pos::IndexedPosVec)
    num_gates = length(gate_pos)

    # avoid using anonymous functions
    sorted_gate_pos = sort!(gate_pos, alg = QuickSort, lt = my_isless)
    if verbose println("-D- sorted 1st plc : \n$sorted_gate_pos") end

    median::Int = div(num_gates, 2)
    return ( sorted_gate_pos[1:median], sorted_gate_pos[median+1:num_gates] )
end

function left_side_containment(verbose   ::Bool,
                               left_pos  ::IndexedPosVec,
                               right_pos ::IndexedPosVec,
                               nets      ::Vector{Net},
                               gates     ::Vector{Gate},
                               pads      ::Vector{Pad})
    num_gates = length(gates)
    left_gates = [deepcopy(gates[left_pos[i][1]]) for i = 1:length(left_pos)]
    left_gates_renumbering = Array{Int}(undef, num_gates)
    fill!(left_gates_renumbering, -1)
    for i = 1:length(left_gates)
        left_gates_renumbering[left_gates[i].id] = i
    end
    left_pads = deepcopy(pads)
    if verbose
        println("-D- left_gates : $left_gates")
        println("-D- left_gates_renumbering : $left_gates_renumbering")
    end
    left_gates_indices  = BitSet([left_pos[i][1] for i = 1:length(left_pos)])
    if verbose println("-D- left gates indices : $left_gates_indices") end
    right_gates_indices = BitSet([right_pos[i][1] for i = 1:length(right_pos)])
    if verbose println("-D- right gates indices : $right_gates_indices") end

    local median::Float64 = 50.0

    num_nets = length(nets)
    left_nets = [Net() for i = 1:num_nets]
    for i = 1:num_nets
        orig_net = nets[i]
        if isempty(intersect(orig_net.gates_set,left_gates_indices))
            if verbose println("-D- the orig_net $orig_net DOESN'T belong to left partition, EXCLUDE it") end
        else
            net_no::UInt = i
            if verbose println("-D- the orig_net $orig_net belongs to left partition, fix it and include") end
            orig_pads_indices = orig_net.pads
            new_pads_indices = UInt[]
            # process orig pads
            for orig_pad_idx in orig_pads_indices
                orig_pad = pads[orig_pad_idx]
                # fix abscissa if needed
                if orig_pad.pos.x <= median # ok, add this pad_idx AS IS
                    push!(new_pads_indices, orig_pad_idx)
                else
                    new_pad = deepcopy(orig_pad)
                    new_pad.pos.x = median
                    push!(left_pads, new_pad)
                    new_pad_idx::UInt = length(left_pads)
                    push!(new_pads_indices, new_pad_idx)
                end
            end
            orig_gates_indices = orig_net.gates
            new_gates_indices = UInt[]
            # process orig gates
            for orig_gate_idx in orig_gates_indices
                if orig_gate_idx in left_gates_indices # Bingo!
                    @assert(left_gates_renumbering[orig_gate_idx] > 0)
                    remapping::UInt = left_gates_renumbering[orig_gate_idx]
                    push!(new_gates_indices, remapping)
                else # gate belongs to right partition, convert it to pad
                    orig_gate = gates[orig_gate_idx]
                    new_pad = Pad(Pos(median, orig_gate.pos.y), net_no)
                    push!(left_pads, new_pad)
                    new_pad_idx::UInt = length(left_pads)
                    push!(new_pads_indices, new_pad_idx)
                end
            end
            the_net = left_nets[net_no]
            the_net.pads = new_pads_indices
            the_net.gates = new_gates_indices
            fill_net_sets(the_net)
        end
    end

    (left_nets, left_gates, left_pads)
end

function right_side_containment(verbose   ::Bool,
                                left_pos  ::IndexedPosVec,
                                right_pos ::IndexedPosVec,
                                nets      ::Vector{Net},
                                gates     ::Vector{Gate},
                                pads      ::Vector{Pad})
    num_gates = length(gates)
    right_gates = [deepcopy(gates[right_pos[i][1]]) for i = 1:length(right_pos)]
    right_pads = deepcopy(pads)
    if verbose println("-D- right_gates : $right_gates") end
    left_gates_indices  = BitSet([left_pos[i][1] for i = 1:length(left_pos)])
    if verbose println("-D- left gates indices : $left_gates_indices") end
    right_gates_indices = BitSet([right_pos[i][1] for i = 1:length(right_pos)])
    if verbose println("-D- right gates indices : $right_gates_indices") end
    right_gates_renumbering = Array{Int}(undef, num_gates)
    fill!(right_gates_renumbering, -1)
    for i = 1:length(right_gates)
        right_gates_renumbering[right_gates[i].id] = i
    end
    if verbose println("-D- right_gates_renumbering : $right_gates_renumbering") end

    local median::Float64 = 50.0

    num_nets = length(nets)
    right_nets = [Net() for i = 1:num_nets]
    for i = 1:num_nets
        orig_net = nets[i]
        if isempty(intersect(orig_net.gates_set,right_gates_indices))
            if verbose println("-D- the orig_net $orig_net DOESN'T belong to right partition, EXCLUDE it") end
        else
            net_no::UInt = i
            if verbose println("-D- the orig_net $orig_net belongs to right partition, fix it and include") end
            orig_pads_indices = orig_net.pads
            new_pads_indices = UInt[]
            # process orig pads
            for orig_pad_idx in orig_pads_indices
                orig_pad = pads[orig_pad_idx]
                # fix abscissa if needed
                if orig_pad.pos.x >= median # ok, add this pad_idx AS IS
                    push!(new_pads_indices, orig_pad_idx)
                else
                    new_pad = deepcopy(orig_pad)
                    new_pad.pos.x = median
                    push!(right_pads, new_pad)
                    new_pad_idx::UInt = length(right_pads)
                    push!(new_pads_indices, new_pad_idx)
                end
            end
            orig_gates_indices = orig_net.gates
            new_gates_indices = UInt[]
            # process orig gates
            for orig_gate_idx in orig_gates_indices
                if orig_gate_idx in right_gates_indices # Bingo!
                    @assert(right_gates_renumbering[orig_gate_idx] > 0)
                    remapping::UInt = right_gates_renumbering[orig_gate_idx]
                    push!(new_gates_indices, remapping)
                else # gate belongs to right partition, convert it to pad
                    orig_gate = gates[orig_gate_idx]
                    new_pad = Pad(Pos(median, orig_gate.pos.y), net_no)
                    push!(right_pads, new_pad)
                    new_pad_idx::UInt = length(right_pads)
                    push!(new_pads_indices, new_pad_idx)
                end
            end
            the_net = right_nets[net_no]
            the_net.pads = new_pads_indices
            the_net.gates = new_gates_indices
            fill_net_sets(the_net)
        end
    end

    (right_nets, right_gates, right_pads)

end

function main()

    local script_name = basename(@__FILE__)
    local doc = """$script_name

Quadratic Placement programming assignment for VLSI course.

Usage:
  $script_name -h | --help
  $script_name [-v | --verbose] [--tol=<tolerance>] [--iterations=<iterations>] <input> [<output>]

Options:
  -h --help                  Show this screen.
  -v --verbose               Adds verbosity.
  --tol=<tolerance>          Specify a tolerance for Conjugate Gradient (CG).
                             sparse symmetric positive definite (SPD) system solver.
  --iterations=<iterations>  Specify a number of iterations; defaults to 200.

"""

    if length(ARGS) < 1
        println(doc)
        exit(0)
    end

    arguments = docopt(doc)

    verbose = arguments["--verbose"]

    if (arguments["--iterations"] != nothing)
        maxiter = parse(Int, arguments["--iterations"])
    else
        maxiter = 200
    end

    if (arguments["--tol"] != nothing)
        tol = parse(Float64, arguments["--tol"])
    else
        tol = 1e-12
    end

    inp_name = arguments["<input>"]
    println("-I- reading from $inp_name")

    (num_gates::UInt, num_nets::UInt, num_pads::UInt, gates, pads) = read_input(inp_name)
    println("-I- num_gates: $num_gates num_nets: $num_nets num_pads: $num_pads")
    if verbose
        println("-D- gates: $gates")
        println("-D- pads: $pads")
    end

    nets = create_nets(num_nets, gates, pads)
    if verbose println("-D- nets: $nets") end
    gate_pos = qp_step(verbose, maxiter, tol, nets, gates, pads)
    if verbose println("-D- 1st plc : \n$gate_pos") end

    # update gates positions after 1st placement
    for i = 1:num_gates
        gates[i].pos = gate_pos[i][2]
    end
    if verbose println("-D- all gates after 1st placement: \n$gates") end

    (left,right) = assignment_cut(verbose, gate_pos)
    @assert(length(left) + length(right) == num_gates)
    if verbose
        @printf("-D- nam_gates %d length of left part %d length of right part %d\n",
                num_gates, length(left), length(right))
        println("-D- left part $left")
        println("-D- right part $right")
    end

    (left_nets::Vector{Net}, left_gates::Vector{Gate}, left_pads::Vector{Pad}) = left_side_containment(verbose, left, right, nets, gates, pads)
    if verbose
        println("-D- left_gates: $left_gates")
        println("-D- left_pads: $left_pads")
    end
    #left_nets = create_nets(new_nets_no, left_gates, left_pads)
    if verbose println("-D- left_nets: $left_nets") end
    gate_pos_2 = qp_step(verbose, maxiter, tol, left_nets, left_gates, left_pads)
    if verbose println("-D- 2nd plc : \n$gate_pos_2") end

    # update gates positions after 2nd placement
    for i = 1:length(left_gates)
        left_gate = left_gates[i]
        global_gates_idx = left_gate.id
        gates[global_gates_idx].pos = gate_pos_2[i][2]
    end

    if verbose println("-D- all gates after 2nd placement: \n$gates") end

    (right_nets::Vector{Net}, right_gates::Vector{Gate}, right_pads::Vector{Pad}) = right_side_containment(verbose, left, right, nets, gates, pads)
    if verbose
        println("-D- right_gates: $right_gates")
        println("-D- right_pads: $right_pads")
        println("-D- right_nets: $right_nets")
    end
    gate_pos_3 = qp_step(verbose, maxiter, tol, right_nets, right_gates, right_pads)
    if verbose println("-D- 3d plc : \n$gate_pos_3") end

    # update gates positions after 3d placement
    for i = 1:length(right_gates)
        right_gate = right_gates[i]
        global_gates_idx = right_gate.id
        gates[global_gates_idx].pos = gate_pos_3[i][2]
    end

    if verbose println("-D- all gates after 3d placement: \n$gates") end

    if arguments["<output>"] != nothing
        out_fname::AbstractString = arguments["<output>"]
        os = open(out_fname, "w")
        for i = 1:num_gates
            gate = gates[i]
            pos = gate.pos
            @printf(os, "%d %f %f\n", i, pos.x, pos.y)
        end
        close(os)
    end
end

@time main()


"""
VshTrack(args...)  -> PyPlot Plot

Return a plot of Vshale logs and optionally Effective porosity in a single track.

 The next table show the list of variables allowed:

 The next table show the list of variables allowed:
|PropertyName|Args|Default|Input|Description
|---|---|---|---|---|
|Depth|Mandatory| -- | Array{Number,1}| 1D Array of Depths|
|Vsh|Mandatory| -- | Array{Number,1}|1D Array of Vshale Values|
|PhiePlot|Optional| -- | Array{Number,1}|1D Array of Phie Values|
|DepthFrom|Optional| minimum(Depth) | Number |Set the Depth upper limit to plot|
|DepthTo|Optional| maximum(Depth) | Number |  Set the Depth lower limit to plot|
|Dtick|Optional|true|Dtick=Bool|Set the Depth ticks On/Off|
"""
function VshPhieTrack(Depth,Vsh;DepthFrom=minimum(Depth),DepthTo=maximum(Depth),Dtick=false,PhiePlot=false,Phiemax=0.30)
vshax=gca()
vshax.plot(Vsh,Depth,linewidth=1,linestyle="--",color="black")
vshax.set_xlim([0,1])
vshax.set_ylim([DepthTo,DepthFrom])
vshax.set_yticks(range(DepthFrom, stop=DepthTo, length=51),minor=true)
vshax.set_yticks(range(DepthFrom, stop=DepthTo, length=11))
vshax.tick_params("both",labelsize=6)
vshax.set_xlabel("VShale")
setp(vshax.get_yticklabels(),visible=Dtick)
vshax.fill_betweenx(Depth,Vsh,1,where=(Vsh .<= 1),color="gold")

if PhiePlot!=false
phieax=vshax.twiny()
phieax.plot(PhiePlot,Depth,linewidth=1,color="black")
phieax.set_xlim([Phiemax,0])
setp(phieax.get_yticklabels(),visible=Dtick)
phieax.set_xlabel("Phie")
phieax.tick_params("both",labelsize=6)

phieax.fill_betweenx(Depth,PhiePlot,0,where=(Vsh .<= 1),color="red")
else
    vshax.xaxis.set_label_position("top")
    vshax.xaxis.tick_top()

end

end
###########################################################
#VshTrack Function
###########################################################
function VshTrack(Depth,Vsh;DepthFrom=minimum(Depth),DepthTo=maximum(Depth),Dtick=false,CutOff=0.7,fill=true)
vshax=gca()
vshax.plot(Vsh,Depth,linewidth=1,linestyle="--",color="black")
vshax.set_xlim([1,0])
vshax.set_ylim([DepthTo,DepthFrom])
vshax.set_yticks(range(DepthFrom, stop=DepthTo, length=51),minor=true)
vshax.set_yticks(range(DepthFrom, stop=DepthTo, length=11))
vshax.tick_params("both",labelsize=6)
vshax.set_xlabel("VShale")
setp(vshax.get_yticklabels(),visible=Dtick)
if fill==true
    vshax.fill_betweenx(Depth,CutOff,Vsh,where=(CutOff .>= Vsh),color="gold")
end
vshax.xaxis.set_label_position("top")
vshax.xaxis.tick_top()
end

function PhieTrack(Depth,Phie;DepthFrom=minimum(Depth),DepthTo=maximum(Depth),Dtick=false,CutOff=0.08,Phiemax=0.30,fill=false)
    phieax=gca()
    phieax.plot(Phie,Depth,linewidth=1,color="black")
    phieax.set_ylim([DepthTo,DepthFrom])
    phieax.set_xlim([0,Phiemax])
    setp(phieax.get_yticklabels(),visible=Dtick)
    phieax.set_xlabel("Phie")
    phieax.tick_params("both",labelsize=6)
    if fill==true
        phieax.fill_betweenx(Depth,CutOff,Phie,where=(CutOff .<= Phie),color="red")
    end
    phieax.xaxis.set_label_position("top")
    phieax.xaxis.tick_top()
end


function ConstantThrustDirectionProvider(arg0::Vector3D)
    return ConstantThrustDirectionProvider((Vector3D,), arg0)
end

function compute_thrust_direction(obj::ConstantThrustDirectionProvider, arg0::PVCoordinatesProvider, arg1::AbsoluteDate, arg2::Frame)
    return jcall(obj, "computeThrustDirection", Vector3D, (PVCoordinatesProvider, AbsoluteDate, Frame), arg0, arg1, arg2)
end


@testset "Each k-mer" begin
    function string_eachkmer(seq::AbstractString, k, step)
        kmers = String[]
        i = 1
        for i in 1:step:length(seq) - k + 1
            subseq = seq[i:i + k - 1]
            if !in('N', subseq)
                push!(kmers, subseq)
            end
        end
        return kmers
    end

    function test_eachkmer(S, seq::AbstractString, k, step)
        xs = [convert(String, x)
              for (i, x) in collect(each(Kmer{eltype(S),k}, S(seq), step))]
        ys = [convert(String, x)
              for (i, x) in collect(eachkmer(S(seq), k, step))]
        zs = string_eachkmer(seq, k, step)
        @test xs == ys == zs
    end

    for k in [0, 1, 3, 16, 32], step in 1:3, len in [1, 2, 3, 5, 10, 100, 1000]
        test_eachkmer(BioSequence{DNAAlphabet{4}}, random_dna(len), k, step)
        test_eachkmer(BioSequence{RNAAlphabet{4}}, random_rna(len), k, step)
        test_eachkmer(ReferenceSequence, random_dna(len), k, step)

        probs = [0.25, 0.25, 0.25, 0.25, 0.00]
        test_eachkmer(BioSequence{DNAAlphabet{2}}, random_dna(len, probs), k, step)
        test_eachkmer(BioSequence{RNAAlphabet{2}}, random_rna(len, probs), k, step)
    end

    @test isempty(collect(each(DNAKmer{1}, dna"")))
    @test isempty(collect(each(DNAKmer{1}, dna"NNNNNNNNNN")))
    @test_throws Exception each(DNAKmer{-1}, dna"ACGT")
    @test_throws Exception each(DNAKmer{33}, dna"ACGT")
    @test collect(each(DNAKmer{3}, dna"AC-TGAG--TGC")) == [(4, DNACodon(DNA_T, DNA_G, DNA_A)),
                                                           (5, DNACodon(DNA_G, DNA_A, DNA_G)),
                                                           (10, DNACodon(DNA_T, DNA_G, DNA_C))]
end