#  Copyright 2016, Iain Dunning, Joey Huchette, Miles Lubin, and contributors
#  This Source Code Form is subject to the terms of the Mozilla Public
#  License, v. 2.0. If a copy of the MPL was not distributed with this
#  file, You can obtain one at http://mozilla.org/MPL/2.0/.
#############################################################################
# JuMP
# An algebraic modeling language for Julia
# See http://github.com/JuliaOpt/JuMP.jl
#############################################################################
# src/solvers.jl
# Handles conversion of the JuMP Model into a format that can be passed
# through the MathProgBase interface to solvers, and ongoing updating of
# that representation if supported by the solver.
#############################################################################

# Analyze a JuMP Model to determine its traits, and thus what solvers can
# be used to solve the problem
type ProblemTraits
    int::Bool  # has integer variables
    lin::Bool  # has only linear objectives and constraints
    qp ::Bool  # has a quadratic objective function
    qc ::Bool  # has a quadratic constraint
    nlp::Bool  # has general nonlinear objective or constraints
    soc::Bool  # has a second-order cone constraint
    sdp::Bool  # has an SDP constraint (or SDP variable bounds)
    sos::Bool  # has an SOS constraint
    conic::Bool  # has an SDP or SOC constraint
end
function ProblemTraits(m::Model; relaxation=false)
    int = !relaxation && any(c-> !(c == :Cont || c == :Fixed), m.colCat)
    qp = !isempty(m.obj.qvars1)
    qc = !isempty(m.quadconstr)
    nlp = m.nlpdata !== nothing
    soc = !isempty(m.socconstr)
    # will need to change this when we add support for arbitrary variable cones
    sdp = !isempty(m.sdpconstr) || !isempty(m.varCones)
    sos = !isempty(m.sosconstr)
    ProblemTraits(int, !(qp|qc|nlp|soc|sdp|sos), qp, qc, nlp, soc, sdp, sos, soc|sdp)
end
function default_solver(traits::ProblemTraits)
    if traits.nlp
        MathProgBase.defaultNLPsolver
    elseif traits.int || traits.sos
        MathProgBase.defaultMIPsolver
    elseif traits.sdp
        MathProgBase.defaultSDPsolver
    elseif traits.conic
        MathProgBase.defaultConicsolver
    elseif traits.qp || traits.qc
        MathProgBase.defaultQPsolver
    else
        MathProgBase.defaultLPsolver
    end
end

function fillConicRedCosts(m::Model)
    bndidx = 0
    numlinconstr = length(m.linconstr)
    vardual = MathProgBase.getvardual(m.internalModel)
    if m.objSense == :Min
        scale!(vardual, -1)
    end
    for i in 1:m.numCols
        lower = false
        upper = false
        lb, ub = m.colLower[i], m.colUpper[i]

        if lb != -Inf && lb != 0.0
            lower = true
            bndidx += 1
        end
        if ub != Inf && ub != 0.0
            upper = true
            bndidx += 1
        end

        if lower && !upper
            m.redCosts[i] = m.conicconstrDuals[numlinconstr + bndidx]
        elseif !lower && upper
            m.redCosts[i] = m.conicconstrDuals[numlinconstr + bndidx]
        elseif lower && upper
            m.redCosts[i] = m.conicconstrDuals[numlinconstr + bndidx]+m.conicconstrDuals[numlinconstr + bndidx-1]
        end
        m.redCosts[i] += vardual[i]
    end
end

function fillConicDuals(m::Model)

    numRows, numCols = length(m.linconstr), m.numCols

    numBndRows = getNumBndRows(m)
    numSOCRows = getNumSOCRows(m)
    m.conicconstrDuals = try
        MathProgBase.getdual(m.internalModel)
    catch
        fill(NaN, numRows+numBndRows+numSOCRows)
    end
    if isfinite(m.conicconstrDuals[1]) # NaN could mean unavailable
        if m.objSense == :Min
            scale!(m.conicconstrDuals, -1)
        end
        m.linconstrDuals = m.conicconstrDuals[1:length(m.linconstr)]
        m.redCosts = zeros(numCols)
        fillConicRedCosts(m)
    end

end

function solve(m::Model; suppress_warnings=false,
                ignore_solve_hook=(m.solvehook===nothing),
                relaxation=false,
                kwargs...)
    # If the user or an extension has provided a solve hook, call
    # that instead of solving the model ourselves
    if !ignore_solve_hook
        return m.solvehook(m; suppress_warnings=suppress_warnings, kwargs...)::Symbol
    end

    isempty(kwargs) || error("Unrecognized keyword arguments: $(join([k[1] for k in kwargs], ", "))")

    # Clear warning counters
    m.getvalue_counter = 0
    m.operator_counter = 0

    # Remember if the solver was initially unset so we can restore
    # it to be unset later
    unset = m.solver == UnsetSolver()

    # Analyze the problems traits to determine what solvers we can use
    traits = ProblemTraits(m, relaxation=relaxation)

    # Build the MathProgBase model from the JuMP model
    build(m, traits=traits, suppress_warnings=suppress_warnings, relaxation=relaxation)

    # If the model is a general nonlinear, use different logic in
    # nlp.jl to solve the problem
    traits.nlp && return solvenlp(m, traits, suppress_warnings=suppress_warnings)

    # Solve the problem
    MathProgBase.optimize!(m.internalModel)
    stat::Symbol = MathProgBase.status(m.internalModel)

    # Extract solution from the solver
    numRows, numCols = length(m.linconstr), m.numCols
    m.objVal = NaN
    m.colVal = fill(NaN, numCols)
    m.linconstrDuals = Array(Float64, 0)

    discrete = !relaxation && (traits.int || traits.sos)
    if stat == :Optimal
        # If we think dual information might be available, try to get it
        # If not, return an array of the correct length
        if !discrete && !traits.conic
            m.redCosts = try
                MathProgBase.getreducedcosts(m.internalModel)[1:numCols]
            catch
                fill(NaN, numCols)
            end

            m.linconstrDuals = try
                MathProgBase.getconstrduals(m.internalModel)[1:numRows]
            catch
                fill(NaN, numRows)
            end
        end
        # conic duals (currently, SOC only)
        if !discrete && traits.soc && !traits.qp && !traits.qc && !traits.sdp
            fillConicDuals(m)
        end
    else
        # Problem was not solved to optimality, attempt to extract useful
        # information anyway
        suppress_warnings || warn("Not solved to optimality, status: $stat")
        # Some solvers provide infeasibility rays (dual) or unbounded
        # rays (primal) for linear problems. Store these as the solution
        # if the exist.
        if traits.lin
            if stat == :Infeasible
                m.linconstrDuals = try
                    infray = MathProgBase.getinfeasibilityray(m.internalModel)
                    @assert length(infray) == numRows
                    infray
                catch
                    suppress_warnings || warn("Infeasibility ray (Farkas proof) not available")
                    fill(NaN, numRows)
                end
            elseif stat == :Unbounded
                m.colVal = try
                    unbdray = MathProgBase.getunboundedray(m.internalModel)
                    @assert length(unbdray) == numCols
                    unbdray
                catch
                    suppress_warnings || warn("Unbounded ray not available")
                    fill(NaN, numCols)
                end
            end
        end
        # conic duals (currently, SOC only)
        if !discrete && traits.soc && !traits.qp && !traits.qc && !traits.sdp
            if stat == :Infeasible
                fillConicDuals(m)
            end
        end
    end

    # If the problem was solved, or if it terminated prematurely, try
    # to extract a solution anyway. This commonly occurs when a time
    # limit or tolerance is set (:UserLimit)
    if !(stat == :Infeasible || stat == :Unbounded)
        try
            objVal = MathProgBase.getobjval(m.internalModel) + m.obj.aff.constant
            colVal = MathProgBase.getsolution(m.internalModel)[1:numCols]
            # Rescale off-diagonal terms of SDP variables
            if traits.sdp
                offdiagvars = offdiagsdpvars(m)
                colVal[offdiagvars] /= sqrt(2)
            end
            # Don't corrupt the answers if one of the above two calls fails
            m.objVal = objVal
            m.colVal = colVal
        end
    end

    # The MathProgBase interface defines a conic problem to always be
    # a minimization problem, so we need to flip the objective before
    # reporting it to the user
    if traits.conic && m.objSense == :Max
        m.objVal *= -1
    end

    # If the solver was initially not set, we will restore this status
    # and drop the internal MPB model. This is important for the case
    # where the solver used changes between solves because the user
    # has changed the problem class (e.g. LP to MILP)
    if unset
        m.solver = UnsetSolver()
        if traits.int
            m.internalModelLoaded = false
        end
    end

    # don't keep relaxed model in memory
    relaxation && (m.internalModelLoaded = false)

    # Return the solve status
    stat
end

function isquadsoc(m::Model)
    # check if all quadratic constraints are actually conic
    all_conic = true
    for qconstr in m.quadconstr
        q = copy(qconstr.terms)
        if qconstr.sense == :(>=)
            q *= -1
        end
        if !(all(t->t==0, q.aff.coeffs) && q.aff.constant == 0)
            all_conic = false
            break
        end
        n_pos_on_diag = 0
        off_diag_idx  = 0
        neg_diag_idx  = 0
        n = length(q.qvars1)
        nz = 0
        for i in 1:n
            q.qcoeffs[i] == 0 && continue
            nz += 1
            if q.qvars1[i].col == q.qvars2[i].col
                if q.qcoeffs[i] == 1
                    n_pos_on_diag += 1
                elseif q.qcoeffs[i] == -1
                    if !(neg_diag_idx == off_diag_idx == 0)
                        all_conic = false; break
                    end
                    neg_diag_idx = i
                else
                    all_conic = false; break
                end
            else
                if q.qcoeffs[i] == -1
                    if !(neg_diag_idx == off_diag_idx == 0)
                        all_conic = false; break
                    end
                    off_diag_idx = i
                else
                    all_conic = false; break
                end
            end
        end
        if n_pos_on_diag == nz-1 && neg_diag_idx > 0
            # Plain SOC
        elseif n_pos_on_diag == nz-1 && off_diag_idx > 0
            # Rotated SOC
        else
            all_conic = false
        end
    end
    return all_conic && length(m.quadconstr) > 0

end

# Converts the JuMP Model into a MathProgBase model based on the
# traits of the model
function build(m::Model; suppress_warnings=false, relaxation=false, traits=ProblemTraits(m,relaxation=relaxation))
    # Set solver based on the model's traits if it hasn't provided
    if isa(m.solver, UnsetSolver)
        m.solver = default_solver(traits)
    end

    # If the model is nonlinear, use different logic in nlp.jl
    # to build the problem
    traits.nlp && return _buildInternalModel_nlp(m, traits)

    ## Temporary hack for Mosek, see https://github.com/JuliaOpt/Mosek.jl/pull/67
    if contains("$(typeof(m.solver))","MosekSolver") && isquadsoc(m)
        traits.conic = true
    end

    if traits.conic
        # If there are semicontinuous/semi-integer variables, we will have to
        # adjust the b vector below to construct a valid relaxation. This seems
        # like a pretty marginal case, so let's punt on it for now.
        if relaxation && any(x -> (x == :SemiCont || x == :SemiInt), m.colCat)
            error("Relaxations of conic problem with semi-integer/semicontinuous variables are not currently supported.")
        end

        traits.qp && error("JuMP does not support quadratic objectives for conic problems")

        # Obtain a fresh MPB model for the solver
        # If the problem is conic, we rebuild the problem from
        # scratch every time
        m.internalModel = MathProgBase.ConicModel(m.solver)

        # Build up the objective, LHS, RHS and cones from the JuMP Model...
        f, A, b, var_cones, con_cones = conicdata(m)
        # ... and pass to the solver
        MathProgBase.loadproblem!(m.internalModel, f, A, b, con_cones, var_cones)
    else
        # Extract objective coefficients and linear constraint bounds
        f, rowlb, rowub = prepProblemBounds(m)
        # If we already have an MPB model for the solver...
        if m.internalModelLoaded
            # ... and if the solver supports updating bounds/objective
            if applicable(MathProgBase.setvarLB!, m.internalModel, m.colLower) &&
               applicable(MathProgBase.setvarUB!, m.internalModel, m.colUpper) &&
               applicable(MathProgBase.setconstrLB!, m.internalModel, rowlb) &&
               applicable(MathProgBase.setconstrUB!, m.internalModel, rowub) &&
               applicable(MathProgBase.setobj!, m.internalModel, f) &&
               applicable(MathProgBase.setsense!, m.internalModel, m.objSense)
                MathProgBase.setvarLB!(m.internalModel, copy(m.colLower))
                MathProgBase.setvarUB!(m.internalModel, copy(m.colUpper))
                MathProgBase.setconstrLB!(m.internalModel, rowlb)
                MathProgBase.setconstrUB!(m.internalModel, rowub)
                MathProgBase.setobj!(m.internalModel, f)
                MathProgBase.setsense!(m.internalModel, m.objSense)
            else
                # The solver doesn't support changing bounds/objective
                # We need to build the model from scratch
                if !suppress_warnings
                    Base.warn_once("Solver does not appear to support hot-starts. Model will be built from scratch.")
                end
                m.internalModelLoaded = false
            end
        end
        # If we don't already have a MPB model
        if !m.internalModelLoaded
            # Obtain a fresh MPB model for the solver
            m.internalModel = MathProgBase.LinearQuadraticModel(m.solver)
            # Construct a LHS matrix from the linear constraints
            A = prepConstrMatrix(m)

            # Load the problem data into the model...
            collb = copy(m.colLower)
            colub = copy(m.colUpper)
            if relaxation
                for i in 1:m.numCols
                    if m.colCat[i] in (:SemiCont,:SemiInt)
                        collb[i] = min(0.0, collb[i])
                        colub[i] = max(0.0, colub[i])
                    end
                end
            end
            MathProgBase.loadproblem!(m.internalModel, A, collb, colub, f, rowlb, rowub, m.objSense)
            # ... and add quadratic and SOS constraints separately
            addQuadratics(m)
            if !relaxation
                addSOS(m)
            end
        end
        # Update solver callbacks, if any
        if !relaxation
            registercallbacks(m)
        end
    end

    # Update the type of each variable
    if applicable(MathProgBase.setvartype!, m.internalModel, Symbol[])
        if relaxation
            MathProgBase.setvartype!(m.internalModel, fill(:Cont, m.numCols))
        else
            colCats = vartypes_without_fixed(m)
            MathProgBase.setvartype!(m.internalModel, colCats)
        end
    elseif traits.int
        # Solver that do not implement anything other than continuous
        # variables do not need to implement this method, so throw an
        # error if the model has anything but continuous
        error("Solver does not support discrete variables")
    end

    # Provide a primal solution to the solve, if the problem is integer
    # and the user has provided one.
    # TODO: change this so you can warm start continuous problems?
    if !relaxation && traits.int && !all(isnan(m.colVal))
        if applicable(MathProgBase.setwarmstart!, m.internalModel, m.colVal)
            MathProgBase.setwarmstart!(m.internalModel, m.colVal)
        else
            suppress_warnings || Base.warn_once("Solver does not appear to support providing initial feasible solutions.")
        end
    end

    # Record that we have a MPB model constructed
    m.internalModelLoaded = true
    nothing
end

# Add the quadratic part of the objective and all quadratic constraints
# to the internal MPB model
function addQuadratics(m::Model)
    # The objective function is always a quadratic expression, but
    # may have no quadratic terms (i.e. be just affine)
    if length(m.obj.qvars1) != 0
        # Check that no coefficients are NaN/Inf
        assert_isfinite(m.obj)
        # Check that quadratic term variables belong to this model
        # Affine portion is checked in prepProblemBounds
        if !(verify_ownership(m, m.obj.qvars1) &&
                verify_ownership(m, m.obj.qvars2))
            error("Variable not owned by model present in objective")
        end
        # Check for solver support for quadratic objectives happens in MPB
        MathProgBase.setquadobjterms!(m.internalModel,
            Cint[v.col for v in m.obj.qvars1],
            Cint[v.col for v in m.obj.qvars2], m.obj.qcoeffs)
    end

    # Add quadratic constraint to solver
    const sensemap = Dict(:(<=) => '<', :(>=) => '>', :(==) => '=')
    for k in 1:length(m.quadconstr)
        qconstr = m.quadconstr[k]::QuadConstraint
        if !haskey(sensemap, qconstr.sense)
            error("Invalid quadratic constraint sense $(qconstr.sense)")
        end
        s = sensemap[qconstr.sense]

        terms::QuadExpr = qconstr.terms
        # Check that no coefficients are NaN/Inf
        assert_isfinite(terms)
        # Check that quadratic and affine term variables belong to this model
        if !(verify_ownership(m, terms.qvars1) &&
                verify_ownership(m, terms.qvars2) &&
                verify_ownership(m, terms.aff.vars))
            error("Variable not owned by model present in quadratic constraint")
        end
        # Extract indices for MPB, and add the constraint (if we can)
        affidx  = Cint[v.col for v in terms.aff.vars]
        var1idx = Cint[v.col for v in terms.qvars1]
        var2idx = Cint[v.col for v in terms.qvars2]
        if applicable(MathProgBase.addquadconstr!, m.internalModel, affidx, terms.aff.coeffs, var1idx, var2idx, terms.qcoeffs, s, -terms.aff.constant)
            MathProgBase.addquadconstr!(m.internalModel,
                affidx, terms.aff.coeffs,           # aᵀx +
                var1idx, var2idx, terms.qcoeffs,    # xᵀQx
                s, -terms.aff.constant)             # ≤/≥ b
        else
            error("Solver does not support quadratic constraints")
        end
    end
    nothing
end

function addSOS(m::Model)
    for i in 1:length(m.sosconstr)
        sos = m.sosconstr[i]
        indices = Int[v.col for v in sos.terms]
        if sos.sostype == :SOS1
            if applicable(MathProgBase.addsos1!, m.internalModel, indices, sos.weights)
                MathProgBase.addsos1!(m.internalModel, indices, sos.weights)
            else
                error("Solver does not support SOS constraints")
            end
        elseif sos.sostype == :SOS2
            if applicable(MathProgBase.addsos2!, m.internalModel, indices, sos.weights)
                MathProgBase.addsos2!(m.internalModel, indices, sos.weights)
            else
                error("Solver does not support SOS constraints")
            end
        end
    end
end

# Returns coefficients for the affine part of the objective and the
# affine constraint lower and upper bounds, all as dense vectors
function prepProblemBounds(m::Model)

    # Create dense objective vector
    objaff::AffExpr = m.obj.aff
    # Check that no coefficients are NaN/Inf
    assert_isfinite(objaff)
    if !verify_ownership(m, objaff.vars)
        error("Variable not owned by model present in objective")
    end
    f = zeros(m.numCols)
    @inbounds for ind in 1:length(objaff.vars)
        f[objaff.vars[ind].col] += objaff.coeffs[ind]
    end

    # Create dense affine constraint bound vectors
    linconstr = m.linconstr::Vector{LinearConstraint}
    numRows = length(linconstr)
    # -Inf means no lower bound, +Inf means no upper bound
    rowlb = fill(-Inf, numRows)
    rowub = fill(+Inf, numRows)
    @inbounds for ind in 1:numRows
        rowlb[ind] = linconstr[ind].lb
        rowub[ind] = linconstr[ind].ub
    end

    return f, rowlb, rowub
end

# Convert all the affine constraints into a sparse column-wise
# matrix of coefficients.
function prepConstrMatrix(m::Model)

    linconstr = m.linconstr::Vector{LinearConstraint}
    numRows = length(linconstr)
    # Calculate the maximum number of nonzeros
    # The actual number may be less because of cancelling or
    # zero-coefficient terms
    nnz = 0
    for c in 1:numRows
        nnz += length(linconstr[c].terms.coeffs)
    end
    # Non-zero row indices
    I = Array(Int,nnz)
    # Non-zero column indices
    J = Array(Int,nnz)
    # Non-zero values
    V = Array(Float64,nnz)

    # Fill it up!
    # Number of nonzeros seen so far
    nnz = 0
    for c in 1:numRows
        # Check that no coefficients are NaN/Inf
        assert_isfinite(linconstr[c].terms)
        coeffs = linconstr[c].terms.coeffs
        vars   = linconstr[c].terms.vars
        # Check that variables belong to this model
        if !verify_ownership(m, vars)
            error("Variable not owned by model present in a constraint")
        end
        # Record all (i,j,v) triplets
        @inbounds for ind in 1:length(coeffs)
            nnz += 1
            I[nnz] = c
            J[nnz] = vars[ind].col
            V[nnz] = coeffs[ind]
        end
    end

    # sparse() handles merging duplicate terms and removing zeros
    A = sparse(I,J,V,numRows,m.numCols)
end

function vartypes_without_fixed(m::Model)
    colCats = copy(m.colCat)
    for i in 1:length(colCats)
        if colCats[i] == :Fixed
            @assert m.colLower[i] == m.colUpper[i]
            colCats[i] = :Cont
        end
    end
    return colCats
end

function collect_expr!(m, tmprow, terms::AffExpr)
    empty!(tmprow)
    assert_isfinite(terms)
    coeffs = terms.coeffs
    vars = terms.vars
    # collect duplicates
    for ind in 1:length(coeffs)
        if !is(vars[ind].m, m)
            error("Variable not owned by model present in constraints")
        end
        addelt!(tmprow,vars[ind].col, coeffs[ind])
    end
    tmprow
end

# Returns a boolean vector indicating if variable in the model
# is an off-diagonal element of an SDP matrix.
# This is needed because we have to rescale coefficients that
# touch these variables.
function offdiagsdpvars(m::Model)
    offdiagvars = falses(m.numCols)
    for (name,idx) in m.varCones
        if name == :SDP
            conelen = length(idx)
            n = round(Int,sqrt(1/4+2*conelen)-1/2)
            @assert n*(n+1)/2 == conelen
            r = 1
            for i in 1:n
                for j in i:n
                    if i != j
                        offdiagvars[idx[r]] = true
                    end
                    r += 1
                end
            end
        end
    end
    return offdiagvars
end

function conicdata(m::Model)
    var_cones = Any[cone for cone in m.varCones]
    con_cones = Any[]
    nnz = 0

    # find starting column indices for sdp matrices
    numSDPRows = 0
    numSymRows = 0
    for c in m.sdpconstr
        n = size(c.terms,1)
        @assert n == size(c.terms,2)
        @assert ndims(c.terms) == 2
        numSDPRows += convert(Int, n*(n+1)/2)
        for i in 1:n, j in i:n
            nnz += length(c.terms[i,j].coeffs)
        end
        if !Compat.issymmetric(c.terms)
            # symmetry constraints
            numSymRows += convert(Int, n*(n-1)/2)
        end
    end

    soc_cones  = Any[]
    rsoc_cones = Any[]
    numQuadRows = 0
    for qconstr in m.quadconstr
        q = copy(qconstr.terms)
        if qconstr.sense == :(>=)
            q *= -1
        end
        if !(all(t->t==0, q.aff.coeffs) && q.aff.constant == 0)
            error("Quadratic constraint $qconstr must be in second-order cone form")
        end
        n_pos_on_diag = 0
        off_diag_idx  = 0
        neg_diag_idx  = 0
        n = length(q.qvars1)
        nz = 0
        for i in 1:n
            q.qcoeffs[i] == 0 && continue
            nz += 1
            if q.qvars1[i].col == q.qvars2[i].col
                if q.qcoeffs[i] == 1
                    n_pos_on_diag += 1
                elseif q.qcoeffs[i] == -1
                    neg_diag_idx == off_diag_idx == 0 || error("Invalid SOC constraint $qconstr")
                    neg_diag_idx = i
                end
            else
                if q.qcoeffs[i] == -1
                    neg_diag_idx == off_diag_idx == 0 || error("Invalid rotated SOC constraint $qconstr")
                    off_diag_idx = i
                    nz += 1
                end
            end
        end
        cone = Array(Int, nz)
        if n_pos_on_diag == nz-1 && neg_diag_idx > 0
            cone[1] = q.qvars1[neg_diag_idx].col
            r = 1
            for i in 1:n
                (q.qcoeffs[i] == 0 || i == neg_diag_idx) && continue
                r += 1
                cone[r] = q.qvars1[i].col
            end
            push!(soc_cones, cone)
        elseif n_pos_on_diag == nz-2 && off_diag_idx > 0
            cone[1] = q.qvars1[off_diag_idx].col
            cone[2] = q.qvars2[off_diag_idx].col
            r = 2
            for i in 1:n
                (q.qcoeffs[i] == 0 || i == off_diag_idx) && continue
                r += 1
                cone[r] = q.qvars1[i].col
            end
            push!(rsoc_cones, cone)
        else
            error("Quadratic constraint $qconstr is not conic representable")
        end
        numQuadRows += length(cone)
    end

    linconstr = m.linconstr::Vector{LinearConstraint}
    numLinRows = length(linconstr)
    numBounds = 0
    nonNeg  = Int[]
    nonPos  = Int[]
    free    = Int[]
    zeroVar = Int[]
    for i in 1:m.numCols
        seen = false
        lb, ub = m.colLower[i], m.colUpper[i]
        for (_,cone) in m.varCones
            if i in cone
                seen = true
                @assert lb == -Inf && ub == Inf
                break
            end
        end

        if !seen
            if lb != -Inf && lb != 0
                numBounds += 1
            end
            if ub != Inf && ub != 0
                numBounds += 1
            end
            if lb == 0 && ub == 0
                push!(zeroVar, i)
            elseif lb == 0
                push!(nonNeg, i)
            elseif ub == 0
                push!(nonPos, i)
            else
                push!(free, i)
            end
        end
    end

    if !isempty(zeroVar)
        push!(var_cones, (:Zero,zeroVar))
    end
    if !isempty(nonNeg)
        push!(var_cones, (:NonNeg,nonNeg))
    end
    if !isempty(nonPos)
        push!(var_cones, (:NonPos,nonPos))
    end
    if !isempty(free)
        push!(var_cones, (:Free,free))
    end

    nnz += numBounds
    for c in 1:numLinRows
        nnz += length(linconstr[c].terms.coeffs)
    end

    numSOCRows = 0
    numNormRows = 0
    for con in m.socconstr
        numNormRows += 1
        numSOCRows += length(con.normexpr.norm.terms) + 1
    end
    numRows = numLinRows + numBounds + numQuadRows + numSOCRows + numSDPRows + numSymRows

    # should maintain the order of constraints in the above form
    # throughout the code c is the conic constraint index
    # TODO: only added linear+bound+soc support, extend to all
    constr_dual_map = Array(Vector{Int}, numLinRows + numBounds + numNormRows)

    b = Array(Float64, numRows)

    I = Int[]
    J = Int[]
    V = Float64[]
    sizehint!(I, nnz)
    sizehint!(J, nnz)
    sizehint!(V, nnz)

    # Fill it up
    nnz = 0
    tmprow = IndexedVector(Float64,m.numCols)
    tmpelts = tmprow.elts
    tmpnzidx = tmprow.nzidx
    nonneg_rows = Int[]
    nonpos_rows = Int[]
    eq_rows     = Int[]
    for c in 1:numLinRows
        if linconstr[c].lb == -Inf
            b[c] = linconstr[c].ub
            push!(nonneg_rows, c)
        elseif linconstr[c].ub == Inf
            b[c] = linconstr[c].lb
            push!(nonpos_rows, c)
        elseif linconstr[c].lb == linconstr[c].ub
            b[c] = linconstr[c].lb
            push!(eq_rows, c)
        else
            error("We currently do not support ranged constraints with conic solvers")
        end

        assert_isfinite(linconstr[c].terms)
        coeffs = linconstr[c].terms.coeffs
        vars = linconstr[c].terms.vars
        # collect duplicates
        for ind in 1:length(coeffs)
            if !is(vars[ind].m, m)
                error("Variable not owned by model present in constraints")
            end
            addelt!(tmprow,vars[ind].col, coeffs[ind])
        end
        nnz = tmprow.nnz
        append!(I, fill(c, nnz))
        indices = tmpnzidx[1:nnz]
        append!(J, indices)
        append!(V, tmpelts[indices])
        empty!(tmprow)
        constr_dual_map[c] = vec(collect(c))
    end

    c = numLinRows
    bndidx = 0
    for idx in 1:m.numCols
        lb = m.colLower[idx]
        if lb != -Inf && lb != 0
            bndidx += 1
            nnz += 1
            c   += 1
            push!(I, c)
            push!(J, idx)
            push!(V, 1.0)
            b[c] = lb
            push!(nonpos_rows, c)
            constr_dual_map[numLinRows + bndidx] = vec(collect(c))
        end
        ub = m.colUpper[idx]
        if ub != Inf && ub != 0
            bndidx += 1
            c   += 1
            push!(I, c)
            push!(J, idx)
            push!(V, 1.0)
            b[c] = ub
            push!(nonneg_rows, c)
            constr_dual_map[numLinRows + bndidx] = vec(collect(c))
        end
    end

    if !isempty(nonneg_rows)
        push!(con_cones, (:NonNeg,nonneg_rows))
    end
    if !isempty(nonpos_rows)
        push!(con_cones, (:NonPos,nonpos_rows))
    end
    if !isempty(eq_rows)
        push!(con_cones, (:Zero,eq_rows))
    end
    @assert c == numLinRows + numBounds

    for cone in soc_cones
        n = length(cone)
        rng = (c+1):(c+n)
        append!(I, rng)
        append!(J, copy(cone))
        append!(V, [-1.0; ones(n-1)])
        push!(con_cones, (:SOC,rng))
        b[rng] = 0
        c += n
    end
    for cone in rsoc_cones
        n = length(cone)
        rng = (c+1):(c+n)
        append!(I, rng)
        append!(J, copy(cone))
        append!(V, [-1/sqrt(2); -1/sqrt(2); ones(n-2)])
        push!(con_cones, (:SOCRotated,rng))
        b[rng] = 0
        c += n
    end
    @assert c == numLinRows + numBounds + numQuadRows

    tmpelts = tmprow.elts
    tmpnzidx = tmprow.nzidx
    socidx = 0
    for con in m.socconstr
        socidx += 1
        expr = con.normexpr
        c += 1
        soc_start = c
        collect_expr!(m, tmprow, expr.aff)
        nnz = tmprow.nnz
        indices = tmpnzidx[1:nnz]
        append!(I, fill(c, nnz))
        append!(J, indices)
        append!(V, tmpelts[indices])
        b[c] = -expr.aff.constant
        for term in expr.norm.terms
            c += 1
            collect_expr!(m, tmprow, term)
            nnz = tmprow.nnz
            indices = tmpnzidx[1:nnz]
            append!(I, fill(c, nnz))
            append!(J, indices)
            append!(V, -expr.coeff*tmpelts[indices])
            b[c] = expr.coeff*term.constant
        end
        push!(con_cones, (:SOC, soc_start:c))
        constr_dual_map[numLinRows + numBounds + socidx] = collect(soc_start:c)
    end
    @assert c == numLinRows + numBounds + numQuadRows + numSOCRows

    numDroppedSym = 0
    for con in m.sdpconstr
        sdp_start = c + 1
        n = size(con.terms,1)
        for i in 1:n, j in i:n
            c += 1
            terms::AffExpr = con.terms[i,j]
            collect_expr!(m, tmprow, terms)
            nnz = tmprow.nnz
            indices = tmpnzidx[1:nnz]
            append!(I, fill(c, nnz))
            append!(J, indices)
            # scale to svec form
            scale = (i == j) ? 1.0 : sqrt(2)
            append!(V, -scale*tmpelts[indices])
            b[c] = scale*terms.constant
        end
        push!(con_cones, (:SDP, sdp_start:c))
        if !Compat.issymmetric(con.terms)
            sym_start = c + 1
            # add linear symmetry constraints
            for i in 1:n, j in 1:(i-1)
                collect_expr!(m, tmprow, con.terms[i,j] - con.terms[j,i])
                nnz = tmprow.nnz
                # if the symmetry-enforcing row is empty, just drop it
                if nnz == 0
                    numDroppedSym += 1
                    continue
                end
                c += 1
                indices = tmpnzidx[1:nnz]
                append!(I, fill(c, nnz))
                append!(J, indices)
                append!(V, tmpelts[indices])
                b[c] = 0
            end
            push!(con_cones, (:Zero, sym_start:c))
        end
    end
    numRows -= numDroppedSym
    resize!(b, numRows)
    @assert c == numRows

    m.constrDualMap = constr_dual_map

    # Use only the objective coefficients from prepProblemBounds
    f,_,_ = prepProblemBounds(m)

    # The conic MPB interface defines conic problems as
    # always being minimization problems, so flip if needed
    m.objSense == :Max && scale!(f, -1.0)

    # Objective coefficients and columns of A matrix are
    # rescaled for SDP variables
    offdiagvars = offdiagsdpvars(m)
    f[offdiagvars] /= sqrt(2)
    for k in 1:length(J)
        if offdiagvars[J[k]]
            V[k] /= sqrt(2)
        end
    end

    A = sparse(I, J, V, numRows, m.numCols)
    # @show full(A), b
    # @show var_cones, con_cones

    # TODO: uncomment these lines when they work with Mosek
    # supported = MathProgBase.supportedcones(m.internalModel)
    # @assert (:NonNeg in supported) && (:NonPos in supported) && (:Free in supported) && (:SDP in supported)
    f, A, b, var_cones, con_cones
end

constraintbounds(m::Model) = constraintbounds(m, ProblemTraits(m))

function constraintbounds(m::Model,traits::ProblemTraits)

    if traits.conic
        error("Not implemented for conic problems")
    elseif traits.sos
        error("Not implemented for SOS constraints")
    end

    linobj, linrowlb, linrowub = prepProblemBounds(m)

    quadrowlb = Float64[]
    quadrowub = Float64[]
    for c::QuadConstraint in m.quadconstr
        if c.sense == :(<=)
            push!(quadrowlb, -Inf)
            push!(quadrowub, 0.0)
        elseif c.sense == :(>=)
            push!(quadrowlb, 0.0)
            push!(quadrowub, Inf)
        elseif c.sense == :(==)
            push!(quadrowlb, 0.0)
            push!(quadrowub, 0.0)
        else
            error("Unrecognized quadratic constraint sense $(c.sense)")
        end
    end

    nlrowlb = Float64[]
    nlrowub = Float64[]

    if traits.nlp
        nldata::NLPData = m.nlpdata
        for c in nldata.nlconstr
            push!(nlrowlb, c.lb)
            push!(nlrowub, c.ub)
        end
    end

    lb = [linrowlb;quadrowlb;nlrowlb]
    ub = [linrowub;quadrowub;nlrowub]

    return lb, ub

end

# returns (unsorted) column indices and coefficient terms for merged vector
# assume that v is zero'd
function merge_duplicates{CoefType,IntType<:Integer}(::Type{IntType},aff::GenericAffExpr{CoefType,Variable}, v::IndexedVector{CoefType}, m::Model)
    resize!(v, m.numCols)
    for ind in 1:length(aff.coeffs)
        var = aff.vars[ind]
        is(var.m, m) || error("Variable does not belong to this model")
        addelt!(v, aff.vars[ind].col, aff.coeffs[ind])
    end
    indices = Array(IntType,v.nnz)
    coeffs = Array(CoefType,v.nnz)
    for i in 1:v.nnz
        idx = v.nzidx[i]
        indices[i] = idx
        coeffs[i] = v.elts[idx]
    end
    empty!(v)

    return indices, coeffs

end


using ArcadeLearningEnvironment
using Logging
using CGP
using Images


"""
Types marked with this trait need to implement the following queries

- [`get_free_values(object)`]
- [`get_fe_space(object)`]

"""
FEFunctionStyle(::Type{T}) where T = Val{false}()

FEFunctionStyle(fe_function) = FEFunctionStyle(typeof(fe_function))

"""
"""
function is_a_fe_function(fe_function)
  v = FEFunctionStyle(fe_function)
  get_val_parameter(v)
end

function is_a_fe_function(::Type)
  @unreachable "is_a_fe_function cannot be called on types"
end

"""
"""
function get_free_values(object)
  @abstractmethod
end

"""
"""
function get_cell_values(object)
  @abstractmethod
end

function get_cell_values(object,cellids::AbstractArray)
  reindex(get_cell_values(object),cellids)
end

function get_cell_values(object,cellids::SkeletonPair)
  vals = reindex(get_cell_values(object),cellids)
  f = get_fe_space(object)
  axs = reindex(get_cell_axes(f),cellids)
  merge_cell_dofs_at_skeleton(vals.left,vals.right,axs.left,axs.right)
end

"""
"""
function get_fe_space(object)
  @abstractmethod
end

"""
"""
function test_fe_function(f)
  @test is_a_fe_function(f)
  free_values = get_free_values(f)
  fe_space = get_fe_space(f)
  @test length(free_values) == num_free_dofs(fe_space)
  cell_vals = get_cell_values(f)
end


function simple_call(fname, args...; kwargs...)
    try
        if endswith(fname, ".")
            fname = chop(fname);
            # f = eval(Main, parse(fname));
            f = funcfind(fname);
            r = broadcast(f, args...);
        else
            # f = eval(Main, parse(fname));
            f = funcfind(fname);
            r = f(args...; kwargs...);
        end
        sexp(r)
    catch e
        Rerror(e, stacktrace(catch_backtrace())).p;
    end;
end

# function simple_call(fname, arg1, arg2)
#     try
#         if endswith(fname, ".")
#             fname = chop(fname);
#             # f = eval(Main, parse(fname));
#             f = funcfind(fname);
#             r = broadcast(f, arg1, arg2);
#         else
#             # f = eval(Main, parse(fname));
#             f = funcfind(fname);
#             r = f(arg1, arg2);
#         end
#         RObject(r).p
#     catch e
#         Rerror(e, stacktrace(catch_backtrace())).p;
#     end;
# end


function hvdc_comparison(
      parent_folder::String,
      output_folder::String,
      objective::String;
      k_cond=[],
      k_ins=[],
      dc_subnet::Int64=-1,
      gen_areas::Array=[],
      area_transfer::Array=[],
      gen_zones::Array=[],
      zone_transfer::Array=[],
      enum_branches::Bool=false,
      plot_best_x::Int64=-1,
      print_results::Bool=false,
      series_labels::Array=[],
      results_folders::Array=[],
      scopf::Bool=false,
      no_converter_loss::Bool=false,
      output_to_files::Bool=true,
      regularize_f::Float64=0.0
   )
   if (length(k_cond)) > 0 && (length(k_ins) > 0)
      if scopf
         dc_params = ([["sn",contingency,"sn",dc_subnet,"k_ins"],["sn",contingency,"sn",dc_subnet,"k_cond"],["sn",contingency,"sn",dc_subnet,"f_max"]],[k_ins, k_cond, zeros(length(k_ins))])
      else
         dc_params = ([["sn",dc_subnet,"k_ins"],["sn",dc_subnet,"k_cond"],["sn",dc_subnet,"f_max"]],[k_ins, k_cond, zeros(length(k_ins))])
      end
   else
      dc_params = ()
   end
   num_series = 0
   if length(dc_params) > 0
      if length(dc_params[1]) > 0
         num_series = length(dc_params[2][1])
      end
   end
   println("num_series: $num_series")
   println("dc_params: $dc_params")
   # Run the opf for the base case then all dc configurations
   results_dict_allplots = []
   n_subnets = 0
   subnet_array = []
   idx_sorted = []
   if length(results_folders) == 0
      # Base case opf
      (results_dict,n_subnets,subnet_array,idx_sorted,series_output_folder,plot_best_x) = run_series(
         parent_folder,
         objective;
         gen_areas,
         area_transfer,
         gen_zones,
         zone_transfer,
         enum_branches,
         plot_best_x,
         print_results,
         suffix=series_labels[1],
         no_converter_loss=no_converter_loss,
         output_to_files=output_to_files,
         regularize_f=regularize_f
      )
      push!(results_dict_allplots, results_dict)

      results_dict_string = JSON.json(results_dict)
      println("Saving results at $(series_output_folder)/all_results_dict.json")
      open("$series_output_folder/all_results_dict.json", "w") do f
         write(f, results_dict_string)
      end
      push!(results_folders, series_output_folder)

      # Iterate through the parameter collections in order
      for param_i in 1:num_series
         override_param = Dict{Any, Any}()
         for (val_idx,value) in enumerate(dc_params[2])
            set_nested!(override_param, string.(dc_params[1][val_idx]), value[param_i])
         end
         (results_dict,n_subnets_dc,subnet_array_dc,idx_sorted_dc,series_output_folder,plot_best_x) = run_series(
            parent_folder,
            objective;
            gen_areas,
            area_transfer,
            gen_zones,
            zone_transfer,
            enum_branches,
            plot_best_x,
            print_results,
            override_param=override_param,
            suffix=series_labels[1+param_i],
            no_converter_loss=no_converter_loss,
            output_to_files=output_to_files,
            regularize_f=regularize_f
         )
         push!(results_dict_allplots, results_dict)

         results_dict_string = JSON.json(results_dict)
         println("Saving results at $(series_output_folder)/all_results_dict.json")
         open("$series_output_folder/all_results_dict.json", "w") do f
            write(f, results_dict_string)
         end
         push!(results_folders, series_output_folder)
      end
      println("results_folders: $results_folders")
   else
      # Use results data that has already been saved
      results_dict_allplots = []
      for (i,output_folder) in enumerate(results_folders)

         results_dict_parsed = JSON.parsefile("$(output_folder)/all_results_dict.json")
         if i == 1
            (n_subnets, subnet_array, idx_sorted) = get_sort_indices(results_dict_parsed, objective, gen_areas, area_transfer, gen_zones, zone_transfer)
         end

         push!(results_dict_allplots, results_dict_parsed)
      end
   end
   if output_to_files
      println("Saving results at $(output_folder)/results_dict_allplots.json")
      results_dict_string = JSON.json(results_dict_allplots)
      if !isdir(output_folder)
         mkpath(output_folder)
      end
      open("$(output_folder)/results_dict_allplots.json", "w") do f
         write(f, results_dict_string)
      end
   end
   plot_results_dicts_bar(results_dict_allplots,n_subnets,subnet_array,idx_sorted,output_folder,plot_best_x,series_labels)

end


# --------------- #
# Display methods #
# --------------- #

function show_extra(me::MomentBasedEstimator{T}) where T<:GMMEstimator
    j, p = J_test(me)
    "\nJ-test: $(round(j; digits = 3)) (P-value: $(round(p, digits = 3)))\n"
end

show_extra(me::MomentBasedEstimator{T}) where T<:MDEstimator = "\n"

function rshow(me::MomentBasedEstimator{T}) where T<:GMMEstimator
    println("GMM")
end

function rshow(me::MomentBasedEstimator{T}) where T<:MDEstimator
    println("MD")
end

function Base.show(io::IO, ::MIME"text/plain", me::T) where T<:MomentBasedEstimator
  s = symbolfy(me)
  println(io, "\n$s [k = $(npar(me)), m = $(nmom(me))]\n")

  if me.status[1] == :Solved
    # Only gives information if the solver
    # converged
    if (status(me) == :Optimal)
      # get coef table and j-test
      ct = coeftable(me)
      # print coefficient table
      println(io, "Coefficients:\n")
      show(io, ct)
      # Then show extra information for this type
      println(io, show_extra(me))
    else
      println("The optimization did not converge to a local solution")
    end
  end
end

function symbolfy(me::T) where T<:MomentBasedEstimator
    s = split(string(T), "MomentBasedEstimator")[2]
    v = split(s, "{")[2]
    Symbol(v)

    # if (v == "GMMEstimator{")
    #     Symbol(v, split(s[4], "{")[1], ", ", split(s[10], ",")[1], ", ", split(s[11], ",")[1], "}")
    # elseif (s[3] == "MDEstimator{")
    #     if (s[4] == "AnaGradMomFun{" || s[4] == "AnaFullMomFun{")
    #       Symbol(s[3], split(s[4], "{")[1], ", ", split(s[7], "},Divergences.")[2], " ", split(s[8], ",")[1], "}")
    #     else
    #       Symbol(s[3], split(s[4], "{")[1], ", ", split(s[9], "},Divergences.")[2], " ", split(s[10], ",")[1], "}")
    #     end
    # end
end


struct WithCache{T}
  data::T
  cache::UInt

end

function with_cache(x::T) where {T}
  h = hash(x)
  return WithCache{T}(x, h)
end

Base.hash(x::WithCache{T}) where {T} = x.cache

struct FacElemWrap{T, S}
  fac::Dict{WithCache{T}, fmpz}
  hash::UInt
  #parent::FacElemMon{S}
end

@inline function Base.iterate(x::FacElemWrap)
  z, st = iterate(x.fac)
  ((z[1]).data => z[2]), st
end

@inline function Base.iterate(x::FacElemWrap, st)
  v = iterate(x.fac, st)
  if v === nothing
    return nothing
  else
    z, st = v
    return ((z[1]).data => z[2]), st
  end
end

function Base.:(*)(x::FacElemWrap, y::FacElemWrap)
  z = copy(x.fac)
  z = copy(x.fac)
  for (a, v) in y.fac
    add_to_key!(z, a, v, remove_zero = true)
  end
  return FacElemWrap(z)
end

function FacElemWrap(x::Dict{T, fmpz}) where {T}
  z = Dict{WithCache{T}, fmpz}(with_cache(k) => p for (k, p) in x)
  return FacElemWrap{T, Nothing}(z, UInt(0))
end


######################################
using DataFrames, CSV, CodecZlib, Mmap

#= Polytope Data from text files: 29×3 DataFrame
│ Row │ Database │ Polytopes │ cumulative_sum │
│     │ Int8     │ Int32     │ Int64          │
├─────┼──────────┼───────────┼────────────────┤
│ 1   │ 5        │ 1561      │ 1561           │
│ 2   │ 6        │ 24189     │ 25750          │
│ 3   │ 7        │ 177446    │ 203196         │
│ 4   │ 8        │ 834638    │ 1037834        │
│ 5   │ 9        │ 2867955   │ 3905789        │
│ 6   │ 10       │ 7725801   │ 11631590       │
│ 7   │ 11       │ 16608387  │ 28239977       │
│ 8   │ 12       │ 29270253  │ 57510230       │
│ 9   │ 13       │ 43458000  │ 100968230      │
│ 10  │ 14       │ 56060584  │ 157028814      │
│ 11  │ 15       │ 64085869  │ 221114683      │
│ 12  │ 16       │ 65615931  │ 286730614      │
│ 13  │ 17       │ 59972682  │ 346703296      │
│ 14  │ 18       │ 48703033  │ 395406329      │
│ 15  │ 19       │ 34847821  │ 430254150      │
│ 16  │ 20       │ 21913680  │ 452167830      │
│ 17  │ 21       │ 12070919  │ 464238749      │
│ 18  │ 22       │ 5826221   │ 470064970      │
│ 19  │ 23       │ 2450720   │ 472515690      │
│ 20  │ 24       │ 898929    │ 473414619      │
│ 21  │ 25       │ 284696    │ 473699315      │
│ 22  │ 26       │ 78468     │ 473777783      │
│ 23  │ 27       │ 18417     │ 473796200      │
│ 24  │ 28       │ 3781      │ 473799981      │
│ 25  │ 29       │ 647       │ 473800628      │
│ 26  │ 30       │ 114       │ 473800742      │
│ 27  │ 31       │ 23        │ 473800765      │
│ 28  │ 32       │ 8         │ 473800773      │
│ 29  │ 33       │ 2         │ 473800775      │   =#

const DIM = 4

# Construct polytope from the polytope coordinates in verts and append to DataFrame
# of all polytopes (Δ²). Note that the length of the polytope and the classification
# number are embedded in the DataFrame.
function MakePolytope!(Δ²::DataFrame, verts, ontology)
    Δ = hcat(verts...)
    size(Δ,1) < size(Δ,2) ? Δ = Δ' : nothing;
    len = convert(Int32,size(Δ, 1))
    delim = zeros(len)
    delim[1] += ontology.v
    delim[2] += ontology.p
    delim[3] += ontology.v_dual
    delim[4] += ontology.p_dual
    Δ = hcat(delim, Δ)
    header = zeros(1 + 4)
    header[1] = len
    header[2] = ontology.h11
    header[3] = ontology.h21
    header[4] = ontology.euler
    Δ = vcat(header', Δ)
    Δ = convert(DataFrame, Δ)
    println(Δ)
    run(`quit`)
    append!(Δ², Δ)
end

# Write Δ² to a .gzip file and compress. Each sub-database
# has a simple integer naming scheme (sub_db_id).
function ExportSubDB(Δ²::DataFrame, sub_db_id::Integer)
    write_path = "/media/share/Dev/CalabiYau/data/polytopes_db_4d/$(sub_db_id).gz"
    open(write_path, "w") do io
        stream = GzipCompressorStream(io)
        CSV.write(stream, Δ²)
        close(stream)
        stream = nothing
        GC.gc()
    end #close .gzf
end

# Grab feature data
function GetOntology(ln::String)
    info =  split(strip(ln))
    Δ⁰_pnts = parse(Int,split(info[3], ":")[2])
    Δ⁰_vrts = parse(Int,info[4])
    Δ_pnts = parse(Int,split(info[5], ":")[2])
    Δ_vrts = parse(Int,info[6])
    H = split(split(info[7], ":")[2], ",")
    h11 = parse(Int,H[1])
    h21 = parse(Int,H[2])
    euler = parse(Int,split(split(info[end], "[")[2],"]")[1])
    return (p_dual = Δ⁰_pnts, v_dual = Δ⁰_vrts,p = Δ_pnts, v =  Δ_vrts,h11= h11,h21= h21,euler= euler)
end

function DoTheThing()#(a::Integer, b::Integer, polytopeCount::Integer, info::DataFrame)
    Δ² = DataFrame()::DataFrame
    ontology = ()
    polytopeCount = 0::Integer
    verts = []
    sub_db_id = 1::Integer
    polyPerDB = 0::Integer
    info = DataFrame(Database = Int8[], Polytopes = Int32[])

    for id in 5:33
        # Initialize path to current database file
        db = "$id"
        if length(db) == 1  db = "0"*db; end
        data_path = "/media/share/Dev/CalabiYau/data/polytopes_db_4d/src/v$db"
        # Open current database file
        open(data_path) do file
            firstLine = true
            polyPerDB = 0
            for ln in eachline(file)
                # Are we on the first line?
                if firstLine
                    ontology = GetOntology(ln)
                    firstLine = false
                    continue
                end
                # Are we on the last line?
                if occursin("#", ln)
                    MakePolytope!(Δ², verts, ontology)
                    verts = nothing
                    polytopeCount += 1
                    polyPerDB += 1
                    verts = []
                    break
                end
                # Have we finished reading a polytope?
                if occursin("[", ln)
                    #= Construct polytope from the polytope coordinates in verts, append
                    to DataFrame of all polytopes (Δ²), and reset relevant variables       =#
                    MakePolytope!(Δ², verts, ontology)
                    verts = nothing
                    polytopeCount += 1
                    polyPerDB += 1
                    verts = []
                    #= If Δ² has reached 1,000,000 polytopes, then write Δ² to file. After
                    compression, iterate the sub-database identifer and parse the next
                    classification number (ontology) from the current line.             =#
                    if polytopeCount == 1000000
                        ExportSubDB(Δ², sub_db_id)
                        Δ² = nothing
                        sub_db_id += 1
                        polytopeCount = 0
                        Δ² = DataFrame()
                    end #end polytope count check
                    ontology = GetOntology(ln)
                else # if reading a coordinate line
                    row = [x for x in parse.(Int16, split(strip(ln)))]
                    push!(verts, row)
                end #end current line
            end #end eachline
            current_db_info = DataFrame(Database = id, Polytopes = polyPerDB)
            append!(info, current_db_info)
            run(`clear`)
            println(info)
        end #close file
    end
    ## Export final (partial) database
    ExportSubDB(Δ², sub_db_id)

    info.cumulative_sum = cumsum(info.Polytopes)
    run(`clear`)
    println(info)
end

DoTheThing()


using Test
using ScottishTaxBenefitModel
using Observables
using .ModelHousehold: count,Household, le_age, ge_age
using .Results: aggregate!, init_household_result
using ScottishTaxBenefitModel.Runner: do_one_run
using .Intermediate: MTIntermediate, make_intermediate
using .UBI: calc_UBI!,make_ubi_post_adjustments!
using .STBParameters
using .STBIncomes
using .ExampleHelpers
using .Monitor: Progress
sys = get_system( scotland=true )
sys.ubi.abolished = false

settings = Settings()

# observer = Observer(Progress("",0,0,0))
tot = 0
obs = Observable( Progress(settings.uuid,"",0,0,0,0))
of = on(obs) do p
    global tot
    println(p)

    tot += p.step
    println(tot)
end

@testset "Basic UBI Tests" begin

    for (hht,hh) in get_all_examples()
        hres = init_household_result( hh )

        calc_UBI!(
            hres,
            hh,
            sys.ubi
        )
        aggregate!( hh, hres )
        numkids = ModelHousehold.count( hh, le_age, sys.ubi.adult_age-1 )
        numpens = ModelHousehold.count( hh, ge_age, sys.ubi.retirement_age)
        numpeeps = length( hh.people )[1]
        numads = numpeeps - numpens - numkids
        ubit = numkids*sys.ubi.child_amount+
            numads*sys.ubi.adult_amount+
            numpens*sys.ubi.universal_pension
        @test hres.income[BASIC_INCOME] ≈ ubit
        println( "$hht : adults=$numads pens=$numpens child=$numkids => ubi=$ubit")
    end

    # todo make_ubi_pre_adjustments
    # toto make_ubi_post_adjustments

end

@testset "UBI Pre Adjustments" begin

    sys = get_system( scotland = true )
    println( typeof(sys.it.non_savings_income))
    sys.ubi.abolished = false
    sys.ubi.mt_bens_treatment = ub_as_is
    make_ubi_pre_adjustments!( sys )
    @test sys.uc.abolished == false
    @test sys.lmt.isa_jsa_esa_abolished == false
    @test sys.lmt.savings_credit.abolished == false
    @test sys.lmt.ctr.abolished == false
    @test sys.lmt.hb.abolished == false
    @test sys.lmt.working_tax_credit.abolished == false
    @test sys.lmt.child_tax_credit.abolished == false

    sys.ubi.mt_bens_treatment = ub_keep_housing
    make_ubi_pre_adjustments!( sys )
    @test sys.lmt.isa_jsa_esa_abolished == true
    @test sys.lmt.savings_credit.abolished == true
    @test sys.lmt.ctr.abolished == false
    @test sys.lmt.hb.abolished == false
    @test sys.lmt.working_tax_credit.abolished == true
    @test sys.lmt.child_tax_credit.abolished == true
    # Worried about inadvertently doing this twice .. should
    # make no difference.
    make_ubi_pre_adjustments!( sys )
    @test sys.lmt.isa_jsa_esa_abolished == true
    @test sys.lmt.savings_credit.abolished == true
    @test sys.lmt.ctr.abolished == false
    @test sys.lmt.hb.abolished == false
    @test sys.lmt.working_tax_credit.abolished == true
    @test sys.lmt.child_tax_credit.abolished == true

    @test BASIC_INCOME in sys.lmt.income_rules.sc_incomes
    @test ! (BASIC_INCOME in sys.it.non_savings_income)

end

@testset "UBI Post Adjustments" begin

    sys = get_system( scotland = true )
    sys.ubi.abolished = false
    res1 = do_one_run(
        settings,
        [sys],
        obs )
    sys.ubi.mt_bens_treatment = ub_keep_housing
    res2 = do_one_run(
        settings,
        [sys],
        obs )
    # dodo finish
end

@testset "example tests" begin
    # TODO FINISH

end


export every, fps, fpswhen, throttle

"""
    throttle(dt, input, f=(acc,x)->x, init=value(input), reinit=x->x)

Throttle a signal to update at most once every dt seconds. By default, the throttled signal holds the last update in the time window.

This behavior can be changed by the `f`, `init` and `reinit` arguments. The `init` and `f` functions are similar to `init` and `f` in `foldp`. `reinit` is called when a new throttle time window opens to reinitialize the initial value for accumulation, it gets one argument, the previous accumulated value.

For example
    y = throttle(0.2, x, push!, Int[], _->Int[])
will create vectors of updates to the integer signal `x` which occur within 0.2 second time windows.

"""
function throttle{T}(dt, node::Signal{T}, f=(acc,x)->x, init=value(node), reinit=x->x; typ=typeof(init))
    output = Signal(typ, init, (node,))
    throttle_connect(dt, output, node, f, init, reinit)
    output
end

# Aggregate a signal producing an update at most once in dt seconds
function throttle_connect(dt, output, input, f, init, reinit)
    let collected = init, timer = Timer(x->x, 0)
        add_action!(input, output) do output, timestep
            collected = f(collected,  value(input))
            close(timer)
            timer = Timer(x -> begin push!(output, collected); collected=reinit(collected) end, dt)
        end
    end
end

"""
    every(dt)

A signal that updates every `dt` seconds to the current timestamp. Consider using `fpswhen` or `fps` before using `every`.
"""
function every(dt)
    n = Signal(time(), ())
    every_connect(dt, n)
    n
end

function every_connect(dt, output)
    outputref = WeakRef(output)
    timer = Timer(x -> _push!(outputref, time(), ()->close(timer)), dt, dt)
    finalizer(output, _->close(timer))
    output
end

"""
    fpswhen(switch, rate)

returns a signal which when `switch` signal is true, updates `rate` times every second. If `rate` is not possible to attain because of slowness in computing dependent signal values, the signal will self adjust to provide the best possible rate.
"""
function fpswhen(switch, rate)
    switch_ons = filter(x->x, false, switch) # only turn-ons
    n = Signal(Float64, 0.0, (switch, switch_ons,))
    fpswhen_connect(rate, switch, switch_ons, n)
    n
end

function setup_next_tick(outputref, switchref, dt, wait_dt)
    if value(switchref.value)
        Timer(t -> if value(switchref.value)
                       _push!(outputref, dt)
                   end, wait_dt)
    end
end

function fpswhen_connect(rate, switch, switch_ons, output)
    let prev_time = time()
        dt = 1.0/rate
        outputref = WeakRef(output)
        switchref = WeakRef(switch)

        for inp in [output, switch_ons]
            add_action!(inp, output) do output, timestep
                start_time = time()
                setup_next_tick(outputref, switchref, start_time-prev_time, dt)
                prev_time = start_time
            end
        end

        setup_next_tick(outputref, switchref, dt, dt)
    end
end

"""
    fps(rate)

Same as `fpswhen(Input(true), rate)`
"""
fps(rate) = fpswhen(Signal(Bool, true, ()), rate)


function DormandPrince853FieldIntegrator(arg0::Field, arg1::jdouble, arg2::jdouble, arg3::Vector{jdouble}, arg4::Vector{jdouble})
    return DormandPrince853FieldIntegrator((Field, jdouble, jdouble, Vector{jdouble}, Vector{jdouble}), arg0, arg1, arg2, arg3, arg4)
end

function DormandPrince853FieldIntegrator(arg0::Field, arg1::jdouble, arg2::jdouble, arg3::jdouble, arg4::jdouble)
    return DormandPrince853FieldIntegrator((Field, jdouble, jdouble, jdouble, jdouble), arg0, arg1, arg2, arg3, arg4)
end

function get_a(obj::DormandPrince853FieldIntegrator)
    return jcall(obj, "getA", Vector{Vector{RealFieldElement}}, ())
end

function get_b(obj::DormandPrince853FieldIntegrator)
    return jcall(obj, "getB", Vector{RealFieldElement}, ())
end

function get_c(obj::DormandPrince853FieldIntegrator)
    return jcall(obj, "getC", Vector{RealFieldElement}, ())
end

function get_order(obj::DormandPrince853FieldIntegrator)
    return jcall(obj, "getOrder", jint, ())
end


module SimpleNLUtils

import SimpleNL: Model, variable, Expression, Expression1, Expression2, Gradient, SparseJacobian, ExpressionSum, Variable, Constant, Parameter, Sink, Field, Field1, num_variables, num_constraints, Constraint, inner, index, fill_sparsity!, non_caching_eval, DUMMY
import SparseArrays: SparseMatrixCSC, sparse
import LinearAlgebra: mul!, norm

export get_terms, get_entries, sparsity, set_KKT_error_evaluator!

include("utils.jl")
include("structured.jl")
include("kkt.jl")

end # module


#!/usr/bin/env julia

# Lists the temporal extents of give RAW or EVR files

using SimradRaw
using EchoviewEvr
using Filetimes
using Base.Iterators

function rawstartend(rawfilename)
    starttime = 0
    endtime = 0
    open(rawfilename, "r") do f
        while !eof(f)
            dgheader= nothing
            body = nothing
            try
                dgheader, body = readencapsulateddatagram(f, datagramreader=readdatagramheader)
            catch ex
                warn("$ex on $f")
                break
            end
            t = SimradRaw.filetime(dgheader.datetime)
            if starttime == 0
                starttime = t
            end
            endtime = t
        end
    end
    return starttime, endtime
end

function evrstartend(evrfilename)
    ps = polygons(evrfilename)

    xs = flatten([x for (x,y) in ps])

    return minimum(xs), maximum(xs)
end

function main(args)

    if length(args) == 0
        x = now()
        println("$x\t$(Filetimes.filetime(x))")
    else
        for arg in args
            if endswith(uppercase(arg), ".RAW")
                starttime, endtime = rawstartend(arg)
                println("$arg\t$starttime\t$endtime\t$(datetime(starttime))\t$(datetime(endtime))")
                continue
            end
            if endswith(uppercase(arg), ".EVR")
                starttime, endtime = evrstartend(arg)
                println("$arg\t$starttime\t$endtime\t$(datetime(starttime))\t$(datetime(endtime))")
                continue
            end

            y = tryparse(Int64,arg)
            if isnull(y)
                println("$arg\t$(Filetimes.filetime(arg))")
            else
                println("$(datetime(get(y)))\t$arg")
            end
        end
    end

end

main(ARGS)


using Adapt
using Statistics
using KernelAbstractions: @kernel, @index, Event
using Oceananigans.Grids
using Oceananigans.BoundaryConditions: fill_halo_regions!

struct ComputedField{X, Y, Z, S, O, A, D, G, T, C} <: AbstractDataField{X, Y, Z, A, G, T}
                   data :: D
           architecture :: A
                   grid :: G
                operand :: O
    boundary_conditions :: C
                 status :: S

    function ComputedField{X, Y, Z}(data::D,
                                    arch::A,
                                    grid::G,
                                    operand::O,
                                    boundary_conditions::C;
                                    recompute_safely=true) where {X, Y, Z, D, A, G, O, C}

        validate_field_data(X, Y, Z, data, grid)

        # Use FieldStatus if we want to avoid always recomputing
        status = recompute_safely ? nothing : FieldStatus(0.0)

        S = typeof(status)
        T = eltype(grid)

        return new{X, Y, Z, S, O, A, D, G, T, C}(data, arch, grid, operand, boundary_conditions, status)
    end
end

# We define special default boundary conditions for ComputedField, because currently
# `DefaultBoundaryCondition` uses ImpenetrableBoundaryCondition() in bounded directions
# and for fields on Faces, which is not what we want in general for ComputedFields.
DefaultComputedFieldBoundaryCondition(::Type{Grids.Periodic}, loc) = PeriodicBoundaryCondition()
DefaultComputedFieldBoundaryCondition(::Type{Flat}, loc) = nothing
DefaultComputedFieldBoundaryCondition(::Type{Bounded}, ::Type{Center}) = NoFluxBoundaryCondition()
DefaultComputedFieldBoundaryCondition(::Type{Bounded}, ::Type{Face}) = nothing

function ComputedFieldBoundaryConditions(grid, loc;
                                           east = DefaultComputedFieldBoundaryCondition(topology(grid, 1), loc[1]),
                                           west = DefaultComputedFieldBoundaryCondition(topology(grid, 1), loc[1]),
                                          south = DefaultComputedFieldBoundaryCondition(topology(grid, 2), loc[2]),
                                          north = DefaultComputedFieldBoundaryCondition(topology(grid, 2), loc[2]),
                                         bottom = DefaultComputedFieldBoundaryCondition(topology(grid, 3), loc[3]),
                                            top = DefaultComputedFieldBoundaryCondition(topology(grid, 3), loc[3]))

    return FieldBoundaryConditions(grid, loc; east=east, west=west, south=south, north=north, bottom=bottom, top=top)
end

"""
    ComputedField(operand [, arch=nothing]; data = nothing, recompute_safely = true,
                  boundary_conditions = ComputedFieldBoundaryConditions(operand.grid, location(operand))

Returns a field whose data is `computed` from `operand`. If `arch`itecture is not supplied it
is inferred from `operand`.

If the keyword argument `data` is not provided, memory is allocated to store
the result. The `arch`itecture of `data` is inferred from `operand`.

If `data` is provided and `recompute_safely=false`, then "recomputation" of the `ComputedField`
is avoided if possible.
"""
function ComputedField(operand, arch=nothing; kwargs...)

    loc = location(operand)
    grid = operand.grid

    return ComputedField(loc..., operand, arch, grid; kwargs...)
end

function ComputedField(LX, LY, LZ, operand, arch, grid;
                       data = nothing,
                       recompute_safely = true,
                       boundary_conditions = ComputedFieldBoundaryConditions(grid, (LX, LY, LZ)))

    # Architecturanigans
    operand_arch = architecture(operand)
    arch = isnothing(operand_arch) ? arch : operand_arch
    isnothing(arch) && error("The architecture must be provided, or inferrable from `operand`!")

    if isnothing(data)
        data = new_data(arch, grid, (LX, LY, LZ))
        recompute_safely = false
    end

    return ComputedField{LX, LY, LZ}(data, arch, grid, operand, boundary_conditions; recompute_safely=recompute_safely)
end

"""
    compute!(comp::ComputedField)

Compute `comp.operand` and store the result in `comp.data`.
"""
function compute!(comp::ComputedField{LX, LY, LZ}, time=nothing) where {LX, LY, LZ}
    compute_at!(comp.operand, time) # ensures any 'dependencies' of the computation are computed first

    arch = architecture(comp)

    workgroup, worksize = work_layout(comp.grid,
                                      :xyz,
                                      include_right_boundaries=true,
                                      location=(LX, LY, LZ))

    compute_kernel! = _compute!(device(arch), workgroup, worksize)

    event = compute_kernel!(comp.data, comp.operand; dependencies=Event(device(arch)))

    wait(device(arch), event)

    fill_halo_regions!(comp, arch)

    return nothing
end

compute_at!(field::ComputedField{LX, LY, LZ, <:FieldStatus}, time) where {LX, LY, LZ} =
    conditional_compute!(field, time)

"""Compute an `operand` and store in `data`."""
@kernel function _compute!(data, operand)
    i, j, k = @index(Global, NTuple)
    @inbounds data[i, j, k] = operand[i, j, k]
end

#####
##### Adapt
#####

Adapt.adapt_structure(to, computed_field::ComputedField) = Adapt.adapt(to, computed_field.data)


# main.jl - Animated Particles
# Modified version of Luxor
push!(LOAD_PATH, "C:\\Users\\mcbaron\\Documents\\Julia")
using Luxor
# Include helper functions
include("deps.jl")

# Constants
num_particles = 50
bounds = [1360 768]
vidlength = 5 # sec
framerate = 32
numFrames = vidlength * framerate

# Instantiate num_particles random particles
liveParts = genScreen(num_particles, bounds)

screenFrames = Movie(bounds[1], bounds[2], "Particles", 0:numFrames)

# animate the whole thing
animate(screenFrames, [
    Scene(screenFrames, backdrop, 0:numFrames)
    Scene(screenFrames, drawScreen, 0:numFrames, lineartween, liveParts)
], creategif=true)

# As of 20190102 this works, but the particles seem to settle as they evolve.


module MimiFUND

using Mimi
using DelimitedFiles
using Random

include("helper.jl")

include("new_marginaldamages.jl")
include("montecarlo/defmcs.jl")
include("montecarlo/run_fund_mcs.jl")
include("montecarlo/run_fund_scc_mcs.jl")

include("components/SocioEconomicComponent.jl")
include("components/PopulationComponent.jl")
include("components/EmissionsComponent.jl")
include("components/GeographyComponent.jl")
include("components/ScenarioUncertaintyComponent.jl")
include("components/ClimateCO2CycleComponent.jl")
include("components/ClimateCH4CycleComponent.jl")
include("components/ClimateN2OCycleComponent.jl")
include("components/ClimateSF6CycleComponent.jl")
include("components/ClimateForcingComponent.jl")
include("components/ClimateDynamicsComponent.jl")
include("components/BioDiversityComponent.jl")
include("components/ClimateRegionalComponent.jl")
include("components/OceanComponent.jl")
include("components/ImpactAgricultureComponent.jl")
include("components/ImpactBioDiversityComponent.jl")
include("components/ImpactCardiovascularRespiratoryComponent.jl")
include("components/ImpactCoolingComponent.jl")
include("components/ImpactDiarrhoeaComponent.jl")
include("components/ImpactExtratropicalStormsComponent.jl")
include("components/ImpactDeathMorbidityComponent.jl")
include("components/ImpactForestsComponent.jl")
include("components/ImpactHeatingComponent.jl")
include("components/ImpactVectorBorneDiseasesComponent.jl")
include("components/ImpactTropicalStormsComponent.jl")
include("components/ImpactWaterResourcesComponent.jl")
include("components/ImpactSeaLevelRiseComponent.jl")
include("components/ImpactAggregationComponent.jl")
include("components/VslVmorbComponent.jl")

export
    getfund # a function that returns a version of fund allowing for different user specifications

const global default_nsteps = 1050
const global default_datadir = joinpath(dirname(@__FILE__), "..", "data")
const global default_params = nothing

function get_model(; nsteps = default_nsteps, datadir = default_datadir, params = default_params)

    # ---------------------------------------------
    # Create model
    # ---------------------------------------------

    m = Model()

    # ---------------------------------------------
    # Set dimensions
    # ---------------------------------------------

    if nsteps != default_nsteps
        if datadir != default_datadir || params != default_params
            set_dimension!(m, :time, collect(1950:1950 + nsteps)) # If the user provided an alternative datadir or params dictionary, then the time dimensions can be reset now
            reset_time_dimension = false
        else
            nsteps > default_nsteps ? error("Invalid `nsteps`: $nsteps. Cannot build a MimiFUND model with more than $default_nsteps timesteps, unless an alternative `datadir` or `params` dictionary is provided.") : nothing
            set_dimension!(m, :time, collect(1950:1950 + default_nsteps)) # start with the default FUND time dimension, so that `set_leftover_params!` will work with the default parameter lengths
            reset_time_dimension = true # then reset the time dimension at the end
        end
    else
        set_dimension!(m, :time, collect(1950:1950 + default_nsteps))    # default FUND time dimension
        reset_time_dimension = false
    end

    set_dimension!(m, :regions, ["USA", "CAN", "WEU", "JPK", "ANZ", "EEU", "FSU", "MDE", "CAM", "LAM", "SAS", "SEA", "CHI", "MAF", "SSA", "SIS"])

    # ---------------------------------------------
    # Create components
    # ---------------------------------------------

    add_comp!(m, scenariouncertainty)
    add_comp!(m, population)          # can't have external params and components with the same name
    add_comp!(m, geography)
    add_comp!(m, socioeconomic)
    add_comp!(m, emissions)
    add_comp!(m, climateco2cycle)
    add_comp!(m, climatech4cycle)
    add_comp!(m, climaten2ocycle)
    add_comp!(m, climatesf6cycle)
    add_comp!(m, climateforcing)
    add_comp!(m, climatedynamics)
    add_comp!(m, biodiversity)
    add_comp!(m, climateregional)
    add_comp!(m, ocean)
    add_comp!(m, impactagriculture)
    add_comp!(m, impactbiodiversity)
    add_comp!(m, impactcardiovascularrespiratory)
    add_comp!(m, impactcooling)
    add_comp!(m, impactdiarrhoea)
    add_comp!(m, impactextratropicalstorms)
    add_comp!(m, impactforests)
    add_comp!(m, impactheating)
    add_comp!(m, impactvectorbornediseases)
    add_comp!(m, impacttropicalstorms)
    add_comp!(m, vslvmorb)
    add_comp!(m, impactdeathmorbidity)
    add_comp!(m, impactwaterresources)
    add_comp!(m, impactsealevelrise)
    add_comp!(m, impactaggregation)

    # ---------------------------------------------
    # Connect parameters to variables
    # ---------------------------------------------

    connect_param!(m, :geography, :landloss, :impactsealevelrise, :landloss)

    connect_param!(m, :population, :pgrowth, :scenariouncertainty, :pgrowth)
    connect_param!(m, :population, :enter, :impactsealevelrise, :enter)
    connect_param!(m, :population, :leave, :impactsealevelrise, :leave)
    connect_param!(m, :population, :dead, :impactdeathmorbidity, :dead)

    connect_param!(m, :socioeconomic, :area, :geography, :area)
    connect_param!(m, :socioeconomic, :globalpopulation, :population, :globalpopulation)
    connect_param!(m, :socioeconomic, :populationin1, :population, :populationin1)
    connect_param!(m, :socioeconomic, :population, :population, :population)
    connect_param!(m, :socioeconomic, :pgrowth, :scenariouncertainty, :pgrowth)
    connect_param!(m, :socioeconomic, :ypcgrowth, :scenariouncertainty, :ypcgrowth)
    connect_param!(m, :socioeconomic, :eloss, :impactaggregation, :eloss)
    connect_param!(m, :socioeconomic, :sloss, :impactaggregation, :sloss)
    connect_param!(m, :socioeconomic, :mitigationcost, :emissions, :mitigationcost)

    connect_param!(m, :emissions, :income, :socioeconomic, :income)
    connect_param!(m, :emissions, :population, :population, :population)
    connect_param!(m, :emissions, :forestemm, :scenariouncertainty, :forestemm)
    connect_param!(m, :emissions, :aeei, :scenariouncertainty, :aeei)
    connect_param!(m, :emissions, :acei, :scenariouncertainty, :acei)
    connect_param!(m, :emissions, :ypcgrowth, :scenariouncertainty, :ypcgrowth)

    connect_param!(m, :climateco2cycle, :mco2, :emissions, :mco2)

    connect_param!(m, :climatech4cycle, :globch4, :emissions, :globch4)

    connect_param!(m, :climaten2ocycle, :globn2o, :emissions, :globn2o)

    connect_param!(m, :climateco2cycle, :temp, :climatedynamics, :temp)

    connect_param!(m, :climatesf6cycle, :globsf6, :emissions, :globsf6)

    connect_param!(m, :climateforcing, :acco2, :climateco2cycle, :acco2)
    connect_param!(m, :climateforcing, :acch4, :climatech4cycle, :acch4)
    connect_param!(m, :climateforcing, :acn2o, :climaten2ocycle, :acn2o)
    connect_param!(m, :climateforcing, :acsf6, :climatesf6cycle, :acsf6)

    connect_param!(m, :climatedynamics, :radforc, :climateforcing, :radforc)

    connect_param!(m, :climateregional, :inputtemp, :climatedynamics, :temp)

    connect_param!(m, :biodiversity, :temp, :climatedynamics, :temp)

    connect_param!(m, :ocean, :temp, :climatedynamics, :temp)

    connect_param!(m, :impactagriculture, :population, :population, :population)
    connect_param!(m, :impactagriculture, :income, :socioeconomic, :income)
    connect_param!(m, :impactagriculture, :temp, :climateregional, :temp)
    connect_param!(m, :impactagriculture, :acco2, :climateco2cycle, :acco2)

    connect_param!(m, :impactbiodiversity, :temp, :climateregional, :temp)
    connect_param!(m, :impactbiodiversity, :nospecies, :biodiversity, :nospecies)
    connect_param!(m, :impactbiodiversity, :income, :socioeconomic, :income)
    connect_param!(m, :impactbiodiversity, :population, :population, :population)

    connect_param!(m, :impactcardiovascularrespiratory, :population, :population, :population)
    connect_param!(m, :impactcardiovascularrespiratory, :temp, :climateregional, :temp)
    connect_param!(m, :impactcardiovascularrespiratory, :plus, :socioeconomic, :plus)
    connect_param!(m, :impactcardiovascularrespiratory, :urbpop, :socioeconomic, :urbpop)

    connect_param!(m, :impactcooling, :population, :population, :population)
    connect_param!(m, :impactcooling, :income, :socioeconomic, :income)
    connect_param!(m, :impactcooling, :temp, :climateregional, :temp)
    connect_param!(m, :impactcooling, :cumaeei, :emissions, :cumaeei)

    connect_param!(m, :impactdiarrhoea, :population, :population, :population)
    connect_param!(m, :impactdiarrhoea, :income, :socioeconomic, :income)
    connect_param!(m, :impactdiarrhoea, :regtmp, :climateregional, :regtmp)

    connect_param!(m, :impactextratropicalstorms, :population, :population, :population)
    connect_param!(m, :impactextratropicalstorms, :income, :socioeconomic, :income)
    connect_param!(m, :impactextratropicalstorms, :acco2, :climateco2cycle, :acco2)

    connect_param!(m, :impactforests, :population, :population, :population)
    connect_param!(m, :impactforests, :income, :socioeconomic, :income)
    connect_param!(m, :impactforests, :temp, :climateregional, :temp)
    connect_param!(m, :impactforests, :acco2, :climateco2cycle, :acco2)

    connect_param!(m, :impactheating, :population, :population, :population)
    connect_param!(m, :impactheating, :income, :socioeconomic, :income)
    connect_param!(m, :impactheating, :temp, :climateregional, :temp)
    connect_param!(m, :impactheating, :cumaeei, :emissions, :cumaeei)

    connect_param!(m, :impactvectorbornediseases, :population, :population, :population)
    connect_param!(m, :impactvectorbornediseases, :income, :socioeconomic, :income)
    connect_param!(m, :impactvectorbornediseases, :temp, :climateregional, :temp)

    connect_param!(m, :impacttropicalstorms, :population, :population, :population)
    connect_param!(m, :impacttropicalstorms, :income, :socioeconomic, :income)
    connect_param!(m, :impacttropicalstorms, :regstmp, :climateregional, :regstmp)

    connect_param!(m, :vslvmorb, :population, :population, :population)
    connect_param!(m, :vslvmorb, :income, :socioeconomic, :income)

    connect_param!(m, :impactdeathmorbidity, :vsl, :vslvmorb, :vsl)
    connect_param!(m, :impactdeathmorbidity, :vmorb, :vslvmorb, :vmorb)
    connect_param!(m, :impactdeathmorbidity, :population, :population, :population)
    connect_param!(m, :impactdeathmorbidity, :dengue, :impactvectorbornediseases, :dengue)
    connect_param!(m, :impactdeathmorbidity, :schisto, :impactvectorbornediseases, :schisto)
    connect_param!(m, :impactdeathmorbidity, :malaria, :impactvectorbornediseases, :malaria)
    connect_param!(m, :impactdeathmorbidity, :cardheat, :impactcardiovascularrespiratory, :cardheat)
    connect_param!(m, :impactdeathmorbidity, :cardcold, :impactcardiovascularrespiratory, :cardcold)
    connect_param!(m, :impactdeathmorbidity, :resp, :impactcardiovascularrespiratory, :resp)
    connect_param!(m, :impactdeathmorbidity, :diadead, :impactdiarrhoea, :diadead)
    connect_param!(m, :impactdeathmorbidity, :hurrdead, :impacttropicalstorms, :hurrdead)
    connect_param!(m, :impactdeathmorbidity, :extratropicalstormsdead, :impactextratropicalstorms, :extratropicalstormsdead)
    connect_param!(m, :impactdeathmorbidity, :diasick, :impactdiarrhoea, :diasick)

    connect_param!(m, :impactwaterresources, :population, :population, :population)
    connect_param!(m, :impactwaterresources, :income, :socioeconomic, :income)
    connect_param!(m, :impactwaterresources, :temp, :climateregional, :temp)

    connect_param!(m, :impactsealevelrise, :population, :population, :population)
    connect_param!(m, :impactsealevelrise, :income, :socioeconomic, :income)
    connect_param!(m, :impactsealevelrise, :sea, :ocean, :sea)
    connect_param!(m, :impactsealevelrise, :area, :geography, :area)

    connect_param!(m, :impactaggregation, :income, :socioeconomic, :income)
    connect_param!(m, :impactaggregation, :heating, :impactheating, :heating)
    connect_param!(m, :impactaggregation, :cooling, :impactcooling, :cooling)
    connect_param!(m, :impactaggregation, :agcost, :impactagriculture, :agcost)
    connect_param!(m, :impactaggregation, :species, :impactbiodiversity, :species)
    connect_param!(m, :impactaggregation, :water, :impactwaterresources, :water)
    connect_param!(m, :impactaggregation, :hurrdam, :impacttropicalstorms, :hurrdam)
    connect_param!(m, :impactaggregation, :extratropicalstormsdam, :impactextratropicalstorms, :extratropicalstormsdam)
    connect_param!(m, :impactaggregation, :forests, :impactforests, :forests)
    connect_param!(m, :impactaggregation, :drycost, :impactsealevelrise, :drycost)
    connect_param!(m, :impactaggregation, :protcost, :impactsealevelrise, :protcost)
    connect_param!(m, :impactaggregation, :entercost, :impactsealevelrise, :entercost)
    connect_param!(m, :impactaggregation, :deadcost, :impactdeathmorbidity, :deadcost)
    connect_param!(m, :impactaggregation, :morbcost, :impactdeathmorbidity, :morbcost)
    connect_param!(m, :impactaggregation, :wetcost, :impactsealevelrise, :wetcost)
    connect_param!(m, :impactaggregation, :leavecost, :impactsealevelrise, :leavecost)

    # ---------------------------------------------
    # Set all external parameter values
    # ---------------------------------------------

    parameters = params === nothing ? load_default_parameters(datadir) : params

    for (name, value) in parameters
        set_param!(m, name, value)
    end

    # Reset the time dimension if needed
    reset_time_dimension ? set_dimension!(m, :time, collect(1950:1950 + nsteps)) : nothing

    return m

end

getfund = get_model

end #module


using HTTP
using JSON
using RCall

export get_lat_lon, vgregion_lat_lon_bounds

#nominatim requires as user agent and will block you otherwise
HTTP.setuseragent!("Mozilla/5.0 (Windows NT 10.0; rv:68.0) Gecko/20100101 Firefox/68.0")

const NominatimSearchPrefix = "https://nominatim.openstreetmap.org/search?q="

# Nominatim will block you if you access more than once per second so keep track
# of time.
struct NominatimSearcher
    lasttime::Float64
    NominatimSearcher() = new(0.0)
end

const DefaultSearcher = NominatimSearcher()

"""
    get_lat_lon(address::String)

Returns (lat,lon) coordinate for address (typically in middle of street)
as well as additional data returned from Nominatim.
"""
function get_lat_lon(address::String, n::NominatimSearcher = DefaultSearcher)
    addressFormatted = join(split(address),"+")
    searchurl = NominatimSearchPrefix * addressFormatted * "&format=json&limit=1"

    if time() - n.lasttime < 1.0
        sleep(1.05 - (time() - n.lasttime))
        n.lasttime = time()
    end

    rawdata = HTTP.get(searchurl)

    metadata = JSON.parse(String(rawdata.body))[1]
    coordinates = parse(Float64, metadata["lat"]), parse(Float64, metadata["lon"])

    return coordinates, metadata
end

# Yttergränser för Västra Götaland från Google Maps 2020-04-22:
const VGRegion = Dict(
    :south => (57.140190, 13.127416), # Gislaved V
    :east  => (58.687169, 14.714468), # Karlsborg N
    :north => (59.260856, 12.208255), # Årjäng SO
    :west  => (58.887149, 10.963946)  # väster om Nordkoster
)

function vgregion_lat_lon_bounds(delta = 0.10)
    lats = map(first, values(VGRegion))
    lons = map(t -> t[2], values(VGRegion))

    # Boundingbox for ggmap given as lowerleftlon, lowerleftlat, upperightlon, upperrightlat
    return (minimum(lons)-delta, minimum(lats)-delta, maximum(lons)+delta, maximum(lats)+delta)
end

# google maptypes: roadmap, terrain, satellite, hybrid
# osm maptypes: none...

#get_stamenmap(bbox = c(left = -95.80204, bottom = 29.38048, right =
#-94.92313, top = 30.14344), zoom = 10, maptype = c("terrain",
#"terrain-background", "terrain-labels", "terrain-lines", "toner",
#"toner-2010", "toner-2011", "toner-background", "toner-hybrid",
#"toner-labels", "toner-lines", "toner-lite", "watercolor"),
#crop = TRUE, messaging = FALSE, urlonly = FALSE,
#color = c("color", "bw"), force = FALSE, where = tempdir(), ...)
function make_stamen_map(boundingbox, filename = "t.png";
    maptype = "toner-lite", color="bw")
    R"library(ggmap)"
    left, bottom, right, top = boundingbox
    R"bbox <- c(left = $left, bottom = $bottom, right = $right, top = $top)"
    R"zoomlevel <- calc_zoom(bbox)"
    R"mymap <- get_stamenmap(bbox, zoom=zoomlevel, maptype=$maptype, crop=FALSE, color=$color)"
    R"p <- ggmap(mymap)"
    R"ggsave($filename, p)"
    @info "Saved to file $filename"
    filename
end


__precompile__()

module LSODA

using Compat, DiffEqBase
import DiffEqBase: solve
using LinearAlgebra, Printf
using LSODA_jll

const warnkeywords =
    (:save_idxs, :d_discontinuities, :isoutofdomain, :unstable_check,
     :calck, :progress, :timeseries_steps, :dense,
     :dtmin, :dtmax,
     :internalnorm, :gamma, :beta1, :beta2, :qmax, :qmin, :qoldinit)

function __init__()
    global warnlist = Set(warnkeywords)
end

abstract type LSODAAlgorithm <: DiffEqBase.AbstractODEAlgorithm end
abstract type LSODAAlgorithm <: DiffEqBase.AbstractODEAlgorithm end
struct lsoda <: LSODAAlgorithm end

export lsoda, lsoda_0, lsoda_opt_t, lsoda_context_t, lsoda_prepare, lsoda_reset, lsoda_opt_t, lsoda_free, lsoda_evolve!, UserFunctionAndData

export LSODAAlgorithm, solve

include("types_and_consts.jl")
include("handle.jl")
include("solver.jl")
include("common.jl")

end # module