################################################################################
#                           SUPPORT ITERATION METHODS
################################################################################
## Form a placeholder parameter reference given the object index
# IndependentParameterIndex
function _temp_parameter_ref(model::InfiniteOpt.InfiniteModel,
    index::InfiniteOpt.IndependentParameterIndex
    )::InfiniteOpt.IndependentParameterRef
    return InfiniteOpt.IndependentParameterRef(model, index)
end

# DependentParametersIndex
function _temp_parameter_ref(model::InfiniteOpt.InfiniteModel,
    index::InfiniteOpt.DependentParametersIndex
    )::InfiniteOpt.DependentParameterRef
    idx = InfiniteOpt.DependentParameterIndex(index, 1)
    return InfiniteOpt.DependentParameterRef(model, idx)
end

# Return the collected supports of an infinite parameter
function _collected_supports(
    pref::Union{InfiniteOpt.IndependentParameterRef, InfiniteOpt.DependentParameterRef}
    )::Vector
    supp_dict = InfiniteOpt._parameter_supports(pref)
    supp_list = collect(keys(supp_dict))
    # append a placeholder NaN support at the end to be used for efficient combinatorics
    return push!(supp_list, map(i -> NaN, first(supp_list)))
end

# Return the collected support labels of an infinite parameter
function _collected_support_labels(
    pref::Union{InfiniteOpt.IndependentParameterRef, InfiniteOpt.DependentParameterRef},
    supports::Vector
    )::Vector{Set{DataType}}
    supp_dict = InfiniteOpt._parameter_supports(pref)
    default = Set{DataType}()
    return map(k -> get(supp_dict, k, default), supports)
end

"""
    set_parameter_supports(trans_model::JuMP.Model,
                           inf_model::InfiniteOpt.InfiniteModel)::Nothing

Collect the infinite parameter supports stored in their respective dictionaries
form `inf_model` and process them into a tuple of vectors where each vector
contains the collected supports of a particular infinite parameter. These support
collections are ordered in accordance with the definition order of the
parameters (i.e., their object numbers). A support collection assocciated with
an independent will be a `Vector{Float64}` and a support collection associated
with a group of dependent parameters will be a `Vector{Vector{Float64}}`. Note
that each collection vector will include an extra final placeholder element
comprised of `NaN`s for convenience in generating support indices via
[`support_index_iterator`](@ref). This also gathers the associated support labels.

Before this is all done, `InfiniteOpt.add_generative_supports` is invoked as needed.
"""
function set_parameter_supports(trans_model::JuMP.Model,
    inf_model::InfiniteOpt.InfiniteModel
    )::Nothing
    # gather the basic information
    param_indices = InfiniteOpt._param_object_indices(inf_model)
    prefs = map(idx -> _temp_parameter_ref(inf_model, idx), param_indices)
    data = transcription_data(trans_model)
    # check and add supports to prefs as needed
    for pref in prefs
        InfiniteOpt.add_generative_supports(pref)
        if InfiniteOpt.has_internal_supports(pref)
            data.has_internal_supports = true
        end
    end
    # build and add the support/label tuples
    supps = Tuple(_collected_supports(pref) for pref in prefs)
    labels = Tuple(_collected_support_labels(pref, supps[i])
                   for (i, pref) in enumerate(prefs))
    data.supports = supps
    data.support_labels = labels
    return
end

################################################################################
#                        VARIABLE INITIALIZATION METHODS
################################################################################
"""
    transcribe_hold_variables!(trans_model::JuMP.Model,
                               inf_model::InfiniteOpt.InfiniteModel)::Nothing

Create a transcription variable (i.e., a JuMP variable) for each `HoldVariable`
stored in `inf_model` and add it to `trans_model`. The variable mapping is
also stored in `TranscriptionData.finvar_mappings` which enables
[`transcription_variable`](@ref) and [`lookup_by_support`](@ref).
"""
function transcribe_hold_variables!(trans_model::JuMP.Model,
    inf_model::InfiniteOpt.InfiniteModel
    )::Nothing
    for (index, object) in InfiniteOpt._data_dictionary(inf_model, InfiniteOpt.HoldVariable)
        hvref = InfiniteOpt._make_variable_ref(inf_model, index)
        vref = JuMP.add_variable(trans_model,
                                 JuMP.ScalarVariable(object.variable.info),
                                 object.name)
        transcription_data(trans_model).finvar_mappings[hvref] = vref
    end
    return
end

# Return a proper scalar variable info object given on with a start value function
function _format_infinite_info(var::InfiniteOpt.InfiniteVariable,
    support::Vector{Float64}
    )::JuMP.VariableInfo
    # generate the start value
    info = var.info
    if var.is_vector_start
        start = info.start(support)
    else
        start = info.start(Tuple(support, var.parameter_refs)...)
    end
    # make the info and return
    return JuMP.VariableInfo(info.has_lb, info.lower_bound, info.has_ub,
                             info.upper_bound, info.has_fix, info.fixed_value,
                             info.has_start, start, info.binary, info.integer)
end

"""
    transcribe_infinite_variables!(trans_model::JuMP.Model,
                                   inf_model::InfiniteOpt.InfiniteModel)::Nothing

Create transcription variables (i.e., JuMP variables) for each `InfiniteVariable`
stored in `inf_model` and add them to `trans_model`. The variable mappings are
also stored in `TranscriptionData.infvar_mappings` in accordance with
`TranscriptionData.infvar_lookup` which enable [`transcription_variable`](@ref)
and [`lookup_by_support`](@ref). Note that the supports will not be generated
until `InfiniteOpt.variable_supports` is invoked via `InfiniteOpt.supports`.
Note that `TranscriptionData.infvar_support_labels` is also populated.
"""
function transcribe_infinite_variables!(trans_model::JuMP.Model,
    inf_model::InfiniteOpt.InfiniteModel
    )::Nothing
    for (index, object) in InfiniteOpt._data_dictionary(inf_model, InfiniteOpt.InfiniteVariable)
        # get the basic variable information
        var = object.variable
        base_name = object.name
        param_nums = var.parameter_nums
        # prepare for iterating over its supports
        supp_indices = support_index_iterator(trans_model, var.object_nums)
        vrefs = Vector{JuMP.VariableRef}(undef, length(supp_indices))
        labels = Vector{Set{DataType}}(undef, length(supp_indices))
        lookup_dict = Dict{Vector{Float64}, Int}()
        # create a variable for each support
        for (counter, i) in enumerate(supp_indices)
            raw_supp = index_to_support(trans_model, i)
            supp = raw_supp[param_nums]
            info = _format_infinite_info(var, supp)
            var_name = string(base_name, "(support: ", counter, ")")
            @inbounds vrefs[counter] = JuMP.add_variable(trans_model,
                                                         JuMP.ScalarVariable(info),
                                                         var_name)
            lookup_dict[supp] = counter
            @inbounds labels[counter] = index_to_labels(trans_model, i)
        end
        # save the transcription information
        ivref = InfiniteOpt._make_variable_ref(inf_model, index)
        data = transcription_data(trans_model)
        data.infvar_lookup[ivref] = lookup_dict
        data.infvar_mappings[ivref] = vrefs
        data.infvar_support_labels[ivref] = labels
    end
    return
end

# Return a proper scalar variable info object given on with a start value function
function _format_derivative_info(d::InfiniteOpt.Derivative,
    support::Vector{Float64}
    )::JuMP.VariableInfo
    # generate the start value
    info = d.info
    if d.is_vector_start
        start = info.start(support)
    else
        prefs = InfiniteOpt.raw_parameter_refs(d.variable_ref)
        start = info.start(Tuple(support, prefs)...)
    end
    # make the info and return
    return JuMP.VariableInfo(info.has_lb, info.lower_bound, info.has_ub,
                             info.upper_bound, info.has_fix, info.fixed_value,
                             info.has_start, start, info.binary, info.integer)
end

"""
    transcribe_derivative_variables!(trans_model::JuMP.Model,
                                     inf_model::InfiniteOpt.InfiniteModel)::Nothing

Create transcription variables (i.e., JuMP variables) for each `Derivative`
stored in `inf_model` and add them to `trans_model`. The variable mappings are
also stored in `TranscriptionData.infvar_mappings` in accordance with
`TranscriptionData.infvar_lookup` which enable [`transcription_variable`](@ref)
and [`lookup_by_support`](@ref). Note that the supports will not be generated
until `InfiniteOpt.variable_supports` is invoked via `InfiniteOpt.supports`. The
futher derivative evaluation constraints are added when
`transcribe_derivative_evaluations!` is invoked. Note that
`TranscriptionData.infvar_support_labels` is also populated.
"""
function transcribe_derivative_variables!(trans_model::JuMP.Model,
    inf_model::InfiniteOpt.InfiniteModel
    )::Nothing
    for (index, object) in InfiniteOpt._data_dictionary(inf_model, InfiniteOpt.Derivative)
        # get the basic variable information
        dref = InfiniteOpt._make_variable_ref(inf_model, index)
        d = object.variable
        base_name = InfiniteOpt.variable_string(JuMP.REPLMode, dispatch_variable_ref(dref))
        param_nums = InfiniteOpt._parameter_numbers(d.variable_ref)
        obj_nums = InfiniteOpt._object_numbers(d.variable_ref)
        # prepare for iterating over its supports
        supp_indices = support_index_iterator(trans_model, obj_nums)
        vrefs = Vector{JuMP.VariableRef}(undef, length(supp_indices))
        labels = Vector{Set{DataType}}(undef, length(supp_indices))
        lookup_dict = Dict{Vector{Float64}, Int}()
        # create a variable for each support
        for (counter, i) in enumerate(supp_indices)
            raw_supp = index_to_support(trans_model, i)
            supp = raw_supp[param_nums]
            info = _format_derivative_info(d, supp)
            deriv_name = string(base_name, "(support: ", counter, ")")
            @inbounds vrefs[counter] = JuMP.add_variable(trans_model,
                                                         JuMP.ScalarVariable(info),
                                                         deriv_name)
            lookup_dict[supp] = counter
            @inbounds labels[counter] = index_to_labels(trans_model, i)
        end
        # save the transcription information
        data = transcription_data(trans_model)
        data.infvar_lookup[dref] = lookup_dict
        data.infvar_mappings[dref] = vrefs
        data.infvar_support_labels[dref] = labels
    end
    return
end

# Setup the mapping for a given reduced variable
function _set_reduced_variable_mapping(trans_model::JuMP.Model,
    var::InfiniteOpt.ReducedVariable,
    rvref::InfiniteOpt.GeneralVariableRef,
    index_type
    )::Nothing
    param_nums = var.parameter_nums
    ivref = var.infinite_variable_ref
    ivref_param_nums = InfiniteOpt._parameter_numbers(ivref)
    eval_supps = var.eval_supports
    # prepare for iterating over its supports
    supp_indices = support_index_iterator(trans_model, var.object_nums)
    vrefs = Vector{JuMP.VariableRef}(undef, length(supp_indices))
    labels = Vector{Set{DataType}}(undef, length(supp_indices))
    lookup_dict = Dict{Vector{Float64}, Int}()
    counter = 1
    # map a variable for each support
    for i in supp_indices
        raw_supp = index_to_support(trans_model, i)
        # ensure this support is valid with the reduced restriction
        if any(!isnan(raw_supp[ivref_param_nums[k]]) && raw_supp[ivref_param_nums[k]] != v
               for (k, v) in eval_supps)
            continue
        end
        # map to the current transcription variable
        supp = raw_supp[param_nums]
        ivref_supp = [haskey(eval_supps, j) ? eval_supps[j] : raw_supp[k]
                      for (j, k) in enumerate(ivref_param_nums)]
        @inbounds vrefs[counter] = lookup_by_support(trans_model, ivref, ivref_supp)
        lookup_dict[supp] = counter
        @inbounds labels[counter] = index_to_labels(trans_model, i)
        counter += 1
    end
    # truncate vrefs if any supports were skipped because of dependent parameter supps
    deleteat!(vrefs, counter:length(vrefs))
    deleteat!(labels, counter:length(vrefs))
    # save the transcription information
    data = transcription_data(trans_model)
    data.infvar_lookup[rvref] = lookup_dict
    data.infvar_mappings[rvref] = vrefs
    data.infvar_support_labels[rvref] = labels
    return
end

# Empty mapping dispatch for infinite parameter functions
function _set_reduced_variable_mapping(trans_model::JuMP.Model,
    var::InfiniteOpt.ReducedVariable,
    rvref::InfiniteOpt.GeneralVariableRef,
    index_type::Type{InfiniteOpt.ParameterFunctionIndex}
    )::Nothing
    return
end

"""
    transcribe_reduced_variables!(trans_model::JuMP.Model,
                                  inf_model::InfiniteOpt.InfiniteModel)::Nothing

Map each `ReducedVariable` in `inf_model` to transcription variables stored in
`trans_model`. The variable mappings are also stored in
`TranscriptionData.infvar_mappings` in accordance with
`TranscriptionData.infvar_lookup` which enable [`transcription_variable`](@ref)
and [`lookup_by_support`](@ref). Note that [`transcribe_infinite_variables!`](@ref)
must be called first. Note that the supports will not be generated
until `InfiniteOpt.variable_supports` is invoked via `InfiniteOpt.supports`.
Note that `TranscriptionData.infvar_support_labels` is also populated.
"""
function transcribe_reduced_variables!(trans_model::JuMP.Model,
    inf_model::InfiniteOpt.InfiniteModel
    )::Nothing
    for (index, object) in InfiniteOpt._data_dictionary(inf_model, InfiniteOpt.ReducedVariable)
        # get the basic variable information
        var = object.variable
        rvref = InfiniteOpt._make_variable_ref(inf_model, index)
        # setup the mappings
        ivref = InfiniteOpt.infinite_variable_ref(rvref)
        _set_reduced_variable_mapping(trans_model, var, rvref,
                                      InfiniteOpt._index_type(ivref))
    end
    return
end

# Override the info of the jump variable with the point variable's if it is different
function _update_point_info(gvref::InfiniteOpt.GeneralVariableRef,
    vref::JuMP.VariableRef
    )::Nothing
    pvref = InfiniteOpt.dispatch_variable_ref(gvref)
    if JuMP.has_lower_bound(pvref)
        if JuMP.is_fixed(vref)
            JuMP.unfix(vref)
        end
        JuMP.set_lower_bound(vref, JuMP.lower_bound(pvref))
    end
    if JuMP.has_upper_bound(pvref)
        if JuMP.is_fixed(vref)
            JuMP.unfix(vref)
        end
        JuMP.set_upper_bound(vref, JuMP.upper_bound(pvref))
    end
    if JuMP.is_fixed(pvref)
        JuMP.fix(vref, JuMP.fix_value(pvref), force = true)
    end
    if JuMP.is_binary(pvref) && !JuMP.is_binary(vref)
        if JuMP.is_integer(vref)
            JuMP.unset_integer(vref)
        end
        JuMP.set_binary(vref)
    elseif JuMP.is_integer(pvref) && !JuMP.is_integer(vref)
        if JuMP.is_binary(vref)
            JuMP.unset_binary(vref)
        end
        JuMP.set_integer(vref)
    end
    if JuMP.start_value(pvref) != JuMP.start_value(vref)
        JuMP.set_start_value(vref, JuMP.start_value(pvref))
    end
    return
end

"""
    transcribe_point_variables!(trans_model::JuMP.Model,
                               inf_model::InfiniteOpt.InfiniteModel)::Nothing

Map each `PointVariable` in `inf_model` to a transcription variable stored in
`trans_model`. The variable mapping is also stored in
`TranscriptionData.finvar_mappings` which enables
[`transcription_variable`](@ref) and [`lookup_by_support`](@ref). Note that
[`transcribe_infinite_variables!`](@ref) must be called first and that the
info constraints associated with the transcription variable will be updated in
accordance with the point variable.
"""
function transcribe_point_variables!(trans_model::JuMP.Model,
    inf_model::InfiniteOpt.InfiniteModel
    )::Nothing
    for (index, object) in InfiniteOpt._data_dictionary(inf_model, InfiniteOpt.PointVariable)
        # get the basic variable information
        var = object.variable
        ivref = var.infinite_variable_ref
        param_nums = InfiniteOpt._parameter_numbers(ivref)
        supp = var.parameter_values
        # find the corresponding variable record the mapping
        vref = lookup_by_support(trans_model, ivref, supp)
        pvref = InfiniteOpt._make_variable_ref(inf_model, index)
        transcription_data(trans_model).finvar_mappings[pvref] = vref
        # update the info constraints as needed
        _update_point_info(pvref, vref)
    end
    return
end

################################################################################
#                       TRANSCRIPTION EXPRESSION METHODS
################################################################################
"""
    transcription_expression(trans_model::JuMP.Model, expr, support::Vector{Float64})

Given the `expr` from an `InfiniteModel`, form its transcripted version in
accordance with the variable mappings available in `trans_model` defined at
`support`. This should only be used once all variables and measures have been
transcribed (e.g., via [`transcribe_hold_variables!`](@ref)).
"""
function transcription_expression(trans_model::JuMP.Model, expr, support)
    error("Unsupported expression type `$(typeof(expr))` for automated " *
          "transcription.")
end

# GeneralVariableRef
function transcription_expression(trans_model::JuMP.Model,
    vref::InfiniteOpt.GeneralVariableRef,
    support::Vector{Float64}
    )
    return transcription_expression(trans_model, vref,
                                    InfiniteOpt._index_type(vref), support)
end

# Infinite variables, infinite parameter functions, and measures
function transcription_expression(trans_model::JuMP.Model,
    vref::InfiniteOpt.GeneralVariableRef,
    index_type::Type{V},
    support::Vector{Float64}
    ) where {V <: Union{InfVarIndex, InfiniteOpt.ParameterFunctionIndex, InfiniteOpt.MeasureIndex}}
    param_nums = InfiniteOpt._parameter_numbers(vref)
    return lookup_by_support(trans_model, vref, index_type, support[param_nums])
end

# Reduced variables
function transcription_expression(trans_model::JuMP.Model,
    vref::InfiniteOpt.GeneralVariableRef,
    index_type::Type{InfiniteOpt.ReducedVariableIndex},
    support::Vector{Float64}
    )
    ivref = InfiniteOpt.infinite_variable_ref(vref)
    if InfiniteOpt._index_type(ivref) == InfiniteOpt.ParameterFunctionIndex
        prefs = InfiniteOpt.raw_parameter_refs(ivref)
        param_nums = InfiniteOpt._parameter_numbers(ivref)
        func = InfiniteOpt.raw_function(ivref)
        return func(Tuple(support[param_nums], prefs)...)
    else
        param_nums = InfiniteOpt._parameter_numbers(vref)
        return lookup_by_support(trans_model, vref, index_type, support[param_nums])
    end
end

# Point variables and hold variables
function transcription_expression(trans_model::JuMP.Model,
    vref::InfiniteOpt.GeneralVariableRef,
    index_type::Type{V},
    support::Vector{Float64}
    )::JuMP.VariableRef where {V <: FinVarIndex}
    return lookup_by_support(trans_model, vref, index_type, support)
end

# Infinite parameters
function transcription_expression(trans_model::JuMP.Model,
    vref::InfiniteOpt.GeneralVariableRef,
    index_type::Type{V},
    support::Vector{Float64}
    )::Float64 where {V <: InfiniteOpt.InfiniteParameterIndex}
    param_num = InfiniteOpt._parameter_number(vref)
    return support[param_num]
end

# Finite parameters
function transcription_expression(trans_model::JuMP.Model,
    vref::InfiniteOpt.GeneralVariableRef,
    index_type::Type{InfiniteOpt.FiniteParameterIndex},
    support::Vector{Float64}
    )::Float64
    return InfiniteOpt.parameter_value(vref)
end

# AffExpr
function transcription_expression(trans_model::JuMP.Model,
    aff::JuMP.GenericAffExpr,
    support::Vector{Float64}
    )
    return JuMP.@expression(trans_model,
                 sum(coef * transcription_expression(trans_model, vref, support)
                     for (coef, vref) in JuMP.linear_terms(aff)) + JuMP.constant(aff))
end

# QuadExpr
function transcription_expression(trans_model::JuMP.Model,
    quad::JuMP.GenericQuadExpr,
    support::Vector{Float64}
    )
    return JuMP.@expression(trans_model,
                 sum(coef * transcription_expression(trans_model, v1, support) *
                     transcription_expression(trans_model, v2, support)
                     for (coef, v1, v2) in JuMP.quad_terms(quad)) +
                 transcription_expression(trans_model, quad.aff, support))
end

# Real Number
function transcription_expression(trans_model::JuMP.Model,
    num::Real,
    support::Vector{Float64}
    )
    return zero(JuMP.AffExpr) + num
end

################################################################################
#                         MEASURE TRANSCRIPTION METHODS
################################################################################
"""
    transcribe_measures!(trans_model::JuMP.Model,
                         inf_model::InfiniteOpt.InfiniteModel)::Nothing

For each `Measure` in `inf_model` expand it via `InfiniteOpt.expand_measure` or
`analytic_expansion` as appropriate and transcribe the expanded expression via
[`transcription_expression`](@ref). Then store the measure to transcripted
expression mappings in `TranscriptionData.measure_mappings` and
`TranscriptionData.measure_lookup` to enable [`transcription_variable`](@ref)
and [`lookup_by_support`](@ref). Note that the supports will not be generated
until `InfiniteOpt.variable_supports` is invoked via `InfiniteOpt.supports`.
Note that `TranscriptionData.measure_support_labels` is also populated.
"""
function transcribe_measures!(trans_model::JuMP.Model,
    inf_model::InfiniteOpt.InfiniteModel
    )::Nothing
    for (index, object) in InfiniteOpt._data_dictionary(inf_model, InfiniteOpt.Measure)
        # get the basic information
        meas = object.measure
        # expand the measure
        if meas.constant_func
            new_expr = InfiniteOpt.analytic_expansion(meas.func, meas.data, trans_model)
        else
            new_expr = InfiniteOpt.expand_measure(meas.func, meas.data, trans_model)
        end
        # prepare to transcribe over the supports
        supp_indices = support_index_iterator(trans_model, meas.object_nums)
        exprs = Vector{JuMP.AbstractJuMPScalar}(undef, length(supp_indices))
        labels = Vector{Set{DataType}}(undef, length(supp_indices))
        lookup_dict = Dict{Vector{Float64}, Int}()
        # map a variable for each support
        for (counter, i) in enumerate(supp_indices)
            raw_supp = index_to_support(trans_model, i)
            @inbounds exprs[counter] = transcription_expression(trans_model,
                                                                new_expr, raw_supp)
            supp = raw_supp[meas.parameter_nums]
            lookup_dict[supp] = counter
            @inbounds labels[counter] = index_to_labels(trans_model, i)
        end
        # save the transcription information
        mref = InfiniteOpt._make_variable_ref(inf_model, index)
        data = transcription_data(trans_model)
        data.measure_lookup[mref] = lookup_dict
        data.measure_mappings[mref] = exprs
        data.measure_support_labels[mref] = labels
    end
    return
end

################################################################################
#                        OBJECTIVE TRANSCRIPTION METHODS
################################################################################
"""
    transcribe_objective!(trans_model::JuMP.Model,
                          inf_model::InfiniteOpt.InfiniteModel)::Nothing

Form the transcripted version of the objective stored in `inf_model` and add it
to `trans_model`. Note that all the variables and measures in `inf_model` must
by transcripted first (e.g., via [`transcribe_infinite_variables!`](@ref)).
"""
function transcribe_objective!(trans_model::JuMP.Model,
    inf_model::InfiniteOpt.InfiniteModel
    )::Nothing
    expr = JuMP.objective_function(inf_model)
    sense = JuMP.objective_sense(inf_model)
    trans_expr = transcription_expression(trans_model, expr, Float64[])
    JuMP.set_objective(trans_model, sense, trans_expr)
    return
end

################################################################################
#                        CONSTRAINT TRANSCRIPTION METHODS
################################################################################
## Given a variable and its set from an info constraint, get the transcribed version
# Lower bound constraint
function _get_info_constr_from_var(trans_model::JuMP.Model,
    vref::InfiniteOpt.GeneralVariableRef,
    set::MOI.GreaterThan,
    support::Vector{Float64}
    )::Union{JuMP.ConstraintRef, Nothing}
    trans_vref = transcription_expression(trans_model, vref, support)
    return JuMP.has_lower_bound(trans_vref) ? JuMP.LowerBoundRef(trans_vref) : nothing
end

# Upper bound constraint
function _get_info_constr_from_var(trans_model::JuMP.Model,
    vref::InfiniteOpt.GeneralVariableRef,
    set::MOI.LessThan,
    support::Vector{Float64}
    )::Union{JuMP.ConstraintRef, Nothing}
    trans_vref = transcription_expression(trans_model, vref, support)
    return JuMP.has_upper_bound(trans_vref) ? JuMP.UpperBoundRef(trans_vref) : nothing
end

# Fix constraint
function _get_info_constr_from_var(trans_model::JuMP.Model,
    vref::InfiniteOpt.GeneralVariableRef,
    set::MOI.EqualTo,
    support::Vector{Float64}
    )::Union{JuMP.ConstraintRef, Nothing}
    trans_vref = transcription_expression(trans_model, vref, support)
    return JuMP.is_fixed(trans_vref) ? JuMP.FixRef(trans_vref) : nothing
end

# Binary constraint
function _get_info_constr_from_var(trans_model::JuMP.Model,
    vref::InfiniteOpt.GeneralVariableRef,
    set::MOI.ZeroOne,
    support::Vector{Float64}
    )::Union{JuMP.ConstraintRef, Nothing}
    trans_vref = transcription_expression(trans_model, vref, support)
    return JuMP.is_binary(trans_vref) ? JuMP.BinaryRef(trans_vref) : nothing
end

# Integer constraint
function _get_info_constr_from_var(trans_model::JuMP.Model,
    vref::InfiniteOpt.GeneralVariableRef,
    set::MOI.Integer,
    support::Vector{Float64}
    )::Union{JuMP.ConstraintRef, Nothing}
    trans_vref = transcription_expression(trans_model, vref, support)
    return JuMP.is_integer(trans_vref) ? JuMP.IntegerRef(trans_vref) : nothing
end

## Extract the parameter bounds from a constr if it has them and collect them
## to return them in a convenient form for iteration
# ScalarConstraint
function _get_constr_bounds(constr::JuMP.ScalarConstraint
    )::Tuple{Vector{Int}, Vector{InfiniteOpt.IntervalSet}}
    return Int[], InfiniteOpt.IntervalSet[]
end

# BoundedScalarConstraint
function _get_constr_bounds(constr::InfiniteOpt.BoundedScalarConstraint
    )::Tuple{Vector{Int}, Vector{InfiniteOpt.IntervalSet}}
    bounds = InfiniteOpt.parameter_bounds(constr)
    prefs = collect(keys(bounds))
    sets = map(p -> bounds[p], prefs)
    indices = map(p -> InfiniteOpt._parameter_number(p), prefs)
    return indices, sets
end

## Determine if a given raw support satisfies constraint parameter bounds
function _support_in_bounds(support::Vector{Float64},
    indices::Vector{Int},
    sets::Vector{InfiniteOpt.IntervalSet}
    )::Bool
    for i in eachindex(indices)
        s = support[indices[i]]
        if !isnan(s) && (s < JuMP.lower_bound(sets[i]) || s > JuMP.upper_bound(sets[i]))
            return false
        end
    end
    return true
end

"""
    transcribe_constraints!(trans_model::JuMP.Model,
                            inf_model::InfiniteOpt.InfiniteModel)::Nothing

For each constraint in `inf_model` form its transcripted version(s) and add them
to `trans_model`. The mappings are stored in `TranscriptionData.constr_mappings`
and the associated supports are stored in `TranscriptionData.constr_supports`
to enable [`transcription_constraint`](@ref) and `InfiniteOpt.constraint_supports`.
Note that variable info constraints are simply mapped to the existing info
constraints already generated along with the transcription variables. Note that
the variables and measures must all first be transcripted (e.g., via
[`transcribe_measures!`](@ref)). Note that
`TranscriptionData.constr_support_labels` is also populated.
"""
function transcribe_constraints!(trans_model::JuMP.Model,
    inf_model::InfiniteOpt.InfiniteModel
    )::Nothing
    param_supps = parameter_supports(trans_model)
    for (index, object) in inf_model.constraints
        # get the basic information
        constr = object.constraint
        func = JuMP.jump_function(constr)
        set = JuMP.moi_set(constr)
        obj_nums = object.object_nums
        # prepare the iteration helpers
        supp_indices = support_index_iterator(trans_model, obj_nums)
        crefs = Vector{JuMP.ConstraintRef}(undef, length(supp_indices))
        supps = Vector{Tuple}(undef, length(supp_indices))
        labels = Vector{Set{DataType}}(undef, length(supp_indices))
        counter = 1
        # iterate over the support indices for the info constraints
        if object.is_info_constraint
            for i in supp_indices
                raw_supp = index_to_support(trans_model, i)
                info_ref = _get_info_constr_from_var(trans_model, func, set,
                                                     raw_supp)
                # not all supports may be defined if overwritten by a point variable
                if info_ref !== nothing
                    @inbounds crefs[counter] = info_ref
                    @inbounds supps[counter] = Tuple(param_supps[j][i[j]]
                                                     for j in obj_nums)
                    @inbounds labels[counter] = index_to_labels(trans_model, i)
                    counter += 1
                end
            end
        # iterate over the supports for regular constraints
        else
            # get basic setup information
            bound_info = _get_constr_bounds(constr)
            name = object.name
            for i in supp_indices
                raw_supp = index_to_support(trans_model, i)
                # ensure the support satisfies parameter bounds and then add it
                if _support_in_bounds(raw_supp, bound_info[1], bound_info[2])
                    new_func = transcription_expression(trans_model, func, raw_supp)
                    trans_constr = JuMP.build_constraint(error, new_func, set)
                    new_name = isempty(name) ? "" : string(name, "(support: ", counter, ")")
                    @inbounds crefs[counter] = JuMP.add_constraint(trans_model,
                                                         trans_constr, new_name)
                    @inbounds supps[counter] = Tuple(param_supps[j][i[j]]
                                                     for j in obj_nums)
                    @inbounds labels[counter] = index_to_labels(trans_model, i)
                    counter += 1
                end
            end
        end
        # truncate the arrays in case not all the supports satisfied the bounds
        deleteat!(crefs, counter:length(crefs))
        deleteat!(supps, counter:length(supps))
        deleteat!(labels, counter:length(supps))
        # add the constraint mappings to the trans model
        cref = InfiniteOpt._make_constraint_ref(inf_model, index)
        data = transcription_data(trans_model)
        data.constr_mappings[cref] = crefs
        data.constr_supports[cref] = supps
        data.constr_support_labels[cref] = labels
    end
    return
end

################################################################################
#                      DERIVATIVE CONSTRAINT TRANSCRIPTION METHODS
################################################################################
"""
    transcribe_derivative_evaluations!(trans_model::JuMP.Model,
                                       inf_model::InfiniteOpt.InfiniteModel)::Nothing

Generate the auxiliary derivative evaluation equations and transcribe them
appropriately for all the derivatives in `inf_model`. These are in turn added to
`trans_model`. Note that no mapping information is recorded since the InfiniteModel
won't have any constraints that correspond to these equations. Also Note that
the variables and measures must all first be transcripted (e.g., via
[`transcribe_derivative_variables!`](@ref)).
"""
function transcribe_derivative_evaluations!(trans_model::JuMP.Model,
    inf_model::InfiniteOpt.InfiniteModel
    )::Nothing
    for (index, object) in InfiniteOpt._data_dictionary(inf_model, InfiniteOpt.Derivative)
        # get the basic variable information
        dref = InfiniteOpt._make_variable_ref(inf_model, index)
        pref = dispatch_variable_ref(object.variable.parameter_ref)
        method = InfiniteOpt.derivative_method(pref)
        # generate the transcription constraints as needed
        if !InfiniteOpt.has_derivative_constraints(dref)
            # generate the evaluation expressions
            exprs = InfiniteOpt.evaluate_derivative(dref, method, trans_model)
            # prepare the iteration helpers
            param_obj_num = InfiniteOpt._object_number(pref)
            obj_nums = filter(!isequal(param_obj_num), InfiniteOpt._object_numbers(dref))
            supp_indices = support_index_iterator(trans_model, obj_nums)
            # transcribe the constraints
            set = MOI.EqualTo(0.0)
            for i in supp_indices
                raw_supp = index_to_support(trans_model, i)
                for expr in exprs
                    new_expr = transcription_expression(trans_model, expr, raw_supp)
                    trans_constr = JuMP.build_constraint(error, new_expr, set)
                    JuMP.add_constraint(trans_model, trans_constr) # TODO maybe add name?
                end
            end
        end
    end
    return
end

################################################################################
#                      INFINITEMODEL TRANSCRIPTION METHODS
################################################################################
"""
    build_transcription_model!(trans_model::JuMP.Model,
                               inf_model::InfiniteOpt.InfiniteModel;
                               [check_support_dims::Bool = true])::Nothing

Given an empty `trans_model` build it using the information stored in `inf_model`.
This is intended for a `TranscriptionModel` that serves as a internal optimizer model
of `inf_model`. This detail is important to correctly enable internally generated
reduced variables during the transcription process such that `inf_model` is not
modified. Note that this will add supports to `inf_model` via
[`InfiniteOpt.fill_in_supports!`](@ref) for infinite parameters that contain
no supports. Also a warning is thrown when the transcription model contains
more than 15,000 support points to alert users when they may naively have
a few independent supports whose product quickly yields a very large grid.
For example having 3 independent parameters with 100 supports each would result
in 1,000,000 supports if all three are together in at least 1 constraint. This
behavior can be overcome using dependent parameters. The warning can be turned off
via `check_support_dims = false`.
"""
function build_transcription_model!(trans_model::JuMP.Model,
    inf_model::InfiniteOpt.InfiniteModel; check_support_dims::Bool = true
    )::Nothing
    # ensure there are supports to add and add them to the trans model
    InfiniteOpt.fill_in_supports!(inf_model, modify = false)
    set_parameter_supports(trans_model, inf_model)
    # check that there isn't a crazy amount of supports from taking the product
    supps = parameter_supports(trans_model)
    num_supps = prod(length, supps)
    if check_support_dims && length(supps) > 1 && num_supps > 15000 # NOTE this is an arbitrary cutoff
        @warn("Due to necessarily considering the combinatorics of independent " *
              "parameter supports, the model will be transcripted over up to $(num_supps) " *
              "supports (if any constraint uses all the infinite parameters) "  *
              "and thus naive solution of the discretized problem may be slow. " *
              "This warning can be turned off via `check_support_dims = false`.")
    end
    # define the variables
    transcribe_hold_variables!(trans_model, inf_model)
    transcribe_infinite_variables!(trans_model, inf_model)
    transcribe_derivative_variables!(trans_model, inf_model)
    transcribe_reduced_variables!(trans_model, inf_model)
    transcribe_point_variables!(trans_model, inf_model)
    transcribe_measures!(trans_model, inf_model)
    # define the objective
    transcribe_objective!(trans_model, inf_model)
    # define the constraints
    transcribe_constraints!(trans_model, inf_model)
    # define the derivative evaluation constraints
    transcribe_derivative_evaluations!(trans_model, inf_model)
    return
end

# TODO make TranscriptionModel(::InfiniteModel) --> have to deal with accessing reduced vars


using JuMP
const MOIT = MOI.Test

MOIU.@model(NoFreeVariable,
            (), (MOI.EqualTo, MOI.LessThan, MOI.GreaterThan), (MOI.Nonnegatives, MOI.Nonpositives, MOI.Zeros, MOI.RotatedSecondOrderCone, MOI.PositiveSemidefiniteConeTriangle), (),
            (), (MOI.ScalarAffineFunction,), (MOI.VectorOfVariables,), (MOI.VectorAffineFunction,))
# No free variables to make sure variable bridges are used to increase coverage
MOI.supports_constraint(::NoFreeVariable, ::Type{MOI.VectorOfVariables}, ::Type{MOI.Reals}) = false

function bridged_mock(mock_optimize!::Function...;
                      model = MOI.Utilities.Model{Float64}())
    mock = MOI.Utilities.MockOptimizer(model)
    bridged = MOI.Bridges.full_bridge_optimizer(mock, Float64)
    MOI.Utilities.set_mock_optimize!(mock, mock_optimize!...)
    return bridged
end

function cached_mock(
    args...; cache = () -> MOIU.UniversalFallback(MOIU.Model{Float64}()), kws...)
    # We want the MOI backend of the JuMP model to be in `MOIU.EMPTY_OPTIMIZER`
    # mode so that it's copied at `optimize!` to test that `copy_to` works.
    # If we just return `cached`, it will be emptied in `_model` and the state
    # will be `MOIU.ATTACHED_OPTIMIZER` which is not what we want. For this
    # reason we return a `JuMP.OptimizerFactory` which returns `cached` instead.
    with_optimizer(() -> begin
        cached = MOIU.CachingOptimizer(cache(), MOIU.AUTOMATIC)
        optimizer = bridged_mock(args...; kws...)
        MOIU.reset_optimizer(cached, optimizer)
        return cached
    end)
end

function mocks(args...; kws...)
    return [bridged_mock(args...; kws...), cached_mock(args...; kws...)]
end


"""
2-D/ 3-D elastic forward modeling using staggered-grid velocity-stress finite-difference formulation.
"""
struct FdtdElastic end
"""
2-D/ 3-D acoustic forward modeling using staggered-grid velocity-stress finite-difference formulation.
"""
struct FdtdAcou end
"""
2-D/ 3-D viscoacoustic forward modeling using staggered-grid velocity-stress finite-difference formulation.
"""
struct FdtdAcouVisco end
"""
2-D Linearized forward modeling using a finite-difference simulation of the acoustic wave-equation.
"""
struct FdtdAcouBorn end

# define supersource, source and receiver structs
struct Srcs
    n::Int
end
Srcs() = Srcs(0)
struct SSrcs
    n::Int
end
SSrcs() = SSrcs(0)
struct Recs
    n::Int
end
Recs() = Recs(0)


module LyceumMuJoCo

using Base: @propagate_inbounds
using Distances
using Distributions
using DocStringExtensions
using LinearAlgebra

using MuJoCo
using MuJoCo.MJCore: mjtNum

using Shapes
using StaticArrays
using Statistics
using Random
using Reexport
using UnsafeArrays

using LyceumBase: @mustimplement, perturb!
@reexport using LyceumBase

const RealVec = AbstractVector{<:Real}

export MJSim, zeroctrl!, zerofullctrl!, forward!
include("mjsim.jl")

export AbstractMuJoCoEnvironment, getsim
include("abstractmujocoenvironment.jl")

####
#### Environments
####

include("suite/pointmass.jl")
include("suite/armhand/armhandpickup.jl")

include("gym/swimmer-v2.jl")
include("gym/hopper-v2.jl")
include("gym/walker2d-v2.jl")

include("dmc/rewards.jl")
include("dmc/cartpole_swingup.jl")

end # module


mutable struct NER
    worddict::Dict
    chardict::Dict
    tagset
    model
end

function NER()
    words = h5read(wordembeds_file, "key")
    worddict = Dict(words[i] => i for i=1:length(words))
    chardict = Dict{String,Int}()
    NER(worddict, chardict, BIO(), nothing)
end

function encode(ner::NER, words::Vector{String})
    worddict = ner.worddict
    chardict = ner.chardict
    unkword = worddict["UNKNOWN"]
    w = map(w -> get!(worddict,lowercase(w),unkword), words)
    cs = map(words) do w
        chars = Vector{Char}(w)
        map(c -> get!(chardict,string(c),length(chardict)+1), chars)
    end
    w, cs
end

function readdata!(ner::NER, path::String)
    data_w, data_c, data_t = Var[], Var[], Var[]
    words, tags = String[], String[]
    lines = open(readlines, path)
    for i = 1:length(lines)
        line = lines[i]
        if isempty(line) || i == length(lines)
            isempty(words) && continue
            w, cs = encode(ner, words)
            t = encode(ner.tagset, tags)
            push!(data_w, Var(w))
            batchdims = map(length, cs)
            c = cat(1, cs...)
            push!(data_c, Var(c,batchdims))
            push!(data_t, Var(t))
            empty!(words)
            empty!(tags)
        else
            items = split(line, "\t")
            push!(words, String(items[1]))
            push!(tags, String(items[2]))
            #word = replace(word, r"[0-9]", '0')
        end
    end
    data_w, data_c, data_t
end

function train(ner::NER, trainfile::String, testfile::String)
    train_w, train_c, train_t = readdata!(ner, trainfile)
    test_w, test_c, test_t = readdata!(ner, testfile)
    info("# Training sentences:\t$(length(train_w))")
    info("# Testing sentences:\t$(length(test_w))")
    info("# Words:\t$(length(ner.worddict))")
    info("# Chars:\t$(length(ner.chardict))")
    info("# Tags:\t$(length(ner.tagset))")

    wordembeds = h5read(wordembeds_file, "value")
    charembeds = randn(Float32, 20, length(ner.chardict)) * 0.01f0
    tagembeds = randn(Float32, 50, length(ner.tagset)) * 0.01f0
    ner.model = Model(wordembeds, charembeds, tagembeds)
    opt = SGD()
    for epoch = 1:50
        println("Epoch:\t$epoch")
        opt.rate = 0.001 / (1 + 0.05*(epoch-1))
        # opt.rate = 0.00075

        train_data = makebatch(16, train_w, train_c, train_t)
        function train_f(data::Tuple)
            w, c, t = data
            y = ner.model(w, c, t)
            #softmax_crossentropy(t, y)
        end
        #train_data = collect(zip(train_w, train_c, train_t))
        loss = minimize!(train_f, opt, collect(zip(train_data...)))
        println("Loss:\t$loss")

        # test
        println("Testing...")
        function test_f(data::Tuple)
            w, c = data
            y = ner.model(w, c)
            vec(argmax(y.data,1))
        end
        test_data = collect(zip(test_w, test_c))
        pred = cat(1, map(test_f, test_data)...)
        gold = cat(1, map(t -> t.data, test_t)...)
        length(pred) == length(gold) || throw("Length mismatch.")

        ranges_p = decode(ner.tagset, pred)
        ranges_g = decode(ner.tagset, gold)
        fscore(ranges_g, ranges_p)
        println()
    end
end

function fscore(golds::Vector{T}, preds::Vector{T}) where T
    set = intersect(Set(golds), Set(preds))
    count = length(set)
    prec = round(count/length(preds), 5)
    recall = round(count/length(golds), 5)
    fval = round(2*recall*prec/(recall+prec), 5)
    println("Prec:\t$prec")
    println("Recall:\t$recall")
    println("Fscore:\t$fval")
end


# Chipmunk's rigid body type. Rigid bodies hold the physical properties of an object like
# it's mass, and position and velocity of it's center of gravity. They don't have an shape on their own.
# They are given a shape by creating collision shapes (cpShape) that point to the body.

@enum(BodyType,
    # A dynamic body is one that is affected by gravity, forces, and collisions.
    # This is the default body type.
dynamic,
    # A kinematic body is an infinite mass, user controlled body that is not affected by gravity, forces or collisions.
    # Instead the body only moves based on it's velocity.
    # Dynamic bodies collide normally with kinematic bodies, though the kinematic body will be unaffected.
    # Collisions between two kinematic bodies, or a kinematic body and a static body produce collision callbacks, but no collision response.
kinematic,
    # A static body is a body that never (or rarely) moves. If you move a static body, you must call one of the cpSpaceReindex*() functions.
    # Chipmunk uses this information to optimize the collision detection.
    # Static bodies do not produce collision callbacks when colliding with other static bodies.
static)

# Allocate a cpBody.
function Body()
    Body(ccall(dlsym(libchipmunk, :cpBodyAlloc), Ptr{Void}, ()))
end

# Allocate and initialize a cpBody.
function Body(mass::Real, moment::Real)
    Body(ccall(dlsym(libchipmunk, :cpBodyNew), Ptr{Void}, (Cdouble, Cdouble,), mass, moment))
end

# Allocate and initialize a cpBody, and set it as a kinematic body.
function KinematicBody()
    Body(ccall(dlsym(libchipmunk, :cpBodyNewKinematic), Ptr{Void}, ()))
end

# Allocate and initialize a cpBody, and set it as a static body.
function StaticBody()
    Body(ccall(dlsym(libchipmunk, :cpBodyNewStatic), Ptr{Void}, ()))
end

# Initialize a cpBody.
function init(body::Body, mass::Real, moment::Real)
    Body(ccall(dlsym(libchipmunk, :cpBodyInit), Ptr{Void}, (Ptr{Void}, Cdouble, Cdouble,), body.ptr, mass, moment))
end

# Destroy and free a cpBody.
function free(body::Body)
    ccall(dlsym(libchipmunk, :cpBodyFree), Void, (Ptr{Void},), body.ptr)
end

# Wake up a sleeping or idle body.
function activate(body::Body)
    ccall(dlsym(libchipmunk, :cpBodyActivate), Void, (Ptr{Void},), body.ptr)
end

# Wake up any sleeping or idle bodies touching a static body.
function activate_static(body::Body, filter::Shape)
    ccall(dlsym(libchipmunk, :cpBodyActivateStatic), Void, (Ptr{Void}, Ptr{Void},), body.ptr, filter.ptr)
end

# Force a body to fall asleep immediately.
function sleep(body::Body)
    ccall(dlsym(libchipmunk, cpBodySleep), Void, (Ptr{Void},), body.ptr)
end

# Force a body to fall asleep immediately along with other bodies in a group.
function sleep_with_group(body::Body, group::Body)
    ccall(dlsym(libchipmunk, :cpBodySleepWithGroup), Void, (Ptr{Void}, Ptr{Void},), body.ptr, group.ptr)
end

# Returns true if the body is sleeping.
function is_sleeping(body::Body)
    Bool(ccall(dlsym(libchipmunk, :cpBodyIsSleeping), Uint8, (Ptr{Void},), body.ptr))
end

# Get the type of the body.
function get_type(body::Body)
    BodyType(ccall(dlsym(libchipmunk, :cpBodyGetType), Int32, (Ptr{Void},), body.ptr))
end

# Set the type of the body.
function set_type(body::Body, body_type::BodyType)
    ccall(dlsym(libchipmunk, :cpBodySetType), Void, (Ptr{Void}, Int32,), body.ptr, body_type)
end

# function accumulate_mass_from_shapes(body::Body)
#   ccall(dlsym(libchipmunk, :cpBodyAccumulateMassFromShapes), Void, (Ptr{Void},), body.ptr)
# end

# Get the space this body is added to.
function get_space(body::Body)
    Space(ccall(dlsym(libchipmunk, :cpBodyGetSpace), Ptr{Void}, (Ptr{Void},), body.ptr))
end

# Get the mass of the body.
function get_mass(body::Body)
    Real(ccall(dlsym(libchipmunk, :cpBodyGetMass), Cdouble, (Ptr{Void},), body.ptr))
end

# Set the mass of the body.
function set_mass(body::Body, mass::Real)
    ccall(dlsym(libchipmunk, :cpBodySetMass), Void, (Ptr{Void}, Cdouble,), body.ptr, mass)
end

# Get the moment of inertia of the body.
function get_moment(body::Body)
    Real(ccall(dlsym(libchipmunk, :cpBodyGetMoment), Cdouble, (Ptr{Void},), body.ptr))
end

# Set the moment of inertia of the body.
function set_moment(body::Body, moment::Real)
    ccall(dlsym(libchipmunk, :cpBodySetMoment), Void, (Ptr{Void}, Cdouble,), body.ptr, moment)
end

# Get the position of a body.
function get_position(body::Body)
    ccall(dlsym(libchipmunk, :cpBodyGetPosition), Vect, (Ptr{Void},), body.ptr)
end

# Set the position of a body.
function get_center_of_gravity(body::Body)
    ccall(dlsym(libchipmunk, :cpBodyGetCenterOfGravity), Vect, (Ptr{Void},), body.ptr)
end

# Get the offset of the center of gravity in body local coordinates.
function set_center_of_gravity(body::Body, cog::Vect)
    ccall(dlsym(libchipmunk, :cpBodySetCenterOfGravity), Void, (Ptr{Void}, Vect,), body.ptr, cog)
end

# Set the offset of the center of gravity in body local coordinates.<F37>
function set_position(body::Body, pos::Vect)
    ccall(dlsym(libchipmunk, :cpBodySetPosition), Void, (Ptr{Void}, Vect,), body.ptr, pos)
end

# Get the velocity of the body.
function get_velocity(body::Body)
    ccall(dlsym(libchipmunk, :cpBodyGetVelocity), Vect, (Ptr{Void},), body.ptr)
end

# Set the velocity of the body.
function set_velocity(body::Body, velocity::Vect)
    ccall(dlsym(libchipmunk, :cpBodySetVelocity), Void, (Ptr{Void}, Vect), body.ptr, velocity)
end

# Get the force applied to the body for the next time step.
function get_force(body::Body)
    ccall(dlsym(libchipmunk, :cpBodyGetForce), Vect, (Ptr{Void},), body.ptr)
end

# Set the force applied to the body for the next time step.
function set_force(body::Body, force::Vect)
    ccall(dlsym(libchipmunk, :cpBodySetForce), Void, (Ptr{Void}, Vect,), body.ptr, force)
end

# Get the angle of the body.
function get_angle(body::Body)
    ccall(dlsym(libchipmunk, :cpBodyGetAngle), Cdouble, (Ptr{Void},), body.ptr)
end

# Set the angle of a body.
function set_angle(body::Body, angle::Real)
    ccall(dlsym(libchipmunk, :cpBodySetAngle), Void, (Ptr{Void}, Cdouble), body.ptr, angle)
end

# Get the angular velocity of the body.
function get_angular_velocity(body::Body)
    ccall(dlsym(libchipmunk, :cpBodyGetAngularVelocity), Cdouble, (Ptr{Void},), body.ptr)
end

# Set the angular velocity of the body.
function set_angular_velocity(body::Body, angular_velocity::Real)
    ccall(dlsym(libchipmunk, :cpBodySetAngularVelocity), Void, (Ptr{Void}, Cdouble,), body.ptr, angular_velocity)
end

# Get the torque applied to the body for the next time step.
function get_torque(body::Body)
    ccall(dlsym(libchipmunk, :cpBodyGetTorque), Cdouble, (Ptr{Void},), body.ptr)
end

# Set the torque applied to the body for the next time step.
function set_torque(body::Body, torque::Real)
    ccall(dlsym(libchipmunk, :cpBodySetTorque), Void, (Ptr{Void}, Cdouble,), body.ptr)
end

# Get the rotation vector of the body. (The x basis vector of it's transform.)
function get_rotation(body::Body)
    ccall(dlsym(libchipmunk, :cpBodyGetRotation), Vect, (Ptr{Void},), body.ptr)
end

# Get the user data pointer assigned to the body.
function get_userdata(body::Body)
    ccall(dlsym(libchipmunk, :cpBodyGetUserData), Ptr{Void}, (Ptr{Void},), body.ptr)
end

# Set the user data pointer assigned to the body.
function set_userdata(body::Body, userdata::Ptr{Void})
    ccall(dlsym(libchipmunk, :cpBodySetUserData), Void, (Ptr{Void}, Ptr{Void},), body.ptr, userdata)
end

# Default velocity integration function.
function update_velocity(body::Body, gravity::Vect, damping::Real, dt::Real)
    ccall(dlsym(libchipmunk, :cpBodyUpdateVelocity), Void, (Ptr{Void}, Vect, Cdouble, Cdouble,), body.ptr, gravity, damping, dt)
end

# Default position integration function.
function update_position(body::Body, dt::Real)
    ccall(dlsym(libchipmunk, :cpBodyUpdatePosition), Void, (Ptr{Void}, Cdouble,), body.ptr, dt)
end

# Convert body relative/local coordinates to absolute/world coordinates.
function local_to_world(body::Body, point::Vect)
    ccall(dlsym(libchipmunk, :cpBodyLocalToWorld), Vect, (Ptr{Void}, Vect,), body.ptr, point)
end

# Convert body absolute/world coordinates to  relative/local coordinates.
function world_to_local(body::Body, point::Vect)
    ccall(dlsym(libchipmunk, :cpBodyWorldToLocal), Vect, (Ptr{Void}, Vect,), body.ptr, point)
end

# Apply a force to a body. Both the force and point are expressed in world coordinates.
function apply_force_at_world_point(body::Body, force::Vect, point::Vect)
    ccall(dlsym(libchipmunk, :cpBodyApplyForceAtWorldPoint), Void, (Ptr{Void}, Vect, Vect,), body.ptr, force, point)
end

# Apply a force to a body. Both the force and point are expressed in body local coordinates.
function apply_force_at_local_point(body::Body, force::Vect, point::Vect)
    ccall(dlsym(libchipmunk, :cpBodyApplyForceAtLocalPoint), Void, (Ptr{Void}, Vect, Vect,), body.ptr, force, point)
end

# Apply an impulse to a body. Both the impulse and point are expressed in world coordinates.
function apply_impulse_at_world_point(body::Body, impulse::Vect, point::Vect)
    ccall(dlsym(libchipmunk, :cpBodyApplyImpulseAtWorldPoint), Void, (Ptr{Void}, Vect, Vect,), body.ptr, impulse, point)
end

# Apply an impulse to a body. Both the impulse and point are expressed in body local coordinates.
function apply_impulse_at_local_point(body::Body, impulse::Vect, point::Vect)
    ccall(dlsym(libchipmunk, :cpBodyApplyImpulseAtLocalPoint), Void, (Ptr{Void}, Vect, Vect,), body.ptr, impulse, point)
end

# Get the velocity on a body (in world units) at a point on the body in world coordinates.
function get_velocity_at_world_point(body::Body, point::Vect)
    ccall(dlsym(libchipmunk, :cpBodyGetVelocityAtWorldPoint), Vect, (Ptr{Void}, Vect,), body.ptr, point)
end

# Get the velocity on a body (in local units) at a point on the body in local coordinates.
function get_velocity_at_local_point(body::Body, point::Vect)
    ccall(dlsym(libchipmunk, :cpBodyGetVelocityAtLocalPoint), Vect, (Ptr{Void}, Vect,), body.ptr, point)
end

# Get the amount of kinetic energy contained by the body.
function kinetic_energy(body::Body)
    ccall(dlsym(libchipmunk, :cpBodyKineticEnergy), Cdouble, (Ptr{Void},), body.ptr)
end

export Body, KinematicBody, StaticBody, BodyType, init, free, is_sleeping, get_type, set_type,
get_space, get_mass, set_mass, get_moment, set_moment, set_position, get_position, get_velocity,
set_velocity, activate, activate_static, sleep, get_angle, set_angle, get_center_of_gravity,
set_center_of_gravity, get_force, set_force, get_angular_velocity, set_angular_velocity,
get_torque, set_torque, get_rotation, get_userdata, set_userdata, update_velocity, update_position,
local_to_world, world_to_local, apply_impulse_at_world_point, apply_force_at_local_point,
apply_impulse_at_world_point, apply_impulse_at_local_point, get_velocity_at_world_point,
get_velocity_at_local_point, kinetic_energy


module FitTools

using LinearAlgebra
using JSON
using DelimitedFiles
using Glob
using Printf

using JuLIP, ACE, ACEtb
using ACEtb.Bonds: get_env
using ACEtb.Utils: read_json, write_json, get_data
using ACEtb.Predictions: predict, train_and_predict

struct FitContext
    base_dir::String
    dftb_path::String
    output_dir::String
    param_basename::String
end

function make_params(mode::Symbol, training_datasets, test_datasets; rcut=12,
    renv=3, zenv=1, deg=5, order=2,
    envdeg=4, pcut=1, unitcell=false,
    method=:QR, rtol=1e-15)

    if mode == :fit
        predict = 1
        fit = 1
    elseif mode == :predict
        predict = 1
        fit = 0
    else
        error("mode should be either :fit or :predict")
    end

    param_dict = Dict(
        "training_datasets" => training_datasets,
        "test_datasets" => test_datasets,
        "model" => Dict(
                "predict" => predict,
                "fit" => fit,
                "cutoff_params" =>
                        Dict("rcut" => rcut,
                            "renv" => renv,
                            "zenv" => zenv,
                            "pcut" => pcut),
                "fit_params" => Dict(
                    "degree" => deg,
                    "order" => order,
                    "env_deg" => envdeg,
                    "method" => method,
                    "rtol" => rtol)),
        "onsite-terms" => Dict(
        "Al" => [
            -5.5708730114137865e+01,
            -4.0432979354580754e+00,
            -4.9611320942331338e-01,
            -2.6243905672312331e+00,
            -2.6243905672300967e+00,
            -2.6241786637179012e+00,
            -4.6415677514380915e-01,
            -4.6415677508608638e-01,
            -4.6415677490900636e-01,
            -2.8478483849360947e-01,
            -2.8478483839241397e-01,
            -2.8478483828616641e-01,
            -1.8885840374342641e-01,
            -1.8885840355491695e-01
        ]))
    return param_dict, params_to_string(param_dict)
end

function params_to_string(params::Dict)
    input_dict = Dict{String}{Any}()
    for (k, v) in params
        if k in ["training_datasets", "test_datasets", "model", "onsite-terms"]
            input_dict[k] = copy(v)
        end
    end
    delete!(input_dict["model"], "fit")
    delete!(input_dict["model"], "predict")
    return @sprintf "%x" hash(input_dict)
end

function do_fit(pot_dict::Dict, elm_names=["Al"])
    trdata = pot_dict["training_datasets"]
    cutoff_params = pot_dict["model"]["cutoff_params"]
    fit_params = pot_dict["model"]["fit_params"]
    @info "│    Fitting..."
    Bint_table, cutf_func, train_dict = train_and_predict(trdata, elm_names, cutoff_params,
                                                          fit_params; MPIproc=1)

    return train_dict, Bint_table, cutf_func
end

function load_model(fc::FitContext, param_hash::String)
    pot_file = joinpath(fc.output_dir, "pot_" * param_hash, "pot.json")
    pot_dict = read_json(pot_file)

    cutoff_params = pot_dict["model"]["cutoff_params"]
    fit_params = pot_dict["model"]["fit_params"]
    @info "│    Loading model..."
    Bint_table, cutf_func = predict(pot_dict, cutoff_params, fit_params; MPIproc=1)
    @info "│    Model is loaded."
    return pot_dict, param_hash, Bint_table, cutf_func
end

function test_error(test, Bint_table)
    errors = []
    for (i, (R0, Renv, V)) in enumerate(test)
        push!(errors, Bint_table(R0, Renv) - V)
    end
    return errors
end

function fit_model(fc::FitContext, params::Dict, training_datasets, test_datasets; band_errors=true)
    pot_dict, param_hash = make_params(:fit, training_datasets, test_datasets; params...)
    pot_dir = joinpath(fc.output_dir, "pot_" * param_hash)
    pot_file = joinpath(pot_dir, "pot.json")

    if isfile(pot_file)
        println("Skipping fit as $pot_file already present")
        pot_dict, param_hash, Bint_table, cutf_func = load_model(fc, param_hash)
    else
        isdir(pot_dir) || mkdir(pot_dir)
        println("Fitting model $param_hash in directory $pot_dir")

        train_dict, Bint_table, cutf_func = do_fit(pot_dict, ["Al"])
        merge!(pot_dict, train_dict)
        @info "Train error $(train_dict["error_train"])"
    end

    pot_dict["model"]["fit"] = 0 # disable fitting for subsequent runs

    if "test_error" ∉ keys(pot_dict) || "test_error_all" ∉ keys(pot_dict)
        # compute test error on independent set
        test_dataset = pot_dict["test_datasets"]
        test_data = ACEtb.Utils.get_data(test_dataset, cutf_func, get_env)
        pot_dict["test_error_all"] = test_error(test_data, Bint_table)
        pot_dict["test_error"] = norm(test_error_all)
        @info "Test error $(pot_dict["test_error"])"
    end

    # save so we can compute Fermi level; will be overwritten later
    open(joinpath(pot_dir, "pot.json"), "w") do json_file
        JSON.print(json_file, pot_dict, 4)
    end

    # compute and save Fermi level of the model for FCC and BCC
    if "fermi_level" ∉ keys(pot_dict)
        fcc_fermi_level = fermi_level(fc, :FCC, param_hash)
        bcc_fermi_level = fermi_level(fc, :BCC, param_hash)
        pot_dict["fermi_level"] = Dict("FCC" => fcc_fermi_level,
                                       "BCC" => bcc_fermi_level)
    end

    # include calculated Fermi level in potential file
    open(joinpath(pot_dir, "pot.json"), "w") do json_file
        JSON.print(json_file, pot_dict, 4)
    end

    # use the new model to compute FCC and BCC band errors
    band_errors && compute_band_errors(fc, param_hash)

    return pot_dict, param_hash, Bint_table, cutf_func
end

function fermi_level(fc::FitContext, phase::Symbol, param_hash::String)
    @assert phase in [:FCC, :BCC]
    println("Calculating Fermi level for $phase with params $param_hash")
    pot_dir = joinpath(fc.output_dir, "pot_" * param_hash)

    cp("dftb_in.hsd-$phase", joinpath(pot_dir, "dftb_in.hsd"), force=true)
    pot_file = joinpath(pot_dir,  "pot.json")
    cp(pot_file, joinpath(pot_dir, fc.param_basename); force=true)
    open(joinpath(pot_dir, "dftb_fermi_$phase.log"), "w") do logfile
        run(pipeline(setenv(`$(fc.dftb_path)/dftb+`, dir=pot_dir), stdout=logfile, stderr=logfile))
    end
    return parse(Float64, split(readchomp(`grep "Fermi level" $pot_dir/detailed.out`))[5])
end

function predict_bands(fc::FitContext, phase::Symbol, param_hash::String)
    @assert phase in [:FCC, :BCC]
    pot_dir = joinpath(fc.output_dir, "pot_" * param_hash)
    band_file = joinpath(pot_dir, "BANDS_$(phase).dat")
    if isfile(band_file)
        println("Skipping band calculation for $phase as $band_file already present")
        return read_bands(fc, phase, param_hash)
    end
    println("Calculating bands for $phase with params $param_hash")
    phase = String(phase)
    pot_file = joinpath(pot_dir, "pot.json")
    cp(pot_file, joinpath(pot_dir, fc.param_basename); force=true)

    # read template with correct band path definition for this phase
    lines = readlines("dftb_in.hsd-$phase-BANDS")

    # set the Fermi level to match value from model
    pot_dict = JSON.parsefile(pot_file)
    fermi_level = pot_dict["fermi_level"][phase]
    new_lines = []
    for line in lines
        m = match(r"(^\s*)FixedFermiLevel.*$", line)
        if m !== nothing
            line = m.captures[1] * "FixedFermiLevel = $(fermi_level)"
        end
        push!(new_lines, line)
    end
    open(joinpath(pot_dir, "dftb_in.hsd"), "w") do dftb_in
        write(dftb_in, join(new_lines, "\n"))
    end

    open(joinpath(pot_dir, "dftb_bands_$phase.log"), "w") do logfile
        run(pipeline(setenv(`$(fc.dftb_path)/dftb+`, dir=pot_dir), stdout=logfile, stderr=logfile))
        run(pipeline(setenv(`$(fc.dftb_path)/dp_bands band.out band`, dir=pot_dir),
                     stdout=logfile, stderr=logfile))
    end
    cp(joinpath(pot_dir, "band_tot.dat"), band_file)

    bands = readdlm(band_file)[:, 2:end]
    return fermi_level, bands
end

function read_bands(fc::FitContext, phase::Symbol, param_hash::String)
    @assert phase in [:FCC, :BCC]
    phase = String(phase)

    pot_dir = joinpath(fc.output_dir, "pot_" * param_hash)
    pot_file = joinpath(pot_dir, "pot.json")
    pot_dict = JSON.parsefile(pot_file)
    fermi_level = get(pot_dict, "fermi_level", missing)
    if fermi_level !== missing
        fermi_level = get(fermi_level, phase, missing)
    end

    band_file = joinpath(pot_dir, "BANDS_$(phase).dat")
    bands = readdlm(band_file)[:, 2:end]

    return fermi_level, bands
end

function read_FHI_bands(fc::FitContext, phase::Symbol)
    @assert phase in [:FCC, :BCC]
    index = phase == :FCC ? 1 : 2
    phase = lowercase(String(phase))
    dict = JSON.parsefile(joinpath(fc.base_dir, "FCC_BCC_BANDS.json"))
    fermi_level = dict["Bands"]["fermi"][phase]
    bands = hcat(dict["Bands"][phase]["TB_FHIaims"][index]...)'
    # undo shift to put back in absolute terms
    bands = bands .+ fermi_level
    return fermi_level, bands
end

kB = 8.6173303e-5
T = 100.0

fermi_dirac(x, μ, T) = 1. / (1 + exp((x - μ) / (kB * T)))

function band_error(fermi_level, bands, fermi_level_ref, bands_ref)
    error_k =  [sum(abs.((bands[n, :] .* fermi_dirac.(bands[n, :], fermi_level_ref, T) -
                          bands_ref[n, :] .* fermi_dirac.(bands_ref[n, :], fermi_level_ref, T))))
                          for n = 1:size(bands, 1)]
    RMSE = sqrt(sum(error_k .^ 2)/length(error_k))
    return RMSE
end

function compute_band_errors(fc::FitContext, param_hash::String)
    pot_file = joinpath(fc.output_dir, "pot_" * param_hash, "pot.json")
    pot_dict = JSON.parsefile(pot_file)

    fcc_fermi_level, fcc_bands = predict_bands(fc, :FCC, param_hash)
    bcc_fermi_level, bcc_bands = predict_bands(fc, :BCC, param_hash)

    fcc_fermi_level_FHI, fcc_bands_FHI = read_FHI_bands(fc, :FCC)
    bcc_fermi_level_FHI, bcc_bands_FHI = read_FHI_bands(fc, :BCC)

    RMSE_FCC = band_error(fcc_fermi_level, fcc_bands, fcc_fermi_level_FHI, fcc_bands_FHI)
    RMSE_BCC = band_error(bcc_fermi_level, bcc_bands, bcc_fermi_level_FHI, bcc_bands_FHI)

    band_error_dict = Dict("FCC" => RMSE_FCC,
                           "BCC" => RMSE_BCC)

    # save results in potential dict
    pot_dict["band_error"] = band_error_dict
    open(pot_file, "w") do json_file
        JSON.print(json_file, pot_dict, 4)
    end

    return band_error_dict
end

export fit_model, load_model, FitContext, read_bands, params_to_string, read_FHI_bands, test_error

end


module PATHSolver

using ForwardDiff
using SparseArrays
using Random
using LinearAlgebra

const depsfile = joinpath(@__DIR__, "..", "deps", "deps.jl")
if isfile(depsfile)
    include(depsfile)
else
    error("PATHSolver not properly installed. Please re-build PATHSolver.")
end

export solveMCP, solveLCP, options

const cached_J = [convert(SparseMatrixCSC{Cdouble, Cint}, zeros(0, 0))]
const cached_J_filled = Ref(false)

const status =
   [  :Solved,                          # 1 - solved
      :StationaryPointFound,            # 2 - stationary point found
      :MajorIterationLimit,             # 3 - major iteration limit
      :CumulativeMinorIterationLimit,   # 4 - cumulative minor iteration limit
      :TimeLimit,                       # 5 - time limit
      :UserInterrupt,                   # 6 - user interrupt
      :BoundError,                      # 7 - bound error (lb is not less than ub)
      :DomainError,                     # 8 - domain error (could not find a starting point)
      :InternalError                    # 9 - internal error
  ]

count_nonzeros(M::AbstractSparseMatrix) = nnz(M)
count_nonzeros(M::AbstractMatrix) = count(x -> x != 0, M) # fallback for dense matrices

###############################################################################
# wrappers for callback functions
###############################################################################
# static int (*f_eval)(int n, double *z, double *f);
# static int (*j_eval)(int n, int nnz, double *z, int *col_start, int *col_len,
      # int *row, double *data);
function f_user_wrap(n::Cint, z_ptr::Ptr{Cdouble}, f_ptr::Ptr{Cdouble})
  z = unsafe_wrap(Array{Cdouble}, z_ptr, Int(n), own=false)
  f = unsafe_wrap(Array{Cdouble}, f_ptr, Int(n), own=false)
  f .= user_f(z)
  return Cint(0)
end

function j_user_wrap(n::Cint, expected_nnz::Cint, z_ptr::Ptr{Cdouble},
                     col_start_ptr::Ptr{Cint}, col_len_ptr::Ptr{Cint},
                     row_ptr::Ptr{Cint}, data_ptr::Ptr{Cdouble})

  z = unsafe_wrap(Array{Cdouble}, z_ptr, Int(n), own=false)
  J::SparseMatrixCSC{Cdouble, Cint} = user_j(z)
  if nnz(J) > expected_nnz
    println("nnz(J) = ", nnz(J))
    println("expected_nnz = ", expected_nnz)
    error("Evaluated jacobian has more nonzero entries than were initially provided in solveMCP(). Try solveMCP(..., nnz=n^2).")
  end
  load_sparse_matrix(J, n, expected_nnz, col_start_ptr, col_len_ptr, row_ptr, data_ptr)
  return Cint(0)
end

function cached_j_user_wrap(n::Cint, expected_nnz::Cint, z_ptr::Ptr{Cdouble},
                     col_start_ptr::Ptr{Cint}, col_len_ptr::Ptr{Cint},
                     row_ptr::Ptr{Cint}, data_ptr::Ptr{Cdouble})
  if !(cached_J_filled[])
    load_sparse_matrix(cached_J[], n, expected_nnz, col_start_ptr, col_len_ptr, row_ptr, data_ptr)
    cached_J_filled[] = true
  end
  return Cint(0)
end

function load_sparse_matrix(J::SparseMatrixCSC, n::Cint, expected_nnz::Cint,
                            col_start_ptr::Ptr{Cint}, col_len_ptr::Ptr{Cint},
                            row_ptr::Ptr{Cint}, data_ptr::Ptr{Cdouble})
  # Transfer data from the computed jacobian into the sparse format that PATH
  # expects. Fortunately, PATH uses a compressed-sparse-column storage which
  # is compatible with Julia's default SparseMatrixCSC format.

  # col_start in PATH corresponds to J.colptr[1:end-1]
  col_start = unsafe_wrap(Array{Cint}, col_start_ptr, Int(n), own=false)
  # col_len in PATH corresponds to diff(J.colptr)
  col_len = unsafe_wrap(Array{Cint}, col_len_ptr, Int(n), own=false)
  # row in PATH corresponds to rowvals(J)
  row = unsafe_wrap(Array{Cint}, row_ptr, Int(expected_nnz), own=false)
  # data in PATH corresponds to nonzeros(J)
  data = unsafe_wrap(Array{Cdouble}, data_ptr, Int(expected_nnz), own=false)

  @inbounds for i in 1:n
    col_start[i] = J.colptr[i]
    col_len[i] = J.colptr[i + 1] - J.colptr[i]
  end

  rv = rowvals(J)
  nz = nonzeros(J)
  num_nonzeros = nnz(J)
  @inbounds for i in 1:num_nonzeros
    row[i] = rv[i]
    data[i] = nz[i]
  end
end

###############################################################################
# solveMCP
###############################################################################

# solveMCP without z0, without j_eval
function solveMCP(f_eval::Function,
                  lb::AbstractVector{T}, ub::AbstractVector{T},
                  var_name::AbstractVector{S}=String[],
                  con_name::AbstractVector{S}=String[];
                  nnz=-1) where {T <: Number, S <: String}

  z0 = (lb + ub) ./ 2
  return solveMCP(f_eval, lb, ub, z0, var_name, con_name; nnz=nnz)
end

# solveMCP without z0, with j_eval
function solveMCP(f_eval::Function, j_eval::Function,
                  lb::AbstractVector{T}, ub::AbstractVector{T},
                  var_name::AbstractVector{S}=String[],
                  con_name::AbstractVector{S}=String[];
                  nnz=-1) where {T <: Number, S <: String}

  z0 = (lb + ub) ./ 2
  return solveMCP(f_eval, j_eval, lb, ub, z0, var_name, con_name; nnz=nnz)
end

# solveMCP with z0, without j_eval
function solveMCP(f_eval::Function,
                  lb::AbstractVector{T}, ub::AbstractVector{T}, z0::AbstractVector{T},
                  var_name::AbstractVector{S}=String[],
                  con_name::AbstractVector{S}=String[];
                  nnz=-1) where {T <: Number, S <: String}

  j_eval = x -> ForwardDiff.jacobian(f_eval, x)
  return solveMCP(f_eval, j_eval, lb, ub, z0, var_name, con_name; nnz=nnz)
end

# Full implementation of solveMCP  / solveMCP with z0, with j_eval
function solveMCP(f_eval::Function, j_eval::Function,
                  lb::AbstractVector{T}, ub::AbstractVector{T}, z0::AbstractVector{T},
                  var_name::AbstractVector{S}=String[],
                  con_name::AbstractVector{S}=String[];
                  nnz=-1) where {T <: Number, S <: String}

  if length(var_name)==0
    var_name = C_NULL
  end

  if length(con_name)==0
    con_name = C_NULL
  end

  global user_f = f_eval
  global user_j = j_eval
  f_user_cb = @cfunction(f_user_wrap, Cint, (Cint, Ptr{Cdouble}, Ptr{Cdouble}))
  j_user_cb = @cfunction(j_user_wrap, Cint, (Cint, Cint, Ptr{Cdouble}, Ptr{Cint}, Ptr{Cint}, Ptr{Cint}, Ptr{Cdouble}))

  n = length(lb)
  z = max.(lb, min.(ub, z0))
  f = zeros(n)

  if nnz == -1
    # overestimating number of nonzeros in Jacobian
    nnz = max( count_nonzeros(j_eval(lb)), count_nonzeros(j_eval(ub)) )
    nnz = max( nnz, count_nonzeros(j_eval(z)) )
    for i in 1:2
      z_rand = max.(lb, min.(ub, rand(Float64, n)))
      nnz_rand = count_nonzeros(j_eval(z_rand))
      nnz = max( nnz, nnz_rand )
    end
    nnz = min( 2 * nnz, n * n )
  end

  t = ccall( (:path_main, libpath47julia), Cint,
          (Cint, Cint,
           Ptr{Cdouble}, Ptr{Cdouble},
           Ptr{Cdouble}, Ptr{Cdouble},
           Ptr{Ptr{Cchar}}, Ptr{Ptr{Cchar}},
           Ptr{Cvoid}, Ptr{Cvoid}),
           n, nnz, z, f, lb, ub, var_name, con_name, f_user_cb, j_user_cb)

  remove_option_file()
  return status[t], z, f
end

###############################################################################
# solveLCP, LCP functions
###############################################################################

# solveLCP without z, without M
function solveLCP(f_eval::Function,
                  lb::AbstractVector{T}, ub::AbstractVector{T},
                  var_name::AbstractVector{S}=String[],
                  con_name::AbstractVector{S}=String[];
                  lcp_check=false) where {T <: Number, S <: String}

    return solveLCP(f_eval, lb, ub, copy(lb), var_name, con_name, lcp_check=lcp_check)
end

# solveLCP with z0, without M
function solveLCP(f_eval::Function,
                  lb::AbstractVector{T}, ub::AbstractVector{T}, z0::AbstractVector{T},
                  var_name::AbstractVector{S}=String[],
                  con_name::AbstractVector{S}=String[];
                  lcp_check=false) where {T <: Number, S <: String}

  J = ForwardDiff.jacobian(f_eval, lb)
  if lcp_check
      Jr = ForwardDiff.jacobian(f_eval, rand(Float64, size(lb)))
      if opnorm(J-Jr, 1) > 1e-8
          error("The problem does not seem linear. Use `solveMCP()` instead.")
      end
  end

  return solveLCP(f_eval, J, lb, ub, z0, var_name, con_name)
end

# solveLCP without z, with M
function solveLCP(f_eval::Function, M::AbstractMatrix,
                  lb::AbstractVector{T}, ub::AbstractVector{T},
                  var_name::AbstractVector{S}=String[],
                  con_name::AbstractVector{S}=String[];
                  lcp_check=false) where {T <: Number, S <: String}

    return solveLCP(f_eval, M, lb, ub, copy(lb), var_name, con_name, lcp_check=lcp_check)
end

# Full implmentation of solveLCP / solveLCP with z0, with M
function solveLCP(f_eval::Function, M::AbstractMatrix,
                  lb::AbstractVector{T}, ub::AbstractVector{T}, z0::AbstractVector{T},
                  var_name::AbstractVector{S}=String[],
                  con_name::AbstractVector{S}=String[];
                  lcp_check=false) where {T <: Number, S <: String}

  if length(var_name)==0
      var_name = C_NULL
  end

  if length(con_name)==0
      con_name = C_NULL
  end

  if lcp_check
      J = ForwardDiff.jacobian(f_eval, lb)
      if opnorm(J-M, 1) > 1e-8
          # warn("The user supplied Jacobian does not match with the result by FowardDiff.jacobian(). It proceeds with the user supplied Jacobian.")
          error("The user supplied Jacobian does not match with the result by FowardDiff.jacobian().")
      end
  end

  global user_f = f_eval
  cached_J[] = M
  cached_J_filled[] = false
  f_user_cb = @cfunction(f_user_wrap, Cint, (Cint, Ptr{Cdouble}, Ptr{Cdouble}))
  j_user_cb = @cfunction(cached_j_user_wrap, Cint, (Cint, Cint, Ptr{Cdouble}, Ptr{Cint}, Ptr{Cint}, Ptr{Cint}, Ptr{Cdouble}))

  n = length(lb)
  z = max.(lb, min.(ub, z0))
  f = zeros(n)

  nnzs = count_nonzeros(M)

  t = ccall( (:path_main, libpath47julia), Cint,
          (Cint, Cint,
           Ptr{Cdouble}, Ptr{Cdouble},
           Ptr{Cdouble}, Ptr{Cdouble},
           Ptr{Ptr{Cchar}}, Ptr{Ptr{Cchar}},
           Ptr{Cvoid}, Ptr{Cvoid}),
           n, nnzs, z, f, lb, ub, var_name, con_name, f_user_cb, j_user_cb)

  remove_option_file()
  return status[t], z, f
end

###############################################################################
# handling PATH options
###############################################################################

function remove_option_file()
  if isfile("path.opt")
    rm("path.opt")
  end
end

function options(;kwargs...)
  opt_file = open("path.opt", "w")
  println(opt_file, "* Generated by PATHSolver.jl. Do not edit.")
  for (key, value) in kwargs
    println(opt_file, key, " ", value)
  end
  close(opt_file)
end

end # Module


module LinOps

include("banded.jl")
include("dense.jl")
include("cuda.jl")
include("linop.jl")

end # module


using Test
import ParallelStencil
using ParallelStencil.ParallelKernel
import ParallelStencil.ParallelKernel: @reset_parallel_kernel, @is_initialized, SUPPORTED_PACKAGES, PKG_CUDA, PKG_THREADS
import ParallelStencil.ParallelKernel: @require, longnameof, @prettyexpand, prettystring
import ParallelStencil.ParallelKernel: checkargs_parallel, checkargs_parallel_indices, parallel, parallel_indices, synchronize
using ParallelStencil.ParallelKernel.Exceptions
@static if (PKG_CUDA in SUPPORTED_PACKAGES) import CUDA end

@static for package in SUPPORTED_PACKAGES  eval(:(
    @testset "$(basename(@__FILE__)) (package: $(nameof($package)))" begin
        @testset "1. parallel macros" begin
            @init_parallel_kernel($package, Float64)
            @require @is_initialized()
            @testset "@parallel" begin
                @static if $package == PKG_CUDA
                    call = prettystring(parallel(:(f(A))))
                    @test occursin("CUDA.@cuda blocks = ParallelStencil.ParallelKernel.compute_nblocks(length.(ParallelStencil.ParallelKernel.promote_ranges(ParallelStencil.ParallelKernel.get_ranges(A))), ParallelStencil.ParallelKernel.compute_nthreads(length.(ParallelStencil.ParallelKernel.promote_ranges(ParallelStencil.ParallelKernel.get_ranges(A))))) threads = ParallelStencil.ParallelKernel.compute_nthreads(length.(ParallelStencil.ParallelKernel.promote_ranges(ParallelStencil.ParallelKernel.get_ranges(A)))) f(A, ParallelStencil.ParallelKernel.promote_ranges(ParallelStencil.ParallelKernel.get_ranges(A)))", call)
                    @test occursin("CUDA.synchronize()", call)
                    call = prettystring(parallel(:ranges, :(f(A))))
                    @test occursin("CUDA.@cuda blocks = ParallelStencil.ParallelKernel.compute_nblocks(length.(ParallelStencil.ParallelKernel.promote_ranges(ranges)), ParallelStencil.ParallelKernel.compute_nthreads(length.(ParallelStencil.ParallelKernel.promote_ranges(ranges)))) threads = ParallelStencil.ParallelKernel.compute_nthreads(length.(ParallelStencil.ParallelKernel.promote_ranges(ranges))) f(A, ParallelStencil.ParallelKernel.promote_ranges(ranges))", call)
                    call = prettystring(parallel(:nblocks, :nthreads, :(f(A))))
                    @test occursin("CUDA.@cuda blocks = nblocks threads = nthreads f(A, ParallelStencil.ParallelKernel.promote_ranges(ParallelStencil.ParallelKernel.compute_ranges(nblocks .* nthreads)))", call)
                    call = prettystring(parallel(:ranges, :nblocks, :nthreads, :(f(A))))
                    @test occursin("CUDA.@cuda blocks = nblocks threads = nthreads f(A, ParallelStencil.ParallelKernel.promote_ranges(ranges))", call)
                    call = prettystring(parallel(:nblocks, :nthreads, :(stream=mystream), :(f(A))))
                    @test occursin("CUDA.@cuda blocks = nblocks threads = nthreads stream = mystream f(A, ParallelStencil.ParallelKernel.promote_ranges(ParallelStencil.ParallelKernel.compute_ranges(nblocks .* nthreads)))", call)
                elseif $package == PKG_THREADS
                    @test prettystring(parallel(:(f(A)))) == "f(A, ParallelStencil.ParallelKernel.promote_ranges(ParallelStencil.ParallelKernel.get_ranges(A)))"
                    @test prettystring(parallel(:ranges, :(f(A)))) == "f(A, ParallelStencil.ParallelKernel.promote_ranges(ranges))"
                    @test prettystring(parallel(:nblocks, :nthreads, :(f(A)))) == "f(A, ParallelStencil.ParallelKernel.promote_ranges(ParallelStencil.ParallelKernel.compute_ranges(nblocks .* nthreads)))"
                    @test prettystring(parallel(:ranges, :nblocks, :nthreads, :(f(A)))) == "f(A, ParallelStencil.ParallelKernel.promote_ranges(ranges))"
                    @test prettystring(parallel(:(stream=mystream), :(f(A)))) == "f(A, ParallelStencil.ParallelKernel.promote_ranges(ParallelStencil.ParallelKernel.get_ranges(A)))"
                end;
            end;
            @testset "@parallel_indices (1D)" begin
                A  = @zeros(4)
                @parallel_indices (ix) function write_indices!(A)
                    A[ix] = ix;
                    return
                end
                @parallel write_indices!(A);
                @test all(Array(A) .== [ix for ix=1:size(A,1)])
            end;
            @testset "@parallel_indices (2D)" begin
                A  = @zeros(4, 5)
                @parallel_indices (ix,iy) function write_indices!(A)
                    A[ix,iy] = ix + (iy-1)*size(A,1);
                    return
                end
                @parallel write_indices!(A);
                @test all(Array(A) .== [ix + (iy-1)*size(A,1) for ix=1:size(A,1), iy=1:size(A,2)])
            end;
            @testset "@parallel_indices (3D)" begin
                A  = @zeros(4, 5, 6)
                @parallel_indices (ix,iy,iz) function write_indices!(A)
                    A[ix,iy,iz] = ix + (iy-1)*size(A,1) + (iz-1)*size(A,1)*size(A,2);
                    return
                end
                @parallel write_indices!(A);
                @test all(Array(A) .== [ix + (iy-1)*size(A,1) + (iz-1)*size(A,1)*size(A,2) for ix=1:size(A,1), iy=1:size(A,2), iz=1:size(A,3)])
            end;
            @testset "@parallel_indices (1D in 3D)" begin
                A  = @zeros(4, 5, 6)
                @parallel_indices (iy) function write_indices!(A)
                    A[1,iy,1] = iy;
                    return
                end
                @parallel 1:size(A,2) write_indices!(A);
                @test all(Array(A)[1,:,1] .== [iy for iy=1:size(A,2)])
            end;
            @testset "@parallel_indices (2D in 3D)" begin
                A  = @zeros(4, 5, 6)
                @parallel_indices (ix,iz) function write_indices!(A)
                    A[ix,end,iz] = ix + (iz-1)*size(A,1);
                    return
                end
                @parallel (1:size(A,1), 1:size(A,3)) write_indices!(A);
                @test all(Array(A)[:,end,:] .== [ix + (iz-1)*size(A,1) for ix=1:size(A,1), iz=1:size(A,3)])
            end;
            @testset "@parallel_async" begin
                @static if $package == PKG_CUDA
                    call = prettystring(parallel(:(f(A)); async=true))
                    @test !occursin("CUDA.synchronize()", call)
                end;
            end;
            @testset "@synchronize" begin
                @static if $package == PKG_CUDA
                    @test prettystring(synchronize()) == "CUDA.synchronize()"
                end;
            end;
            @reset_parallel_kernel()
        end;
        @testset "2. Exceptions" begin
            @init_parallel_kernel($package, Float64)
            @require @is_initialized
            @testset "arguments @parallel" begin
                @test_throws ArgumentError checkargs_parallel();                                                    # Error: isempty(args)
                @test_throws ArgumentError checkargs_parallel(:(f()), :(something));                                # Error: last arg is not function call.
                @test_throws ArgumentError checkargs_parallel(:(f()=99));                                           # Error: last arg is not function call.
                @test_throws ArgumentError checkargs_parallel(:ranges, :nblocks, :nthreads, :something, :(f()));    # Error: length(posargs) > 3
                @test_throws KeywordArgumentError checkargs_parallel(:(blocks=blocks), :(f()));                     # Error: blocks keyword argument is not allowed
                @test_throws KeywordArgumentError checkargs_parallel(:(threads=threads), :(f()));                   # Error: threads keyword argument is not allowed
            end;
            @testset "arguments @parallel_indices" begin
                @test_throws ArgumentError checkargs_parallel_indices();                                            # Error: length(args) != 2
                @test_throws ArgumentError checkargs_parallel_indices(:(f()=99));                                   # Error: length(args) != 2
                @test_throws ArgumentError checkargs_parallel_indices(:((ix,iy,iz)), :(f()=99), :(something));      # Error: length(args) != 2
                @test_throws ArgumentError checkargs_parallel_indices(:ix, :iy, :iz, :(f()=99));                    # Error: length(args) != 2
                @test_throws ArgumentError checkargs_parallel_indices(:(f()=99), :((ix,iy,iz)));                    # Error: last arg is not function.
                @test_throws ArgumentError checkargs_parallel_indices(:((ix,iy,iz)), :(f()));                       # Error: last arg is not function.
                @test_throws ArgumentError parallel_indices(:((ix,iy,iz)), :(f()=99))                               # Error: no return statement in function.
                @test_throws ArgumentError parallel_indices(:((ix,iy,iz)), :(f()=(99; return something)))           # Error: function does not return nothing.
                @test_throws ArgumentError parallel_indices(:((ix,iy,iz)), :(f()=(99; if x return y end; return)))  # Error: function contains more than one return statement.
            end;
            @reset_parallel_kernel()
        end;
    end;
)) end == nothing || true;


using BVHFiles
using Test

@testset "interpolation" begin
    println("Starting test of 'interpolation'")

    g = load("Example.bvh") |>
        interpolate!(1)

    @test frames(g) == 481

    g = load("Example.bvh") |>
    interpolate!(2)

    @test frames(g) == 721

    g = load("Example.bvh") |>
    interpolate!(3)

    @test frames(g) == 961
end


abstract type AbstractHashNode{T, N, p} <: AbstractNode{T, N, p} end

Base.IndexStyle(::Type{<: AbstractHashNode}) = IndexLinear()

@inline function Base.getindex(x::AbstractHashNode, i::Int)
    @boundscheck checkbounds(x, i)
    isnull(x) && return null(childtype(x))
    # data can haskey even if the mask is false
    # so trying to return stored data
    # in this case, need to handle stored data very carefully because
    # we can change its data unexpectedly even if the mask is false
    get(x.data, i, null(childtype(x)))
end

@inline function Base.setindex!(x::AbstractHashNode, v, i::Int)
    @boundscheck checkbounds(x, i)
    isnull(v) && throw(ArgumentError("trying to set null node, call `delete!(node, i)` instead"))
    @inbounds x.data[i] = v
    x
end

@inline function Base.setindex!(x::AbstractHashNode, ::Nothing, i::Int)
    @boundscheck checkbounds(x, i)
    @inbounds begin
        if isactive(x, i)
            x.data[i] = nothing # make `mask` `false` at `i`
            deactivate!(unsafe_getindex(x, i))
        end
    end
    x
end

@inline function deactivate!(x::AbstractHashNode)
    isnull(x) && return x
    fill!(getmask(x), false)
    foreach(deactivate!, values(x.data))
    x
end

function Base.delete!(x::AbstractHashNode, i)
    @boundscheck checkbounds(x, i)
    @inbounds begin
        # only this case allows to set null node in `x`
        x.data[i] = nothing # make mask false
        delete!(x.data, i) # not set null, just delete it
    end
    x
end

function allocate!(x::AbstractHashNode{T}, i::Int) where {T}
    @boundscheck checkbounds(x, i)
    @inbounds begin
        if haskey(x.data, i)
            childnode = unsafe_getindex(x, i)
        else
            childnode = T()
        end
        x[i] = childnode # activated in setindex!
    end
    Activated(x, i)
end

isallocated(x::AbstractHashNode, i::Int) = haskey(x.data, i)

struct HashNode{T <: AbstractNode, N, p} <: AbstractHashNode{T, N, p}
    data::MaskedHashArray{T, N}
    HashNode{T, N, p}(data) where {T, N, p} = new(data)
    HashNode{T, N, p}(::UndefInitializer) where {T, N, p} = new()
end

function HashNode{T, N, p}() where {T, N, p}
    dims = size(HashNode{T, N, p})
    data = MaskedHashArray{T}(undef, dims)
    HashNode{T, N, p}(data)
end

Base.size(x::HashNode) = size(typeof(x))

struct DynamicHashNode{T <: AbstractNode, N} <: AbstractHashNode{T, N, Dynamic()}
    data::MaskedHashArray{T, N}
    dims::NTuple{N, Int}
    DynamicHashNode{T, N}(data, dims) where {T, N} = new(data, dims)
    DynamicHashNode{T, N}(::UndefInitializer) where {T, N} = new()
end

function DynamicHashNode{T, N}(dims::Int...) where {T, N}
    data = MaskedHashArray{T}(undef, dims)
    DynamicHashNode{T, N}(data, dims)
end

Base.size(x::DynamicHashNode) = x.dims


# This file is a part of Julia. License is MIT: https://julialang.org/license

types = Any[
    Bool,
    Int8, UInt8, Int16, UInt16, Int32, UInt32, Int64, UInt64, Float32, Float64,
    Rational{Int8}, Rational{UInt8}, Rational{Int16}, Rational{UInt16},
    Rational{Int32}, Rational{UInt32}, Rational{Int64}, Rational{UInt64}
]
vals = vcat(
    typemin(Int64),
    -Int64(maxintfloat(Float64)) .+ Int64[-4:1;],
    typemin(Int32),
    -Integer(maxintfloat(Float32)) .+ (-4:1),
    -2:2,
    Integer(maxintfloat(Float32)) .+ (-1:4),
    typemax(Int32),
    Int64(maxintfloat(Float64)) .+ Int64[-1:4;],
    typemax(Int64),
)

function coerce(T::Type, x)
    if T<:Rational
        convert(T, coerce(typeof(numerator(zero(T))), x))
    elseif !(T<:Integer)
        convert(T, x)
    else
        x % T
    end
end

for T = types[2:end],
    x = vals,
    a = coerce(T, x)
    @test hash(a,zero(UInt)) == invoke(hash, Tuple{Real, UInt}, a, zero(UInt))
end

for T = types,
    S = types,
    x = vals,
    a = coerce(T, x),
    b = coerce(S, x)
    #println("$(typeof(a)) $a")
    #println("$(typeof(b)) $b")
    @test isequal(a,b) == (hash(a)==hash(b))
    # for y=vals
    #     println("T=$T; S=$S; x=$x; y=$y")
    #     c = convert(T,x//y)
    #     d = convert(S,x//y)
    #     @test !isequal(a,b) || hash(a)==hash(b)
    # end
end

# issue #8619
@test hash(nextfloat(2.0^63)) == hash(UInt64(nextfloat(2.0^63)))
@test hash(prevfloat(2.0^64)) == hash(UInt64(prevfloat(2.0^64)))

# issue #9264
@test hash(1//6,zero(UInt)) == invoke(hash, Tuple{Real, UInt}, 1//6, zero(UInt))
@test hash(1//6) == hash(big(1)//big(6))
@test hash(1//6) == hash(0x01//0x06)

# hashing collections (e.g. issue #6870)
vals = Any[
    [1,2,3,4], [1 3;2 4], Any[1,2,3,4], [1,3,2,4],
    [1.0, 2.0, 3.0, 4.0], BigInt[1, 2, 3, 4],
    [1,0], [true,false], BitArray([true,false]),
    # Irrationals
    Any[1, pi], [1, pi], [pi, pi], Any[pi, pi],
    # Overflow with Int8
    Any[Int8(127), Int8(-128), -383], 127:-255:-383,
    # Loss of precision with Float64
    Any[-2^53-1, 0.0, 2^53+1], [-2^53-1, 0, 2^53+1], (-2^53-1):2^53+1:(2^53+1),
    Set([1,2,3,4]),
    Set([1:10;]),                # these lead to different key orders
    Set([7,9,4,10,2,3,5,8,6,1]), #
    Dict(42 => 101, 77 => 93), Dict{Any,Any}(42 => 101, 77 => 93),
    (1,2,3,4), (1.0,2.0,3.0,4.0), (1,3,2,4),
    ("a","b"), (SubString("a",1,1), SubString("b",1,1)),
    # issue #6900
    Dict(x => x for x in 1:10),
    Dict(7=>7,9=>9,4=>4,10=>10,2=>2,3=>3,8=>8,5=>5,6=>6,1=>1),
    [], [1], [2], [1, 1], [1, 2], [1, 3], [2, 2], [1, 2, 2], [1, 3, 3],
    zeros(2, 2), spzeros(2, 2), Matrix(1.0I, 2, 2), sparse(1.0I, 2, 2),
    sparse(ones(2, 2)), ones(2, 2), sparse([0 0; 1 0]), [0 0; 1 0],
    [-0. 0; -0. 0.], SparseMatrixCSC(2, 2, [1, 3, 3], [1, 2], [-0., -0.]),
    # issue #16364
    1:4, 1:1:4, 1:-1:0, 1.0:4.0, 1.0:1.0:4.0, linspace(1, 4, 4),
    'a':'e', ['a', 'b', 'c', 'd', 'e'],
    # check that hash is still consistent with heterogeneous arrays for which - is defined
    # for some pairs and not others
    ["a", "b", 1, 2], ["a", 1, 2], ["a", "b", 2, 2], ["a", "a", 1, 2], ["a", "b", 2, 3]
]

for a in vals, b in vals
    @test isequal(a,b) == (hash(a)==hash(b))
    if a isa AbstractArray
        @test hash(a) == hash(collect(a)) == hash(collect(Any, a))
    end
end

vals = Any[
    Int[], Float64[],
    [0], [1], [2],
    # test vectors starting with ranges
    [1, 2], [1, 2, 3], [1, 2, 3, 4], [1, 2, 3, 4, 5], [1, 2, 3, 4, 5, 6],
    [2, 1], [3, 2, 1], [4, 3, 2, 1], [5, 4, 3, 2, 1], [5, 4, 3, 2, 1, 0, -1],
    # test vectors starting with ranges which trigger overflow with Int8
    [124, 125, 126, 127], [124, 125, 126, 127, -128], [-128, 127, -128],
    # test vectors including ranges
    [2, 1, 2, 3], [2, 3, 2, 1], [2, 1, 2, 3, 2], [2, 3, 2, 1, 2],
    # test various sparsity patterns
    [0, 0], [0, 0, 0], [0, 1], [1, 0],
    [0, 0, 1], [0, 1, 0], [1, 0, 0], [0, 1, 2],
    [0 0; 0 0], [1 0; 0 0], [0 1; 0 0], [0 0; 1 0], [0 0; 0 1],
    [5 1; 0 0], [1 1; 0 1], [0 2; 3 0], [0 2; 4 6], [4 0; 0 1],
    [0 0 0; 0 0 0], [1 0 0; 0 0 1], [0 0 2; 3 0 0], [0 0 7; 6 1 2],
    [4 0 0; 3 0 1], [0 2 4; 6 0 0],
    # various stored zeros patterns
    sparse([1], [1], [0]), sparse([1], [1], [-0.0]),
    sparse([1, 2], [1, 1], [-0.0, 0.0]), sparse([1, 2], [1, 1], [0.0, -0.0]),
    sparse([1, 2], [1, 1], [-0.0, 0.0], 3, 1), sparse([1, 2], [1, 1], [0.0, -0.0], 3, 1),
    sparse([1, 3], [1, 1], [-0.0, 0.0], 3, 1), sparse([1, 3], [1, 1], [0.0, -0.0], 3, 1),
    sparse([1, 2, 3], [1, 1, 1], [-1, 0, 1], 3, 1), sparse([1, 2, 3], [1, 1, 1], [-1.0, -0.0, 1.0], 3, 1),
    sparse([1, 3], [1, 1], [-1, 0], 3, 1), sparse([1, 2], [1, 1], [-1, 0], 3, 1)
]

for a in vals
    b = Array(a)
    @test hash(convert(Array{Any}, a)) == hash(b)
    @test hash(convert(Array{supertype(eltype(a))}, a)) == hash(b)
    @test hash(convert(Array{Float64}, a)) == hash(b)
    @test hash(sparse(a)) == hash(b)
    if !any(x -> isequal(x, -0.0), a)
        @test hash(convert(Array{Int}, a)) == hash(b)
        if all(x -> typemin(Int8) <= x <= typemax(Int8), a)
            @test hash(convert(Array{Int8}, a)) == hash(b)
        end
    end
end

# Test that overflow does not give inconsistent hashes with heterogeneous arrays
@test hash(Any[Int8(1), Int8(2), 255]) == hash([1, 2, 255])
@test hash(Any[Int8(127), Int8(-128), 129, 130]) ==
    hash([127, -128, 129, 130]) != hash([127,  128, 129, 130])

# Test hashing sparse matrix with type which does not support -
struct CustomHashReal
    x::Float64
end
Base.hash(x::CustomHashReal, h::UInt) = hash(x.x, h)
Base.:(==)(x::CustomHashReal, y::Number) = x.x == y
Base.:(==)(x::Number, y::CustomHashReal) = x == y.x
Base.zero(::Type{CustomHashReal}) = CustomHashReal(0.0)

let a = sparse([CustomHashReal(0), CustomHashReal(3), CustomHashReal(3)])
    @test hash(a) == hash(Array(a))
end

vals = Any[
    0.0:0.1:0.3, 0.3:-0.1:0.0,
    0:-1:1, 0.0:-1.0:1.0, 0.0:1.1:10.0, -4:10,
    'a':'e', 'b':'a',
    linspace(1, 1, 1), linspace(0.3, 1.0, 3),  linspace(1, 1.1, 20)
]

for a in vals
    @test hash(collect(a)) == hash(a)
end

@test hash(SubString("--hello--",3,7)) == hash("hello")
@test hash(:(X.x)) == hash(:(X.x))
@test hash(:(X.x)) != hash(:(X.y))

@test hash([1,2]) == hash(view([1,2,3,4],1:2))

let a = QuoteNode(1), b = QuoteNode(1.0)
    @test (hash(a)==hash(b)) == (a==b)
end

let a = Expr(:block, TypedSlot(1, Any)),
    b = Expr(:block, TypedSlot(1, Any)),
    c = Expr(:block, TypedSlot(3, Any))
    @test a == b && hash(a) == hash(b)
    @test a != c && hash(a) != hash(c)
    @test b != c && hash(b) != hash(c)
end

@test hash(Dict(),hash(Set())) != hash(Set(),hash(Dict()))

# issue 15659
for prec in [3, 11, 15, 16, 31, 32, 33, 63, 64, 65, 254, 255, 256, 257, 258, 1023, 1024, 1025],
    v in Any[-0.0, 0, 1, -1, 1//10, 2//10, 3//10, 1//2, pi]
    setprecision(prec) do
        x = convert(BigFloat, v)
        @test precision(x) == prec
        num, pow, den = Base.decompose(x)
        y = num*big(2.0)^pow/den
        @test precision(y) == prec
        @test isequal(x, y)
    end
end

# issue #20744
@test hash(:c, hash(:b, hash(:a))) != hash(:a, hash(:b, hash(:c)))

# issue #5849, object_id of types
@test Vector === (Array{T,1} where T)
@test (Pair{A,B} where A where B) !== (Pair{A,B} where B where A)
let vals_expr = :(Any[Vector, (Array{T,1} where T), 1, 2, Union{Int, String}, Union{String, Int},
                      (Union{String, T} where T), Ref{Ref{T} where T}, (Ref{Ref{T}} where T),
                      (Vector{T} where T<:Real), (Vector{T} where T<:Integer),
                      (Vector{T} where T>:Integer),
                      (Pair{A,B} where A where B), (Pair{A,B} where B where A)])
    vals_a = eval(vals_expr)
    vals_b = eval(vals_expr)
    for (i, a) in enumerate(vals_a), (j, b) in enumerate(vals_b)
        @test i != j || (a === b)
        @test (a === b) == (object_id(a) == object_id(b))
    end
end