export PathTracker,
    track!,
    track,
    PathTrackerStatus,
    is_at_infinity,
    is_success,
    is_tracking,
    is_failed,
    is_terminated_callback,
    is_invalid_startvalue,
    pathtracker,
    pathtracker_startsolutions,
    PathResult,
    solution,
    start_solution,
    accuracy,
    residual,
    winding_number,
    multiplicity,
    condition_jacobian,
    path_number,
    is_finite,
    is_singular,
    is_nonsingular,
    is_real,
    is_projective,
    is_affine

const pathtracker_supported_keywords = [
    :at_infinity_check,
    :endgame_start,
    :endgame_start_callback,
    :max_winding_number,
    :min_accuracy,
    :min_cond_eg,
    :min_step_size_before_eg,
    :min_step_size_eg,
    :s_always_consider_valuation,
    :samples_per_loop,
    :precision_strategy,
]

############
## STATUS ##
############

module PathTrackerStatus
"""
    enum PathTrackerStatus

The possible states a `PathTracker` can be in:

* `PathTrackerStatus.tracking`
* `PathTrackerStatus.success`
* `PathTrackerStatus.at_infinity`
* `PathTrackerStatus.excess_solution`
* `PathTrackerStatus.post_check_failed`
* `PathTrackerStatus.terminated_accuracy_limit`
* `PathTrackerStatus.terminated_ill_conditioned`
* `PathTrackerStatus.terminated_invalid_startvalue`
* `PathTrackerStatus.terminated_max_winding_number`
* `PathTrackerStatus.terminated_max_iters`
* `PathTrackerStatus.terminated_step_size_too_small`
"""
@enum states begin
    tracking
    success
    at_infinity
    terminated_accuracy_limit
    terminated_invalid_startvalue
    terminated_ill_conditioned
    terminated_max_iters
    terminated_max_winding_number
    terminated_step_size_too_small
    post_check_failed
    excess_solution
end
end

"""
    path_tracker_status(code::CoreTrackerStatus.states)

Construct a [`PathTrackerStatus.states`](@ref) from a [`CoreTrackerStatus.states`](@ref).
"""
function path_tracker_status(code::CoreTrackerStatus.states)
    if code == CoreTrackerStatus.success
        return PathTrackerStatus.success
    elseif code == CoreTrackerStatus.terminated_invalid_startvalue
        return PathTrackerStatus.terminated_invalid_startvalue
    elseif code == CoreTrackerStatus.terminated_maximal_iterations
        return PathTrackerStatus.terminated_max_iters
    elseif code == CoreTrackerStatus.terminated_step_size_too_small
        return PathTrackerStatus.terminated_step_size_too_small
    elseif code == CoreTrackerStatus.terminated_ill_conditioned
        return PathTrackerStatus.terminated_ill_conditioned
    else
        return PathTrackerStatus.tracking
    end
end

"""
    is_success(status::PathTrackerStatus.states)

Returns `true` if `status` indicates a success in tracking.
"""
is_success(status::PathTrackerStatus.states) = status == PathTrackerStatus.success

"""
    is_success(status::PathTrackerStatus.states)

Returns `true` if `status` indicates that a path diverged towards infinity.
"""
is_at_infinity(status::PathTrackerStatus.states) = status == PathTrackerStatus.at_infinity

"""
    is_tracking(status::PathTrackerStatus.states)

Returns `true` if `status` indicates the tracking is not going on.
"""
is_tracking(status::PathTrackerStatus.states) = status == PathTrackerStatus.tracking

"""
    is_invalid_startvalue(status::PathTrackerStatus.states)

Returns `true` if the provided start value was not valid.
"""
is_invalid_startvalue(status::PathTrackerStatus.states) =
    status == PathTrackerStatus.terminated_invalid_startvalue

"""
    is_terminated_callback(status::PathTrackerStatus.states)

Returns `true` if the provided callback indicated a termination of the path.
"""
is_terminated_callback(status::PathTrackerStatus.states) =
    status == PathTrackerStatus.terminated_callback

#############
## Options ##
#############
@enum PrecisionStrategy begin
    PREC_STRATEGY_NEVER
    PREC_STRATEGY_FINITE
    PREC_STRATEGY_ALWAYS
end

mutable struct PathTrackerOptions
    endgame_start::Float64
    at_infinity_check::Bool
    min_accuracy::Float64
    min_cond_eg::Float64
    min_step_size_before_eg::Float64
    min_step_size_eg::Float64
    precision_strategy::PrecisionStrategy
    s_always_consider_valuation::Float64
end

Base.show(io::IO, opts::PathTrackerOptions) = print_fieldnames(io, opts)
Base.show(io::IO, ::MIME"application/prs.juno.inline", opts::PathTrackerOptions) = opts

###########
## STATE ##
###########

mutable struct PathTrackerState{AV<:AbstractVector}
    status::PathTrackerStatus.states
    valuation::Valuation
    prediction::AV
    solution::AV
    intermediate_solution::AV
    solution_accuracy::Float64
    solution_cond::Float64
    solution_residual::Float64
    winding_number::Union{Nothing,Int}
    max_winding_number_hit::Bool
    s::Float64
    eg_started::Bool
end

function PathTrackerState(x::AbstractVector; at_infinity_check::Bool = true)
    status = PathTrackerStatus.tracking
    valuation = Valuation(x; affine = at_infinity_check)
    prediction = copy(x)
    solution = copy(x)
    intermediate_solution = copy(x)
    solution_accuracy = NaN
    solution_cond = NaN
    solution_residual = NaN
    winding_number = nothing
    max_winding_number_hit = false
    s = 0.0
    eg_started = false
    PathTrackerState(
        status,
        valuation,
        prediction,
        solution,
        intermediate_solution,
        solution_accuracy,
        solution_cond,
        solution_residual,
        winding_number,
        max_winding_number_hit,
        s,
        eg_started,
    )
end

Base.show(io::IO, S::PathTrackerState) = print_fieldnames(io, S)
Base.show(io::IO, ::MIME"application/prs.juno.inline", S::PathTrackerState) = S

function init!(state::PathTrackerState, s::Float64)
    state.status = PathTrackerStatus.tracking
    init!(state.valuation)
    state.prediction .= 0.0
    state.solution .= 0.0
    state.intermediate_solution .= NaN
    state.solution_accuracy = NaN
    state.solution_cond = NaN
    state.solution_residual = NaN
    state.winding_number = nothing
    state.max_winding_number_hit = false
    state.s = s
    state.eg_started = false
end

##################
## PATH TRACKER ##
##################

abstract type AbstractPathTracker end
Base.broadcastable(T::AbstractPathTracker) = Ref(T)

"""
    PathTracker

The `PathTracker` combines the path tracking (with [`CoreTracker`](@ref)) with an *endgame*
routine. The *endgame*  is the name for special algorithms the end of the path tracking.
These enable to detect if a path is diverging or if it ends in a singular solution.
The `PathTracker` is more opinionated than the `CoreTracker` and implements additional logic
to handle numerical difficult solutions. In particular it always reparametrizes a solution
path to use a logarithmic time scale, i.e., ``x(t) → x(e^{-s})`` and we track for ``s`` from
``0`` to ``∞``.

In order to construct a `PathTracker` it is recommended to use the [`pathtracker`](@ref) and
[`pathtracker_startsolutions`](@ref) helper functions.
With a `PathTracker` constructed you can track a single path using the [`track`](@ref) method.
The result of this will be a [`PathResult`](@ref).

The `PathTracker` assumes that the provided homotopy `H` is defined in such a way that
`H(x,1)` is the start system and `H(x,0)` the target system.
During the path tracking an approximation of the valuation of a Puiseux series expansion of
the solution is computed. This is used to decide whether a path is diverging.
Before a certain treshold (`s_always_consider_valuation`) this approximation is only trusted
if a path gives some sign of ill-conditioning.
To compute singular solutions the *Cauchy endgame* is used which is based on
Cauchy's integral formula. For this we have to track solutions along a loop around the origin.
The number of loops necessary to arrive back at the start point is called the *winding number*.

## Options

The path tracker accepts all options which [`CoreTracker`](@ref) accepts. Furthermore the
following options are accepted:

* `at_infinity_check`: This is true if the provided start system is an affine polynomial system.
* `endgame_start` (default `2.0`): The value of `s` where the endgame is started earliest.
* `max_winding_number` (default `12`): The maximal winding number tried in the Cauchy endgame.
* `min_accuracy` (default `1e-5`): The `PathTracker` automatically lowers the desired
  accuracy automatically to `min_accuracy` if path tracking would fail othwerwise (since
  the desired cannot be reached)
* `min_cond_eg` (default `1e5`): The minimal condition number before the Cauchy endgame is
  started or a path is cut off.
* `min_step_size_before_eg` (default `exp2(-40)`): The minimal allowed step size before the
  endgame starts.
* `min_step_size_eg` (default `exp2(-120)`): The minimal allowed step size during the endgame.
  This is also control what is considered `∞` for the path tracking.
* `s_always_consider_valuation` (default `-log(1e-16)`) A threshold after which we always consider
  the valuation.
* `samples_per_loop` (default `8`): The number of samples used during the endgame.
* `precision_strategy` (default `:adaptive_finite`): This controls whether `H(x,t)` is
  possibly evaluated with higher than machine precision during the endgame
  (the Jacobian is always computed with machine precision). The `:adaptive_finite` allows
  this only if we are optimistic that we can still obtain a finite solution.
  Other options are `:adaptive_never` where this is never allowed and `:adaptive_always`
  where it is always enabled.
"""
struct PathTracker{
    AV<:AbstractVector{Complex{Float64}},
    Prob<:AbstractProblem,
    CT<:CoreTracker{Float64,AV},
} <: AbstractPathTracker
    problem::Prob
    core_tracker::CT
    endgame::CauchyEndgame{AV}
    state::PathTrackerState{AV}
    options::PathTrackerOptions
    # We modify options of the core tracker dynamically. Therefore we need to be able
    # so start fresh for a new path
    default_ct_options::CoreTrackerOptions
end

function PathTracker(
    prob::AbstractProblem,
    x::AbstractVector{<:Number};
    min_step_size = 1e-30,
    kwargs...,
)

    core_tracker_kwargs, options = splitkwargs(kwargs, coretracker_supported_keywords)
    core_tracker = CoreTracker(
        prob,
        x;
        log_transform = true,
        predictor = MixPredictor(Pade21(), Heun()),
        min_step_size = min_step_size,
        core_tracker_kwargs...,
    )
    PathTracker(prob, core_tracker; options...)
end

function PathTracker(
    prob::Problem,
    core_tracker::CoreTracker;
    at_infinity_check = default_at_infinity_check(prob),
    endgame_start::Float64 = -log(0.1),
    max_winding_number::Int = 12,
    min_accuracy::Float64 = max(core_tracker.options.accuracy, 1e-5),
    min_cond_endgame::Float64 = 1e4,
    min_step_size_before_eg::Float64 = exp2(-40),
    min_step_size_eg = exp2(-120),
    s_always_consider_valuation::Float64 = -log(1e-16),
    samples_per_loop::Int = 8,
    precision_strategy::Symbol = :adaptive_finite,
)
    state = PathTrackerState(current_x(core_tracker); at_infinity_check = at_infinity_check)
    endgame = CauchyEndgame(
        current_x(core_tracker);
        samples_per_loop = samples_per_loop, max_winding_number = max_winding_number,
    )
    default_ct_options = copy(core_tracker.options)

    options = PathTrackerOptions(
        endgame_start,
        at_infinity_check,
        min_accuracy,
        min_cond_endgame,
        min_step_size_before_eg,
        min_step_size_eg,
        make_precision_strategy(precision_strategy),
        s_always_consider_valuation,
    )

    PathTracker(prob, core_tracker, endgame, state, options, default_ct_options)
end

default_at_infinity_check(prob::Problem{AffineTracking}) = true
default_at_infinity_check(prob::Problem{ProjectiveTracking}) = homvars(prob) !== nothing

always_true(_, _) = true

function make_precision_strategy(precision_strategy::Symbol)
    if precision_strategy == :adaptive_finite
        PREC_STRATEGY_FINITE
    elseif precision_strategy == :adaptive_never
        PREC_STRATEGY_NEVER
    elseif precision_strategy == :adaptive_always
        PREC_STRATEGY_ALWAYS
    else
        ArgumentError(
            "Unknown argument `precision_strategy = $(precision_strategy)`." *
            "Possible values are `:adaptive_finite`, `:adaptive_never`" *
            " or `:adaptive_always`.",
        ) |> throw
    end
end

Base.show(io::IO, ::MIME"application/prs.juno.inline", PT::PathTracker) = PT
function Base.show(io::IO, S::PathTracker{AV}) where {AV}
    println(io, "PathTracker with solution type $AV")
end

seed(PT::PathTracker) = seed(PT.problem)

"""
    init!(tracker::PathTracker, x)

Prepare the `PathTracker`` `tracker` to track a path with start solution `x`.
"""
function init!(
    tracker::PathTracker,
    x;
    accuracy::Union{Nothing,Float64} = nothing,
    max_corrector_iters::Union{Nothing,Int} = nothing,
)
    options, ct_options = tracker.options, tracker.core_tracker.options
    copy!(ct_options, tracker.default_ct_options)
    if accuracy !== nothing
        ct_options.accuracy = accuracy
    end
    if max_corrector_iters !== nothing
        ct_options.max_corrector_iters = max_corrector_iters
    end
    if options.precision_strategy == PREC_STRATEGY_ALWAYS ||
       options.precision_strategy == PREC_STRATEGY_FINITE
        ct_options.precision = PRECISION_ADAPTIVE
    end
    if tracker.core_tracker.predictor isa MixPredictorCache
        tracker.core_tracker.predictor.predictor1_active[] = true
    end

    init!(tracker.core_tracker, x, 0.0, options.endgame_start)
    init!(tracker.state, 0.0)

    # before endgame don't track the condition number
    # and we only update the limiting accuracy etc after a step failed
    ct_options.track_cond = false
    ct_options.steps_jacobian_info_update = -1
    ct_options.min_step_size = options.min_step_size_before_eg

    tracker
end

"""
    step!(tracker::PathTracker)

Perform a single step of the `PathTracker`.
"""
function step!(tracker::PathTracker)
    @unpack core_tracker, state, options, endgame = tracker
    step!(core_tracker)

    state.s = s = real(current_t(core_tracker))
    ct_status = status(core_tracker)

    # If s > 18.42 we have exp(-18.42) ≈ 1e-8 and we get to the range where numerical
    # differentation with the Padé predictor can become tricky.
    # Therefore we switch to the other predictor
    s > 18.42 && activate_predictor2!(core_tracker)

    if is_tracking(ct_status)
        state.eg_started || return state.status
        # If we didn't move forward there is nothing to do
        core_tracker.state.last_step_failed && return state.status

        # update the valuation
        update!(state.valuation, core_tracker)

        # We only care about the valuation if the path is starting to get worse
        # or we passed a certain threshold.
        cond_bad = cond(core_tracker) > options.min_cond_eg
        consider_always = s > options.s_always_consider_valuation
        if !(consider_always || cond_bad)
            return state.status
        end

        # Judge the current valuation to determine how to proceed
        verdict = judge(state.valuation; tol = 1e-2, tol_at_infinity = 1e-4)
        if verdict == VAL_AT_INFINITY &&
           (consider_always || cond_bad) && options.at_infinity_check
            state.status = PathTrackerStatus.at_infinity
        # If we expect a finite value and there is some ill conditioning let's do the
        # Cauchy endgame
        elseif verdict == VAL_FINITE && cond_bad
            # Perform endgame to estimate singular solution
            @label run_cauchy_eg
            activate_predictor2!(core_tracker)
            retcode, m, p_accuracy = predict!(state.prediction, core_tracker, endgame)

            if retcode == CAUCHY_SUCCESS
                # We only accept a result of the Cauchy endgame only if two consecutive
                # loops resulted in the same number of loop
                m′ = state.winding_number
                if m′ === nothing
                    state.winding_number = m
                    state.solution .= state.prediction
                elseif m′ == m &&
                       norm(core_tracker)(state.solution, state.prediction) ≤ 4p_accuracy
                    @label cauchy_eg_success
                    state.status = PathTrackerStatus.success
                    state.solution_accuracy = p_accuracy
                    converged, s_accuracy, s_cond, s_res = check_converged!(
                        state.solution,
                        core_tracker,
                        state.prediction,
                        Inf;
                        tol = tracker.default_ct_options.accuracy,
                    )
                    state.solution_cond = s_cond
                    state.solution_residual = s_res
                    if converged && s_cond < 1e8 && s_accuracy < p_accuracy
                        state.solution_accuracy = s_accuracy
                    else
                        state.solution .= state.prediction
                    end
                else
                    state.solution .= state.prediction
                    state.winding_number = m
                end

            elseif retcode == CAUCHY_TERMINATED_ACCURACY_LIMIT
                if core_tracker.options.precision == PRECISION_FIXED_64
                    core_tracker.options.precision = PRECISION_ADAPTIVE
                    @goto run_cauchy_eg
                end

            elseif options.precision_strategy == PREC_STRATEGY_NEVER
                state.status = PathTrackerStatus.terminated_accuracy_limit
                # If we are here, we have a precision strategy which allows to use
                # higher precision. Therefore switch to it if this not yet happened
                # and retry to cauchy eg

            elseif retcode == CAUCHY_TERMINATED_MAX_WINDING_NUMBER
                # Terminate if we hit twice max winding number and make one step pause
                if state.max_winding_number_hit
                    state.status = PathTrackerStatus.terminated_max_winding_number
                else
                    state.max_winding_number_hit = true
                end
            elseif retcode == CAUCHY_TERMINATED_MAX_ITERS
                state.status = PathTrackerStatus.terminated_max_iters
            else
                state.status = PathTrackerStatus.terminated_ill_conditioned
            end
        end

    # We split the tracking in two parts. Pre endgame and endgame. Therefore we have to
    # handle the success case twice
    elseif is_success(ct_status) && !state.eg_started
        state.intermediate_solution .= current_x(core_tracker)
        state.eg_started = true
        # set min_step size
        core_tracker.options.min_step_size = options.min_step_size_eg
        # setup path tracker to continue tracking
        t₀ = -log(4 * options.min_step_size_eg)
        init!(
            tracker.core_tracker,
            current_x(core_tracker),
            options.endgame_start,
            t₀;
            loop = true,
        )
        # start keeping track of the condition number during the endgame
        tracker.core_tracker.options.track_cond = true
        # update the limiting accuracy and cond every other step
        tracker.core_tracker.options.steps_jacobian_info_update = 2
        # update the valuation
        update!(state.valuation, core_tracker)

        # If our strategy is to allow adaptive precision only for finite values
        # then we start with not allowing adaptive precision during the endgame
        # and wait to hit the accuracy limit.
        if options.precision_strategy == PREC_STRATEGY_FINITE
            tracker.core_tracker.options.precision = PRECISION_FIXED_64
        end

    elseif is_success(ct_status) ||
           ct_status == CoreTrackerStatus.terminated_step_size_too_small
        converged, s_acc, s_cond, s_res = check_converged!(
            state.solution,
            core_tracker,
            current_x(core_tracker),
            Inf;
            tol = tracker.default_ct_options.accuracy,
        )
        if converged
            state.status = PathTrackerStatus.success
            state.solution_accuracy = s_acc
            state.solution_cond = s_cond
            state.solution_residual = s_res
            state.winding_number = nothing
            state.s = Inf
        elseif ct_status == CoreTrackerStatus.terminated_step_size_too_small
            state.status = PathTrackerStatus.terminated_step_size_too_small
        else
            state.status = PathTrackerStatus.post_check_failed
        end

    elseif ct_status == CoreTrackerStatus.terminated_accuracy_limit
        # First update the valuation  and check whether we are good
        update!(state.valuation, core_tracker)
        # we put less requirement on tol_at_infinity
        verdict = judge(state.valuation; tol = 1e-3, tol_at_infinity = 1e-4)
        # We also terminate at this point if the valuation indicates that we are
        # substantially less than 0
        if verdict == VAL_AT_INFINITY && options.at_infinity_check
            state.status = PathTrackerStatus.at_infinity
        # Now, we have to differentiate 3 different cases:
        # 1) Current accuracy is larger than the defined minimal accuracy
        #     -> decrease accuracy to minimal accuracy
        # 2) We are already at minimal accuracy and allow adaptive precision for finite
        #    values
        #  a) it looks like we could end up with something finite
        #     -> enable adaptive precision
        #  b) else
        #     -> terminate
        # 3) We are already at minimal accuracy and don't allow adaptive precision
        #     -> terminate
        # 1)
        elseif core_tracker.options.accuracy > options.min_accuracy
            # TODO: Currently we never increase this again
            core_tracker.options.accuracy = options.min_accuracy
            core_tracker.state.status = CoreTrackerStatus.tracking
        # 2)
        elseif options.precision_strategy == PREC_STRATEGY_FINITE
            # 2.a) We say that a path could still become finite if it is not classified
            #      as going to infinity for a loose tolerance
            #      Also if the current valuation is less than -1 (so significantly less than 0)
            #      we do not continue the tracking
            loose_verdict = judge(state.valuation; tol = 1e-2, tol_at_infinity = 1e-2)
            if loose_verdict != VAL_AT_INFINITY
                core_tracker.options.precision = PRECISION_ADAPTIVE
                core_tracker.state.status = CoreTrackerStatus.tracking
            # 2.b)
            else
                state.status = PathTrackerStatus.terminated_accuracy_limit
            end
        # 3)
        else
            state.status = PathTrackerStatus.terminated_accuracy_limit
        end
    elseif state.winding_number !== nothing
        p_accuracy = 1e-5
        @goto cauchy_eg_success
    else
        state.status = path_tracker_status(ct_status)
    end

    if !is_success(state.status) && !is_tracking(state.status)
        state.solution .= current_x(core_tracker)
        state.solution_cond = LA.cond(core_tracker)
    end

    state.status
end

"Update the valuation with the current state."
update!(val::Valuation, T::CoreTracker) =
    update!(val, T.state.x, T.state.ẋ, real(current_t(T)), T.predictor)

function activate_predictor2!(core_tracker::CoreTracker)
    if core_tracker.predictor isa MixPredictorCache
        core_tracker.predictor.predictor1_active[] = false
    end
    nothing
end

"""
    check_converged!(y, tracker::CoreTracker, x::AbstractVector, t)

Check that the CoreTracker is converged also converged at `(x,t)`. Returns a named tuple
`(converged, accuracy, cond)` and stores a (possibly) refined solution into `y`.
"""
function check_converged!(
    y,
    tracker::CoreTracker,
    x::AbstractVector,
    t::Number;
    tol::Float64 = 1e-6,
)
    init!(tracker.state.norm, x)
    # Make sure to evaluate the Jacobian only *once*. Otherwise it can happen at singuar
    # solutions that we bounce away from a good solution.
    result =
        correct!(y, tracker, x, t; tol = tol, min_iters = 2, max_iters = 3, full_steps = 1)
    if is_converged(result)
        tracker.state.residual .= abs.(tracker.corrector.r)
    else
        evaluate!(tracker.corrector.r, tracker.homotopy, x, t)
        tracker.state.residual .= abs.(tracker.corrector.r)
    end
    res = LA.norm(tracker.state.residual, InfNorm())
    # Fix the residual to the norm otherwise we can scale away zero rows.
    tracker.state.residual .= res
    # If the accuracy is NaN we assume that we dived by 0 -> Jacobian singular
    if isnan(result.accuracy)
        cond_jac = Inf
    else
        cond_jac = cond!(tracker.state.jacobian, tracker.state.norm, tracker.state.residual)
    end
    (
        converged = is_converged(result),
        accuracy = result.accuracy,
        cond = cond_jac,
        res = res,
    )
end

function Base.iterate(tracker::PathTracker, state::Int = 0)
    if is_tracking(tracker.state.status)
        step!(tracker)
        tracker, state + 1
    else
        nothing
    end
end

####################
## QUERYING STATE ##
####################

status(tracker::PathTracker) = tracker.state.status

"""
    solution(tracker::PathTracker)

Obtain the solution computed by the `PathTracker`.
"""
solution(tracker::PathTracker) = pull_back(tracker.problem, tracker.state.solution)

"""
    norm(tracker::PathTracker)

Obtain the norm used by the `PathTracker`.
"""
LA.norm(tracker::PathTracker) = LA.norm(tracker.core_tracker)

"""
    winding_number(tracker::PathTracker)

Obtain the estimate of the winding number computed by the `PathTracker`. Returns `nothing`
if the Cauchy endgame was not run.
"""
winding_number(tracker::PathTracker) = winding_number(tracker.state)
winding_number(state::PathTrackerState) = state.winding_number

###############################
## Convencience constructors ##
###############################

const pathtracker_startsolutions_supported_keywords = [
    problem_startsolutions_supported_keywords
    coretracker_supported_keywords
    pathtracker_supported_keywords
]

"""
    pathtracker_startsolutions(args...; kwargs...)

Construct a [`PathTracker`](@ref) and start solutions in the same way [`solve`](@ref) does it.
This also takes the same input arguments as `solve`. This is convenient if you want
to investigate single paths.
"""
function pathtracker_startsolutions(args...; kwargs...)
    invalid = invalid_kwargs(kwargs, pathtracker_startsolutions_supported_keywords)
    check_kwargs_empty(invalid, pathtracker_startsolutions_supported_keywords)
    supported, rest = splitkwargs(kwargs, problem_startsolutions_supported_keywords)
    prob, startsolutions = problem_startsolutions(args...; supported...)
    construct_tracker(prob, startsolutions; rest...), startsolutions
end

function construct_tracker(prob::Problem, startsolutions; kwargs...)
    PathTracker(prob, start_solution_sample(startsolutions); kwargs...)
end

"""
    pathtracker(args...; kwargs...)

Construct a [`PathTracker`](@ref) in the same way [`solve`](@ref) does it.
This also takes the same input arguments as `solve` with the exception that you do not need
to specify startsolutions.

## Examples

### Obtain single solution
We want to construct a path tracker to track a parametrized system `f` with parameters `p`
from the parameters `a` to `b`.
```julia
tracker = pathtracker(f, parameters=p, p₁=a, p₀=b)
```
You then can obtain a single solution at `b` by using
```julia
x_b = solution(track(tracker, x_a))
```
"""
pathtracker(args...; kwargs...) = first(pathtracker_startsolutions(args...; kwargs...))

#################
## PATH RESULT ##
#################

"""
    PathResult{V<:AbstractVector}

A `PathResult` is the result of tracking of a path with [`track`](@ref) using a
[`PathTracker`](@ref).

# Fields
* `return_code`: See the list of return codes below.
* `solution::V`: The solution vector.
* `t::Float64`: The value of `t` at which `solution` was computed. Note that if
  `return_code` is `:at_infinity`, then `t` is the value when this was decided.
* `accuracy::Float64`: An approximation of ``||x-x̄||`` where ``x`` is the computed solution
  and ``x̄`` is the true solution.
* `residual::Float64`: The value of the infinity-norm of `H(solution, 0)`.
* `multiplicity::Union{Nothing, Int}` is the multiplicity of the `solution`. This is
  only assigned by [`singular`](@ref).
* `condition_jacobian::Union{Nothing, Float64}`: This is the condition number of the
  row-equilibrated Jacobian at the solution. A high condition number indicates a singularity.
* `winding_number:Union{Nothing, Int}`: The estimated winding number. This is a lower bound
  on the multiplicity of the solution.
* `start_solution::Union{Nothing, Int}`: The start solution of the path.
* `accepted_steps::Int`: The number of accepted steps during the path tracking.
* `rejected_steps::Int`: The number of rejected steps during the path tracking.
* `valuation::Union{Nothing, Vector{Float64}}`: An approximation of the valuation of the
  Puiseux series expansion of `x(t)`.
* `valuation_accuracy::Union{Nothing, Vector{Float64}}`: An estimate of the accuracy of the
  valuation of the Puiseux series expansion of `x(t)`.

# Return codes

These is the list of possible return codes:

* `:success`: The `PathTracker` obtained a solution.
* `:at_infinity`: The `PathTracker` stopped the tracking of the path since it determined
  that that path is diverging towards infinity.
* `:terminated_callback`: One of the optional `PathTracker` callbacks terminated the tracking.
* `:terminated_max_iters`: The `PathTracker` terminated since it reached the limit accuracy.
* `:terminated_invalid_startvalue`: The `PathTracker` terminated since the provided start
  value is invalid.
* `:terminated_step_size_too_small`: The `PathTracker` terminated since the step size
  became smaller than the provided threshold.
* `:terminated_accuracy_limit`: The `PathTracker` terminated since the problem was too
  ill-conditioned to be tracked further with the desired minimal accuracy.
* `:terminated_ill_conditioned`: The `PathTracker` terminated since the Jacobian of the
  homotopy was too ill-conditioned.
* `:post_check_failed`: The verification of a non-singular solution failed.

# Constructors

     PathResult(tracker::PathTracker,
                start_solution=nothing,
                path_number::Union{Nothing,Int}=nothing;
                details=:default)

Construct a `PathResult` using the current state of the `PathTracker`.
Possible values for `details` are `:minimal` (minimal details), `:default` (default) and
`:extensive` (all information possible).
"""
struct PathResult{V<:AbstractVector}
    return_code::Symbol
    solution::V
    t::Float64
    accuracy::Float64
    residual::Float64
    condition_jacobian::Float64
    winding_number::Union{Nothing,Int}
    path_number::Base.RefValue{Union{Nothing,Int}}
    start_solution::Union{Nothing,V} # level 1+
    intermediate_solution::Union{Nothing,V}
    multiplicity::Base.RefValue{Union{Nothing,Int}} # only assigned by singular(Result)
    # performance stats
    accepted_steps::Int
    rejected_steps::Int
    valuation::Union{Nothing,Vector{Float64}} # level 2+
    valuation_accuracy::Union{Nothing,Vector{Float64}} # level 2+
end

function PathResult(
    tracker::PathTracker,
    start_solution::Union{Nothing,AbstractVector} = nothing,
    path_number::Union{Nothing,Int} = nothing;
    details::Symbol = :default,
)
    @unpack state, core_tracker = tracker
    details_level = detailslevel(details)

    return_code = Symbol(state.status)
    x = solution(tracker)
    t = return_code == :success ? 0.0 : exp(-state.s)
    accuracy = state.solution_accuracy
    residual = state.solution_residual
    condition_jac = state.solution_cond
    windingnumber = state.winding_number
    # this needs to be assigned manually
    multiplicity = Base.RefValue{Union{Nothing,Int}}(nothing)

    startsolution = nothing
    if details_level ≥ 1 && start_solution !== nothing
        # mimic the behaviour in track! to get a start solution of the same type as x
        embed!(core_tracker.state.x, tracker.problem, start_solution)
        startsolution = pull_back(tracker.problem, core_tracker.state.x; regauge = false)
    end
    intermediate_solution = nothing
    if !isnan(state.intermediate_solution[1])
        intermediate_solution = pull_back(tracker.problem, state.intermediate_solution)
    end

    accepted_steps = core_tracker.state.accepted_steps
    rejected_steps = core_tracker.state.rejected_steps

    valuation = valuation_accuracy = nothing
    if details_level == 2
        valuation = copy(tracker.state.valuation.ν)
        valuation_accuracy = abs.(tracker.state.valuation.ν̇)
    end

    PathResult(
        return_code,
        x,
        t,
        accuracy,
        residual,
        condition_jac,
        windingnumber,
        Base.RefValue{Union{Nothing,Int}}(path_number),
        startsolution,
        intermediate_solution,
        multiplicity,
        accepted_steps,
        rejected_steps,
        valuation,
        valuation_accuracy,
    )
end

function detailslevel(details::Symbol)
    if details == :minimal
        return 0
    elseif details == :extensive
        return 2
    else
        return 1
    end
end

"""
    result_type(tracker::PathTracker)

Returns the type of result `track` will return.
"""
result_type(tracker::PathTracker) = PathResult{typeof(solution(tracker))}

function Base.show(io::IO, r::PathResult{AV}) where {AV}
    iscompact = get(io, :compact, false)
    if iscompact || haskey(io, :typeinfo)
        println(io, "• return_code → :$(r.return_code)")
        if r.return_code ≠ :success
            println(io, " • t → $(r.t)")
        end
        println(io, " • solution → ", r.solution)
        r.accuracy !== nothing && println(
            io,
            " • accuracy → $(Printf.@sprintf "%.3e" r.accuracy)",
        )
        winding_number(
            r,
        ) !== nothing && println(io, " • winding_number → $(winding_number(r))")
        if multiplicity(r) !== nothing
            println(io, " • multiplicity → $(multiplicity(r))")
        end
        path_number(r) !== nothing && println(io, " • path_number: ", path_number(r))
    else
        header = "PathResult{$AV}"
        compact_io = IOContext(io, :compact => true)
        println(io, "PathResult{$AV}")
        println(io, "="^length(header))
        path_number(r) !== nothing && println(io, " • path_number → ", path_number(r))
        println(io, " • return_code → :$(r.return_code)")
        if r.return_code ≠ :success
            println(compact_io, " • t → $(r.t)")
        end
        println(compact_io, " • solution → ", r.solution)
        r.accuracy !== nothing && println(compact_io, " • accuracy → ", r.accuracy)
        r.residual !== nothing && println(compact_io, " • residual → ", r.residual)
        if r.condition_jacobian !== nothing
            println(compact_io, " • condition_jacobian → ", r.condition_jacobian)
        end

        if multiplicity(r) !== nothing
            println(io, " • multiplicity → $(multiplicity(r))")
        end
        if winding_number(r) !== nothing
            println(io, " • winding_number → $(winding_number(r))")
        end
        if r.valuation !== nothing
            println(compact_io, " • valuation → ", r.valuation)
        end
        if r.valuation_accuracy !== nothing
            println(compact_io, " • valuation_accuracy → ", r.valuation_accuracy)
        end
    end
end
Base.show(io::IO, ::MIME"application/prs.juno.inline", x::PathResult) = x

"""
    solution(r::PathResult)

Get the solution of the path.
"""
solution(r::PathResult) = r.solution

"""
    start_solution(r::PathResult)

Get the start solution of the path.
"""
start_solution(r::PathResult) = r.start_solution

"""
    accuracy(r::PathResult)

Get the accuracy of the solution. This is an estimate of the (relative) distance to the
true solution.
"""
accuracy(r::PathResult) = r.accuracy

"""
    residual(r::PathResult)

Get the residual of the solution ``x`` of the path, i.e., ``||H(x, 0)||₂``.
"""
residual(r::PathResult) = r.residual

"""
    winding_number(r::PathResult)

Get the winding number of the solution of the path. Returns `nothing` if it wasn't computed.
"""
winding_number(r::PathResult) = r.winding_number

"""
    multiplicity(P::PathResult)

Returns the multiplicity of `P`.
"""
multiplicity(P::PathResult) = P.multiplicity[]

"""
    condition_jacobian(r::PathResult)

Return the condition number of the Jacobian of the result.
"""
condition_jacobian(r::PathResult) = r.condition_jacobian

"""
    cond(r::PathResult)

Return the condition number of the Jacobian of the result.
"""
LA.cond(r::PathResult) = r.condition_jacobian

"""
    path_number(r::PathResult)

The number of the path.
"""
path_number(r::PathResult) = r.path_number[]

"""
    is_success(r::PathResult)

Checks whether the path is successfull.
"""
is_success(r::PathResult) = r.return_code == :success

"""
    is_failed(r::PathResult)

Checks whether the path failed.
"""
is_failed(r::PathResult) = !(r.return_code == :at_infinity || r.return_code == :success)

"""
    is_at_infinity(r::PathResult)

Checks whether the path goes to infinity.
"""
is_at_infinity(r::PathResult) = r.return_code == :at_infinity

"""
    is_finite(r::PathResult)

Checks whether the path result is finite.
"""
is_finite(r::PathResult) = r.return_code == :success

# Base fallback
Base.isfinite(r::PathResult) = is_finite(r)

"""
    is_singular(r::PathResult; tol=1e10)

Checks whether the path result is singular. This is true if
the multiplicity is larger than  1 or if the condition number of the Jacobian
is larger than `tol`.
"""
is_singular(r::PathResult; tol = 1e10) = is_singular(r, tol)
function is_singular(r::PathResult, tol::Real)
    (
        unpack(r.condition_jacobian, 1.0) > tol ||
        unpack(multiplicity(r), 1) > 1 || unpack(winding_number(r), 1) > 1
    ) && is_success(r)
end

"""
    is_nonsingular(r::PathResult; tol=1e10)

Checks whether the path result is non-singular. This is true if
it is not singular.
"""
is_nonsingular(r::PathResult; kwargs...) = !is_singular(r; kwargs...) && is_success(r)
is_nonsingular(r::PathResult, tol::Real) = !is_singular(r, tol) && is_success(r)

"""
    is_real(r::PathResult; tol=1e-6)

We consider a result as `real` if the 2-norm of the imaginary part of the solution is at most `tol`.
"""
is_real(r::PathResult; tol = 1e-6) = is_real(r, tol)
is_real(r::PathResult, tol::Real) = is_real_vector(r.solution, tol)
# provide fallback since this in in Base
Base.isreal(r::PathResult, tol) = is_real(r, tol)
Base.isreal(r::PathResult; kwargs...) = is_real(r; kwargs...)

"""
    is_projective(r::PathResult)

Return`s true if the solution is a projective vector.
"""
is_projective(r::PathResult) = false
is_projective(r::PathResult{<:PVector}) = true

"""
    is_affine(pathresult)

Return`s true if the solution is an affine vector.
"""
is_affine(r::PathResult) = !is_projective(r)

###########
## track ##
###########

"""
    track(tracker::PathTracker, x₁;
            path_number=nothing,
            details::Symbol=:default,
            start_parameters = nothing,
            target_parameters = nothing)

Track the path `x(t)` with start solution `x₁` from ``1`` towards ``0``.
Returns a [`PathResult`](@ref).

The `details` options controls the level of details of the information available
in the [`PathResult`](@ref).
If `tracker` uses a parameter homotopy you can set the start and target parameters
by setting the corresponding fields.
To investigate the behaviour of a particular take a look at [`path_info`](@ref).
"""
function track(
    tracker::PathTracker,
    x,
    t₁ = nothing,
    t₀ = nothing;
    path_number::Union{Int,Nothing} = nothing,
    details::Symbol = :default,
    accuracy::Union{Nothing,Float64} = nothing,
    max_corrector_iters::Union{Nothing,Int} = nothing,
    start_parameters = nothing,
    target_parameters = nothing,
)
    if t₁ !== nothing
        @warn(
            "track(tracker, x, t₁, t₀) is deprecated. `t₁` is always assumed to be `1` " *
            " and `t₀` set automatically.",
        )
    end

    start_parameters !== nothing && start_parameters!(tracker, start_parameters)
    target_parameters !== nothing && target_parameters!(tracker, target_parameters)

    track!(tracker, x; accuracy = accuracy, max_corrector_iters = max_corrector_iters)
    PathResult(tracker, x, path_number; details = details)
end

"""
    start_parameters!(tracker::PathTracker, p)

Set the start parameters of the homotopy in in `tracker` to `p`.
"""
start_parameters!(T::PathTracker, p) = start_parameters!(T.core_tracker, p)

"""
    target_parameters!(tracker::PathTracker, p)

Set the target parameters of the homotopy in in `tracker` to `p`.
"""
target_parameters!(T::PathTracker, p) = target_parameters!(T.core_tracker, p)

"""
    track!(tracker::PathTracker, x₁)::PathTrackerStatus.states

Track the path `x(t)` with start solution `x₁` from ``1`` towards ``0``.
Returns a [`PathTrackerStatus.states`](@ref).
"""
function track!(
    tracker::PathTracker,
    x;
    accuracy::Union{Nothing,Float64} = nothing,
    max_corrector_iters::Union{Nothing,Int} = nothing,
)
    init!(tracker, x; accuracy = accuracy, max_corrector_iters = max_corrector_iters)
    while is_tracking(tracker.state.status)
        step!(tracker)
    end
    tracker.state.status
end


type CuDNNSoftmaxState
  inputs_desc  :: Vector{CuDNN.Tensor4dDescriptor}
  outputs_desc :: Vector{CuDNN.Tensor4dDescriptor}
end

function setup_etc(backend::GPUBackend, layer::SoftmaxLayer, dims::Vector{Int}, data_type, inputs)
  inputs_desc = Array{CuDNN.Tensor4dDescriptor}(length(inputs))
  outputs_desc = Array{CuDNN.Tensor4dDescriptor}(length(inputs))
  for i = 1:length(inputs)
    dim_sp, dim_prob, dim_num = split_dims(inputs[i], dims[i])
    inputs_desc[i] = CuDNN.create_tensor4d_descriptor(data_type, (1,dim_sp,dim_prob,dim_num))
    outputs_desc[i] = CuDNN.create_tensor4d_descriptor(data_type, (1,dim_sp,dim_prob,dim_num))
  end
  etc = CuDNNSoftmaxState(inputs_desc, outputs_desc)
  return etc
end
function shutdown(backend::GPUBackend, state::SoftmaxLayerState)
  map(destroy, state.blobs)
  map(CuDNN.destroy_tensor4d_descriptor, state.etc.inputs_desc)
  map(CuDNN.destroy_tensor4d_descriptor, state.etc.outputs_desc)
end

function forward(backend::GPUBackend, state::SoftmaxLayerState, inputs::Vector{Blob})
  alpha = one(eltype(inputs[1]))
  beta = zero(eltype(inputs[1]))
  for i = 1:length(inputs)
    CuDNN.softmax_forward(backend.cudnn_ctx, CuDNN.CUDNN_SOFTMAX_ACCURATE,
        CuDNN.CUDNN_SOFTMAX_MODE_CHANNEL, alpha, state.etc.inputs_desc[i], inputs[i].ptr,
        beta, state.etc.outputs_desc[i], state.blobs[i].ptr)
  end
end

function backward(backend::GPUBackend, state::SoftmaxLayerState, inputs::Vector{Blob}, diffs::Vector{Blob})
  alpha = one(eltype(inputs[1]))
  beta = zero(eltype(inputs[1]))

  for i = 1:length(inputs)
    diff = diffs[i]
    if !isa(diff, NullBlob)
      CuDNN.softmax_backward(backend.cudnn_ctx, CuDNN.CUDNN_SOFTMAX_ACCURATE,
          CuDNN.CUDNN_SOFTMAX_MODE_CHANNEL, alpha, state.etc.outputs_desc[i], state.blobs[i].ptr,
          state.etc.outputs_desc[i], state.blobs_diff[i].ptr, beta, state.etc.inputs_desc[i], diff.ptr)
    end
  end
end


# Different ways to simulate molecules

export
    VelocityVerlet,
    StormerVerlet,
    simulate!

# Forces are often expressed per mol but this dimension needs removing for use in the integrator
function remove_molar(x)
    fx = first(x)
    if dimension(fx) == u"𝐋 * 𝐍^-1 * 𝐓^-2"
        T = typeof(ustrip(fx))
        return x / T(Unitful.Na)
    else
        return x
    end
end

"""
    VelocityVerlet(; <keyword arguments>)

The velocity Verlet integrator.

# Arguments
- `dt::T`: the time step of the simulation.
- `coupling::C=NoCoupling()`: the coupling which applies during the simulation.
"""
struct VelocityVerlet{T, C}
    dt::T
    coupling::C
end

VelocityVerlet(; dt, coupling=NoCoupling()) = VelocityVerlet(dt, coupling)

"""
    StormerVerlet(; <keyword arguments>)

The Störmer-Verlet integrator.
In this case the `velocities` given to the simulator act as the previous step
coordinates for the first step.
Does not currently work with units or thermostats.

# Arguments
- `dt::T`: the time step of the simulation.
- `coupling::C=NoCoupling()`: the coupling which applies during the simulation.
"""
struct StormerVerlet{T, C}
    dt::T
    coupling::C
end

StormerVerlet(; dt, coupling=NoCoupling()) = StormerVerlet(dt, coupling)

"""
    simulate!(system, simulator, n_steps; parallel=true)
    simulate!(system, simulator; parallel=true)

Run a simulation on a system according to the rules of the given simulator.
Custom simulators should implement this function.
"""
function simulate!(sys::System{D, S, false},
                    sim::VelocityVerlet,
                    n_steps::Integer;
                    parallel::Bool=true) where {D, S}
    # See https://www.saylor.org/site/wp-content/uploads/2011/06/MA221-6.1.pdf for
    #   integration algorithm - used shorter second version
    neighbors = find_neighbors(sys, sys.neighbor_finder; parallel=parallel)
    accels_t = accelerations(sys, neighbors; parallel=parallel)
    accels_t_dt = zero(accels_t)

    @showprogress for step_n in 1:n_steps
        run_loggers!(sys, neighbors, step_n)

        # Update coordinates
        for i in 1:length(sys)
            sys.coords[i] += sys.velocities[i] * sim.dt + remove_molar(accels_t[i]) * (sim.dt ^ 2) / 2
            sys.coords[i] = wrap_coords.(sys.coords[i], sys.box_size)
        end

        accels_t_dt = accelerations(sys, neighbors; parallel=parallel)

        # Update velocities
        for i in 1:length(sys)
            sys.velocities[i] += remove_molar(accels_t[i] + accels_t_dt[i]) * sim.dt / 2
        end

        apply_coupling!(sys, sim, sim.coupling)
        neighbors = find_neighbors(sys, sys.neighbor_finder, neighbors, step_n; parallel=parallel)

        accels_t = accels_t_dt
    end
    return sys
end

function simulate!(sys::System{D, S, true},
                    sim::VelocityVerlet,
                    n_steps::Integer;
                    parallel::Bool=true) where {D, S}
    if length([inter for inter in values(sys.general_inters) if !inter.nl_only]) > 0
        neighbors_all = all_neighbors(length(sys))
    else
        neighbors_all = nothing
    end
    neighbors = find_neighbors(sys, sys.neighbor_finder)
    accels_t = accelerations(sys, sys.coords, sys.atoms, neighbors, neighbors_all)
    accels_t_dt = zero(accels_t)

    for step_n in 1:n_steps
        run_loggers!(sys, neighbors, step_n)

        sys.coords += sys.velocities .* sim.dt .+ (remove_molar.(accels_t) .* sim.dt ^ 2) ./ 2
        sys.coords = wrap_coords_vec.(sys.coords, (sys.box_size,))
        accels_t_dt = accelerations(sys, sys.coords, sys.atoms, neighbors, neighbors_all)
        sys.velocities += remove_molar.(accels_t .+ accels_t_dt) .* sim.dt / 2

        apply_coupling!(sys, sim, sim.coupling)
        neighbors = find_neighbors(sys, sys.neighbor_finder, neighbors, step_n)

        accels_t = accels_t_dt
    end
    return sys
end

function simulate!(sys::System,
                    sim::StormerVerlet,
                    n_steps::Integer;
                    parallel::Bool=true)
    neighbors = find_neighbors(sys, sys.neighbor_finder; parallel=parallel)

    @showprogress for step_n in 1:n_steps
        run_loggers!(sys, neighbors, step_n)

        accels_t = accelerations(sys, neighbors; parallel=parallel)

        # Update coordinates
        coords_copy = sys.coords
        for i in 1:length(sys)
            sys.coords[i] = sys.coords[i] + vector(sys.velocities[i], sys.coords[i], sys.box_size) + remove_molar(accels_t[i]) * sim.dt ^ 2
            sys.coords[i] = wrap_coords.(sys.coords[i], sys.box_size)
        end
        sys.velocities = coords_copy

        apply_coupling!(sys, sim, sim.coupling)
        neighbors = find_neighbors(sys, sys.neighbor_finder, neighbors, step_n; parallel=parallel)
    end
    return sys
end


JSON.lower(a::Aggregator) = non_nothing_dict(a)

"""
    Count(output_name)

Count number of rows.
"""
struct Count <: Aggregator
    type::String
    name::String
    Count(name) = new("count", name)
end

"""
    SingleField(type, field, output_name)

Apply `type` aggregator to the `field` and get result as `output_name`.

Available `type`s: "longSum", "doubleSum", "floatSum", "longMin", "doubleMin",
"floatMin", "longMax", "doubleMax", "floatMax", "longFirst", "doubleFirst",
"floatFirst", "longLast", "doubleLast", "floatLast", "longAny", "doubleAny",
"floatAny", "doubleMean"
"""
struct SingleField <: Aggregator
    type::String
    fieldName::String
    name::String
    function SingleField(type, of, as)
        type ∈ [
            "longSum", "doubleSum", "floatSum",
            "longMin", "doubleMin", "floatMin",
            "longMax", "doubleMax", "floatMax",
            "longFirst", "doubleFirst", "floatFirst",
            "longLast", "doubleLast", "floatLast",
            "longAny", "doubleAny", "floatAny",
            "doubleMean"
        ] || error("Unknown SingleField aggregation")
        new(type, of, as)
    end
end

"""
    StringAgg(type, field, output_name; maxStringBytes=nothing)

Apply `type` string aggregator to `field` and get result as `output_name`.

Available `type`s: "first", "last", "any"

See Druid documentation for `maxStringBytes`.
"""
struct StringAgg <: Aggregator
    type::String
    fieldName::String
    name::String
    maxStringBytes
    function StringAgg(type, of, as; maxStringBytes=nothing)
        type = lowercase(type)
        type ∈ ["first", "last", "any"] || error("Unknown StringAgg aggregation")
        maxStringBytes === nothing || (isa(maxStringBytes, Integer) && maxStringBytes >= 0) || error("maxStringBytes must be a positive integer")
        new("string" * titlecase(type), of, as, maxStringBytes)
    end
end

"""
    Grouping(groupings, name)

Grouping aggregator
"""
struct Grouping <: Aggregator
    type::String
    groupings::Vector{String}
    name::String
    Grouping(groupings, name) = new("grouping", groupings, name)
end

"""
    Filtered(filter, aggregator)

An aggregator that combines a filter with the `aggregator`.
"""
struct Filtered <: Aggregator
    type::String
    filter::Filter
    aggregator::Aggregator
    Filtered(filter, aggregator) = new("filtered", filter, aggregator)
end


using DiffRules, SpecialFunctions, NaNMath

for (M, f, arity) in DiffRules.diffrules()
  arity == 1 || continue
  @eval begin
    @adjoint $M.$f(x::Real) = $M.$f(x),
      Δ -> (Δ * $(DiffRules.diffrule(M, f, :x)),)
  end
end

for (M, f, arity) in DiffRules.diffrules()
  arity == 2 || continue
  da, db = DiffRules.diffrule(M, f, :a, :b)
  @eval begin
    @adjoint $M.$f(a::Real, b::Real) = $M.$f(a, b),
      Δ -> (Δ * $da, Δ * $db)
  end
end

@adjoint Base.convert(T::Type{<:Real}, x::Real) = convert(T, x), Δ -> (nothing, Δ)

@adjoint Base.:+(xs...) = +(xs...), Δ -> map(_ -> Δ, xs)

@adjoint function sincos(x)
  s, c = sincos(x)
  (s, c), ((s̄, c̄),) -> (s̄*c - c̄*s,)
end


using Pkg
Pkg.activate(".")
verbose = true
using Revise
if verbose   println("# Loading RvSpecML")    end
using RvSpectML
if verbose   println("# Loading other packages")    end
using DataFrames, Query

# TODO: USER:  Either the paths that specify where datafiles are stored here or in examples/data_paths.jl
if verbose   println("# Creating manifest of files to process.")    end
if isdir("examples")
   cd("examples")
end
solar_data_path = "20190918"
ancilary_solar_data_path = "."
 if isfile("data_paths.jl")
    include("data_paths.jl")
 end
 solar_data_path

num_spectra_to_use = 20
 bjd_first_good = 2458745.1296134139
 bjd_last_good = 2458745.283
 df_files = NEID.make_manifest(solar_data_path)

df_files_use = df_files |>
  @filter( _.target == "Solar" ) |>
  @filter(bjd_first_good <= _.bjd < bjd_last_good) |>
#  @take(num_spectra_to_use) |>
  DataFrame

if verbose println("# Reading in FITS files.")  end
@time solar_data = map(NEID.read_solar_data,eachrow(df_files_use))

if verbose println("# Applying wavelength corrections.")  end
NEID.read_drift_corrections!(joinpath(ancilary_solar_data_path,"SolarRV20190918_JD_SciRV_CalRV.txt"), df_files_use)

NEID.read_barycentric_corrections!(joinpath(ancilary_solar_data_path,"SolarTelescope2019-09-18_inclGravRedshiftAirMassAltitude.csv"), df_files_use)

NEID.read_differential_extinctions!(joinpath(ancilary_solar_data_path,"20190918_diff_ex_full_fixed.txt"), df_files_use)

apply_doppler_boost!(solar_data,df_files_use)


function OneWayAnova()
    return OneWayAnova(())
end

function anova_f_value(obj::OneWayAnova, arg0::Collection)
    return jcall(obj, "anovaFValue", jdouble, (Collection,), arg0)
end

function anova_p_value(obj::OneWayAnova, arg0::Collection)
    return jcall(obj, "anovaPValue", jdouble, (Collection,), arg0)
end

function anova_p_value(obj::OneWayAnova, arg0::Collection, arg1::jboolean)
    return jcall(obj, "anovaPValue", jdouble, (Collection, jboolean), arg0, arg1)
end

function anova_test(obj::OneWayAnova, arg0::Collection, arg1::jdouble)
    return jcall(obj, "anovaTest", jboolean, (Collection, jdouble), arg0, arg1)
end


# import Pkg;

# Pkg.add("JLD")
# Pkg.add("PlotlyJS")
# Pkg.add("JSON")
# Pkg.add("JuMP")
# Pkg.add("YAML")
# Pkg.add("CSV")
# Pkg.add("DataFrames")
# Pkg.add("Query")
# Pkg.add("OrderedCollections")
# Pkg.add("Missings")

import YAML

# using JLD
using OrderedCollections
using CSV
using DataFrames
using Dates
using Query
using Statistics
using JuMP
using Missings

yamlFile = YAML.load_file("../mintsDefinitionsV2.yaml"; dicttype=OrderedDict{String,Any})      # loading yaml file and arrange in order
dirctory = yamlFile["dataFolder"]                            # taking the directory of data stored
sensorNames = yamlFile["liveStack"]                         # all the sensor names
#sensorNames = ["PPD42NSDuo"]

# only 11th node data is saved
nodeIDs_array = yamlFile["nodeIDs"]                         # node's IDs array
node11 = nodeIDs_array[11]                                  # Taking 11th node
nodeId = node11["nodeID"]                                   # taking 11th nodeID

path = dirctory*"/"*nodeId

############# anaylzing and averaging data
function ss(dd, mm, yy, sensr)
    CSV_fileName_arry = readdir(path*"/"*yy*"/"*mm*"/"*dd*"/")       # taking all the file names from the folder
    num_files = length(CSV_fileName_arry)                            # number of files

    CSV_file = "MINTS_"*nodeId*"_"*sensr*"_"*yy*"_"*mm*"_"*dd*".csv"            # CSV file name
    if (CSV_file in CSV_fileName_arry)                                          # check CSV fime exist or not
        df = CSV.read(path*"/"*yy*"/"*mm*"/"*dd*"/"*CSV_file, DataFrame)        # If the CSV file exist then take the data into a dataframe
        #df.dateTime =  SubString.(string.(df.dateTime), 1, 19)                  # Removing last three decimal numbers in dateTime column

        if sensr == "GPSGPGGA2"
            df = select!(df, Not(:timestamp))
            df = select!(df, Not(:latitudeDirection))
            df = select!(df, Not(:longitudeDirection))
            df = select!(df, Not(:altitudeUnits))
            df = select!(df, Not(:undulationUnits))
            df = select!(df, Not(:age))
            df = select!(df, Not(:stationID))
        end

        if sensr == "GPSGPRMC2"
            df = select!(df, Not(:timestamp))
            df = select!(df, Not(:status))
            df = select!(df, Not(:latitudeDirection))
            df = select!(df, Not(:longitudeDirection))
            df = select!(df, Not(:dateStamp))
            df = select!(df, Not(:magVariation))
            df = select!(df, Not(:magVariationDirection))
            df = select!(df, Not(:trueCourse))
        end

        df1 = df
        df2 = df[1:2,:]

        try
            #df1 = df
            df1.dateTime =  SubString.(string.(df1.dateTime), 1, 19)
            df1.dateTime = DateTime.(df1.dateTime,"yyyy-mm-dd HH:MM:SS")

            df1.dateTime = map((x) -> round(x, Dates.Second(30)), df1.dateTime)       # Rounding dateTime for 30 seconds
            gdf1 = groupby(df1, :dateTime)                                            # making groups by same dateTime
            cgdf1 = combine(gdf1, valuecols(gdf1) .=> mean)                            # Calculate the mean of each group and then combine the groups

            colFullName = names(cgdf1)                                               # Take the column names of dataFrame into a array
            for x in 2:length(colFullName)                                          # Starting from 2nd column and go through all the columns
                num_let = length(colFullName[x])                                    # taking number of characters in one column name
                colName = SubString.(string.(colFullName[x]), 1, num_let-5)         # removing last five characters of each column name (that is _mean)
                rename!(cgdf1,colFullName[x] => sensr*"_"*colName)                   # adding sensor name to the column name
            end

            return cgdf1                             # return the dataFrame to for loop

        catch e
            println("Error on "*CSV_file)
            #df2 = df[1:10,:]

            if sensr == "GPSGPGGA2"
                clName = names(df2)
                if "Column18" in clName
                    df2 = select!(df2, Not(:Column18))
                end
                if "Column19" in clName
                    df2 = select!(df2, Not(:Column19))
                end

                try
                    df2.latitudeCoordinate = parse.(Float64, df2.latitudeCoordinate)
                    df2.longitudeCoordinate = parse.(Float64, df2.longitudeCoordinate)
                    df2.numberOfSatellites = parse.(Float64, df2.numberOfSatellites)
                catch e
                end
            end

            if sensr == "GPSGPRMC2"
                clName = names(df2)
                if "Column15" in clName
                    df2 = select!(df2, Not(:Column15))
                end
                if "Column16" in clName
                    df2 = select!(df2, Not(:Column16))
                end
                if "Column17" in clName
                    df2 = select!(df2, Not(:Column17))
                end
                if "Column18" in clName
                    df2 = select!(df2, Not(:Column18))
                end
                if "Column19" in clName
                    df2 = select!(df2, Not(:Column19))
                end
                if "Column20" in clName
                    df2 = select!(df2, Not(:Column20))
                end
                if "Column21" in clName
                    df2 = select!(df2, Not(:Column21))
                end

                try
                    #df2.latitudeCoordinate = parse.(Float64, df2.latitudeCoordinate)
                    #df2.longitudeCoordinate = parse.(Float64, df2.longitudeCoordinate)
                    df2.speedOverGround = parse.(Float64, df2.speedOverGround)
                    df2.longitude = parse.(Float64, df2.longitude)
                catch e
                end
            end

            if sensr == "PPD42NSDuo"
                clName = names(df2)
                if "Column10" in clName
                    df2 = select!(df2, Not(:Column10))
                end
                if "Column11" in clName
                    df2 = select!(df2, Not(:Column11))
                end

                try
                    df2.LPOPmMid = parse.(Float64, df2.LPOPmMid)
                catch e
                end
            end

            println(df2)
            df2.dateTime =  SubString.(string.(df2.dateTime), 1, 19)
            df2.dateTime = DateTime.(df2.dateTime,"yyyy-mm-dd HH:MM:SS")

            df2.dateTime = map((x) -> round(x, Dates.Second(30)), df2.dateTime)       # Rounding dateTime for 30 seconds
            gdf2 = groupby(df2, :dateTime)                                            # making groups by same dateTime
            cgdf2 = combine(gdf2, valuecols(gdf2) .=> mean)                            # Calculate the mean of each group and then combine the groups

            colFullName = names(cgdf2)                                               # Take the column names of dataFrame into a array
            for x in 2:length(colFullName)                                          # Starting from 2nd column and go through all the columns
                num_let = length(colFullName[x])                                    # taking number of characters in one column name
                colName = SubString.(string.(colFullName[x]), 1, num_let-5)         # removing last five characters of each column name (that is _mean)
                rename!(cgdf2,colFullName[x] => sensr*"_"*colName)                   # adding sensor name to the column name
            end

            return cgdf2
        end
    end
end

#global sensr_exsist = false
n = 0

for sensr in sensorNames                                                # Go through all the sensor names
    k = 0
    years = readdir(path)                                               # taking all the year folder names

    for y in years                                                      #
        months = readdir(path*"/"*y)

        for m in months
            days = readdir(path*"/"*y*"/"*m)

            for d in days
                filess = readdir(path*"/"*y*"/"*m*"/"*d)
                CSV_file = "MINTS_"*nodeId*"_"*sensr*"_"*y*"_"*m*"_"*d*".csv"

                if (CSV_file in filess)
                    println("Date: "*y*"_"*m*"_"*d)
                    println(sensr)
                    averaged_df = ss(d, m, y, sensr)
                    df2 = averaged_df
                    k = k + 1

                    println("done")
                    global sensr_exsist = true

                    if (k > 1)

                        #try
                        global new_df = outerjoin(new_df, df2, on = intersect(names(new_df), names(df2)))
                        #catch e
                            #println("Error of adding: "*"MINTS_"*nodeId*"_"*sensr*"_"*y*"_"*m*"_"*d*".csv")
                        #end

                    else
                        new_df = averaged_df
                    end

                else
                    println("No data of "*sensr)
                    sensr_exsist = false
                    #continue
                end
            end
        end
    end

    global n = n + 1

    if sensr_exsist

        if (n > 1)
            println("******************************************************")
            global final_df = outerjoin(final_df, new_df, on = :dateTime, makeunique=true)
        else
            final_df = new_df
        end

    end

end

final_df2 = sort!(final_df)
CSV.write("/mnt/3fbf694d-d8e0-46c0-903d-69d994241206/mintsData/avJlData/"*nodeId*"_df.csv", final_df2)


oggm_config()

## Retrieving gdirs and climate for the following glaciers
rgi_ids = ["RGI60-11.03638", "RGI60-11.01450", "RGI60-08.00213", "RGI60-04.04351", "RGI60-01.02170",
"RGI60-02.05098", "RGI60-01.01104", "RGI60-01.09162", "RGI60-01.00570", "RGI60-04.07051",
"RGI60-07.00274", "RGI60-07.01323", "RGI60-03.04207", "RGI60-03.03533", "RGI60-01.17316"]

gdirs = init_gdirs(rgi_ids, force=false)
tspan = (0.0,5.0) # period in years for simulation
gdirs_climate = get_gdirs_with_climate(gdirs, tspan, overwrite=false, plot=false)

# Load input data for the SIA PDE simulation
# gdirs_climate = JLD.load(joinpath(ODINN.root_dir, "test/data/gdirs_climate.jld"), "gdirs_climate")
# Load reference values for the simulation
PDE_refs = load(joinpath(ODINN.root_dir, "data/PDE_refs.jld2"))

# Run the forward PDE simulation
H_refs, V̄x_refs, V̄y_refs = @time generate_ref_dataset(gdirs_climate, tspan)

@test H_refs ≈ PDE_refs["H_refs"]
@test V̄x_refs ≈ PDE_refs["V̄x_refs"]
@test V̄y_refs ≈ PDE_refs["V̄y_refs"]