"""
    PlaybackPolicy{A<:AbstractArray, P<:Policy, V<:AbstractArray{<:Real}}
a policy that applies a fixed sequence of actions until they are all used and then falls back onto a backup policy until the end of the episode.

Constructor:

    `PlaybackPolicy(actions::AbstractArray, backup_policy::Policy; logpdfs::AbstractArray{Float64, 1} = Float64[])`

# Fields
- `actions::Vector{A}` a vector of actions to play back
- `backup_policy::Policy` the policy to use when all prescribed actions have been taken but the episode continues
- `logpdfs::Vector{Float64}` the log probability (density) of actions
- `i::Int64` the current action index
"""
mutable struct PlaybackPolicy{A<:AbstractArray, P<:Policy, V<:AbstractArray{<:Real}} <: Policy
    actions::A
    backup_policy::P
    logpdfs::V
    i::Int64
end

# Constructor for the PlaybackPolicy
PlaybackPolicy(actions::AbstractArray, backup_policy::Policy; logpdfs::AbstractArray{<:Real} = Float64[]) = PlaybackPolicy(actions, backup_policy, logpdfs, 1)

# Action selection for the PlaybackPolicy
function POMDPs.action(p::PlaybackPolicy, s)
    a = p.i <= length(p.actions) ? p.actions[p.i] : action(p.backup_policy, s)
    p.i += 1
    a
end

# Get the logpdf of the history from the playback policy and the backup policy
function Distributions.logpdf(p::PlaybackPolicy, h)
    N = min(length(p.actions), length(h))
    # @assert all(collect(action_hist(h))[1:N] .== p.actions[1:N])
    @assert length(p.actions) == length(p.logpdfs)
    if length(h) > N
        return sum(p.logpdfs) + sum(logpdf(p.backup_policy, view(h, N+1:length(h))))
    else
        return sum(p.logpdfs[1:N])
    end
end


export VaxInt16, VaxInt32

struct VaxInt16 <: VaxInt
    x::UInt16

    VaxInt16(x::UInt16) = new(htol(x))
end

VaxInt16(x::Signed) = VaxInt16(trunc(Int16,x) % UInt16)

Base.convert(::Type{Int16}, x::VaxInt16) = ltoh(x.x) % Int16
function Base.convert(::Type{T}, x::VaxInt16) where T <: Union{Int32,Int64,Int128,BigInt,AbstractFloat}
    return convert(T, convert(Int16, x))
end

struct VaxInt32 <: VaxInt
    x::UInt32

    VaxInt32(x::UInt32) = new(htol(x))
end

VaxInt32(x::Signed) = VaxInt32(trunc(Int32,x) % UInt32)

Base.convert(::Type{Int32}, x::VaxInt32) = ltoh(x.x) % Int32
function Base.convert(::Type{T}, x::VaxInt32) where T <: Union{Int16,Int64,Int128,AbstractFloat}
    return convert(T,convert(Int32,x))
end


function JuMP.parse_constraint_call(_error::Function, vectorized::Bool,
                                            ::Val{:⊂}, lhs, rhs)
    _error("Unrecognized symbol ⊂ you mean ⊆ ?")
end
function JuMP.parse_constraint_call(_error::Function, vectorized::Bool,
                                            ::Val{:⊆}, lhs, rhs)
    parse_code = :()
    if vectorized
        build_call = :(JuMP.build_constraint.($_error, $(esc(lhs)), $(esc(:(SetProg.PowerSet.($rhs))))))
    else
        build_call = :(JuMP.build_constraint($_error, $(esc(lhs)), $(esc(:(SetProg.PowerSet($rhs))))))
    end
    return parse_code, build_call
end
function JuMP.parse_constraint_call(_error::Function, vectorized::Bool,
                                            ::Val{:⊃}, lhs, rhs)
    _error("Unrecognized symbol ⊃, did you mean ⊇ ?")
end
function JuMP.parse_constraint_call(_error::Function, vectorized::Bool,
                                            ::Val{:⊇}, lhs, rhs)
    parse_one_operator_constraint(_error, vectorized, Val(:⊆), rhs, lhs)
end


using Oceananigans.TimeSteppers: ab2_step_field!

import Oceananigans.TimeSteppers: ab2_step!

combine_events(::Nothing, tracer_events) = MultiEvent(tuple(tracer_events...))
combine_events(free_surface_event, tracer_events) = MultiEvent(tuple(free_surface_event, tracer_events...))

function ab2_step!(model::HydrostaticFreeSurfaceModel, Δt, χ)

    workgroup, worksize = work_layout(model.grid, :xyz)

    barrier = Event(device(model.architecture))

    step_field_kernel! = ab2_step_field!(device(model.architecture), workgroup, worksize)

    # Launch velocity update kernels

    velocities_events = []

    for name in (:u, :v)
        Gⁿ = model.timestepper.Gⁿ[name]
        G⁻ = model.timestepper.G⁻[name]
        velocity_field = model.velocities[name]

        event = step_field_kernel!(velocity_field, Δt, χ, Gⁿ, G⁻,
                                   dependencies=Event(device(model.architecture)))

        push!(velocities_events, event)
    end

    # Launch tracer update kernels

    tracer_events = []

    for name in tracernames(model.tracers)
        Gⁿ = model.timestepper.Gⁿ[name]
        G⁻ = model.timestepper.G⁻[name]
        tracer_field = model.tracers[name]

        event = step_field_kernel!(tracer_field, Δt, χ, Gⁿ, G⁻,
                                   dependencies=Event(device(model.architecture)))

        push!(tracer_events, event)
    end

    velocities_update = MultiEvent(Tuple(velocities_events))

    # Update the free surface if not using a rigid lid once the velocities have finished updating.
    free_surface_event = ab2_step_free_surface!(model.free_surface, velocities_update, model, χ, Δt)

    tracer_and_free_surface_events = combine_events(free_surface_event, tracer_events)

    wait(device(model.architecture), tracer_and_free_surface_events)

    return nothing
end

#####
##### Free surface time-stepping: explicit, implicit, rigid lid ?
#####

ab2_step_free_surface!(free_surface::ExplicitFreeSurface, velocities_update, model, χ, Δt) =
    explicit_ab2_step_free_surface!(free_surface, velocities_update, model, χ, Δt)

function explicit_ab2_step_free_surface!(free_surface, velocities_update, model, χ, Δt)

    event = launch!(model.architecture, model.grid, :xy,
                    _ab2_step_free_surface!,
                    model.free_surface.η,
                    χ,
                    Δt,
                    model.timestepper.Gⁿ.η,
                    model.timestepper.G⁻.η,
                    dependencies=Event(device(model.architecture)))

    return event
end

@kernel function _ab2_step_free_surface!(η, χ::FT, Δt, Gηⁿ, Gη⁻) where FT
    i, j = @index(Global, NTuple)

    @inbounds begin
        η[i, j, 1] += Δt * ((FT(1.5) + χ) * Gηⁿ[i, j, 1] - (FT(0.5) + χ) * Gη⁻[i, j, 1])
    end
end

function ab2_step_free_surface!(free_surface::ImplicitFreeSurface, velocities_update, model, χ, Δt)

    ##### Implicit solver for η

    ## Need to wait for u* and v* to finish
    wait(device(model.architecture), velocities_update)
    fill_halo_regions!(model.velocities, model.architecture, model.clock, fields(model) )

    ## Leaving this here for now. There may be some scenarios where stepping forward η and then using
    ## the stepped forward value as a guess is helpful.
    ## η_save = deepcopy(free_surface.η)
    ## event = explicit_ab2_step_free_surface!(free_surface, velocities_update, model, χ, Δt)
    ## wait(device(model.architecture), event)

    ## We need vertically integrated U,V
    event = compute_vertically_integrated_volume_flux!(free_surface, model)
    wait(device(model.architecture), event)
    u=free_surface.barotropic_volume_flux.u
    v=free_surface.barotropic_volume_flux.v
    fill_halo_regions!(u.data ,u.boundary_conditions, model.architecture, model.grid, model.clock, fields(model) )
    fill_halo_regions!(v.data ,v.boundary_conditions, model.architecture, model.grid, model.clock, fields(model) )

    ## Compute volume scaled divergence of the barotropic transport and put into solver RHS
    event = compute_volume_scaled_divergence!(free_surface, model)
    wait(device(model.architecture), event)

    ## Include surface pressure term into RHS
    RHS = free_surface.implicit_step_solver.solver.settings.RHS
    RHS .= RHS/(model.free_surface.gravitational_acceleration*Δt)
    η = free_surface.η
    fill_halo_regions!(RHS   , η.boundary_conditions, model.architecture, model.grid)
    fill_halo_regions!(η.data, η.boundary_conditions, model.architecture, model.grid)
    ##  need to subtract Azᵃᵃᵃ(i, j, 1, grid)*η[i,j, 1]/(g*Δt^2)
    event = add_previous_free_surface_contribution(free_surface, model, Δt )
    wait(device(model.architecture), event)
    fill_halo_regions!(RHS   , η.boundary_conditions, model.architecture, model.grid)
    ## RHS .= RHS .+ free_surface.η.data/Δt

    ## Then we can invoke solve_for_pressure! on the right type via calculate_pressure_correction!
    x  = free_surface.implicit_step_solver.solver.settings.x
    x .= η.data
    fill_halo_regions!(x ,η.boundary_conditions, model.architecture, model.grid)
    solve_poisson_equation!(free_surface.implicit_step_solver.solver, RHS, x; Δt=Δt, g=free_surface.gravitational_acceleration)
    fill_halo_regions!(x ,η.boundary_conditions, model.architecture, model.grid)
    free_surface.η.data .= x

    ## The explicit form of this function defaults to returning an event, we do the same for now.
    return event
end


# mri/z-list.jl

include("kspace.jl")
export ir_mri_kspace_ga_radial

include("sensemap-sim.jl")
export ir_mri_sensemap_sim


"""
    to_linear_operator(lind::GraphLindbladian, [T=eltype(lind)])

Converts the Lindbladian to a `KLocalOperatorSum` with eltype `T` for the non
Hermitian Hamiltonian and a set of KLocalOperator(Sum) for the dissipators.

This structure has minimal memory cost, and is NEEDED when the need to represent
systems in very big lattices arises. Evidently, there is a small performance
price to pay when using those structures.
"""
to_linear_operator(lind::GraphLindbladian, T=nothing) =
    to_linear_operator(hamiltonian(lind), jump_operators_graph(lind), T)

function to_linear_operator(ham::GraphOperator, c_ops::Vector, T::Union{Nothing, Type{<:Number}}=nothing)

    ham_locs = ham.LocalOperators

    # default type
    T = isnothing(T) ?  eltype(first(ham_locs).data) : T

    op_loc = KLocalOperatorRow(T, [1], [length(basis(first(ham_locs)))],
                               first(ham_locs).data)

    H = KLocalOperatorSum(op_loc)

    # Add the local hamiltonian terms
    for (i, h_loc) = enumerate(ham.LocalOperators)
        i == 1 && continue

        dim = length(basis(h_loc))
        sum!(H, KLocalOperatorRow(T, [i], [dim], h_loc.data))
    end

    # Hoppings
    for edge=edges(graph(ham))
        connections = ham.EdgeOpList[edge]
        for (coeff, l_id, r_id)=connections
            hl = ham.HopOperatorsList[l_id]
            hr = ham.HopOperatorsList[r_id]

            dim_l = length(basis(hl))
            dim_r = length(basis(hr))
            sum!(H, KLocalOperatorRow(T, [edge.src, edge.dst],
                                      [dim_l, dim_r],
                                      coeff*((hl⊗hr).data)))
        end
    end

    # store the vector of local-acting, individual dissipators
    loss_ops   = []
    loss_ops_t = []

    # dissipations
    # TODO support nonlocal dissipators
    for (i, L) = enumerate(c_ops)

        # Find local basis where the operator is non-zero
        nz_sites = Int[]
        for (j, L_j) = enumerate(L.LocalOperators)
            length(L_j.data.nzval) == 0 && continue
            push!(nz_sites, j)
        end

        isempty(nz_sites) && continue

        hilb_dims = length.(basis.(L.LocalOperators[nz_sites]))

        L_nz      = tensor(L.LocalOperators[nz_sites]...)

        op        = KLocalOperatorRow(T, nz_sites, hilb_dims, L_nz.data)

        op_hnh    = KLocalOperatorRow(T, nz_sites, hilb_dims,
                                      -im/2*(L_nz'*L_nz).data)

        sum!(H, op_hnh)
        push!(loss_ops, op)
        push!(loss_ops_t, transpose(op))
    end

    T = isempty(loss_ops) ? Nothing : typeof(first(loss_ops))

    loss_ops   = Vector{T}(loss_ops)
    loss_ops_t = Vector{T}(loss_ops_t)

    return (H, loss_ops, loss_ops_t)
end

"""
    to_linear_operator(lind::GraphLindbladian, [T=eltype(lind)])

Converts the Hamiltonian to a `KLocalOperatorSum` with eltype `T`.

This structure has minimal memory cost, and is NEEDED when the need to represent
systems in very big lattices arises. Evidently, there is a small performance
price to pay when using those structures.
"""
function to_linear_operator(op::GraphOperator, T::Union{Nothing, Type{<:Number}}=nothing)

    op_locs = op.LocalOperators

    # default type
    T = isnothing(T) ?  eltype(first(op_locs).data) : T

    op_loc = KLocalOperatorRow(T, [1], [length(basis(first(op_locs)))],
                        first(op_locs).data)

    res_op = KLocalOperatorSum(op_loc)

    # Add the local hamiltonian terms
    for (i, op_loc) = enumerate(op.LocalOperators)
        i == 1 && continue # the first one was already added by the constructor.

        dim = length(basis(op_loc))
        sum!(res_op, KLocalOperatorRow(T, [i], [dim], op_loc.data))
    end

    # Hoppings
    for edge=edges(graph(op))
        edge ∉ keys(op.EdgeOpList) && continue

        connections = op.EdgeOpList[edge]
        for (coeff, l_id, r_id)=connections
            hl = op.HopOperatorsList[l_id]
            hr = op.HopOperatorsList[r_id]

            dim_l = length(basis(hl))
            dim_r = length(basis(hr))
            sum!(res_op, KLocalOperatorRow(T, [edge.src, edge.dst],
                                           [dim_l, dim_r],
                                           coeff*((hl⊗hr).data)))
        end
    end

    return res_op
end


#=##############################################################################################
Filename: plot_lift_moment_distribution.jl
Author: Ryan Anderson
Contact: rymanderson@gmail.com
README: `<: Action` function plots the lift distribution of all lifting surfaces at the specified steps
=###############################################################################################

"""
    plot_lift_moment_distribution(aircraft, parameters, freestream, environment, step_range, stepi, step_symbol) <: Action

# Arguments:

* `aircraft::Aircraft`: `Aircraft` system object
* `parameters<:Parameters`: `Parameters` struct
* `freestream::Freestream`: `Freestream` object
* `environment::Environment` `Environment` object
* `step_range::AbstractArray`: array of step for which the simulation is run
* `stepi::Int`: index of the current step
* `step_symbol::String`: defines the step, e.g. `alpha` or `time`

`parameters <: Parameters` requires the following elements:

* `surfacenames::Vector{String}`: names of each lifting surface to be shown in the legend
* `cfs::Vector{Array{Float64,2}}`: vector with length equal to the number of lifting surfaces, each member containing an array of size (3,ns) of force coefficients cd, cy, cl
* `cms::Vector{Array{Float64,2}}`: vector with length equal to the number of lifting surfaces, each member containing an array of size (3,ns) of moment coefficients cmx, cmy, cmz
* `plot_directory::String`: path to the folder where plots will be saved
* `plot_base_name::String`: first part of saved figure file names
* `plot_extension::String`: extension of saved figure file names
* `plotstepi::Vector{Int}`: which steps at which to plot
"""
function plot_lift_moment_distribution(aircraft, parameters, freestream, environment, step_range, stepi, step_symbol)

    # extract plot indices
    plotstepi = parameters.plotstepi
    basename = parameters.plot_base_name

    # cycle through styles
    styles = ["-", "--", "--^", "--v"]

    # extract main wing span
    b = aircraft.wing_system.system.surfaces[1][end].rtr[2] * 2

    if stepi in plotstepi

        # extract info
        surfacenames = parameters.surfacenames
        cfs = parameters.cfs
        cms = parameters.cms
        lifting_line_rs = aircraft.wing_system.lifting_line_rs
        plot_directory = parameters.plot_directory
        plot_base_name = parameters.plot_base_name
        plot_extension = parameters.plot_extension
        # cl_ylim = parameters.cl_ylim
        # cd_ylim = parameters.cd_ylim
        # cy_ylim = parameters.cy_ylim

        # create axes
        fig_lift_distribution = plt.figure(basename * "_lift_distribution")
        fig_cf_distribution = plt.figure(basename * "_cf_distribution")
        nsurfaces = length(surfacenames)
        nsubplotbase = 300 + 10 * nsurfaces
        if stepi == parameters.plotstepi[1]
            fig_lift_distribution.clear()
            fig_lift_distribution.add_subplot(111, ylabel = L"c_l", xlabel = L"2y/b")
            fig_cf_distribution.clear()
            for isurface = 1:nsurfaces
                fig_cf_distribution.add_subplot(nsubplotbase + 2 + (isurface-1) * nsurfaces, ylabel = L"c_d") # drag ylim = cd_ylim,
                fig_cf_distribution.add_subplot(nsubplotbase + 3 + (isurface-1) * nsurfaces, ylabel = L"c_y", xlabel = L"y [m]") # side force ylim = cy_ylim,
                fig_cf_distribution.add_subplot(nsubplotbase + 1 + (isurface-1) * nsurfaces, ylabel = L"c_l", title = surfacenames[isurface]) # lift ylim = cl_ylim,
                # fig_cf_distribution.suptitle("t = $(step_range[stepi]), ti = $stepi") # add title

                # set axes
                # axs[1].set_ylim(cd_ylim)
                # axs[1].set_ylabel(L"c_d")
                # # axs[1].legend()
                # axs[2].set_ylim(cy_ylim)
                # axs[2].set_ylabel(L"c_y")
                # axs[2].set_xlabel(L"y [m]")
                # axs[2].legend()
                # axs[3].set_ylim(cl_ylim)
                # axs[3].set_ylabel(L"c_l")
            end
        end
        ax_cl = fig_lift_distribution.get_axes()[1]
        axs_cf = fig_cf_distribution.get_axes()

        # get color
        cratio = findfirst((x)->x==stepi,plotstepi) / length(plotstepi)

        # plot
        for (isurface,cf) in enumerate(cfs[stepi])
            rs_plot = get_midpoints(lifting_line_rs[isurface][2,:])
            ax_cl.plot(rs_plot ./ b * 2, cf[3,:], styles[isurface], color = (0.05, 0.85-cratio*0.7, 0.15 + 0.75 * cratio), label="$(surfacenames[isurface]), $step_symbol = $(round(step_range[stepi],digits=3))")
            for icf = 1:3
                axs_cf[(isurface - 1) * nsurfaces + icf].plot(rs_plot, cf[icf,:], color=(0.05, 0.85-cratio*0.7, 0.15 + 0.75 * cratio), label="$step_symbol = $(round(step_range[stepi],digits=3))")
            end
        end

        # save
        if stepi == plotstepi[end] # last step
            ax_cl.legend(loc="upper left", bbox_to_anchor=(1.01,1))
            fig_lift_distribution.set_size_inches(10, 6, forward=true)
            fig_lift_distribution.tight_layout()
            savepath_cl = joinpath(plot_directory, plot_base_name * "_lift_distribution" * plot_extension)
            fig_lift_distribution.savefig(savepath_cl, bbox_inches="tight")
            axs_cf[3].legend(loc="upper left", bbox_to_anchor=(1.01,1)) # set legend
            fig_cf_distribution.set_size_inches(6, 10, forward=true)
            fig_cf_distribution.tight_layout() # clean up white space
            savepath_cf = joinpath(plot_directory, plot_base_name * "_cf_distribution" * plot_extension)
            fig_cf_distribution.savefig(savepath_cf, bbox_inches="tight")
        end

        # cms plot
        fig_moment_distribution = plt.figure(basename * "_moment_distribution")
        nsurfaces = length(surfacenames)
        nsubplotbase = 300 + 10 * nsurfaces
        if stepi == parameters.plotstepi[1]
            fig_moment_distribution.clear()
            for isurface = 1:nsurfaces
                fig_moment_distribution.add_subplot(nsubplotbase + 2 + (isurface-1) * nsurfaces, ylabel = L"c_{mx}") # drag ylim = cd_ylim,
                fig_moment_distribution.add_subplot(nsubplotbase + 1 + (isurface-1) * nsurfaces, ylabel = L"c_{my}", title = surfacenames[isurface]) # lift ylim = cl_ylim,
                fig_moment_distribution.add_subplot(nsubplotbase + 3 + (isurface-1) * nsurfaces, ylabel = L"c_{mz}", xlabel = L"y [m]") # side force ylim = cy_ylim,
                # fig_moment_distribution.suptitle("t = $(step_range[stepi]), ti = $stepi") # add title

                # set axes
                # axs[1].set_ylim(cd_ylim)
                # axs[1].set_ylabel(L"c_d")
                # # axs[1].legend()
                # axs[2].set_ylim(cy_ylim)
                # axs[2].set_ylabel(L"c_y")
                # axs[2].set_xlabel(L"y [m]")
                # axs[2].legend()
                # axs[3].set_ylim(cl_ylim)
                # axs[3].set_ylabel(L"c_l")
            end
        end

        # plot
        axs = fig_moment_distribution.get_axes()
        for (isurface,cm) in enumerate(cms[stepi])
            rs_plot = get_midpoints(lifting_line_rs[isurface][2,:])
            for icf = 1:3
                axs[(isurface - 1) * nsurfaces + icf].plot(rs_plot, cm[icf,:], color=(0.05, 0.85-cratio*0.7, 0.15 + 0.75 * cratio), label="$step_symbol = $(round(step_range[stepi],digits=3))")
            end
        end

        # save
        if stepi == plotstepi[end]
            axs[2].legend(loc="upper left", bbox_to_anchor=(1.01,1)) # set legend
            fig_moment_distribution.set_size_inches(6, 10, forward=true)
            fig_moment_distribution.tight_layout() # clean up white space
            savepath = joinpath(plot_directory, plot_base_name * "_moment_distribution" * plot_extension)
            fig_moment_distribution.savefig(savepath, bbox_inches="tight")
        end
    end

    return false
end

"""
plot_lift_moment_distribution(aircraft, step_range)

Method returns initialized elements required for the `parameters <: Parameters` struct during simulation.

# Arguments:

* `aircraft::Aircraft`: system to be simulated
* `step_range::AbstractArray`: defines each step of the simulation

# Returns:

* `surfacenames::Vector{String}`: names of each lifting surface to be shown in the legend
* `cfs::Vector{String}`: vector with length equal to the number of lifting surfaces, each member containing an array of size (3,ns) of force coefficients
* `plot_directory::String`: path to the folder where plots will be saved
* `plot_base_name::String`: first part of saved figure file names
* `plot_extension::String`: extension of saved figure file names
"""
function plot_lift_moment_distribution(aircraft, step_range)

    nwings = length(aircraft.wing_system.surfaces)
    surfacenames = Vector{String}(undef,nwings)
    cfs = [[zeros(3, length(aircraft.wing_system.surfaces[i])) for i in 1:nwings] for j in 1:length(step_range)]
    cms = deepcopy(cfs)
    plot_directory = ""
    plot_base_name = "default"
    plot_extension = ".pdf"
    plotstepi = deepcopy(step_range)

    return surfacenames, cfs, cms, plot_directory, plot_base_name, plot_extension, plotstepi#, cl_ylim, cd_ylim, cy_ylim
end


using Test
using SymbolicUtils

# == / != syntax is nice, let's use it in tests
macro eqtest(expr)
    @assert expr.head == :call && expr.args[1] in [:(==), :(!=)]
    if expr.args[1] == :(==)
        :(@test isequal($(expr.args[2]), $(expr.args[3])))
    else
        :(@test !isequal($(expr.args[2]), $(expr.args[3])))
    end |> esc
end
SymbolicUtils.show_simplified[] = false

include("basics.jl")
include("order.jl")
include("rewrite.jl")
include("rulesets.jl")
include("nf.jl")
include("interface.jl")
include("fuzz.jl")

if haskey(ENV, "CI")
    include("benchmark.jl")
    using Pkg
    Pkg.add(url="https://github.com/SciML/ModelingToolkit.jl.git", rev="master")
    Pkg.test("ModelingToolkit")
end


# This file is a part of LegendDataTypes.jl, licensed under the MIT License (MIT).

# TODO: Replace by true custom types to avoid type piracy (e.g. with plotting
# recipes) on NamedTuple and TypedTables.Table (even though the types aliased
# here are quite specific, so the risk of a conflict is low).

const WaveformSamples = AbstractVector{<:RealQuantity}

"""
    ChannelWaveform

Fields:

* `evtno::Integer`: channel-event number
* `chno::Integer`: channel number
* `samplerate::RealQuantity`: sample rate
* ``
"""
const ChannelWaveform = NamedTuple{
    (:evtno, :chno, :delta_t, :offset, :samples),
    <:Tuple{Integer, Integer, RealQuantity, RealQuantity, WaveformSamples}
}
export ChannelWaveform

const ChannelWaveforms = TypedTables.Table{<:ChannelWaveform}
export ChannelWaveforms

timeaxis(waveform::ChannelWaveform) = waveform.delta_t .* (waveform.offset - 1 .+ axes(waveform.samples, 1))
export timeaxis

#=
function StatsBase.Histogram(waveforms::ChannelWaveforms)
    samples = ...

    xedge = axes(samples, 1)
    ph = Histogram((xedge, yedge), Float64, :left)
    @inbounds for evtno in axes(samples, 2)
        for i in axes(samples, 1)
            push!(ph, (i, samples[i, evtno]))
        end
    end
    ph.weights .= log10orNaN.(ph.weights)
    ph
end
=#


module CellArrayApply

using Gridap
using Gridap.Helpers
using Gridap.CachedArrays
using Gridap.Kernels: _compute_T, _compute_N
using Gridap.CellNumberApply: _checks
using Gridap.CellNumberApply: _getvalues
using Gridap.Kernels: _size_for_broadcast

import Gridap: apply
import Base: iterate
import Base: length
import Base: size
import Base: getindex
import Base: IndexStyle
import Gridap.CellNumberApply: _apply

function apply(k::ArrayKernel,v::Vararg{<:CellValue})
  CellArrayFromKernel(k,v...)
end

function apply(k::ArrayKernel,v::Vararg{<:IndexCellValue})
  IndexCellArrayFromKernel(k,v...)
end

function _apply(f,v,::Val{true})
  k = ArrayKernelFromBroadcastedFunction(f)
  apply(k,v...)
end

struct CellArrayFromKernel{T,N,K,V} <: IterCellArray{T,N,CachedArray{T,N,Array{T,N}}}
  kernel::K
  cellvalues::V
end

function CellArrayFromKernel(k::ArrayKernel,v::Vararg{<:CellValue})
  _checks(v)
  T = _compute_T(k,v)
  N = _compute_N(k,v)
  K = typeof(k)
  V = typeof(v)
  CellArrayFromKernel{T,N,K,V}(k,v)
end

function length(self::CellArrayFromKernel)
  vi, = self.cellvalues
  length(vi)
end

@inline function iterate(self::CellArrayFromKernel{T,N}) where {T,N}
  zipped = zip(self.cellvalues...)
  znext = iterate(zipped)
  v = CachedArray(T,N)
  _iterate(self,znext,zipped,v)
end

@inline function iterate(self::CellArrayFromKernel,state)
  zipped, zstate, v = state
  znext = iterate(zipped,zstate)
  _iterate(self,znext,zipped,v)
end

@inline function _iterate(self::CellArrayFromKernel,znext,zipped,v)
  if znext === nothing; return nothing end
  a, zstate = znext
  s = _compute_sizes(a...)
  z = compute_size(self.kernel,s...)
  setsize!(v,z)
  compute_value!(v,self.kernel,a...)
  state = (zipped,zstate,v)
  (v, state)
end

const _bs = _size_for_broadcast

# TODO use a generated function?
_compute_sizes(a...) = @notimplemented
_compute_sizes(a1) = (_bs(a1),)
_compute_sizes(a1,a2) = (_bs(a1),_bs(a2))
_compute_sizes(a1,a2,a3) = (_bs(a1),_bs(a2),_bs(a3))
_compute_sizes(a1,a2,a3,a4) = (_bs(a1),_bs(a2),_bs(a3),_bs(a4))
_compute_sizes(a1,a2,a3,a4,a5) = (_bs(a1),_bs(a2),_bs(a3),_bs(a4),_bs(a5))
_compute_sizes(a1,a2,a3,a4,a5,a6) = (_bs(a1),_bs(a2),_bs(a3),_bs(a4),_bs(a5),_bs(a6))

struct IndexCellArrayFromKernel{T,N,K,V} <: IndexCellArray{T,N,CachedArray{T,N,Array{T,N}},1}
  kernel::K
  cellvalues::V
  cache::CachedArray{T,N,Array{T,N}}
end

function IndexCellArrayFromKernel(k::ArrayKernel,v::Vararg{<:CellValue})
  _checks(v)
  T = _compute_T(k,v)
  N = _compute_N(k,v)
  K = typeof(k)
  V = typeof(v)
  cache = CachedArray(T,N)
  IndexCellArrayFromKernel{T,N,K,V}(k,v,cache)
end

function length(self::IndexCellArrayFromKernel)
  vi, = self.cellvalues
  length(vi)
end

size(self::IndexCellArrayFromKernel) = (length(self),)

function getindex(self::IndexCellArrayFromKernel,i::Integer)
  a = _getvalues(i,self.cellvalues...)
  s = _compute_sizes(a...)
  z = compute_size(self.kernel,s...)
  v = self.cache
  setsize!(v,z)
  compute_value!(v,self.kernel,a...)
  v
end

end # module


export Prostate

"""
    Prostate

The dataset is from the following reference:

> Dinesh Singh, Phillip G. Febbo, Kenneth Ross, Donald G. Jackson, Judith Manola,
 Christine Ladd, Pablo Tamayo, Andrew A. Renshaw, Anthony V. D’Amico, Jerome P. Richie,
 Eric S. Lander, Massimo Loda, Philip W. Kantoff, Todd R. Golub, and William R. Sellers.
 Gene expression correlates of clinical prostate cancer behavior.
 Cancer cell, 1(2): 203–209, 2002.

See the following monograph for further illustrations
of empirical Bayes methods on this dataset:

> Bradley Efron. Large-scale inference: Empirical Bayes methods for estimation, testing,
and prediction. Cambridge University Press, 2012.
"""
module Prostate

using CSV
using ..Empirikos: MultinomialSummary, StandardNormalSample

const URL = "https://web.stanford.edu/~hastie/CASI_files/DATA/prostz.txt"

function load_table()
    CSV.File(download(URL), header=false)
end

function ebayes_samples()
    tbl = load_table()
    Zs = StandardNormalSample.(tbl.Column1)
end

end


module LogisticRegression
    using Pluto

    include("logistic_regression.jl")

    run_notebook_logistic_regression() = Pluto.run(notebook=joinpath(@__DIR__, "../notebook/logistic_regression.jl"))
    run_notebook_candy_contain_chocolate() = Pluto.run(notebook=joinpath(@__DIR__, "../notebook/candy_contain_chocolate.jl"))
end


module ACL

export
    floorsum

include("math/floorsum.jl")

end


@system Planting(Stage) begin
    planting_date ~ hold

    planted(planting_date, t=calendar.time) => (t >= planting_date) ~ flag
end


"""
    molar_mass(c::AbstractEoSComponent)

Return the molar mass of a component.
"""
function molar_mass end

"""
    carbon_number(c::AbstractEoSComponent)

Return the number of carbons atoms in the hydrocarbon chain.
"""
function carbon_number end

"""
    name(c::AbstractEoSComponent)

Return the component species name.
"""
function name end

"""
    describe(c::AbstractEoSComponent)

Return `Dict` of parameters. Useful for logging.
"""
function describe end

"""
    load(::Type{T}; name::AbstractString, databases) where {T<:AbstractEoSComponent}

Load component `::T` by its `name` by joining the data on this species in `databases`.
"""
function load end

Base.length(m::AbstractEoSMixture) = m.number_of_components
Base.show(io::IO, x::AbstractEoSMixture) = print(io, "$(typeof(x))($(name(x)))")

components(m::AbstractEoSMixture) = m.components
name(m::AbstractEoSMixture) = join(map(name, components(m)), " + ")
describe(m::AbstractEoSMixture) = Dict{String,Any}("noparameters" => NaN)
load(::Type{T}; names, physdb) where {T<:AbstractEoSMixture} = error("NotImpemented")

function describe(x::BrusilovskyEoSComponent)
    return Dict{String,Any}(
        "data structure" => repr(x),
        "name" => name(x),
        "critical pressure [Pa]" => x.Pc,
        "critical temperature [K]" => x.Tc,
        "pitzer acentric factor" => x.acentric_factor,
        "molar mass [kg mol⁻¹]" => x.molar_mass,
        "number of carbons atoms" => x.carbon_number,
        "eos" => "brusilovsky",
        "eos param: ac [?]" => x.ac,
        "eos param: b [?]"  => x.b,
        "eos param: c [?]"  => x.c,
        "eos param: d [?]"  => x.d,
    )
end


# TODO: add waveguide sites for matched conditions
"""
	module Simulations

Package data used to define simulation and generate the relevant differential operators.
Exports are `Simulation`, `smooth!`, `update_dielectric!`, `update_pump`
"""
module Simulations

export Simulation
export smooth!
export update_dielectric!
export update_pump!

files = (
	"1D/Simulations1D.jl",
	"2D/Symmetric/Simulations2D.jl",
	# "3D/Simulations3D.jl"
	)

using ..Curlcurls
using ..Dispersions
using ..Domains
using ..Laplacians
using ..Lattices
using ..Points
using ..SelfEnergies
using ..Shapes
using ..VectorFields
# using LinearAlgebra
using SparseArrays
using Statistics
using RecipesBase

import ..AbstractDomain
import ..Symmetric, ..Unsymmetric

"""
	Simulation(ω₀, domains...; [k₂₀, k₃₀, k₁₀]) -> simulation

Build a simulation object which can be passed to the various solvers to find
resonance frequencies, etc.

There are three types of `domains`: `LatticeDomain`, `NondispersiveDomain`,
`DispersiveDomain`, which can be provided in any order, though at least one
`LatticeDomain` must be provided.
Where domains of the same type overlap, the one that appears first takes priority.
"""
struct Simulation{N,CLASS,T,TLDOM,TNDOM,TDDOM,TSE}
	lattice_domains::TLDOM
	nondispersive_domains::TNDOM
	dispersive_domains::TDDOM

	x::Vector{Point{N,Cartesian}}
	lattice_domain_indices::Vector{Int}
	nondispersive_domain_indices::Vector{Int}
	dispersive_domain_indices::Vector{Int}
	ε::Vector{ComplexF64}
	F::Vector{Float64}
	χ::Vector{AbstractDispersion}
	Fs::Vector{Vector{Float64}}

	laplacian::Laplacian{N,CLASS}
	# curlcurl::Curlcurl{N,CLASS}
	self_energy::TSE
	α::NTuple{N,Vector{ComplexF64}}
	σ::NTuple{N,Vector{ComplexF64}}
	x_half::NTuple{N,Vector{Point{N,Cartesian}}}
	σ_half::NTuple{N,Vector{ComplexF64}}
	ω₀::Float64
	k₁₀::Float64
	k₂₀::Float64
	k₃₀::Float64
	smoothed::Base.RefValue{Bool}
end

# for arbitrary domain order
function Simulation(
			ω₀::Real,
			domains::Vararg{AbstractDomain{N}};
			kwargs...
			) where N

	lattice_domains = _get_lattice_domains(domains...)
	nondispersive_domains = _get_nondispersive_domains(domains...)
	dispersive_domains = _get_dispersive_domains(domains...)

	# check
	length(lattice_domains)>0 || throw(ArgumentError("must provide at least one `LatticeDomain`, but 0 provided instead"))
	return Simulation(ω₀, lattice_domains, nondispersive_domains, dispersive_domains; kwargs...)
end

# functions to extract lattice_domains, etc, from arbitrary list of domains
fnames = (:_get_lattice_domains, :_get_nondispersive_domains, :_get_dispersive_domains)
types = (LatticeDomain, NondispersiveDomain, DispersiveDomain)
for i ∈ eachindex(types)
	@eval begin
		$(fnames[i])(ldom::$(types[i]), args...) = (ldom,$(fnames[i])(args...)...)
		$(fnames[i])(arg, args...) = $(fnames[i])(args...)
		$(fnames[i])(ldom1::$(types[i]), ldom2::$(types[i]), args...) = (ldom1,ldom2,$(fnames[i])(args...)...)
		$(fnames[i])(arg, ldom::$(types[i]), args...) = (ldom,$(fnames[i])(args...)...)
		$(fnames[i])(ldom::$(types[i]), arg, args...) = (ldom,$(fnames[i])(args...)...)
		$(fnames[i])(arg1, arg2, args...) = $(fnames[i])(args...)
		$(fnames[i])() = ()
	end
end

Base.length(sim::Simulation) = length(sim.x)
Base.ndims(sim::Simulation{N}) where N = N

# Base.conj(sim::Simulation) = Simulation(sim.ω₀,map(conj,sim.domains)...)

import ..NUM_SUBPIXELS

"""
	smooth!(sim,[num_sub_pixel])

Use sub-pixel sampling along domain walls to smooth the dieletric and pump profiles.

`num_sub_pixel` defaults to `NUM_SUB_PIXEL` in module `Defaults`, which is currently set to 5.
"""
function smooth!(sim::Simulation, num_sub_pixel::Integer = NUM_SUBPIXELS)
    smooth_dielectric!(sim, num_sub_pixel)
	smooth_pump!(sim, num_sub_pixel)
	sim.smoothed[] = true
    return sim
end

"""
	update!(sim)

update simulation `sim` by recomputing the dielectric and pump functions
"""
function VectorFields.update!(sim::Simulation)
	update_dielectric!(sim)
	update_pump!(sim)
	return nothing
end

"""
	update_dielectric!(sim,[ε])

update simulation `sim` by recomputing the dielectric function or by giving it an explicit array `ε`
"""
function update_dielectric!(sim::Simulation)
	simulation_dielectric!(sim.ε, sim, sim.x)
	sim.smoothed[] = false
	return nothing
end
function update_dielectric!(sim::Simulation,ε::AbstractVecOrMat)
	copyto!(sim.ε,ε)
	sim.smoothed[] = false
	return nothing
end

"""
	update_pump!(sim,[F])

update simulation `sim` by recomputing the pump function or by giving it an explicit array `F`
"""
function update_pump!(sim::Simulation)
	simulation_pump!(sim.F, sim, sim.x)
	sim.smoothed[] = false
	return nothing
end
function update_pump!(sim::Simulation,F::AbstractVecOrMat)
	copyto!(sim.F, F)
	sim.smoothed[] = false
	return nothing
end

# the dielectric as a function of position (x,y)
# function simulation_dielectric(sim::Simulation, x::Vector)
	# ε = Vector{ComplexF64}(undef,length(x))
	# simulation_dielectric!(ε, sim , x)
	# return ε
# end
function simulation_dielectric!(ε::Vector, sim::Simulation, x::Vector, backgrounds=map(d->d.ε, sim.lattice_domains), dielectrics=map(d->d.dielectric, sim.nondispersive_domains))
    for i ∈ eachindex(x) ε[i] = simulation_dielectric(sim, x[i], backgrounds, dielectrics) end
    return nothing
end
@inline function simulation_dielectric(sim::Simulation, x::Point, backgrounds=map(d->d.ε, sim.lattice_domains), dielectrics=map(d->d.dielectric, sim.nondispersive_domains))
    lattice_index = which_domain(sim.lattice_domains, x)
	nondispersive_index = which_domain(sim.nondispersive_domains, x)
	if nondispersive_index==0
		return backgrounds[lattice_index]
	else
	    return dielectrics[nondispersive_index](x)
	end
end

# the dielectric as a function of position (x,y)
# function simulation_pump(sim::Simulation, x::Vector)
# 	F = Vector{Float64}(undef,length(x))
# 	simulation_pump!(F, sim , x)
# 	return F
# end
function simulation_pump!(F::Vector, sim::Simulation, x::Vector, pumps=map(d->d.pump, sim.dispersive_domains))
    for i ∈ eachindex(x) F[i] = simulation_pump(sim, x[i], pumps) end
    return nothing
end
@inline function simulation_pump(sim::Simulation, x::Point, pumps=map(d->d.pump, sim.dispersive_domains))
	dispersive_index = which_domain(sim.dispersive_domains, x)
	if dispersive_index==0
		return 0.0
	else
	    return pumps[dispersive_index](x)
	end
end

################################################################################
# utilities

# identify which domain is associated with a given spatial position (x)
function which_domains(domains, x::Vector)
    indices = Vector{Int}(undef, length(x))
	shapes = map(z->z.shape, domains)
    which_domains!(indices, domains, shapes, x)
    return indices
end
function which_domains!(indices::Vector{Int}, domains, shapes, x)
    @inbounds for i ∈ eachindex(x) indices[i] = which_domain(domains, shapes, x[i]) end
    return nothing
end
# above is for arrays of points, below is for single points
function which_domain(domains,x::Point)
	shapes = map(z->z.shape,domains)
    return which_domain(domains, shapes, x)
end
@inline function which_domain(domains, shapes, x::Point)
	index = findfirst(z->z(x),shapes)
    isnothing(index) ? index=0 : nothing
    iszero(index) ? nothing : (isvoid(domains[index]) ? index=0 : nothing)
    return index
end

isvoid(domain::AbstractDomain) = domain.type==:Void

smooth_dielectric!() = nothing
smooth_pump!() = nothing

foreach(include,files)

# take individually defined domains and concatenate all relevant data
function append_domains!(domain_index,nnm,nnp,name,x,indices,ε,F,interior,bulk,surface,corner,domains)
	count = 1
	for d ∈ domains
		append!(domain_index,fill(count,length(d.x)))
		for i ∈ eachindex(d.nnm)
			append!(nnm[i],d.nnm[i].+length(nnm[i]))
			append!(nnp[i],d.nnp[i].+length(nnp[i]))
		end
		append!(name,fill(d.name,length(d.x)))
		append!(x,d.x)
		append!(indices,d.indices)
		append!(ε,d.ε)
		append!(F,d.F)
		append!(interior,d.interior)
		append!(bulk,d.bulk)
		append!(surface,d.surface)
		append!(corner,d.corner)
		count += 1
	end
	return nothing
end

# identify which sites are to be removed by determining which sites belong to the topmost domain.
get_sites_to_remove(domains,dom,x) = which_domains(domains,x).!==dom

# remove the pre-computed nearest neighbors, which makes the bulk nn calculation very fast
function trim_nn!(nnm,nnp,removed,bulk)
	cr = cumsum(removed)
	for j ∈ eachindex(nnm)
		for i ∈ eachindex(nnm[j])
			if bulk[i]
				nnm[j][i] -= cr[nnm[j][i]]
				nnp[j][i] -= cr[nnp[j][i]]
			end
		end
	end
end

# remove the sites determined by get_sites_to_remove
remove_sites!(sites::Tuple,removed) = map(z->remove_sites!(z,removed),sites)
remove_sites!(site::NTuple{N,Array},removed) where N = map(z->deleteat!(z,findall(removed)),site)
remove_sites!(site,removed) = deleteat!(site,findall(removed))

################################################################################

VectorField{M}(sim::Simulation{N}, args...) where {N,M} = VectorField{N,M}(sim.x, args...)
VectorField{N,M}(sim::Simulation{N}, args...) where {N,M} = VectorField{N,M}(sim.x, args...)

################################################################################
#Pretty Printing

import ..PRINTED_COLOR_GOOD
import ..PRINTED_COLOR_WARN
import ..PRINTED_COLOR_DARK

function Base.show(io::IO,sim::Simulation{N,CLASS}) where {N,CLASS}
	suffix = length(sim.domains)>1 ? "s" : ""
	if sim.smoothed[]
		printstyled(io,"Smoothed ",color=PRINTED_COLOR_GOOD)
		print(io,"$(N)D $CLASS ")
		printstyled(io," Simulation",color=PRINTED_COLOR_DARK)
		println(io, " with ", length(sim.domains)," domain",suffix)
	else
		printstyled(io,"Unsmoothed ",color=PRINTED_COLOR_WARN)
		print(io,N, "D ", CLASS)
		printstyled(io," Simulation",color=PRINTED_COLOR_DARK)
		println(io, " with ", length(sim.domains)," domain",suffix)
	end
    print(io,"\tn sites: ")
	printstyled(io,length(sim),"\n",color=:light_cyan)
	println(io,"\t\t====================")
	for d ∈ eachindex(sim.domains)
		printstyled(io,"\t\tDomain ",color=PRINTED_COLOR_DARK)
		print(io, d," (",sim.domains[d].type,"): ")
		print(io,sim.domains[d].name)
		d < length(sim.domains) ? println(io) : nothing
	end
end

@recipe f(v::VectorField, sim::Simulation) = sim, v
@recipe f(e::VectorField,sim::Simulation,by::Function) = sim,e,by
@recipe f(sim::Simulation,by::Function,e::VectorField) = sim,e,by
@recipe f(e::VectorField,by::Function,sim::Simulation) = sim,e,by
@recipe f(by::Function,sim::Simulation,e::VectorField) = sim,e,by
@recipe f(by::Function,e::VectorField,sim::Simulation) = sim,e,by

end # module


# Eric S. Tellez <eric.tellez@infotec.mx>

#Pkg.add("GZip")
#Pkg.add("Glob")
#Pkg.add("JSON")

import JSON
import Glob

include("textmodel.jl")
include("io.jl")

function normpatterns(patlist::Vector{Vector{String}}, config::TextConfig)
    A = Vector{String}[]
    for patterns in patlist
        B = String[]
        for pat in patterns
            T = tokenize(pat, config)
            length(T) > 0 && push!(B, join(T, ' '))
        end
        length(B) > 0 && push!(A, B)
    end

    Set(A)
end

function normpatternsset(patlist::Vector{Vector{String}}, config::TextConfig)
    A = Vector{Set{String}}[]
    for patterns in patlist
        B = Set{String}[]
        for pat in patterns
            T = tokenize(pat, config)
            length(T) > 0 && push!(B, Set(T))
        end
        length(B) > 0 && push!(A, B)
    end

    Set(A)
end

function intersectmatch(fun, input, config::TextConfig, sep, key, patlist::Vector{Vector{String}})
    patlist = normpatternsset(patlist, config)

    while !eof(input)
        arr = split(rstrip(readline(input)), sep)
        if key < 0
            tokens = tokenize(arr[length(arr)+key+1], config) |> Set
        else
            tokens = tokenize(arr[key], config) |> Set
        end

        matches = 0
        for (i, patterns) in enumerate(patlist)
            for pat in patterns
                iset = intersect(tokens, pat)
                if length(iset) == length(pat)
                    matches += 1
                    break
                end
            end
            matches != i && break
        end

        matches == length(patlist) && fun(arr)
    end
end

function kerrorsmatch{T1 <: Any, T2 <: Any}(a::T1, b::T2, errors::Int)::Bool
    # if length(a) < length(b)
    #     a, b = b, a
    # end

    alen::Int = length(a)
    blen::Int = length(b)

    alen == 0 && return alen == blen
    blen == 0 && return true

    C::Vector{Int} = Vector{Int}(0:blen)

    for i in 1:alen
	prevA::Int = 0
	prevC::Int = C[1]
	j::Int = 1

	while j <= blen
	    cost::Int = 1
	    @inbounds if a[i] == b[j]
		cost = 0
	    end
	    @inbounds C[j] = prevA
	    j += 1
	    @inbounds prevA = min(C[j]+1, prevA+1, prevC+cost)
	    @inbounds prevC = C[j]
	end
	@inbounds C[j] = prevA
        if prevA <= errors
            return true
        end
    end

    return false
end

function kerrorsmatch(fun, input, config::TextConfig, sep, key, patlist::Vector{Vector{String}}, maxerrors::Int=1)
    # config.stem = false
    patlist = normpatterns(patlist, config)

    while !eof(input)
        arr = split(rstrip(readline(input)), sep)
        if key < 0
            text = normtext(arr[length(arr)+key+1], config)
        else
            text = normtext(arr[key], config)
        end

        matches = 0
        for (i, patterns) in enumerate(patlist)
            for pat in patterns
                if kerrorsmatch(text, pat, maxerrors)
                    matches += 1
                    break
                end
            end
            matches != i && break
        end

        matches == length(patlist) && fun(arr)
    end
end

function substringmatch(fun, input, config::TextConfig, sep, key, patlist::Vector{Vector{String}})
    # patlist = [[normtext(pat, config) for pat in patterns] for patterns in patlist]
    patlist = normpatterns(patlist, config)

    while !eof(input)
        arr = split(rstrip(readline(input)), sep)
        if key < 0
            text = normtext(arr[length(arr)+key+1], config)
        else
            text = normtext(arr[key], config)
        end

        matches = 0
        for (i, patterns) in enumerate(patlist)
            for pat in patterns
                if contains(text, pat)
                    matches += 1
                    break
                end
            end
            matches != i && break
        end

        matches == length(patlist) && fun(arr)
    end
end

function harvesine(p1::Vector{Float64}, p2::Vector{Float64})
    # Tweets use GEOJson format: [longitude, latitude]
    lon1, lat1 = p1
    lon2, lat2 = p2
    2 * 6372.8 * asin(sqrt(sind((lat2-lat1)/2)^2 + cosd(lat1) * cosd(lat2) * sind((lon2 - lon1)/2)^2))
end

function radius(fun, input, sep, key, coordinates)
    center = [x |> Float64 for x in coordinates["coordinates"]]
    distance = coordinates["radius"]

    while !eof(input)
        arr = split(rstrip(readline(input)), sep)
        if key < 0
            _coord = arr[length(arr)+key+1]
        else
            _coord = arr[key]
        end

        _coord == "{}" && continue
        p = JSON.parse(_coord)

        coord = [x |> Float64 for x in p["coordinates"]]

        if harvesine(center, coord) <= distance
            fun(arr)
        end
    end
end

function vocextraction(fun, input, config::TextConfig, sep, key)
    # patlist = [[normtext(pat, config) for pat in patterns] for patterns in patlist]
    thesaurus = Dict{String,Int}()
    while !eof(input)
        arr = split(rstrip(readline(input)), sep)
        if key < 0
            tokens = tokenize(arr[length(arr)+key+1], config)
        else
            tokens = tokenize(arr[key], config)
        end

        for tok in tokens
            thesaurus[tok] = get(thesaurus, tok, 0) + 1
        end
    end

    L = [(v, k) for (k,v) in thesaurus]
    sort!(L, by=(x) -> x[1])
    for x in L
        fun(x)
    end
end

config = TextConfig()
patlist = Vector{String}[]
stopwords = ""
kind = ""
key = -1
sep = "\t"
coordinates = nothing
errors = 1

for arg in ARGS
    arg = strip(arg)
    if arg[1] == '{'
        _config = JSON.parse(arg)
        for (k, v) in JSON.parse(arg)
            setfield!(config, Symbol(k), v)
        end
    elseif arg[1] == '['
        push!(patlist, JSON.parse(arg))
    elseif startswith(arg, "file:")
        push!(patlist, readlines(arg[6:end]))
    elseif startswith(arg, "stopwords:")
        stopwords = arg[length("stopwords:")+1:end]
    else
        if startswith(arg, "substring")
            kind = "substring"
        elseif startswith(arg, "intersect")
            kind = "intersect"
        elseif startswith(arg, "kerror")
            kind = "kerrors"
            if ':' in kind
                errors = parse(Int, split(arg, ':')[2])
            end
        elseif startswith(arg, "voc")
            kind = "voc"
        elseif startswith(arg, "key")
            key = parse(Int, split(arg, ':')[2])
        elseif startswith(arg, "sep")
            key = split(arg, ':')[2]
        elseif startswith(arg, "radius")
            kind = "radius"
            coordinates = JSON.parse(arg[8:end])  ## radius:{"coordinates":[long,lat],"radius":radius}
        else
            error("Unknown command $arg")
        end
    end
end

if length(stopwords) > 0
    config.stopwords = Set([join(normchars(strip(line), config)) for line in readlines(stopwords)])
end

if kind == "substring"
    substringmatch(STDIN, config, sep, key, patlist) do arr
        write(STDOUT, join(arr, sep), '\n')
    end
elseif kind == "intersect"
    intersectmatch(STDIN, config, sep, key, patlist) do arr
        write(STDOUT, join(arr, sep), '\n')
    end
elseif kind == "kerrors"
    kerrorsmatch(STDIN, config, sep, key, patlist, errors) do arr
        write(STDOUT, join(arr, sep), '\n')
    end
elseif kind == "radius"
    radius(STDIN, sep, key, coordinates) do arr
        write(STDOUT, join(arr, sep), '\n')
    end
elseif kind == "voc"
    vocextraction(STDIN, config, sep, key) do arr
        write(STDOUT, join(arr, '\t'), '\n')
    end
else
    info("""
    Usage: julia PatternSearch/matches.jl KIND SEP KEY CONFIG STOPWORDS PAT1 PAT2 ...
    Where

    KIND is substring|intersect|kerrors|voc|radius
    - in the case of kerrors you can specify the maximum number of errors using the syntax kerrors:numerrors
    - the default kind is not specified, it is a mandatory argument
    - radius dumps records intersecting a ball; the center and radius of the ball (in kilometers). It uses the harvesing formulae to compute distances. The syntax is radius:JSON-coordinates
        - JSON-coordinates are of the form {"coordinates": [longitude, latitute], "radius": kilometers}
        - for example,
             - radius:'{"coordinates":[-102.295914,21.918859], "radius": 0.3}'
    - voc dumps the vocabulary's histogram sorted by increasing frequency

    SEP is the separator string
    - the syntax is sep:sepstring, e.g.,
        - sep:"<SEP>" -- a <SEP> string
        - sep:,       -- a single comma
        - sep:\$'\\t'  -- a single tab
    - the tabulator is the default separator

    KEY indicates the id of the column
    - the syntax is key:numcolumn, e.g., key:1
    - negative keys are mapped to numcolumns-abs(key)+1 (so, the last column, -1, is just numcolumns)
    - the default key is the last one, i.e., key:-1

    CONFIG is a json-dictionary representing a TextConfig object, it controls how the text is preprocessed for the match
    - please check PatternSearch/textmodel.jl to check the fields and default values
    - the default value is created with the default constructor of TextConfig

    STOPWORDS indicates the path of the stopwords file
    - the syntax is stopwords:filename

    PAT is a json-list of strings, each string is a pattern in the list
    - PatternSearch/matches.jl basically evaluates a intersection-union tree
    - each PAT defines a list of patterns, the text needs to match at least one to be a candidate (union)
    - a real match is a candidate matching all given PATs (intersection)

    The input is always read from STDIN

    Note: This tool is intended to be used with unix pipes and command line tools like awk, perl, grep, PatternSearch/matches.jl, etc.;
    so a lot of possible features will never be implemented.

    """)
    error("please read the help")
end


using SimilaritySearch, Random

function main(res, n=30_000_000)
    Random.seed!(0)
    @info "===========", typeof(res), n
    st = initialstate(res)
    st = push!(res, st, 1, rand())
    st = push!(res, st, 2, rand())

    for i in 3:n
        if length(res, st) > 0 && rand() < 0.001
            _, st = pop!(res, st)
        end

        if length(res, st) > 0 && rand() < 0.001
            _, st = popfirst!(res, st)
        end
        st = push!(res, st, i, rand())
    end

    res
end

ksearch = 10
n = 1000
main(KnnResultMatrix(ksearch), n)
main(KnnResult(ksearch), n)

n = 30_000_000
@timev main(KnnResultMatrix(ksearch), n)
@timev main(KnnResult(ksearch), n)


using Oceananigans.Architectures
using Oceananigans.BoundaryConditions
using Oceananigans.Utils

"""
    HorizontalAverage{F, R, P, I, Ω} <: AbstractDiagnostic

A diagnostic for computing horizontal average of a field.
"""
mutable struct HorizontalAverage{F, R, P, I, Ω, G} <: AbstractDiagnostic
          field :: F
         result :: P
      frequency :: Ω
       interval :: I
       previous :: Float64
    return_type :: R
           grid :: G
end

"""
    HorizontalAverage(model, field; frequency=nothing, interval=nothing, return_type=Array)

Construct a `HorizontalAverage` of `field`.

After the horizontal average is computed it will be stored in the `result` property.

The `HorizontalAverage` can be used as a callable object that computes and returns the
horizontal average.

A `frequency` or `interval` (or both) can be passed to indicate how often to run this
diagnostic if it is part of `model.diagnostics`. `frequency` is a number of iterations
while `interval` is a time interval in units of `model.clock.time`.

A `return_type` can be used to specify the type returned when the `HorizontalAverage` is
used as a callable object. The default `return_type=Array` is useful when running a GPU
model and you want to save the output to disk by passing it to an output writer.
"""
function HorizontalAverage(field; frequency=nothing, interval=nothing, return_type=Array)
    arch = architecture(field)
    result = zeros(arch, field.grid, 1, 1, field.grid.Tz)
    return HorizontalAverage(field, result, frequency, interval, 0.0, return_type, field.grid)
end

# Normalize a horizontal sum to get the horizontal average.
normalize_horizontal_sum!(havg, grid) = havg.result /= (grid.Nx * grid.Ny)

"""
    run_diagnostic(model, havg::HorizontalAverage{NTuple{1}})

Compute the horizontal average of `havg.field` and store the result in `havg.result`.
"""
function run_diagnostic(model, havg::HorizontalAverage)
    zero_halo_regions!(parent(havg.field), model.grid)
    sum!(havg.result, parent(havg.field))
    normalize_horizontal_sum!(havg, model.grid)
    return nothing
end

function (havg::HorizontalAverage{F, Nothing})(model) where F
    run_diagnostic(model, havg)
    return havg.result
end

function (havg::HorizontalAverage)(model)
    run_diagnostic(model, havg)
    return havg.return_type(havg.result)
end


# 2.4 plotting
   tmpexpr = :(
      xlabel("Year"); grid(true);
      xlim([ idx_year2plot[1] - 1, idx_year2plot[end] + 1 ]);
   )
   figure(figsize = (13,8))
      subplot(2,2,1)  # gap / exp
         for tmpzeta in 1:length(reform_zeta_levs)
            plot( Dt[:Year][idx_plot], res_ContriRatRise["gap/exp"][tmpzeta], reform_zeta_stys[tmpzeta] )
         end
         eval(tmpexpr); ylabel("Gap / Pool Expenditure (%)");
         legend( string.("ζ = ",reform_zeta_levs) , loc = "lower right")
      subplot(2,2,2)  # gap / in
         for tmpzeta in 1:length(reform_zeta_levs)
            plot( Dt[:Year][idx_plot], res_ContriRatRise["gap/in"][tmpzeta], reform_zeta_stys[tmpzeta] )
         end
         eval(tmpexpr); ylabel("Gap / Pool Income (%)");
         # legend( string.("ζ = ",reform_zeta_levs) , loc = "upper left")
      subplot(2,2,3)  # gap / gdp
         for tmpzeta in 1:length(reform_zeta_levs)
            plot( Dt[:Year][idx_plot], res_ContriRatRise["gap/gdp"][tmpzeta], reform_zeta_stys[tmpzeta] )
         end
         eval(tmpexpr); ylabel("Gap / GDP (%)");
         # legend( string.("ζ = ",reform_zeta_levs) , loc = "upper left")
      subplot(2,2,4)  # gap / taxrev
         for tmpzeta in 1:length(reform_zeta_levs)
            plot( Dt[:Year][idx_plot], res_ContriRatRise["gap/taxrev"][tmpzeta], reform_zeta_stys[tmpzeta] )
         end
         eval(tmpexpr); ylabel("Gap / Tax Revenues (%)");
         # legend( string.("ζ = ",reform_zeta_levs) , loc = "upper left")
      tight_layout()
# 2.5 save figure
savefig( "./output/ContributionRateRise.pdf", format = "pdf" )

#


#ДАНО: Робот - в произвольной клетке
#ограниченного прямоугольного поля,
# на котором могут находиться также внутренние
# прямоугольные перегородки
#(все перегородки изолированы друг от друга,
#прямоугольники могут вырождаться в отрезки)
#РЕЗУЛЬТАТ: Робот - в исходном положении и
# в углах поля стоят маркеры
function movement!(r,side)
    while !isborder(r,side)
        move!(r,side)
    end
end
function go!(r)
    a1 = 0
    a2 = 0
    a3 = 0
    a4 = 0
    while (a1 != 1) || (a2 != 1) || (a3 != 1) || (a4 != 1)
        for side in (Ost,Sud,West,Nord)
            movement!(r,side)
            if (isborder(r,West) && isborder(r,Sud) && (a1!=1))==true
                putmarker!(r)
                a1=1
            end
            if (isborder(r,West) && isborder(r,Nord) && (a2!=1))==true
                putmarker!(r)
                a2=1
            end
            if (isborder(r,Ost) && isborder(r,Nord) && (a3!=1))==true
                putmarker!(r)
                a3=1
            end
            if (isborder(r,Ost) && isborder(r,Sud) && (a4!=1))==true
                putmarker!(r)
                a4=1
            end
        end
    end
end


# currently supported packages
const SUPPORTED_EXPONENTIAL_PACKAGES = [:ExponentialUtilities, :Expokit]

# =============
# Global option
# =============

global exponential_backend = missing  # global state of the exponential backend

function set_exponential_backend!(backend::Module)
    global exponential_backend = Val(Symbol(backend))
end

function get_exponential_backend()
    ismissing(exponential_backend) && error("no exponential backend is loaded; " *
        "load one of these packages: $SUPPORTED_EXPONENTIAL_PACKAGES")
    return exponential_backend
end

# ====================
# ExponentialUtilities
# ====================

function load_exponentialutilities()
return quote

if ismissing(exponential_backend)
    set_exponential_backend!(ExponentialUtilities)
end

end end  # quote / load_exponentialutilities

function _expmv(::Val{:ExponentialUtilities}, t, A, b)
    return ExponentialUtilities.expv(t, A, b)
end

# =======
# Expokit
# =======

function load_expokit()
return quote

if ismissing(exponential_backend)
    set_exponential_backend!(Expokit)
end

end end  # quote / load_expokit

function _expmv(::Val{:Expokit}, t, A, b::Vector)
    return Expokit.expmv(t, A, b)
end

function _expmv(backend::Val{:Expokit}, t, A, b)
    b = Vector(b)  # Expokit requires a dense vector
    return _expmv(backend, t, A, b)
end


#! /usr/bin/julia

# Rosetta Code, Find common directory path

function commonpath{T<:String}(ds::Array{T,1}, delim::Char='/')
    0 < length(ds) || return convert(T, "")
    1 < length(ds) || return ds[1]
    p = split(ds[1], delim)
    mcnt = length(p)
    for d in ds[2:end]
        q = split(d, delim)
        mcnt = min(mcnt, length(q))
        hits = findfirst(p[1:mcnt] .== q[1:mcnt], false)
        hits != 0 || continue
        mcnt = hits - 1
        mcnt != 0 || return convert(T, "")
    end
    1 < mcnt || p[1] != "" || return convert(T, string(delim))
    convert(T, join(p[1:mcnt], delim))
end

test = ["/home/user1/tmp/coverage/test",
        "/home/user1/tmp/covert/operator",
        "/home/user1/tmp/coven/members"]

println("Comparing")
for s in test
    println("    ", s)
end
println("for their common directory path yields:")
println("    ", commonpath(test))


#####
##### Game trace
#####

"""
    Trace{Game, State}

An object that collects all states visited during a game, along with the
rewards obtained at each step and the successive player policies to be used
as targets.

# Constructor

    Trace{Game}(initial_state)

"""
mutable struct Trace{Game, State}
  states :: Vector{State}
  policies :: Vector{Vector{Float64}}
  rewards :: Vector{Float32}
  function Trace{G}(init_state) where G
    return new{G, GI.State(G)}([init_state], [], [])
  end
end

GameType(::Trace{Game}) where Game = Game

function trace_invariant(t::Trace)
  return length(t.policies) == length(t.rewards) == length(t.states) - 1
end

"""
    Base.push!(t::Trace, π, r, s)

Add a (target policy, reward, new state) triple to a trace.
"""
function Base.push!(t::Trace, π, r, s)
  push!(t.states, s)
  push!(t.policies, π)
  push!(t.rewards, r)
end

function Base.length(t::Trace)
  return length(t.rewards)
end

function total_reward(t::Trace, gamma=1.)
  return sum(gamma^(i-1) * r for (i, r) in enumerate(t.rewards))
end

function debug_trace(t::Trace{Game}) where Game
  n = length(t)
  for i in 1:n
    println("Transition $i:")
    game = Game(t.states[i])
    GI.render(game)
    for (a, p) in zip(GI.available_actions(game),  t.policies[i])
      print("$(GI.action_string(Game, a)): $(fmt(".3f", p))  ")
    end
    println("")
    println("Obtained reward of: $(t.rewards[i]).")
    println("")
  end
  println("Showing final state:")
  GI.render(Game(t.states[n + 1]))
end


using Test
import SharedVertexMesh

const STL = joinpath("data", "nist.stl")

mesh = SharedVertexMesh.from(STL)
@test length(mesh.triangles) == 7392
@test length(mesh.vertices) == 3688


@time @testset "Reachability.options" begin include("unit_options.jl") end
@time @testset "Reachability.solve" begin include("unit_solve.jl") end


using Images, SIUnits.ShortUnits
using Base.Test

if VERSION.minor == 2
    macro mytest_throws(extype, arg)
        :(@test_throws $(esc(arg)))
    end
else
    macro mytest_throws(extype, arg)
        :(@test_throws $(esc(extype)) $(esc(arg)))
    end
end

B = rand(1:20,3,5)
cmap = uint8(repmat(linspace(12,255,20),1,3))
img = ImageCmap(copy(B),cmap,["colorspace" => "RGB", "pixelspacing" => [2.0, 3.0], "spatialorder" => Images.yx])
imgd = convert(Image, img)
imgd["pixelspacing"] = [2.0mm, 3.0mm]
@test eltype(img) == Uint8
@test eltype(imgd) == Uint8

# basic information
@test size(img) == (3,5)
@test size(imgd) == (3,5,3)
@test ndims(img) == 2
@test ndims(imgd) == 3

# copy/similar
A = randn(3,5,3)
imgc = copy(img)
@test imgc.data == img.data
imgc = copy(imgd, A)
@test imgc.data == A
img2 = similar(img)
@test isa(img2, ImageCmap)
@test img2.data != img.data
img2 = similar(imgd)
@test isa(img2, Image)

# ref/assign
prev = img[4]
@test prev == B[4]
img[4] = prev+1
@test img.data[4] == prev+1
@test img[4] == prev+1

# properties
@test colorspace(img) == "RGB"
@test colordim(img) == 0
@test colordim(imgd) == 3
@test timedim(img) == 0
@test limits(img) == (12,255)
@test limits(imgd) == (0,255)
@test pixelspacing(img) == [2.0, 3.0]
@test pixelspacing(imgd) == [2.0mm, 3.0mm]
@test spacedirections(img) == Vector{Float64}[[2.0, 0], [0, 3.0]]
@test spacedirections(imgd) == Vector{SIUnits.SIQuantity{Float64,1,0,0,0,0,0,0}}[[2.0mm, 0.0mm], [0.0mm, 3.0mm]]

@test sdims(img) == sdims(imgd)
@test coords_spatial(img) == coords_spatial(imgd)
@test size_spatial(img) == size_spatial(imgd)

@test storageorder(img) == Images.yx
@test storageorder(imgd) == [Images.yx, "color"]

A = rand(4,4,3)
@test colordim(A) == 3
Aimg = permutedims(convert(Image, A), [3,1,2])
@test colordim(Aimg) == 1

# sub/slice
s = sub(img, 2, 1:4)
@test ndims(s) == 2
@test sdims(s) == 2
@test size(s) == (1,4)
s = subim(img, 2, 1:4)
@test ndims(s) == 2
@test sdims(s) == 2
@test size(s) == (1,4)
s = sliceim(img, 2, 1:4)
@test ndims(s) == 1
@test sdims(s) == 1
@test size(s) == (4,)
s = sliceim(imgd, 2, 1:4, 1:3)
@test ndims(s) == 2
@test sdims(s) == 1
@test colordim(s) == 2
@test spatialorder(s) == ["x"]
s = sliceim(imgd, 2:2, 1:4, 1:3)
@test ndims(s) == 3
@test sdims(s) == 2
@test colordim(s) == 3
@test spatialorder(s) == ["y","x"]

# reslicing
D = randn(3,5,4)
sd = SliceData(D, 2)
C = slice(D, sd, 2)
@test C == reshape(D[1:end, 2, 1:end], size(C))
reslice!(C, sd, 3)
@test C == reshape(D[1:end, 3, 1:end], size(C))
sd = SliceData(D, 3)
C = slice(D, sd, 2)
@test C == reshape(D[1:end, 1:end, 2], size(C))

sd = SliceData(imgd, 2)
s = sliceim(imgd, sd, 2)
@test colordim(s) == 2
@test colorspace(s) == "RGB"
@test spatialorder(s) == ["y"]
@test s.data == reshape(imgd[:,2,:], size(s))
sd = SliceData(imgd, 3)
s = sliceim(imgd, sd, 2)
@test colordim(s) == 0
@test colorspace(s) == "Unknown"
@test spatialorder(s) == Images.yx
@test s.data == imgd[:,:,2]
reslice!(s, sd, 3)
@test s.data == imgd[:,:,3]

# named indexing
@test dimindex(imgd, "color") == 3
@test dimindex(imgd, "y") == 1
@test dimindex(imgd, "z") == 0
imgdp = permutedims(imgd, [3,1,2])
@test dimindex(imgdp, "y") == 2
@test img["y", 2, "x", 4] == B[2,4]
@test img["x", 4, "y", 2] == B[2,4]
chan = imgd["color", 2]
Blookup = reshape(cmap[B[:],1], size(B))
@test chan == Blookup

sd = SliceData(imgd, "x")
s = sliceim(imgd, sd, 2)
@test spatialorder(s) == ["y"]
@test s.data == reshape(imgd[:,2,:], size(s))
sd = SliceData(imgd, "y")
s = sliceim(imgd, sd, 2)
@test spatialorder(s) == ["x"]
@test s.data == reshape(imgd[2,:,:], size(s))
sd = SliceData(imgd, "x", "y")
s = sliceim(imgd, sd, 2, 1)
@test s.data == reshape(imgd[1,2,:], 3)

# spatial order, width/height, and permutations
@test spatialpermutation(Images.yx, imgd) == [1,2]
@test widthheight(imgd) == (5,3)
C = convert(Array, imgd)
@test C == imgd.data
imgd["spatialorder"] = ["x", "y"]
@test spatialpermutation(Images.xy, imgd) == [1,2]
@test widthheight(imgd) == (3,5)
C = convert(Array, imgd)
@test C == permutedims(imgd.data, [2,1,3])
imgd.properties["spatialorder"] = ["y", "x"]
@test spatialpermutation(Images.xy, imgd) == [2,1]
imgd.properties["spatialorder"] = ["x", "L"]
@test spatialpermutation(Images.xy, imgd) == [1,2]
imgd.properties["spatialorder"] = ["L", "x"]
@test spatialpermutation(Images.xy, imgd) == [2,1]
@test spatialpermutation(Images.xy, A) == [2,1]
@test spatialpermutation(Images.yx, A) == [1,2]

imgd.properties["spatialorder"] = Images.yx
imgp = permutedims(imgd, ["x", "y", "color"])
@test imgp.data == permutedims(imgd.data, [2,1,3])
imgp = permutedims(imgd, ("color", "x", "y"))
@test imgp.data == permutedims(imgd.data, [3,2,1])
@test pixelspacing(imgp) == [3.0mm, 2.0mm]
imgc = copy(imgd)
imgc["spacedirections"] = spacedirections(imgc)
delete!(imgc, "pixelspacing")
imgp = permutedims(imgc, ["x", "y", "color"])
@test spacedirections(imgp) == Vector{SIUnits.SIQuantity{Float64,1,0,0,0,0,0,0}}[[0.0mm, 3.0mm],[2.0mm, 0.0mm]]
@test pixelspacing(imgp) == [3.0mm, 2.0mm]

## Convenience constructors
# colorim
@test colordim(colorim(rand(Uint8, 3, 5, 5))) == 1
@test colordim(colorim(rand(Uint8, 5, 5, 3))) == 3
@test spatialorder(colorim(rand(Uint8, 3, 5, 5))) == ["x", "y"]
@test spatialorder(colorim(rand(Uint8, 5, 5, 3))) == ["y", "x"]
@test colordim(colorim(rand(Uint8, 4, 5, 5), "RGBA")) == 1
@test colordim(colorim(rand(Uint8, 4, 5, 5), "ARGB")) == 1
@test colordim(colorim(rand(Uint8, 5, 5, 4), "RGBA")) == 3
@test colordim(colorim(rand(Uint8, 5, 5, 4), "ARGB")) == 3
@test spatialorder(colorim(rand(Uint8, 4, 5, 5), "ARGB")) == ["x", "y"]
@test spatialorder(colorim(rand(Uint8, 5, 5, 4), "ARGB")) == ["y", "x"]
@mytest_throws ErrorException colorim(rand(Uint8, 3, 5, 3))
@mytest_throws ErrorException colorim(rand(Uint8, 4, 5, 5))
@mytest_throws ErrorException colorim(rand(Uint8, 5, 5, 4))
@mytest_throws ErrorException colorim(rand(Uint8, 4, 5, 4), "ARGB")
@mytest_throws ErrorException colorim(rand(Uint8, 5, 5, 5), "foo")
@mytest_throws ErrorException colorim(rand(Uint8, 2, 2, 2), "bar")


# Definition of a sample which we fit the phase-type distribution to.
struct Sample
    obs::Vector{Float64}
    obsweight::Vector{Float64}
    cens::Vector{Float64}
    censweight::Vector{Float64}
    int::Matrix{Float64}
    intweight::Vector{Float64}

    function Sample(obs::Vector{Float64}, obsweight::Vector{Float64},
            cens::Vector{Float64}, censweight::Vector{Float64},
            int::Matrix{Float64}, intweight::Vector{Float64})
        cond = all(obs .>= 0) && all(obsweight .>= 0) && all(cens .>= 0) &&
                all(censweight .>= 0) && all(int .>= 0) && all(intweight .>= 0)
        if ~cond
            error("Require non-negativity of observations and weights")
        end

        new(obs, obsweight, cens, censweight, int, intweight)
    end
end

function Sample(;obs::Vector{Float64}=Vector{Float64}(),
        obsweight::Vector{Float64}=Vector{Float64}(),
        cens::Vector{Float64}=Vector{Float64}(),
        censweight::Vector{Float64}=Vector{Float64}(),
        int::Matrix{Float64}=Matrix{Float64}(undef, 0, 0),
        intweight::Vector{Float64}=Vector{Float64}())

    if ~isempty(obs) && isempty(obsweight)
        obsweight = ones(length(obs))
    end
    if ~isempty(cens) && isempty(censweight)
        censweight = ones(length(cens))
    end
    if ~isempty(int) && isempty(intweight)
        intweight = ones(length(int))
    end

    Sample(obs, obsweight, cens, censweight, int, intweight)
end


##############################################################################
##
## sumby...
##
##############################################################################

"""
Perform sum by group
```julia
sumby(df::Union{AbstractDataFrame,NDSparse}, by::Symbol, val::Symbol)
sumby(by::AbstractVector  val::AbstractVector)
```
### Arguments
* `df` : an AbstractDataFrame/NDSparse from which to extract the by and val columns
* `by` : data table column to group by
* `val`: data table column to sum
### Returns
* `::Dict` : A Dict that maps unqiues values of by to sum of val

### Examples
```julia
using FastGroupBy
using DataFrames, IndexedTables, Compat, BenchmarkTools
import DataFrames.DataFrame

const N = 10_000_000; const K = 100

# sumby is faster for DataFrame without missings
srand(1);
@time df = DataFrame(id = rand(1:Int(round(N/K)), N), val = rand(round.(rand(K)*100,4), N));
@belapsed DataFrames.aggregate(df, :id, sum)
@belapsed sumby(df, :id, :val)

```
"""
sumby(by, val, alg = :auto) = sumby!(copy(by), copy(val), alg)

# sumby(by, val) = sumby!(copy(by), copy(val))

function sumby!(by::AbstractVector{T},  val::AbstractVector{S}; alg = :auto)::Dict{T,S} where {T, S<:Number}
    l = length(by)

    l == length(val) || throw(ErrorException("length of by and val must be the same"))

    if l == 0
        return Dict{T,S}()
    elseif l == 1
        return Dict{T,S}(by[1], val[1])
    elseif issorted(by)
        return sumby_contiguous(by, val)
    elseif l <= 2^16
        return sumby_sortperm(by, val)
    elseif !isbits(T) || alg == :dict
        return sumby_dict(by, val)
    elseif nthreads() > 1
        return sumby_multi_rs(by, val)
    elseif l <= 50_000_000
        return sumby_radixsort!(by, val)
    else
        # return sumby_radixgroup(by, val)
        return sumby_radixsort!(by, val)
    end
end

"sumby by using radix and counting sort to group by; it's only a partial sort. It's faster for large by"
function sumby_radixgroup!(by::AbstractVector{T},  val::AbstractVector{S}; cutsize = 2048)::Dict{T,S} where {T, S<:Number}
    by_sim = similar(by)
    val1=similar(val)
    o = Forward

    l = length(by)

    if l == 0
        return Dict{T,S}()
    elseif l == 1
        return Dict{T,S}(by[1] => val[1])
    end

    # Make sure we're sorting a bits type
    if !isbits(T)
      error("sumby_nosort on works on bits types (got $T)")
    end

    # Init
    iters = ceil(Integer, sizeof(T)*8/RADIX_SIZE)

    # distinct range to groupby over; initially the whole block is one range
    nextranges = Tuple{Int, Int}[]

    new_small_grps = similar(nextranges)

    push!(nextranges, (1,l))

    for j = 1:iters
        #println(string(j,":",length(nextranges)))
        ranges = nextranges::Array{Tuple{Int, Int}}
        nextranges = Tuple{Int,Int}[]
        if length(ranges) > 0
            for (lo, hi) in ranges
                bin = zeros(UInt32, 2^RADIX_SIZE)

                # Histogram for each element, radix
                for i = lo:hi
                    #v = uint_mapping(o, by[i])
                    idx = Int((by[i] >> (j-1)*RADIX_SIZE) & RADIX_MASK) + 1
                    @inbounds bin[idx] += 1
                end

                # Sort!
                len = hi - lo + 1

                # Unroll first data iteration, check for degenerate case
                #v = uint_mapping(o, by[hi])
                idx = Int((by[hi] >> (j-1)*RADIX_SIZE) & RADIX_MASK) + 1

                # are all values the same at this radix?
                if bin[idx] == len
                    nextranges = vcat(nextranges, [(lo, hi)])::Array{Tuple{Int, Int}}
                    @inbounds by_sim[lo:hi] = by[lo:hi]
                    @inbounds val1[lo:hi] = val[lo:hi]
                else
                    if lo > 1;  bin[1] += UInt32(lo-1);  end
                    cbin = cumsum(bin)

                    # compute the new ranges for next round
                    tmpr = sort(setdiff(unique(cbin), lo-1))::Vector{Int}
                    tmpnextranges = vcat(nextranges, [(lo,Int(tmpr[1]))], [(Int(a)+1,Int(b)) for (a,b) in zip(tmpr[1:end-1],tmpr[2:end])])::Array{Tuple{Int, Int}}
                    nextranges = filter(x -> x[2] - x[1] + 1 > cutsize, tmpnextranges)::Array{Tuple{Int, Int}}

                    if length(tmpnextranges) != length(nextranges)
                        # this is a small grp just sort it
                        new_small_grps = filter(x -> x[2] - x[1] + 1 <= cutsize, tmpnextranges)
                    end

                    ci = cbin[idx]
                    by_sim[ci] = by[hi]
                    val1[ci] = val[hi]
                    cbin[idx] -= 1

                    # Finish the loop...
                    @inbounds for i in hi-1:-1:lo
                      #v = uint_mapping(o, by[i])
                      idx =Int((by[i] >> (j-1)*RADIX_SIZE) & RADIX_MASK) + 1
                      ci = cbin[idx]
                      by_sim[ci] = by[i]
                      val1[ci] = val[i]
                      cbin[idx] -= 1
                    end
                end
            end
            if length(new_small_grps) > 0
                for (lo_nsg, hi_nsg) in new_small_grps
                    @inbounds bytmp = by_sim[lo_nsg:hi_nsg]
                    sp = sortperm(bytmp)
                    bytmp_sorted = bytmp[sp]
                    val1_sorted = val1[lo_nsg:hi_nsg][sp]
                    @inbounds by[lo_nsg:hi_nsg] = bytmp_sorted
                    @inbounds val[lo_nsg:hi_nsg] = val1_sorted
                    @inbounds by_sim[lo_nsg:hi_nsg] = bytmp_sorted
                    @inbounds val1[lo_nsg:hi_nsg] = val1_sorted
                end
                new_small_grps = Int[]
            end
            by, by_sim = by_sim, by
            val, val1 = val1, val
        end
    end

    return sumby_contiguous(by, val)
end

"sumby by sorting the by column using radixsort"
function sumby_radixsort!(by::AbstractVector{T},  val::AbstractVector{S})::Dict{T,S} where {T, S<:Number}

  hi = length(by)

  if hi == 0;  return Dict{T,S}();
  elseif hi == 1;  return Dict{T,S}(by[1] => val[1]);  end

  by_sim = similar(by)
  val1=similar(val)
  lo = 1

  # Make sure we're sorting a bits type
  #TT = Base.Order.ordtype(o, by)
  if !isbits(T)
      throw(error("Radix sort only sorts bits types (got $T)"))
  end

  # Init
  iters = ceil(Integer, sizeof(T)*8/RADIX_SIZE)
  bin = zeros(UInt32, 2^RADIX_SIZE, iters)
  if lo > 1;  bin[1,:] = lo-1;  end

  # Histogram for each element, radix
  for i = lo:hi
    v = by[i]
    for j = 1:iters
        idx = Int((v >> (j-1)*RADIX_SIZE) & RADIX_MASK) + 1
        @inbounds bin[idx,j] += 1
    end
  end

  # Sort!
  # swaps = 0
  len = hi-lo+1
  for j = 1:iters
      # Unroll first data iteration, check for degenerate case
      v = by[hi]
      idx = Int((v >> (j-1)*RADIX_SIZE) & RADIX_MASK) + 1

      # are all values the same at this radix?
      if bin[idx,j] == len;  continue;  end

      cbin = cumsum(bin[:,j])
      ci = cbin[idx]
      by_sim[ci] = by[hi]
      val1[ci] = val[hi]

      cbin[idx] -= 1

      # Finish the loop...
      @inbounds for i in hi-1:-1:lo
          v = by[i]
          idx = Int((v >> (j-1)*RADIX_SIZE) & RADIX_MASK) + 1
          ci = cbin[idx]
          by_sim[ci] = by[i]
          val1[ci] = val[i]
          cbin[idx] -= 1
      end
      by,by_sim = by_sim,by
      val,val1 = val1,val
    #   swaps += 1
  end

  # @inbounds if isodd(swaps)
  #   by,by_sim = by_sim,by
  #   val,val1 = val1,val
  #   for i = lo:hi
  #     by[i] = by_sim[i]
  #     val[i] = val1[i]
  #   end
  # end

  sumby_contiguous(by, val)
end

"sumby assuming that the elements are organised contiguously; it does not perform a check"
function sumby_contiguous(by_sorted::AbstractVector{T},  val::AbstractVector{S})::Dict{T,S} where {T, S<:Number}
  res = Dict{T,S}()
  @inbounds tmp_val = val[1]
  @inbounds last_byi = by_sorted[1]
  @inbounds for i in 2:length(by_sorted)
    if by_sorted[i] == last_byi
      tmp_val += val[i]
    else
      res[last_byi] = tmp_val
      tmp_val = val[i]
      last_byi = by_sorted[i]
    end
  end

  @inbounds res[last_byi] = tmp_val

  res
end

#sumby_contiguous2 is too slow
# function sumby_contiguous2{T,S}(by_sorted::AbstractVector{T},  val::AbstractVector{S})
#   res = Dict{T,S}()
#   @inbounds last_byi = by_sorted[1]
#   lo = 1
#   hi = 1
#   @inbounds for i in 2:length(by_sorted)
#     if val[i] == last_byi
#       hi += 1
#     else
#       res[last_byi] = sum(val[lo:hi])
#       last_byi = by_sorted[i]
#       lo, hi = hi + 1, hi + 1
#     end
#   end
#
#   @inbounds res[last_byi] = sum(val[lo:hi])
#
#   res
# end

"This is faster for smaller by and also doesn't change the input"
function sumby_sortperm(by::AbstractVector{T},  val::AbstractVector{S})::Dict{T,S} where {T, S<:Number}
    sp = sortperm(by, alg = RadixSort)
    sumby_contiguous(view(by, sp), view(val,sp))
end
# function sumby_sortperm2{T, S<:Number}(by::AbstractVector{T},  val::AbstractVector{S})::Dict{T,S}
#     sp = sortperm(by)
#     sumby_contiguous(by[sp], val[sp])
# end

"sumby using Dict - can be quite slow due to slow hash table operations"
function sumby_dict(by::AbstractVector{T}, val::AbstractVector{S})::Dict{T,S} where {T,S<:Number}
  res = Dict{T, S}()
  # resize the Dict to a larger size
  for (byi, vali) in zip(by, val)
    index = ht_keyindex2!(res, byi)
    if index > 0
      @inbounds res.vals[index] += vali
    else
      @inbounds _setindex!(res, vali, byi, -index)
    end
  end
  return res
end

#Optimized sumby for PooledArrays
function sumby!(by::Union{PooledArray, CategoricalArray}, val::AbstractVector{S}) where {S<:Number}
  l = length(by.pool)
  res = zeros(S, l)

  for (i, v) in zip(by.refs, val)
    @inbounds res[i] += v
  end
  return Dict(by.pool[i] => res[i] for i in S(1):S(l))
end

sumby(by::Union{PooledArray, CategoricalArray}, val::AbstractVector{S}) where {S<:Number} = sumby!(by, val)

sumby!(dt::Union{AbstractDataFrame, NDSparse}, by::Symbol, val::Symbol) = sumby!(column(dt,by), column(dt,val))


using SparseArrays
abstract type abstract_nlp end

mutable struct Class_cache
    fval::Float64
    cons::Array{Float64,1}
    grad::Array{Float64,1}
    J::SparseMatrixCSC{Float64,Int64}
    J_T::SparseMatrixCSC{Float64,Int64}
    H::SparseMatrixCSC{Float64,Int64}
    fval_updated::Bool
    cons_updated::Bool
    grad_updated::Bool
    J_updated::Bool
    H_updated::Bool

    function Class_cache()
      return new()
    end
end

function is_updated(c::Class_cache)
    return c.fval_updated && c.cons_updated && c.grad_updated && c.J_updated && c.H_updated
end

function is_updated_correction(c::Class_cache)
    return c.fval_updated && c.cons_updated && c.grad_updated && c.J_updated
end

mutable struct Class_local_info
    delta::Float64
    radius::Float64
    prev_step::Symbol
    function Class_local_info()
      return new(0.0,0.0,:blah)
    end
end

mutable struct Class_iterate
    primal_residual_intial::Array{Float64,1}
    point::Class_point
    nlp::abstract_nlp
    cache::Class_cache
    local_info::Class_local_info
    ncon::Int64
    nvar::Int64
    frac_bd::Array{Float64,1}
    frac_bd_predict::Array{Float64,1}

    # regularizer info
    a_norm_penalty_par::Float64

    function Class_iterate()
        return new();
    end

    function Class_iterate(intial_point::Class_point, nlp::abstract_nlp, local_info::Class_local_info, timer::class_advanced_timer, pars::Class_parameters)
      this = new() # zeros(length(intial_point.x)), intial_point, nlp, Class_cache(), local_info)
      this.primal_residual_intial = zeros(length(intial_point.x))
      this.point = intial_point
      this.nlp = nlp
      this.local_info = local_info
      this.nvar = length(intial_point.x)
      this.ncon = length(intial_point.y)
      this.a_norm_penalty_par = pars.a_norm_penalty;
      this.frac_bd = pars.ls.fraction_to_boundary * ones(this.ncon)
      this.frac_bd_predict = pars.ls.fraction_to_boundary_predict * ones(this.ncon)

      @assert(this.ncon == length(intial_point.y))

      this.cache = Class_cache()
      update!(this, timer, pars)
      #update_cons!(this, timer, pars)
      #update_obj!(this, timer, pars)
      #update_H!(this, timer, pars)
      #update_J!(this, timer, pars)

      #update_primal_cache!(this, timer)

      this.primal_residual_intial = get_primal_res(this);
      return this
    end
end

function get_original_x(it::Class_iterate)
     _cute_x(it.nlp, it.point.x)
end

function get_constrduals(it::Class_iterate)
    return get_constrduals(it.nlp,it.point.y)
end

function get_reducedcosts(it::Class_iterate)
    return get_reducedcosts(it.nlp,it.point.y)
end

function validate(it::Class_iterate)
    @assert(it.nvar == length(it.point.x))
    @assert(it.ncon == length(it.point.y))
    @assert(it.ncon == length(it.point.s))
end

import Base.copy

function copy(it::Class_iterate, timer::class_advanced_timer)
   validate(it)

   new_it = Class_iterate()
   new_it.point = deepcopy(it.point)

   new_it.primal_residual_intial = it.primal_residual_intial
   new_it.nlp = it.nlp
   new_it.local_info = it.local_info
   new_it.nvar = it.nvar
   new_it.ncon = it.ncon

   new_it.a_norm_penalty_par = it.a_norm_penalty_par;
   new_it.frac_bd = it.frac_bd
   new_it.frac_bd_predict = it.frac_bd_predict

   new_it.cache = it.cache
   copy_cache!(new_it, timer)

   return new_it
end

function finite_diff_check(it::Class_iterate)
   approx_grad = grad(it.nlp.eval_f, it.point.x)
   actual_grad = it.nlp.eval_grad_lag(it.point.x, 0.0, zeros(length(it.point.y)))
   @assert(LinearAlgebra.norm(approx_grad -  actual_grad, 2) < 1e-3)

   actual_jac = it.nlp.eval_jac(it.point.x)
   for i = 1:length(it.point.s)
       a_i = x::Vector -> it.nlp.eval_a(x)[i]
       approx_grad_a_i = grad(a_i, it.point.x)
       @assert(LinearAlgebra.norm(actual_jac[i,:] - approx_grad_a_i,2) < 1e-3)
   end
end

function feasible_ls(it::Class_iterate, dir::Class_point, pars::Class_parameters, timer::class_advanced_timer)

ccifg(io_err, n, icon, x, ci, gci, grad)

end

function dim(it::Class_iterate)
    return length(it.point.x)
end

function ncon(it::Class_iterate)
    @assert(length(length(it.point.s)) == length(length(it.point.y)))
    return length(it.point.s)
end

function get_delta(it::Class_iterate)
    return it.local_info.delta
end

function set_delta(it::Class_iterate, delta::Float64)
    it.local_info.delta = delta;
end

function get_radius(it::Class_iterate)
    return it.local_info.radius
end

function set_prev_step(it::Class_iterate, prev_step::Symbol)
    it.local_info.prev_step = prev_step;
end

function get_prev_step(it::Class_iterate)
    return it.local_info.prev_step
end
function set_radius(it::Class_iterate, radius::Float64)
    it.local_info.radius = radius;
end

function get_x(it::Class_iterate)
    return it.point.x
end

function get_y(it::Class_iterate)
    return it.point.y
end

function get_s(it::Class_iterate)
    return it.point.s
end

function get_mu(it::Class_iterate)
    return it.point.mu
end

function copy_cache!(it::Class_iterate, timer::class_advanced_timer)
    nlp = it.nlp
    start_advanced_timer(timer, "CACHE")
    start_advanced_timer(timer, "CACHE/copy")
    cache = it.cache

    new_cache = Class_cache()
    new_cache.J = deepcopy(cache.J)
    new_cache.J_T = deepcopy(cache.J_T)
    new_cache.J_updated = false
    new_cache.H = deepcopy(cache.H)
    new_cache.H_updated = false
    new_cache.cons = deepcopy(cache.cons)
    new_cache.cons_updated = false
    new_cache.fval = deepcopy(cache.fval)
    new_cache.fval_updated = false
    new_cache.grad = deepcopy(cache.grad)
    new_cache.grad_updated = false

    it.cache = new_cache

    pause_advanced_timer(timer, "CACHE/copy")
    pause_advanced_timer(timer, "CACHE")
end

function update!(it::Class_iterate, timer::class_advanced_timer, pars::Class_parameters)
    success = update_cons!(it, timer, pars) && update_obj!(it, timer, pars) && update_H!(it, timer, pars) && update_grad!(it, timer, pars) && update_J!(it, timer, pars)
    return success
end

function getbad(something::Array{Float64,1})
    for val in something
      if isbad(val)
        return val
      end
    end

    return 0.0
end

function update_cons!(it::Class_iterate, timer::class_advanced_timer, pars::Class_parameters)
    start_advanced_timer(timer, "CACHE")
    start_advanced_timer(timer, "CACHE/update_cons")
    it.cache.cons = eval_a(it.nlp, it.point.x)
    it.cache.cons_updated = true
    if isbad(it.cache.cons)
        println("NaN or Inf in cons")
        if pars.throw_error_nans
          throw(Eval_NaN_error(getbad(it.cache.cons), it.point.x,"cons"))
        else
          no_nan = false
        end
    else
      no_nan = true
    end
    pause_advanced_timer(timer, "CACHE/update_cons")
    pause_advanced_timer(timer, "CACHE")

    return no_nan
end

function update_obj!(it::Class_iterate, timer::class_advanced_timer, pars::Class_parameters)
    start_advanced_timer(timer, "CACHE")
    start_advanced_timer(timer, "CACHE/update_obj")
    it.cache.fval = eval_f(it.nlp, it.point.x)
    it.cache.fval_updated = true
    if isbad(it.cache.fval)
        println("$(it.cache.fval) in obj")
        if pars.throw_error_nans
          throw(Eval_NaN_error(it.cache.fval, it.point.x, "obj"))
        else
          no_nan = false
        end
    else
      no_nan = true
    end
    pause_advanced_timer(timer, "CACHE/update_obj")
    pause_advanced_timer(timer, "CACHE")

    return no_nan
end

function update_H!(it::Class_iterate, timer::class_advanced_timer, pars::Class_parameters)
    start_advanced_timer(timer, "CACHE")
    start_advanced_timer(timer, "CACHE/update_H")

    # regularizer
    begin

      lambda = it.point.mu #* use_prox(it)
      y_rx = lambda * it.a_norm_penalty_par

      x = it.point.x

      it.cache.H = eval_lag_hess(it.nlp, x, it.point.y .+ y_rx, 1.0)
      #it.cache.H = eval_lag_hess(it.nlp, it.point.x, it.point.y, 1.0)
      it.cache.H_updated = true
    end

    nzH = nonzeros(it.cache.H)
    if isbad(nzH)
        println("NaN or Inf in H")
        if pars.throw_error_nans
          throw(Eval_NaN_error(getbad(nzH), it.point.x, "H"))
        else
          no_nan = false
        end
    else
      no_nan = true
    end
    pause_advanced_timer(timer, "CACHE/update_H")
    pause_advanced_timer(timer, "CACHE")

    return no_nan
end

function update_grad!(it::Class_iterate, timer::class_advanced_timer, pars::Class_parameters)
    start_advanced_timer(timer, "CACHE")
    start_advanced_timer(timer, "CACHE/update_grad")
    it.cache.grad = eval_grad_f(it.nlp, it.point.x)
    it.cache.grad_updated = true
    if isbad(it.cache.grad)
        println("NaN or Inf in grad")
        if pars.throw_error_nans
          throw(Eval_NaN_error(getbad(it.cache.grad), it.point.x, "grad"))
        else
          no_nan = false
        end
    else
      no_nan = true
    end
    pause_advanced_timer(timer, "CACHE/update_grad")
    pause_advanced_timer(timer, "CACHE")

    return no_nan
end

function update_J!(it::Class_iterate, timer::class_advanced_timer, pars::Class_parameters)
    start_advanced_timer(timer, "CACHE")
    start_advanced_timer(timer, "CACHE/update_J")
    @assert(length(it.point.x) == length(it.cache.grad))
    it.cache.J = eval_jac(it.nlp, it.point.x)
    it.cache.J_T = it.cache.J'
    it.cache.J_updated = true
    if isbad(nonzeros(it.cache.J))
        println("NaN or Inf in J")
        if pars.throw_error_nans
          throw(Eval_NaN_error(getbad(it.cache.J), it.point.x, "J"))
        else
          no_nan = false
        end
    else
      no_nan = true
    end
    pause_advanced_timer(timer, "CACHE/update_J")
    pause_advanced_timer(timer, "CACHE")

    return no_nan
end