module CubedSpheres

export ConformalCubedSphereGrid, face, faces, λnodes, φnodes

include("cubed_sphere_utils.jl")
include("conformal_cubed_sphere_grid.jl")
include("cubed_sphere_exchange_bcs.jl")
include("cubed_sphere_faces.jl")
include("cubed_sphere_halo_filling.jl")
include("cubed_sphere_kernel_launching.jl")
include("immersed_conformal_cubed_sphere_grid.jl")

#####
##### Validating cubed sphere stuff
#####

import Oceananigans.Fields: validate_field_data, validate_boundary_conditions, validate_indices
import Oceananigans.Models.HydrostaticFreeSurfaceModels: validate_vertical_velocity_boundary_conditions

validate_indices(indices, loc, grid::ConformalCubedSphereGrid) = indices

function validate_field_data(loc, data, grid::ConformalCubedSphereGrid, indices)

    for (face_data, face_grid) in zip(data.faces, grid.faces)
        validate_field_data(loc, face_data, face_grid, indices)
    end

    return nothing
end

# We don't support validating cubed sphere boundary conditions at this time
validate_boundary_conditions(loc, grid::ConformalCubedSphereGrid, bcs::CubedSphereFaces) = nothing
validate_boundary_conditions(loc, grid::ConformalCubedSphereFaceGrid, bcs) = nothing

validate_vertical_velocity_boundary_conditions(w::AbstractCubedSphereField) =
    [validate_vertical_velocity_boundary_conditions(w_face) for w_face in faces(w)]

#####
##### Regularizing field boundary conditions
#####

import Oceananigans.BoundaryConditions: regularize_field_boundary_conditions

function regularize_field_boundary_conditions(bcs::CubedSphereFaces,
                                              grid::AbstractGrid,
                                              field_name::Symbol,
                                              prognostic_field_names)

    faces = Tuple(regularize_field_boundary_conditions(face_bcs, face_grid, field_name, prognostic_field_names)
                  for (face_bcs, face_grid) in zip(bcs.faces, grid.faces))

    return CubedSphereFaces{typeof(faces[1]), typeof(faces)}(faces)
end

function regularize_field_boundary_conditions(bcs::FieldBoundaryConditions,
                                              grid::ConformalCubedSphereGrid,
                                              field_name::Symbol,
                                              prognostic_field_names)

    faces = Tuple(
        inject_cubed_sphere_exchange_boundary_conditions(
            regularize_field_boundary_conditions(bcs, face_grid, field_name, prognostic_field_names),
            face_index,
            grid.face_connectivity
        )
        for (face_index, face_grid) in enumerate(grid.faces)
    )

    return CubedSphereFaces{typeof(faces[1]), typeof(faces)}(faces)
end

#####
##### Applying flux boundary conditions
#####

import Oceananigans.Models.HydrostaticFreeSurfaceModels: apply_flux_bcs!

function apply_flux_bcs!(Gcⁿ::AbstractCubedSphereField, events, c::AbstractCubedSphereField, arch, barrier, args...)

    for (face_index, Gcⁿ_face) in enumerate(faces(Gcⁿ))
        apply_flux_bcs!(Gcⁿ_face, events, get_face(c, face_index), arch, barrier,
                        Tuple(get_face(a, face_index) for a in args)...)
    end

    return nothing
end

import Oceananigans.Models.HydrostaticFreeSurfaceModels: top_tracer_boundary_conditions

function face_tracers(tracers, face)
    tracer_names = propertynames(tracers)
    return NamedTuple(name => get_face(tracers[name], face) for name in tracer_names)
end

function top_tracer_boundary_conditions(grid::ConformalCubedSphereGrid, tracers)
    tracer_names = propertynames(tracers)
    top_tracer_bcs = Tuple(top_tracer_boundary_conditions(get_face(grid, i), face_tracers(tracers, i))
                           for i = 1:length(grid.faces))
    return CubedSphereFaces(top_tracer_bcs)
end

#####
##### Forcing functions on the cubed sphere
#####

using Oceananigans.Forcings: user_function_arguments
import Oceananigans.Forcings: ContinuousForcing

@inline function (forcing::ContinuousForcing{LX, LY, LZ})(i, j, k, grid::ConformalCubedSphereFaceGrid, clock, model_fields) where {LX, LY, LZ}

    args = user_function_arguments(i, j, k, grid, model_fields, forcing.parameters, forcing)

    λ = λnode(LX(), LY(), LZ(), i, j, k, grid)
    φ = φnode(LX(), LY(), LZ(), i, j, k, grid)
    z = znode(LX(), LY(), LZ(), i, j, k, grid)

    return @inbounds forcing.func(λ, φ, z, clock.time, args...)
end

#####
##### NaN checker for cubed sphere fields
#####

import Oceananigans.Simulations: hasnan

function hasnan(field::AbstractCubedSphereField)
    for (face_index, face_field) in enumerate(faces(field))
        nanface = hasnan(face_field)
        if nanface
            @info "NaN found on face $face_index..."
            return true
        end
    end
    return false
end

#####
##### CFL for cubed sphere fields
#####

import Oceananigans.Diagnostics: accurate_cell_advection_timescale

function accurate_cell_advection_timescale(grid::ConformalCubedSphereGrid, velocities)

    min_timescale_on_faces = []

    for (face_index, grid_face) in enumerate(grid.faces)
        velocities_face = get_face(velocities, face_index)
        min_timescale_on_face = accurate_cell_advection_timescale(grid_face, velocities_face)
        push!(min_timescale_on_faces, min_timescale_on_face)
    end

    return minimum(min_timescale_on_faces)
end

#####
##### compute...
#####

import Oceananigans.Fields: compute!
using Oceananigans.AbstractOperations: _compute!
using Oceananigans.Fields: compute_at!

const CubedSphereComputedField{LX, LY, LZ} = Field{LX, LY, LZ,
                                                   <:AbstractOperation,
                                                   <:ConformalCubedSphereGrid} where {LX, LY, LZ}

function compute!(comp::CubedSphereComputedField, time=nothing)
    # First compute `dependencies`:
    compute_at!(comp.operand, time)

    arch = architecture(comp)
    events = Tuple(launch!(arch, c.grid, size(c), _compute!, c.data, c.operand, c.indices)
                   for c in faces(comp))

    wait(device(arch), MultiEvent(events))

    fill_halo_regions!(comp)

    return comp
end

#####
##### Output writing for cubed sphere fields
#####

import Oceananigans.OutputWriters: fetch_output

function fetch_output(field::AbstractCubedSphereField, model, field_slicer)
    compute!(field)
    return Tuple(fetch_output(face_field, model, field_slicer) for face_field in faces(field))
end

#####
##### StateChecker for each face is useful for debugging
#####

import Oceananigans.Diagnostics: state_check

function state_check(field::AbstractCubedSphereField, name, pad)
    face_fields = faces(field)
    Nf = length(face_fields)
    for (face_index, face_field) in enumerate(face_fields)
        face_str = " face $face_index"
        state_check(face_field, string(name) * face_str, pad + length(face_str))

        # Leave empty line between fields for easier visual inspection.
        face_index == Nf && @info ""
    end
end

end # module


module JLRasterization

using Colors, ColorVectorSpace, StaticArrays
using GeometryTypes, Interpolations
using ..Visualize: DepthBuffer, ColorBuffer, Area, Scene, AbstractWindow, Window, Canvas, JLCanvas, AbstractCanvas
import ..Visualize: rasterizer

using FieldTraits
using FieldTraits: ComposableLike, @composed, Partial
import FieldTraits: convertfor, default
import Interpolations: interpolate

@composed type JLWindow <: AbstractWindow
    <: Window
    Canvas
end

default(::Type{Canvas}, window::Partial{JLWindow}) = JLCanvas(window.val)

@inline function edge_function(a, b, c)
    (c[1] - a[1]) * (b[2] - a[2]) - (c[2] - a[2]) * (b[1] - a[1])
end
function src_alpha{T <: Colorant}(c::T)
    a = alpha(c)
    a == 0.0 ? zero(T) : c ./ a
end
one_minus_alpha{T <: Colorant}(c::T) = one(T) .- src_alpha(c)
blend(source, dest, src_func, dest_func) = clamp01(src_func(source) .+ dest_func(dest))
ColorTypes.alpha(x::StaticVector) = x[4]
function standard_transparency{T}(source, dest::T)
    (alpha(source) .* source) .+ ((one(eltype(T)) - alpha(source)) .* dest)
end

type FixedGeomView{GeomOut, VT}
    buffer::Vector{GeomOut}
    view::VT
    idx::Int
end

function FixedGeomView(T, max_primitives, primitive_in, primitive_out)
    buffer = Vector{Tuple{Point4f0, T}}(max_primitives)
    # TODO implement primitive_in and out correctly
    # this is for triangle_strip and 4 max_primitives
    if max_primitives != 4 || primitive_out == :triangle_strip
        error("Not implemented for max_primitives == $max_primitives and primitive_out == $primitive_out.")
    end
    geometry_view = if primitive_in == :triangle_strip
        view(buffer, [Face(1, 2, 3), Face(3, 2, 4)])
    else
        error("$primitive_in not supported. Only :triangle_strip supported right now")
    end
    FixedGeomView(buffer, geometry_view, 1)
end

function reset!(A::FixedGeomView)
    A.idx = 1
end
function Base.push!(A::FixedGeomView, element)
    if A.idx > length(A.buffer)
        error("Emit called more often than max_primitives. max_primitives: $(length(A.buffer))")
    end
    A.buffer[A.idx] = element
    A.idx += 1
    return
end

immutable JLRasterizer{Vertex, Args, FragN, VS, FS, GS, GV, EF}
    vertexshader::VS
    fragmentshader::FS

    geometryshader::GS
    geometry_view::GV
    emit::EF
end
function (::Type{JLRasterizer{Vertex, Args, FragN}}){Vertex, Args, FragN, VS, FS, GS, GV, EF}(
        vertexshader::VS,
        fragmentshader::FS,
        geometryshader::GS,
        geometry_view::GV,
        emit::EF
    )
    JLRasterizer{Vertex, Args, FragN, VS, FS, GS, GV, EF}(
        vertexshader,
        fragmentshader,
        geometryshader,
        geometry_view,
        emit
    )
end

function geometry_return_type(vertex_array, vertexshader, geometryshader, uniforms)
    typ = Any
    emit_t{T}(position, ::T) = (typ = T)
    face1 = first(vertex_array)
    vertex_stage = map(reverse(face1)) do f
        vertexshader(f, uniforms...)
    end
    geometryshader(emit_t, vertex_stage, uniforms...) # figure out typ
    typ
end

arglength{T <: Tuple}(::Type{T}) = length(T.parameters)
arglength{T <: AbstractArray}(::Type{T}) = 1
arglength{T}(::Type{T}) = nfields(T)

function rasterizer(
        window::AbstractWindow,
        vertexarray::AbstractArray,
        uniforms::Tuple,
        vertexshader::Function,
        fragmentshader::Function;
        geometryshader = nothing,
        max_primitives = 4,
        primitive_in = :points,
        primitive_out = :triangle_strip,
    )

    emit, geometry_view = nothing, nothing
    fragment_in_ndim = if geometryshader != nothing
        T = geometry_return_type(vertexarray, vertexshader, geometryshader, uniforms)
        geometry_view = FixedGeomView(T, max_primitives, primitive_in, primitive_out)
        emit = (position, fragment_args) -> push!(geometry_view, (position, fragment_args))
        arglength(T)
    else
        # when we don't have a geometry shader, vertex shader will feed fragment shader
        T = Base.Core.Inference.return_type(vertexshader, Tuple{eltype(vertexarray), map(typeof, uniforms)...})
        if T <: Tuple
            # TODO error handling
            arglength(T.parameters[2])
        else # if not a tuple, vertex shader doesn't pass any arguments to fragment shader
            0
        end
    end

    raster = JLRasterizer{eltype(vertexarray), typeof(uniforms), fragment_in_ndim}(
        vertexshader,
        fragmentshader,
        geometryshader,
        geometry_view,
        emit
    )
    raster, (vertexarray, uniforms)
end

Base.@pure Next{N}(::Val{N}) = Val{N - 1}()
@inline function interpolate{N, T}(bary, face::NTuple{N, T}, vn::Val{0}, aggregate)
    if T <: Tuple
        aggregate
    else
        T(aggregate...)
    end
end
@inline function interpolate{N}(bary, face, vn::Val{N}, aggregate = ())
    @inbounds begin
        res = (
            bary[1] * getfield(face[1], N) .+
            bary[2] * getfield(face[2], N) .+
            bary[3] * getfield(face[3], N)
        )
    end
    interpolate(bary, face, Next(vn), (res, aggregate...))
end

broadcastmin(a, b) = min.(a, b)
broadcastmax(a, b) = max.(a, b)

function clip2pixel_space(position, resolution)
    clipspace = position / position[4]
    p = clipspace[Vec(1, 2)]
    (((p + 1f0) / 2f0) .* (resolution - 1f0)) + 1f0
end

function (r::JLRasterizer{Vert, Args, FragN}){Vert, Args, FragN}(
        canvas, vertex_array::AbstractArray{Vert}, uniforms::Args
    )
    framebuffers = canvas[ColorBuffer]; depthbuffer = canvas[DepthBuffer]
    resolution = Vec2f0(size(framebuffers[1]))
    # hoisting out functions... Seems to help inference a bit. Or not?
    vshader = r.vertexshader
    gshader = r.geometryshader
    fshader = r.fragmentshader
    FragNVal = Val{FragN}()
    fragments_drawn = 0
    for face in vertex_array
        vertex_stage = map(reverse(face)) do f
            vshader(f, uniforms...)
        end
        geom_stage = if isa(r.geometryshader, Void)
            (vertex_stage,)
        else
            reset!(r.geometry_view)
            gshader(r.emit, vertex_stage, uniforms...)
            r.geometry_view.view
        end
        for geom_face in geom_stage
            fdepth = map(geom_face) do vert
                fv = first(vert)
                p = clip2pixel_space(fv, resolution)
                p[Vec(1, 2)], p[3]
            end
            f = map(first, fdepth)
            depths = map(last, fdepth)
            vertex_out = map(last, geom_face)
            # Bounding rectangle
            mini = max.(reduce(broadcastmin, f), 1f0)
            maxi = min.(reduce(broadcastmax, f), resolution)
            area = edge_function(f[1], f[2], f[3])
            for y = mini[2]:maxi[2]
                for x = mini[1]:maxi[1]
                    p = Vec(x, y)
                    w = Vec(
                        edge_function(f[2], f[3], p),
                        edge_function(f[3], f[1], p),
                        edge_function(f[1], f[2], p)
                    )
                    yi, xi = round(Int, y), round(Int, x)
                    if all(w-> w <= 0f0, w) && checkbounds(Bool, framebuffers[1], yi, xi)
                        bary = w / area
                        depth = bary[1] * depths[1] + bary[2] * depths[2] + bary[3] * depths[3]

                        if depth <= depthbuffer[yi, xi]
                            depthbuffer[yi, xi] = depth
                            fragment_in = interpolate(bary, vertex_out, FragNVal)
                            fragment_out = fshader(fragment_in, uniforms...)
                            for i = eachindex(fragment_out)
                                src_color = framebuffers[i][yi, xi]
                                dest_color = fragment_out[i]
                                fragments_drawn += 1
                                framebuffers[i][yi, xi] = standard_transparency(
                                    src_color,
                                    RGBA{Float32}(dest_color[1], dest_color[2], dest_color[3], dest_color[4])
                                )
                            end
                        end
                    end
                end
            end
        end
    end
    println("fragments drawn: ", fragments_drawn)
    return
end

end


@inline ngens(R::Nemo.Generic.MPolyRing) = R.num_vars

function _prod(A, b)
  for a in A
    b = b * a
  end
  return b
end

#aparently, should be called evaluate, talk to Bill...
#definitely non-optimal, in particular for automorphisms
function msubst(f::Generic.MPoly{T}, v::Array{NfRelElem{T}, 1}) where T
  k = base_ring(parent(f))
  n = length(v)
  @assert n == ngens(parent(f))
  r = zero(parent(v[1]))
  L = parent(v[1])
  for i=1:length(f)
    #@show prod(v[j]^f.exps[j, i] for j=1:n)
    s = _prod((v[j]^f.exps[n - j + 1, i] for j=1:n), one(L))
    r += f.coeffs[i]* s
  end
  return r
end
function msubst(f::Generic.MPoly{T}, v::Array{NfRelNSElem{T}, 1}) where T
  k = base_ring(parent(f))
  n = length(v)
  @assert n == ngens(parent(f))
  r = zero(k)
  for i=1:length(f)
    r += f.coeffs[i]*prod(v[j]^f.exps[n - j + 1, i] for j=1:n)
  end
  return r
end

################################################################################
#
#  Permutation group from NfRelNSToNfRelNSMor
#
################################################################################

function _get_poly_from_elem(a::NfRelNSElem{nf_elem}, Qxy)
  K = base_field(parent(a))
  Qx = parent(K.pol)
  p = change_base_ring(a.data, x -> evaluate(Qx(x), gen(Qxy, nvars(Qxy))))
  p1 = evaluate(p, [i for i in 1:ngens(parent(a))], gens(Qxy)[1:end-1])
  res = coeff(p1, [0 for i = 1:nvars(parent(p1))])
  return res
end

function multivariate_from_tower(f::Generic.MPoly{nf_elem}, Qxy)
  M = MPolyBuildCtx(Qxy)
  K = base_ring(f)
  Qx = parent(K.pol)
  cvzip = zip(coefficients(f), exponent_vectors(f))
  for (c, v) in cvzip
    pc = Qx(c)
    for i = degree(pc):-1:0
      cpc = coeff(pc, i)
      if iszero(cpc)
        continue
      end
      vn = vcat(v, i)
      push_term!(M, cpc, vn)
    end
  end
  return finish(M)::fmpq_mpoly
end

function (Rxy::NmodMPolyRing)(f::fmpq_mpoly)
  R = base_ring(Rxy)
  res = change_base_ring(f, x -> divexact(R(numerator(x)), R(denominator(x))), Rxy)
  return res
end

function _get_polys_from_auto(f::NfRelNSToNfRelNSMor_nf_elem, Qxy)
  res = Vector{elem_type(Qxy)}(undef, nvars(Qxy))
  if isdefined(f.image_data.base_field_map_data, :prim_image)
    # ap is a constant, but there is no easy way to coerce to the base field
    ap = f.image_data.base_field_map_data.prim_image.data.coeffs[1]
  else
    ap = gen(base_field(codomain(f)))
  end
  K = base_field(codomain(f))
  res[nvars(Qxy)] = evaluate(parent(K.pol)(ap), gen(Qxy, nvars(Qxy)))
  for i = 1:nvars(Qxy)-1
    res[i] = multivariate_from_tower(image_generators(f)[i].data, Qxy)
  end
  return res
end

function Base.hash(f::nmod_mpoly, h::UInt)
  return UInt(1)
end

function permutation_group1(G::Vector{NfRelNSToNfRelNSMor_nf_elem})
  L = domain(G[1])
  K = base_field(L)
  dK = absolute_degree(L)
  d1 = numerator(discriminant(L, FlintQQ))
  p = 2
  while divisible(d1, p)
    p = next_prime(p)
  end
  R = ResidueRing(FlintZZ, p, cached = false)
  Rm, gRm = PolynomialRing(R, ngens(L)+1, cached = false)
  fmod = Vector{nmod_mpoly}(undef, ngens(L)+1)
  RQm, gRQm = PolynomialRing(FlintQQ, ngens(L)+1, cached = false)
  p1 = K.pol
  p1Q = evaluate(p1, gRQm[end])
  fmod[1] = Rm(p1Q)
  for i = 1:ngens(L)
    pp = L.pol[i]
    pp1 = multivariate_from_tower(pp, RQm)
    fmod[i+1] = Rm(pp1)
  end
  permutations = Array{Array{Int, 1},1}(undef, length(G))
  for i = 1:length(G)
    permutations[i] = Vector{Int}(undef, dK)
  end
  pols = Vector{Vector{nmod_mpoly}}(undef, dK)
  pols[1] = gRm
  ind = 2
  gpols = nmod_mpoly[Rm(elel) for elel in _get_polys_from_auto(G[1], RQm)]
  if gpols != gRm
    pols[ind] = gpols
    ind += 1
    gpol = nmod_mpoly[compose_mod(gpols[i], [j for j = 1:nvars(Rm)], pols[2], fmod) for i = 1:length(gpols)]
    while gRm != gpol
      pols[ind] = gpol
      ind += 1
      gpol = nmod_mpoly[compose_mod(gpol[i], [j for j in 1:nvars(Rm)], pols[2], fmod) for i = 1:length(gpols)]
    end
  end
  for i in 2:length(G)
    pi = nmod_mpoly[Rm(x) for x in _get_polys_from_auto(G[i], RQm)]
    if !(pi in pols[1:ind-1])
      previous_order = ind - 1
      pols[ind] = pi
      ind += 1
      for j in 2:previous_order
        pols[ind] = nmod_mpoly[compose_mod(pols[j][s], Int[z for z in 1:nvars(Rm)], pi, fmod) for s = 1:length(pi)]
        ind += 1
      end
      if ind - 1 == dK
        break
      end
      rep_pos = previous_order + 1
      while rep_pos <= ind - 1
        for k in 1:i
          po = nmod_mpoly[Rm(elel) for elel in _get_polys_from_auto(G[k], RQm)]
          att = nmod_mpoly[compose_mod(pols[rep_pos][s], Int[i for i in 1:nvars(Rm)], po, fmod) for s = 1:length(pols[rep_pos])]
          if !(att in pols[1:ind-1])
            pols[ind] = att
            ind += 1
            for j in 2:previous_order
              pols[ind] = nmod_mpoly[compose_mod(pols[j][s], Int[z for z in 1:nvars(Rm)], att, fmod) for s = 1:length(pols[j])]
              ind += 1
            end
            if ind - 1 == dK
              break
            end
          end
        end
        rep_pos = rep_pos + previous_order
      end
    end
  end
  #Now, I have the images mod p
  Dcreation = Vector{Tuple{Vector{nmod_mpoly}, Int}}(undef, length(pols))
  for i = 1:length(pols)
    Dcreation[i] = (pols[i], i)
  end

  gen_pols = Vector{nmod_mpoly}[nmod_mpoly[Rm(y) for y in _get_polys_from_auto(x, RQm)] for x in G]
  D = Dict(Dcreation)
  for s = 1:length(G)
    permutations[s][1] = D[gen_pols[s]]
    for i = 2:length(pols)
      permutations[s][i] = D[nmod_mpoly[compose_mod(gen_pols[s][t], Int[i for i in 1:nvars(Rm)], pols[i], fmod) for t = 1:length(gen_pols[s])]]
    end
  end
  return permutations
end

@doc Markdown.doc"""
    compose_mod(a::AbstractAlgebra.MPolyElem{T}, vars::Vector{Int}, vals::Vector{MPolyElem{T}}, mod::Vector{MPolyElem{T}}) where T <: FieldElement
Evaluate the polynomial by substituting in the supplied values in the array `vals` for
the corresponding variables with indices given by the array `vars`. The evaluation will
succeed if multiplication is defined between elements of the coefficient ring of $a$ and
elements of `vals`. The result will be reduced modulo "mod". If "mod" is a Groebner basis for the ideal
the elements generate.
"""
function compose_mod(a::S, vars::Vector{Int}, vals::Vector{S}, mod::Vector{S}) where S <:MPolyElem{T} where T <: RingElem
  unique(vars) != vars && error("Variables not unique")
  length(vars) != length(vals) && error("Number of variables does not match number of values")
  for i = 1:length(vars)
    if vars[i] < 1 || vars[i] > nvars(parent(a))
      error("Variable index not in range")
    end
  end
  if length(vars) == 0
    return a
  end
  powers = Dict{Int, S}[Dict{Int, S}() for i in 1:length(vals)]
  return _compose_mod(a, vars, vals, powers, mod)::S
end

function powermod(a::S, i::Union{Int, fmpz}, modu::Vector{S}) where S <:MPolyElem{T} where T <: RingElem
  if i == 0
    return one(parent(a))
  end
  if i == 1
    b = divrem(a, modu)[2]
    return b
  end
  if mod(i, 2) == 0
    j = div(i, 2)
    b = powermod(a, j, modu)
    b = b*b
    b = divrem(b, modu)[2]
    return b
  end
  b = divrem(a * powermod(a, i - 1, modu), modu)[2]
  return b
end

function mulmod(a::S, b::S, mod::Vector{S}) where S <:MPolyElem{T} where T <: RingElem
  return divrem(a*b, mod)[2]
end

function _compose_mod(a, vars, vals, powers, modu)
  S = parent(a)
  r = AbstractAlgebra.Generic.geobucket(S)
  cvzip = zip(coefficients(a), exponent_vectors(a))
  for (c, v) in cvzip
    t = one(S)
    for j = 1:length(vars)
      varnum = vars[j]
      exp = v[varnum]
      if !haskey(powers[j], exp)
        powers[j][exp] = powermod(vals[j], exp, modu)
      end
      t = mulmod(t, powers[j][exp], modu)
      v[varnum] = 0
    end
    M = MPolyBuildCtx(S)
    push_term!(M, c, v)
    push!(r, mulmod(t, finish(M), modu))
  end
  return finish(r)
end

function change_base_ring(p::MPolyElem{T}, g, new_polynomial_ring) where {T <: RingElement}
  cvzip = zip(coefficients(p), exponent_vectors(p))
  M = MPolyBuildCtx(new_polynomial_ring)
  for (c, v) in cvzip
    res = g(c)
    if !iszero(res)
      push_term!(M, g(c), v)
    end
  end
  return finish(M)::elem_type(new_polynomial_ring)
end

function Base.:(*)(f::Hecke.NfToNfRel, g::Hecke.NfRelToNfRelNSMor_nf_elem)
  @assert codomain(f) === domain(g)
  return hom(domain(f), codomain(g), g(f(gen(domain(f)))))
end
#
#function hom(K::AnticNumberField, L::NfRelNS{nf_elem}, img_gen::NfRelNSElem{nf_elem})
#  return Hecke.NfToNfRelNSMor(K, L, img_gen)
#end
#
#function image(f::Hecke.NfToNfRelNSMor, a::nf_elem)
#  K = parent(a)
#  Qx = parent(K.pol)
#  return evaluate(Qx(a), f.img_gen)
#end
#
#function preimage(phi::Hecke.NfToNfRelNSMor, a::NfRelNSElem{nf_elem})
#  @assert isdefined(phi, :preimg_base_field) && isdefined(phi, :preimgs)
#  f = data(a)
#  K = codomain(phi)
#  k = base_field(K)
#  R = parent(k.pol)
#  g = map_coefficients(x -> evaluate(R(x), phi.preimg_base_field), f)
#  return evaluate(g, phi.preimgs)
#end
#
#
function degrees(L::NfRelNS)
  return Int[total_degree(x) for x in L.pol]
end

function automorphisms(L::NfRelNS{T}) where T
  c = get_special(L, :automorphisms)
  if c !== nothing
    return c
  end
  auts = _automorphisms(L)
  Hecke.set_special(L, :automorphisms => auts)
  return auts
end

function _automorphisms(L::NfRelNS{T}) where T
  Kx, _ = PolynomialRing(base_field(L), "x", cached = false)
  rts = Vector{elem_type(L)}[roots(to_univariate(Kx, x), L) for x in L.pol]
  auts = Vector{morphism_type(L)}(undef, prod(length(x) for x in rts))
  ind = 1
  it = cartesian_product_iterator([1:length(rts[i]) for i in 1:length(rts)], inplace = true)
  for i in it
    embs = NfRelNSElem{T}[rts[j][i[j]] for j = 1:length(rts)]
    auts[ind] = hom(L, L, embs)
    ind += 1
  end
  return auts
end


function _initialization_attempt(mcmc::MCMC{S, M},
                                 max_lposterior::Float64) where {
                                 S <: DiseaseStateSequence,
                                 M <: ILM}
  events = generate(Events, mcmc)
  rparams = generate(RiskParameters, mcmc)
  lprior = logprior(rparams, mcmc.risk_priors)
  if M <: PhyloILM
    sm = generate(mcmc.substitution_model, mcmc.substitution_model_prior)
    lprior += logprior(sm, mcmc.substitution_model_prior)
  end
  llikelihood, tr, tx = loglikelihood(rparams,
                                      mcmc.risk_functions,
                                      events,
                                      mcmc.population,
                                      mcmc.starting_states,
                                      early_decision_value = max_lposterior - lprior)
  lposterior = llikelihood + lprior
  if lposterior > max_lposterior
    network = generate(TransmissionNetwork, tr, mcmc, tx)
    if M <: PhyloILM
      tree, obs_nodes = generate(Tree, events, mcmc.event_observations, network)
      leaf_data = Dict{Int64, RNASeq}()
      for k = eachindex(obs_nodes)
        if !isnothing(obs_nodes[k])
          leaf_data[obs_nodes[k]] = mcmc.event_observations.seq[k]
        end
      end
      lposterior += PhyloModels.loglikelihood(tree, sm, leaf_data)
    end
  end
  if lposterior > max_lposterior
    if M <: PhyloILM
      return MarkovChain{S, M}(events, network, rparams, sm, lposterior)
    elseif M <: TNILM
      return MarkovChain{S, M}(events, network, rparams, lposterior)
    end
  else
    return nothing
  end
end

function initialize(::Type{MarkovChain},
                    mcmc::MCMC{S, M},
                    progress_channel::RemoteChannel;
                    attempts::Int64=1000) where {
                    S <: DiseaseStateSequence,
                    M <: ILM}
  if attempts <= 0
    throw(ErrorException("Must have at least 1 initialization attempt"))
  end
  local markov_chain
  max_lposterior = -Inf
  for i in 1:attempts
    initialization = _initialization_attempt(mcmc, max_lposterior)
    if initialization !== nothing
      markov_chain = initialization
      max_lposterior = markov_chain.log_posterior[1]
    end
    put!(progress_channel, true)
  end
  if max_lposterior == -Inf
    @error "Failed to initialize Markov Chain"
    markov_chain = nothing
  end
  return markov_chain
end

function initialize(::Type{MarkovChain},
                    mcmc::MCMC{S, M};
                    attempts::Int64=1000) where {
                    S <: DiseaseStateSequence,
                    M <: ILM}
  if attempts <= 0
    throw(ErrorException("Must have at least 1 initialization attempt"))
  end
  max_lposterior = -Inf
  local markov_chain
  pmeter = Progress(attempts, "Initialization progress")
  for i in 1:attempts
    initialization = _initialization_attempt(mcmc, max_lposterior)
    if initialization !== nothing
      markov_chain = initialization
      max_lposterior = markov_chain.log_posterior[1]
    end
    next!(pmeter)
  end
  finish!(pmeter)
  if max_lposterior == -Inf
    @error "Failed to initialize Markov Chain"
    markov_chain = nothing
  end
  return markov_chain
end


using SolidStateDetectors

include("Plotting.jl")
include("ReadGeant4Hits.jl")
include("EventSimulation.jl")

defaultDir = "plots/"

T = Float32

function BiasVariation(configFile::String, initVoltage::Real, finalVoltage::Real, length::Integer, CCDName::String="")::AbstractDict{Real, Simulation}
    biasRange = range(initVoltage, stop=finalVoltage, length=length)

    @info "Constructing base detector"
    sim_base = Simulation(configFile)
    sim_dict = Dict{Real, Simulation}()

    if CCDName != ""
        #include(CCDName)  This line needs to instead be put into the main script you are running
        sim_base.detector.semiconductors[1].charge_density_model = ccdm
    end

    @info "Calculating weighting potentials"
    for i in 1:size(sim_base.detector.contacts, 1)
        calculate_weighting_potential!(sim_base, i,
        max_refinements=2,
        convergence_limit=1e-3
        #init_grid_spacing = T.( (2e-4, 2e-4, 2e-4) )
        )
    end

    for i in 1:size(biasRange, 1)
        @info "Setting bias voltage"
        det = deepcopy(sim_base.detector)
        det.contacts[1].potential = -biasRange[i]

        sim = Simulation(det)
        calculate_electric_potential!(sim,
        depletion_handling=true,
        max_refinements=2,
        convergence_limit=1e-3
        #init_grid_spacing = T.( (2e-4, 2e-4, 2e-4) )
        )
        calculate_electric_field!(sim)

        sim.weighting_potentials = sim_base.weighting_potentials

        set_charge_drift_model!(sim, ADLChargeDriftModel())
        calculate_drift_fields!(sim)

        sim_dict[biasRange[i]] = sim
    end

    return sim_dict
end

function GetEligibleFiles(dataDir::String,
    minEnergy::Real=0., maxEnergy::Real=Inf,
    minAngle::Real=0., maxAngle::Real=Inf)::Vector{String}

    fileList = readdir(dataDir, join=true)

    filteredList = Vector{String}()

    regex = r"(\d+)+"

    for s in fileList
         m = collect(eachmatch(regex, s))
         E = parse(Int64, m[end-1].match)
         A = parse(Int64, m[end].match)
         if E >= minEnergy && E <= maxEnergy && A >= minAngle && A <= maxAngle
             push!(filteredList, s)
         end
    end

    return filteredList
end

function PlotSingleParticleDetectorResponse(sim::Simulation, fullPath::String,
    impactPosition::CartesianPoint=CartesianPoint{T}(0, 0, 0), prefix::String="",
    suffix::String="", dir::String=defaultDir, stepLimiter::Integer=1000, qe=x->1)::Vector{Event}

    fileName = split(fullPath, "/")[end]
    strippedName = split(fileName, ".")[1]
    suffix = "_" * strippedName * suffix

    gdf = GetHitInformation(fullPath)
    events = DriftGeant4Events(gdf, sim, impactPosition, stepLimiter=stepLimiter, qe=qe)
    PlotEvents(events, dir, prefix, suffix)
    return events
end

function PlotParticleDetectorResponse(sims::AbstractDict{Real, Simulation}, dataDir::String;
    impactPosition::CartesianPoint=CartesianPoint{T}(0, 0, 0), minEnergy::Real=0., maxEnergy::Real=Inf,
    minAngle::Real=0., maxAngle::Real=Inf,
    prefix::String="", dir::String=defaultDir, maxEvents::Integer=100, qe=x->1)::Nothing

    println(impactPosition)

    prefix = prefix * "particle_"
    biasRange = collect(keys(sims))
    fileList = GetEligibleFiles(dataDir, minEnergy, maxEnergy, minAngle, maxAngle)

    @showprogress for i in 1:size(biasRange, 1)
        sim = sims[biasRange[i]]
        suffix = @sprintf "_%.1fV" biasRange[i]
        @showprogress for s in fileList
            PlotSingleParticleDetectorResponse(sim, s, impactPosition, prefix, suffix, dir, maxEvents, qe)
        end
    end
end

function PlotDetectorPerformance(sims::AbstractDict{Real, Simulation}; prefix::String="", dir::String=defaultDir)::Nothing
    biasRange = collect(keys(sims))
    prefix = prefix * "detector_"
    PlotGeometry(sims[biasRange[1]], dir, prefix)
    Plot2DWeightingPotential(sims[biasRange[1]], dir, prefix)
    Plot1DWeightingPotential(sims[biasRange[1]], dir, prefix)
    PlotMaterialProperties(sims[biasRange[1]], dir, prefix)

    for i in 1:size(biasRange, 1)
        suffix = @sprintf "_%.1fV" biasRange[i]
        PlotElectricPotential(sims[biasRange[i]], dir, prefix, suffix)
        PlotElectricField(sims[biasRange[i]], dir, prefix, suffix)
        PlotPointType(sims[biasRange[i]], dir, prefix, suffix)
    end

    PlotCV(collect(values(sims)), dir, prefix)
end


include("./src/GBDTsfuzzy.jl")
include("./MyGrammar.jl")

using .GBDTsfuzzy
using .MyGrammar
using ArgParse
using MultivariateTimeSeries
using Printf
import JLD
using Random; Random.seed!(1)

println("Ready to accept")
while !eof(stdin)
    input = readline(stdin)
    splitted = split(input)
    data = String(splitted[1])
    model_path = String(splitted[2])
    println("Reading data")
    X, y = read_data_labeled(data)
    println("Finished reading")

    println("Reading model")

    m = JLD.load(model_path)

    println(m)
    global v = m["v"]
    model = m["model"]
    mode = m["mode"]

    println("Model loaded")

    println("Classify")
#    if mode == "normal"
#        pred = classify(model, X)
#    else
    pred = classifyfuzzy(model, X)
#    end

    println("Classification done")
    println(pred)

    T = Vector{Float64}(undef, length(X))

#    if mode == "normal"
#        for i in eachindex(T)
#            if pred[i] == model.void_label
#                T[i] = 1.0
#            else
#                T[i] = 0.0
#            end
#        end
#    else
    for i in eachindex(T)
        val = get(pred[i], model.void_label, 0.0)
        T[i] = val
    end
 #   end

    println("Wrongness:")
    println(T)
    println("Done")
end


module Taarruz
using Knet
using LinearAlgebra

_etype = gpu() >= 0 ? Float32 : Float64
_atype = gpu() >= 0 ? KnetArray{_etype} : Array{_etype}

"""
    FloatArray
A KnetArray or Array. Its elements should be Float32 or Float64.
"""
const FloatArray = Union{KnetArray{F}, Array{F}} where F <: AbstractFloat

include("attacks.jl"); export FGSM, IterativeFGSM, MomentumIterativeFGSM
include("lenet.jl"); export Lenet

end # module


# using Revise
using ShaleDrillingLikelihood
using ShaleDrillingLikelihood.SDLParameters

# detect if using SLURM
if "SLURM_JOBID" in keys(ENV)
    SLURM_JOBID = ENV["SLURM_JOBID"]
    using ClusterManagers
else
    SLURM_JOBID = ""
end

using CountPlus, Distributed, JLD2
using ShaleDrillingLikelihood: value_function, EVobj, cost
using SparseArrays: nonzeros
using Formatting: generate_formatter
using Dates: today
using RCall

num_nonzeros(x) = length(nonzeros(x))
intstr = generate_formatter("%'d")

using ShaleDrillingLikelihood: learn, royalty, constr, tech, tax,
    PerfectInfo, MaxLearning, drill,
    simulationPrimitives_information,
    doSimulations, average_cost_df,
    Theta_NoTech, theta_revenue,
    time_idx, DateQuarter

# ------------------- number of simulations ----------------------

pargs = parse_commandline_counterfactuals()
print_parsed_args_counterfactuals(pargs)

DO_PAR         = !pargs["noPar"]
JLD2FILE       = pargs["jld2"]
DATE_STOP      = pargs["dateStop"]
TECH_YEAR_ZERO = pargs["techYearZero"]
RFILEDIR       = pargs["rFileDir"]

# --------------- create data ---------------

if "SLURM_JOBID" in keys(ENV)
    DATADIR = "/home/magerton/haynesville/intermediate_data"
else
    DATADIR = "E:/projects/haynesville/intermediate_data"
end

# JLD2FILE = "E:/projects/ShaleDrillingResults/2020-01/17493674/estimation-results-17493674-no-rigs-data_all_leases.jld2"
# JLD2FILE = "E:/projects/ShaleDrillingResults/2020-01/17493743/estimation-results-17493743-WITH-rigs-data_all_leases.jld2"
# JLD2FILE = "E:/projects/ShaleDrillingResults/2020-02/18273309/estimation-results-18273309-no-rigs-dbl-timefe-data_all_leases.jld2"

println_time_flush("Loading results from $JLD2FILE")
file = jldopen(JLD2FILE, "r")
    DATAPATH = file["DATAPATH"]
    M        = file["M"]
    ddmnovf  = file["ddm_novf"]
    theta    = file["theta1"]
close(file)
println_time_flush("Done!")

REWARD = reward(ddmnovf)
ddm = DynamicDrillModel(ddmnovf, REWARD)

# --------------- create data ---------------

# load in data from disk
rdatapath = joinpath(DATADIR, DATAPATH)
println_time_flush("loading $rdatapath")

data_royalty    = DataRoyalty(REWARD, rdatapath)
data_produce    = DataProduce(REWARD, rdatapath)
data_drill_prim = DataDrillPrimitive(REWARD, rdatapath)
data_drill      = DataDrill(data_drill_prim, ddm)

evdims = size(EVobj(value_function(ddm)))
println_time_flush("EV dimension is $evdims, implying $(intstr(prod(evdims))) states")

# full dataset
dataset_full = DataSetofSets(data_drill, data_royalty, data_produce, CoefLinks(data_drill))

println_time_flush("Data created")

let thts = split_thetas(dataset_full, theta)
    println("theta_drill = $(round.(thts[1]; digits=3))")
    println("theta_roy   = $(round.(thts[2]; digits=3))")
    println("theta_pdxn  = $(round.(thts[3]; digits=3))")
end

# ------------------- simulations -----------------------

@show wp = statespace(ddm)
PP = PerpetualProblem(wp)

# ------------------- average costs -----------------------

cost_df = average_cost_df(dataset_full, theta)
# using Plots, StatsPlots
# @df cost_df plot(:date, -3.843351*[:cost1, :cost2], ylims=(0,Inf))

# ------------------- simulation info -----------------------
R = ShaleDrillingLikelihood.revenue(REWARD)
C = ShaleDrillingLikelihood.cost(REWARD)
E = ShaleDrillingLikelihood.extend(REWARD)
S = ShaleDrillingLikelihood.scrap(REWARD)

DrillRev(lrn, roy) = DrillingRevenue(constr(R), tech(R), tax(R), lrn, roy)
DrillRwrd(lrn, roy) = DrillReward(DrillRev(lrn,roy), C, E, S)

theta_notech = Theta_NoTech(dataset_full, theta, TECH_YEAR_ZERO)
TSTOP = time_idx(zchars(drill(dataset_full)), DateQuarter(DATE_STOP))

simlist = [
    (DrillRwrd( learn(R),      royalty(R) ),   wp, theta, ),           # "Baseline"
    (DrillRwrd( learn(R),      royalty(R) ),   wp, theta_notech, ),    # "Where to drill"
    (DrillRwrd( NoLearn(),     royalty(R) ),   wp, theta, ),           # "No update ($\\psi_{it} = \\psi_i^0$)"
    (DrillRwrd( PerfectInfo(), royalty(R) ),   wp, theta, ),           # "Perfect information ($\\psi_{it} = \\psi_i^1$)"
    (DrillRwrd( MaxLearning(), royalty(R) ),   wp, theta, ),           # "Uninformative signals ($\\rho = 0$)"
    (DrillRwrd( learn(R),      NoRoyalty()),   wp, theta, ),           # "No royalty"
    (DrillRwrd( learn(R),      royalty(R) ),   PP, theta, ),           # "No expiration"
    (DrillRwrd( learn(R),      NoRoyalty()),   PP, theta, ),           # "Ownership"
    (DrillRwrd(NoLearn(),      NoRoyalty()  ), PP, theta_notech, ),    # "Price only"
    (DrillRwrd(NoLearn(),      NoRoyalty()  ), PP, theta, ),           # "Price + tech (no distortions)"
    (DrillRwrd(NoLearn(),      WithRoyalty()), wp, theta_notech, ),    # "Price + leases"
    (DrillRwrd(Learn(),        NoRoyalty()  ), PP, theta_notech, ),    # "Price + learning"
    (DrillRwrd(NoLearn(),      WithRoyalty()), PP, theta, ),           # "How to drill"
]

simlist_meta = [(a,b,copy(theta_revenue(dataset_full, c))) for (a,b,c) in simlist]

simulations_meta_info = simulationPrimitives_information(simlist_meta)

# ------------------- simulations -----------------------

if DO_PAR
    pids = start_up_workers(ENV)
    @everywhere using ShaleDrillingLikelihood
    println_time_flush("Library loaded on workers")
end

println_time_flush("Starting simulations")
df_d, df_D = doSimulations(dataset_full, simlist, TSTOP, M)
println_time_flush("Simulations done")

# ------------------- save -----------------------

dpath_no_rdata = replace(DATAPATH, ".RData" => "")
filenm = "simulations-$(SLURM_JOBID)-$(dpath_no_rdata)-$(today()).RData"
filepath = joinpath(RFILEDIR, filenm)

println_time_flush("Saving simulations to $filepath")

THETA = theta
FILEPATH = filepath

@rput df_d df_D cost_df simulations_meta_info THETA FILEPATH TSTOP

R"""
library(data.table)
setDT(df_D)
setDT(df_d)
setDT(cost_df)
setDT(simulations_meta_info)

save(simulations_meta_info, df_d, df_D, cost_df, THETA, TSTOP, file=FILEPATH)
rm(  simulations_meta_info, df_d, df_D, cost_df, THETA, TSTOP,      FILEPATH)
gc()
"""

println_time_flush("Mischief managed. :)")


@get_units @with_kw struct SimpleReservoir{T}
    Δt::T | "s"                                         # Model time step [s]
    maxvolume::Vector{T} | "m3"                         # maximum storage (above which water is spilled) [m³]
    area::Vector{T} | "m2"                              # reservoir area [m²]
    maxrelease::Vector{T} | "m3 s-1"                    # maximum amount that can be released if below spillway [m³ s⁻¹]
    demand::Vector{T} | "m3 s-1"                        # minimum (environmental) flow requirement downstream of the reservoir [m³ s⁻¹]
    targetminfrac::Vector{T} | "-"                      # target minimum full fraction (of max storage) [-]
    targetfullfrac::Vector{T} | "-"                     # target fraction full (of max storage) [-]
    volume::Vector{T} | "m3"                            # reservoir volume [m³]
    inflow::Vector{T} | "m3"                            # total inflow into reservoir [m³]
    outflow::Vector{T} | "m3 s-1"                       # outflow from reservoir [m³ s⁻¹]
    totaloutflow::Vector{T} | "m3"                      # total outflow from reservoir [m³]
    percfull::Vector{T} | "-"                           # fraction full (of max storage) [-]
    demandrelease::Vector{T} | "m3 s-1"                 # minimum (environmental) flow released from reservoir [m³ s⁻¹]
    precipitation::Vector{T}                            # average precipitation for reservoir area [mm Δt⁻¹]
    evaporation::Vector{T}                              # average evaporation for reservoir area [mm Δt⁻¹]

    function SimpleReservoir{T}(args...) where {T}
        equal_size_vectors(args)
        return new(args...)
    end
end

statevars(::SimpleReservoir) = (:volume,)

function initialize_simple_reservoir(config, nc, inds_riv, nriv, pits, Δt)
    # read only reservoir data if reservoirs true
    # allow reservoirs only in river cells
    # note that these locations are only the reservoir outlet pixels
    reslocs = ncread(
        nc,
        config.input,
        "lateral.river.reservoir.locs";
        optional = false,
        sel = inds_riv,
        type = Int,
        fill = 0,
    )

    # this holds the same ids as reslocs, but covers the entire reservoir
    rescoverage_2d = ncread(
        nc,
        config.input,
        "lateral.river.reservoir.areas";
        optional = false,
        allow_missing = true,
    )
    # for each reservoir, a list of 2D indices, needed for getting the mean precipitation
    inds_res_cov = Vector{CartesianIndex{2}}[]

    rev_inds_reservoir = zeros(Int, size(rescoverage_2d))

    # construct a map from the rivers to the reservoirs and
    # a map of the reservoirs to the 2D model grid
    resindex = fill(0, nriv)
    inds_res = CartesianIndex{2}[]
    rescounter = 0
    for (i, ind) in enumerate(inds_riv)
        res_id = reslocs[i]
        if res_id > 0
            push!(inds_res, ind)
            rescounter += 1
            resindex[i] = rescounter
            rev_inds_reservoir[ind] = rescounter

            # get all indices related to this reservoir outlet
            # done in this loop to ensure that the order is equal to the order in the
            # SimpleReservoir struct
            res_cov = findall(isequal(res_id), rescoverage_2d)
            push!(inds_res_cov, res_cov)
        end
    end

    resdemand = ncread(
        nc,
        config.input,
        "lateral.river.reservoir.demand";
        optional = false,
        sel = inds_res,
        type = Float,
        fill = 0,
    )
    resmaxrelease = ncread(
        nc,
        config.input,
        "lateral.river.reservoir.maxrelease";
        optional = false,
        sel = inds_res,
        type = Float,
        fill = 0,
    )
    resmaxvolume = ncread(
        nc,
        config.input,
        "lateral.river.reservoir.maxvolume";
        optional = false,
        sel = inds_res,
        type = Float,
        fill = 0,
    )
    resarea = ncread(
        nc,
        config.input,
        "lateral.river.reservoir.area";
        optional = false,
        sel = inds_res,
        type = Float,
        fill = 0,
    )
    res_targetfullfrac = ncread(
        nc,
        config.input,
        "lateral.river.reservoir.targetfullfrac";
        optional = false,
        sel = inds_res,
        type = Float,
        fill = 0,
    )
    res_targetminfrac = ncread(
        nc,
        config.input,
        "lateral.river.reservoir.targetminfrac";
        optional = false,
        sel = inds_res,
        type = Float,
        fill = 0,
    )

    # for surface water routing reservoir locations are considered pits in the flow network
    # all upstream flow goes to the river and flows into the reservoir
    pits[inds_res] .= true

    n = length(resarea)
    @info "Read `$n` reservoir locations."
    reservoirs = SimpleReservoir{Float}(
        Δt = Δt,
        demand = resdemand,
        maxrelease = resmaxrelease,
        maxvolume = resmaxvolume,
        area = resarea,
        targetfullfrac = res_targetfullfrac,
        targetminfrac = res_targetminfrac,
        volume = res_targetfullfrac .* resmaxvolume,
        inflow = fill(mv, n),
        outflow = fill(mv, n),
        totaloutflow = fill(mv, n),
        percfull = fill(mv, n),
        demandrelease = fill(mv, n),
        precipitation = fill(mv, n),
        evaporation = fill(mv, n),
    )

    return reservoirs,
    resindex,
    (
        indices_outlet = inds_res,
        indices_coverage = inds_res_cov,
        reverse_indices = rev_inds_reservoir,
    ),
    pits
end

"""
Update a single reservoir at position `i`.

This is called from within the kinematic wave loop, therefore updating only for a single
element rather than all at once.
"""
function update(res::SimpleReservoir, i, inflow, timestepsecs)

    vol = max(
        0.0,
        (
            res.volume[i] +
            (inflow * timestepsecs) +
            (res.precipitation[i] * (timestepsecs / res.Δt) / 1000.0) * res.area[i] -
            (res.evaporation[i] * (timestepsecs / res.Δt) / 1000.0) * res.area[i]
        ),
    )

    percfull = vol / res.maxvolume[i]
    # first determine minimum (environmental) flow using a simple sigmoid curve to scale for target level
    fac = scurve(percfull, res.targetminfrac[i], Float(1.0), Float(30.0))
    demandrelease = min(fac * res.demand[i] * timestepsecs, vol)
    vol = vol - demandrelease

    wantrel = max(0.0, vol - (res.maxvolume[i] * res.targetfullfrac[i]))
    # Assume extra maximum Q if spilling
    overflow_q = max((vol - res.maxvolume[i]), 0.0)
    torelease = min(wantrel, overflow_q + res.maxrelease[i] * timestepsecs - demandrelease)
    vol = vol - torelease
    outflow = torelease + demandrelease
    percfull = vol / res.maxvolume[i]

    # update values in place
    res.outflow[i] = outflow / timestepsecs
    res.inflow[i] += inflow * timestepsecs
    res.totaloutflow[i] += outflow
    res.demandrelease[i] = demandrelease / timestepsecs
    res.percfull[i] = percfull
    res.volume[i] = vol

    return res
end

@get_units @with_kw struct NaturalLake{T}
    Δt::T | "s"                             # Model time step [s]
    lowerlake_ind::Vector{Int} | "-"        # Index of lower lake (linked lakes)
    area::Vector{T} | "m2"                  # lake area [m²]
    threshold::Vector{T} | "m"              # water level threshold H₀ [m] below that level outflow is zero
    storfunc::Vector{Int} | "-"             # type of lake storage curve, 1: S = AH, 2: S = f(H) from lake data and interpolation
    outflowfunc::Vector{Int} | "-"          # type of lake rating curve, 1: Q = f(H) from lake data and interpolation, 2: General Q = b(H - H₀)ᵉ, 3: Case of Puls Approach Q = b(H - H₀)²
    b::Vector{T} | "m3/2 s-1 (if e=3/2)"    # rating curve coefficient
    e::Vector{T} | "-"                      # rating curve exponent
    sh::Vector{Union{SH,Missing}}           # data for storage curve
    hq::Vector{Union{HQ,Missing}}           # data for rating curve
    waterlevel::Vector{T} | "m"             # waterlevel H [m] of lake
    inflow::Vector{T} | "m3"                # inflow to the lake [m³]
    storage::Vector{T} | "m3"               # storage lake [m³]
    outflow::Vector{T} | "m3 s-1"           # outflow lake [m³ s⁻¹]
    totaloutflow::Vector{T} | "m3"          # total outflow lake [m³]
    precipitation::Vector{T}                # average precipitation for lake area [mm Δt⁻¹]
    evaporation::Vector{T}                  # average evaporation for lake area [mm Δt⁻¹]

    function NaturalLake{T}(args...) where {T}
        equal_size_vectors(args)
        return new(args...)
    end
end

function initialize_natural_lake(config, nc, inds_riv, nriv, pits, Δt)
    # read only lake data if lakes true
    # allow lakes only in river cells
    # note that these locations are only the lake outlet pixels
    lakelocs_2d = ncread(
        nc,
        config.input,
        "lateral.river.lake.locs";
        optional = false,
        type = Int,
        fill = 0,
    )
    lakelocs = lakelocs_2d[inds_riv]

    # this holds the same ids as lakelocs, but covers the entire lake
    lakecoverage_2d = ncread(
        nc,
        config.input,
        "lateral.river.lake.areas";
        optional = false,
        allow_missing = true,
    )
    # for each lake, a list of 2D indices, needed for getting the mean precipitation
    inds_lake_cov = Vector{CartesianIndex{2}}[]

    rev_inds_lake = zeros(Int, size(lakecoverage_2d))

    # construct a map from the rivers to the lakes and
    # a map of the lakes to the 2D model grid
    lakeindex = fill(0, nriv)
    inds_lake = CartesianIndex{2}[]
    lakecounter = 0
    for (i, ind) in enumerate(inds_riv)
        lake_id = lakelocs[i]
        if lake_id > 0
            push!(inds_lake, ind)
            lakecounter += 1
            lakeindex[i] = lakecounter
            rev_inds_lake[ind] = lakecounter

            # get all indices related to this lake outlet
            # done in this loop to ensure that the order is equal to the order in the
            # NaturalLake struct
            lake_cov = findall(isequal(lake_id), lakecoverage_2d)
            push!(inds_lake_cov, lake_cov)
        end
    end

    lakearea = ncread(
        nc,
        config.input,
        "lateral.river.lake.area";
        optional = false,
        sel = inds_lake,
        type = Float,
        fill = 0,
    )
    lake_b = ncread(
        nc,
        config.input,
        "lateral.river.lake.b";
        optional = false,
        sel = inds_lake,
        type = Float,
        fill = 0,
    )
    lake_e = ncread(
        nc,
        config.input,
        "lateral.river.lake.e";
        optional = false,
        sel = inds_lake,
        type = Float,
        fill = 0,
    )
    lake_threshold = ncread(
        nc,
        config.input,
        "lateral.river.lake.threshold";
        optional = false,
        sel = inds_lake,
        type = Float,
        fill = 0,
    )
    linked_lakelocs = ncread(
        nc,
        config.input,
        "lateral.river.lake.linkedlakelocs";
        sel = inds_lake,
        defaults = 0,
        type = Int,
        fill = 0,
    )
    lake_storfunc = ncread(
        nc,
        config.input,
        "lateral.river.lake.storfunc";
        optional = false,
        sel = inds_lake,
        type = Int,
        fill = 0,
    )
    lake_outflowfunc = ncread(
        nc,
        config.input,
        "lateral.river.lake.outflowfunc";
        optional = false,
        sel = inds_lake,
        type = Int,
        fill = 0,
    )
    lake_waterlevel = ncread(
        nc,
        config.input,
        "lateral.river.lake.waterlevel";
        optional = false,
        sel = inds_lake,
        type = Float,
        fill = 0,
    )

    # for surface water routing lake locations are considered pits in the flow network
    # all upstream flow goes to the river and flows into the lake
    pits[inds_lake] .= true

    # This is currently the same length as all river cells, but will be the
    # length of all lake cells. To do that we need to introduce a mapping.
    n_lakes = length(inds_lake)
    lakelocs = lakelocs_2d[inds_lake]

    @info "Read `$n_lakes` lake locations."

    sh = Vector{Union{SH,Missing}}(missing, n_lakes)
    hq = Vector{Union{HQ,Missing}}(missing, n_lakes)
    lowerlake_ind = fill(0, n_lakes)
    # lake CSV parameter files are expected in the same directory as path_static
    path = dirname(input_path(config, config.input.path_static))

    for i = 1:n_lakes
        lakeloc = lakelocs[i]
        if linked_lakelocs[i] > 0
            lowerlake_ind[i] = only(findall(x -> x == linked_lakelocs[i], lakelocs))
        end

        if lake_storfunc[i] == 2
            csv_path = joinpath(path, "lake_sh_$lakeloc.csv")
            sh[i] = read_sh_csv(csv_path)
        end

        if lake_outflowfunc[i] == 1
            csv_path = joinpath(path, "lake_hq_$lakeloc.csv")
            hq[i] = read_hq_csv(csv_path)
        end

        if lake_outflowfunc[i] == 3 && lake_storfunc[i] != 1
            @warn(
                "For the modified pulse approach (LakeOutflowFunc = 3) the LakeStorFunc should be 1"
            )
        end
    end
    n = length(lakearea)
    lakes = NaturalLake{Float}(
        Δt = Δt,
        lowerlake_ind = lowerlake_ind,
        area = lakearea,
        threshold = lake_threshold,
        storfunc = lake_storfunc,
        outflowfunc = lake_outflowfunc,
        b = lake_b,
        e = lake_e,
        waterlevel = lake_waterlevel,
        sh = sh,
        hq = hq,
        inflow = fill(mv, n),
        storage = initialize_storage(lake_storfunc, lakearea, lake_waterlevel, sh),
        outflow = fill(mv, n),
        totaloutflow = fill(mv, n),
        precipitation = fill(mv, n),
        evaporation = fill(mv, n),
    )

    return lakes,
    lakeindex,
    (
        indices_outlet = inds_lake,
        indices_coverage = inds_lake_cov,
        reverse_indices = rev_inds_lake,
    ),
    pits
end

"Determine the initial storage depending on the storage function"
function initialize_storage(storfunc, area, waterlevel, sh)
    storage = similar(area)
    for i in eachindex(storage)
        if storfunc[i] == 1
            storage[i] = area[i] * waterlevel[i]
        else
            storage[i] = interpolate_linear(waterlevel[i], sh[i].H, sh[i].S)
        end
    end
    return storage
end

statevars(::NaturalLake) = (:waterlevel,)

"""
    statevars(::SBM; snow=false)
    statevars(::LateralSSF)
    statevars(::SurfaceFlow)
    statevars(::NaturalLake)
    statevars(::SimpleReservoir)

Get a tuple of symbols representing the fields that are model states.
"""
function statevars end

function interpolate_linear(x, xp, fp)
    if x <= minimum(xp)
        return minimum(fp)
    elseif x >= maximum(xp)
        return maximum(fp)
    else
        idx = findall(i -> i <= x, xp)
        i1 = last(idx)
        i2 = i1 + 1
        return fp[i1] * (1.0 - (x - xp[i1]) / (xp[i2] - xp[i1])) +
               fp[i2] * (x - xp[i1]) / (xp[i2] - xp[i1])
    end
end

"""
Update a single lake at position `i`.

This is called from within the kinematic wave loop, therefore updating only for a single
element rather than all at once.
"""
function update(lake::NaturalLake, i, inflow, doy, timestepsecs)

    lo = lake.lowerlake_ind[i]
    has_lowerlake = lo != 0

    col = min(doy, 365)
    ### Modified Puls Approach (Burek et al., 2013, LISFLOOD) ###
    # outflowfunc = 3
    # Calculate lake factor and SI parameter
    if lake.outflowfunc[i] == 3
        lakefactor = lake.area[i] / (timestepsecs * pow(lake.b[i], 0.5))
        si_factor =
            (
                lake.storage[i] +
                (lake.precipitation[i] - lake.evaporation[i]) *
                (timestepsecs / lake.Δt) *
                lake.area[i] / 1000.0
            ) / timestepsecs + inflow
        #Adjust SIFactor for ResThreshold != 0
        si_factor_adj = si_factor - lake.area[i] * lake.threshold[i] / timestepsecs
        #Calculate the new lake outflow/waterlevel/storage

        if si_factor_adj > 0.0
            outflow = pow(
                -lakefactor + pow((pow(lakefactor, 2.0) + 2.0 * si_factor_adj), 0.5),
                2.0,
            )
        else
            outflow = 0.0
        end
        storage = (si_factor - outflow) * timestepsecs
        waterlevel = storage / lake.area[i]
    end

    ### Linearisation for specific storage/rating curves ###
    if lake.outflowfunc[i] == 1 || lake.outflowfunc[i] == 2

        diff_wl = has_lowerlake ? lake.waterlevel[i] - lake.waterlevel[lo] : 0.0

        if lake.outflowfunc[i] == 1
            outflow =
                interpolate_linear(lake.waterlevel[i], lake.hq[i].H, lake.hq[i].Q[:, col])
        else
            if diff_wl >= 0.0
                if lake.waterlevel[i] > lake.threshold[i]
                    outflow =
                        lake.b[i] * pow((lake.waterlevel[i] - lake.threshold[i]), lake.e[i])
                else
                    outflow = Float(0)
                end
            else
                if lake.waterlevel[lo] > lake.threshold[i]
                    outflow =
                        -1.0 *
                        lake.b[i] *
                        pow((lake.waterlevel[lo] - lake.threshold[i]), lake.e[i])
                else
                    outflow = Float(0)
                end
            end
        end

        storage =
            lake.storage[i] +
            inflow * timestepsecs +
            (lake.precipitation[i] / 1000.0) * (timestepsecs / lake.Δt) * lake.area[i] -
            (lake.evaporation[i] / 1000.0) * (timestepsecs / lake.Δt) * lake.area[i] -
            outflow * timestepsecs

        waterlevel = if lake.storfunc[i] == 1
            lake.waterlevel[i] + (storage - lake.storage[i]) / lake.area[i]
        else
            interpolate_linear(storage, lake.sh[i].S, lake.sh[i].H)
        end

        # update lower lake (linked lakes) in case flow from lower lake to upper lake occurs
        if diff_wl < 0.0
            lowerlake_storage = lake.storage[lo] + outflow * timestepsecs

            lowerlake_waterlevel = if lake.storfunc[lo] == 1
                lake.waterlevel[lo] + (lowerlake_storage - lake.storage[lo]) / lake.area[lo]
            else
                interpolate_linear(lowerlake_storage, lake.sh[lo].S, lake.sh[lo].H)
            end

            # update values for the lower lake in place
            lake.outflow[lo] = -outflow
            lake.storage[lo] = lowerlake_storage
            lake.waterlevel[lo] = lowerlake_waterlevel
        end

    end

    # update values in place
    lake.outflow[i] = max(outflow, 0.0) # for a linked lake flow can be negative
    lake.waterlevel[i] = waterlevel
    lake.inflow[i] += inflow * timestepsecs
    lake.totaloutflow[i] += outflow * timestepsecs
    lake.storage[i] = storage

    return lake
end


# Base.include(Main, "dataStruct.jl")
# Base.MainInclude.include("../../RawFileofflineProc/dataStruct.jl")
function readin!(cfgdata::cfgStruct,CfgArray::Array{UInt16,1})
    cfgdata.meterperSeg =CfgArray[1] /5000
    cfgdata.chn2SegBegIdx =CfgArray[2]
    cfgdata.reflectorFactor =CfgArray[3] /10000
    cfgdata.attenutionFactor =CfgArray[4] /10000
    cfgdata.scanrate = CfgArray[5]
    cfgdata.calSamples = CfgArray[6]
    cfgdata.SegNumber = CfgArray[7]
end

function show( dataObj::RNG )
    println( "RNG Data :-----------------------------------" )
    println( " AlrmProcStartID:$(dataObj.AlrmProcStartID)")
    println( " AlrmProcEndtID1:$(dataObj.AlrmProcEndtID1)")
    println( " startID1:$(dataObj.startID1)")
    println( " endID1:$(dataObj.endID1)")
    println( " chnid:$(dataObj.chnid)")
    println( " ProtectSpaceWidth:$(dataObj.ProtectSpaceWidth)")
    println("----------------------")
end
###################
#############
function seg2Meters( CfgInfoObj::cfgStruct,segID::Int  )
    meter = CfgInfoObj.meterperSeg
    chn2Pos = CfgInfoObj.chn2SegBegIdx
    ## write the title s: 通道号下的光缆 时间、各段的米标 。
    # print( "时间" ,"," )
    # for segID = CumSumDataobj.AlrmProcStartID : CumSumDataobj.AlrmProcEndtID
    if segID  >= chn2Pos;
        return ( round( UInt,Float64(segID-chn2Pos)* meter) ,2 )
    else
        return (round( UInt,Float64( segID  )* meter ),1)
    end
end

function getRNG!( x::RNG,cfgObj::cfgStruct, startMeters::Int ,endmeters::Int ,ProtectMeters::Int,chnid::Int )
    ProtectSegs =  round(Int, Float64( ProtectMeters ) / cfgObj.meterperSeg)+1
    AlrmProcStartID =  round(Int, Float64( startMeters ) / cfgObj.meterperSeg)
    AlrmProcEndtID1 =   round(Int, Float64( endmeters ) / cfgObj.meterperSeg)

    if chnid == 1 ;
        AlrmProcStartID < 1 ? AlrmProcStartID = 1 : 0  ;
        AlrmProcEndtID1 >=  cfgObj.chn2SegBegIdx  ? AlrmProcEndtID1 = cfgObj.chn2SegBegIdx -1 : 0  ;

        startID1 = AlrmProcStartID - ProtectSegs
        endID1 = AlrmProcEndtID1 + ProtectSegs

        startID1 < 1 ? startID1 = 1 :  0 ;
        endID1 >=  cfgObj.chn2SegBegIdx  ? endID1 = cfgObj.chn2SegBegIdx : 0 ;
    else
        AlrmProcEndtID1 +=  cfgObj.chn2SegBegIdx
        AlrmProcStartID += cfgObj.chn2SegBegIdx

        AlrmProcEndtID1 >  cfgObj.SegNumber  ? AlrmProcEndtID1 = cfgObj.SegNumber :  0 ;
        AlrmProcStartID >  cfgObj.SegNumber  ? AlrmProcStartID = cfgObj.SegNumber :  0 ;

        startID1 = AlrmProcStartID - ProtectSegs
        endID1 = AlrmProcEndtID1 + ProtectSegs

        startID1 < cfgObj.chn2SegBegIdx ? startID1 = cfgObj.chn2SegBegIdx : 0 ;
        endID1 >  cfgObj.SegNumber  ? endID1 = cfgObj.SegNumber :  0 ;
        # println( AlrmProcEndtID1 ,"=======$AlrmProcStartID===$endID1 ===$startID1====")
    end

    x.ProtectSpaceWidth = ProtectSegs

    x.AlrmProcStartID = AlrmProcStartID
    x.AlrmProcEndtID1 = AlrmProcEndtID1
    x.startID1 = startID1
    x.endID1 = endID1
    x.chnid = chnid >= 2 ? chnid =2 : chnid =1;

end

function setValue!( x::cfgStruct  , y::CfgDataStruct)
    x.meterperSeg     = y.SpatialResolution
    x.chn2SegBegIdx   = y.LenCh1 +1
    x.reflectorFactor = y.Refractivity
    x.attenutionFactor=y.ATT
    x.scanrate        =y.ScanRate
    x.calSamples      = y.EnergyCnt
    x.SegNumber       = y.LenCh1 +y.LenCh2
end
function show( dataObj::CfgDataStruct   )
    println( "Config Data :-----------------------------------" )
    println( " SpatialResolution:$(dataObj.SpatialResolution)")
    println( " Refractivity:$(dataObj.Refractivity)")
    println( " OffsetM:$(dataObj.OffsetM)")
    println( " ATT:$(dataObj.ATT)")
    println( " ScanRate:$(dataObj.ScanRate)")
    println( " EnergyCnt:$(dataObj.EnergyCnt)")
    println( " LcLimitHigh:$(dataObj.LcLimitHigh)")
    println( " LcLimitLow:$(dataObj.LcLimitLow)")
    println( " StartCh1:$(dataObj.StartCh1)")
    println( " LenCh1:$(dataObj.LenCh1)")
    println( " StartCh2:$(dataObj.StartCh2)")
    println( " LenCh2:$(dataObj.LenCh2)")
    # println( "SpatialResolution:$(dataObj.SpatialResolution)")

    println("----------------------")
end
function show(cfgdata::cfgStruct)
    println(" meterPerSeg = $(cfgdata.meterperSeg) ")
    println(" chn2SegBegIdx = $(cfgdata.chn2SegBegIdx )")
    println(" reflectorFactor = $(cfgdata.reflectorFactor) ")
    println(" attenutionFactor = $(cfgdata.attenutionFactor) ")
    println(" calSamples = $(cfgdata.calSamples) ")
    println(" scanrate = $(cfgdata.scanrate) ")
    println(" SegNumber = $(cfgdata.SegNumber) ")
end


# Subtyping AbstractVector{ElemT} make Base think that RepIterator implements indexing e.g. for copy!
abstract type AbstractRepIterator{N, T, ElemT, PT} end

# For a RepIterator with only one representation, we fallback to the indexing interface, the index being the iterator state
# A representation can overwrite this if it can do something more efficient or if it simply does not support indexing
const SingleRepIterator{N, T, ElemT, RepT} = AbstractRepIterator{N, T, ElemT, Tuple{RepT}}
# With MapRepIterator, we could have RepT<:Rep{N', T'} with N' != N or T' != T, with eachindex, we need to replace everything with N', T'
Base.eachindex(it::SingleRepIterator{<:Any, <:Any, ElemT, RepT}) where {N, T, ElemT, RepT<:Rep{N, T}} = Indices{N, T, similar_type(ElemT, FullDim{N}(), T)}(it.ps[1])
Base.start(it::SingleRepIterator{<:Any, <:Any, ElemT, RepT})     where {N, T, ElemT, RepT<:Rep{N, T}} = start(eachindex(it))::Index{N, T, similar_type(ElemT, FullDim{N}(), T)}
Base.done(it::SingleRepIterator, idx::Index) = done(eachindex(it), idx)
function Base.next(it::SingleRepIterator, idx::Index)
    #mapitem(it, 1, get(it.ps[1], idx)), nextindex(it.ps[1], idx)
    x = get(it.ps[1], idx)
    a = mapitem(it, 1, x)
    b = nextindex(it.ps[1], idx)
    a, b
end

# If there are multiple representations, we need to iterate.
# Builds a SingleRepIterator{ElemT} from p
function checknext(it::AbstractRepIterator, i::Int, state)
    while i <= length(it.ps) && (i == 0 || done(repit(it, i), state))
        i += 1
        if i <= length(it.ps)
            state = start(repit(it, i))
        end
    end
    i > length(it.ps) ? (i, nothing) : (i, state)
end
Base.start(it::AbstractRepIterator) = checknext(it, 0, nothing)
Base.done(it::AbstractRepIterator, state) = state[1] > length(it.ps)

function Base.next(it::AbstractRepIterator, state)
    i, substate = state
    item, newsubstate = next(repit(it, i), substate)
    newstate = checknext(it, i, newsubstate)
    item, newstate
end

# RepIterator
struct RepIterator{N, T, ElemT, PT<:Tuple{Vararg{Rep{N, T}}}} <: AbstractRepIterator{N, T, ElemT, PT}
    ps::PT
    function RepIterator{N, T, ElemT}(ps::PT) where {N, T, ElemT, PT<:Tuple{Vararg{Rep{N, T}}}}
        new{N, T, ElemT, PT}(ps)
    end
end
repit(it::RepIterator{N, T, ElemT}, i::Int) where {N, T, ElemT} = RepIterator{N, T, ElemT}((it.ps[i],))
mapitem(it::RepIterator, i, item) = item

# MapRepIterator
struct MapRepIterator{N, T, ElemT, PT} <: AbstractRepIterator{N, T, ElemT, PT}
    ps::PT
    f::Function
    function MapRepIterator{N, T, ElemT}(ps::PT, f::Function) where {N, T, ElemT, PT<:Tuple}
        new{N, T, ElemT, PT}(ps, f)
    end
end
# N, T are the output dim and coeftype, it can be different that the ones of is.ps[i]
repit(it::MapRepIterator{N, T, ElemT}, i::Int) where {N, T, ElemT} = MapRepIterator{N, T, ElemT}((it.ps[i],), (j, item) -> it.f(i, item)) # j should be 1
mapitem(it::MapRepIterator, i, item) = it.f(i, item)

function Base.first(it::AbstractRepIterator)
    next(it, start(it))[1]
end
FullDim(it::AbstractRepIterator{N}) where N = FullDim{N}()
Base.eltype(it::AbstractRepIterator{N, T, ElemT}) where {N, T, ElemT} = ElemT
function typed_map(f, d::FullDim{N}, ::Type{T}, it::RepIterator{Nin, Tin, ElemT}) where {N, T, Nin, Tin, ElemT}
    MapRepIterator{N, T, similar_type(ElemT, d, T)}(it.ps, f)
end

function RepIterator{N, T}(it::RepIterator) where {N, T}
    typed_map((i,x) -> similar_type(typeof(x), T)(x), FullDim{N}(), T, it)
end

# FIXME the variables need to be defined outside of the local scope of for
#       for Julia to know them inside the """ ... """ of the docstrings
singular = HorV = HorVRep = horvrep = singularlin = plural = plurallin = lenp = isnotemptyp = repexem = listexem = :_

for (isVrep, elt, loop_singular) in [#(true, :SymPoint, :sympoint),
                                     (true, :AbstractPoint, :point),
                                     (true, :Line, :line), (true, :Ray, :ray),
                                     (false, :HyperPlane, :hyperplane), (false, :HalfSpace, :halfspace)]
    global singular = loop_singular
    if isVrep
        vectortype = :vvectortype
        global HorV = :V
        global HorVRep = :VRep
        global horvrep = :vrep
    else
        vectortype = :hvectortype
        global HorV = :H
        global HorVRep = :HRep
        global horvrep = :hrep
    end
    typename = :(AbstractRepIterator{N, T, <:$elt})
    singularstr = string(singular)
    elemtype = Symbol(singularstr * "type")
    donep = Symbol("done" * singularstr)
    startp = Symbol("start" * singularstr)
    nextp = Symbol("next" * singularstr)
    pluralstr = singularstr * "s"
    global plural = Symbol(pluralstr)
    global lenp = Symbol("n" * pluralstr)
    global isnotemptyp = Symbol("has" * pluralstr)
    mapit = Symbol("map" * pluralstr)
    inc = Symbol("incident" * pluralstr)
    incidx = Symbol("incident" * singularstr * "indices")

    @eval begin
        export $plural, $lenp, $isnotemptyp, $startp, $donep, $nextp, $elemtype
        export $inc, $incidx

        """
            $plural($horvrep::$HorVRep)

        Returns an iterator over the $plural of the $HorV-representation `$horvrep`.
        """
        function $plural end

        """
            incident$plural(p::Polyhedron, idx)

        Returns the list of $plural incident to idx for the polyhedron `p`.
        """
        $inc(p::Polyhedron{N, T}, idx) where {N, T} = get(p, IncidentElements{N, T, $elemtype(p)}(p, idx))

        """
            incident$(singular)indices(p::Polyhedron, idx)

        Returns the list of the indices of $plural incident to idx for the polyhedron `p`.
        """
        $incidx(p::Polyhedron{N, T}, idx) where {N, T} = get(p, IncidentIndices{N, T, $elemtype(p)}(p, idx))

        """
            $lenp($horvrep::$HorVRep)

        Returns the number of $plural of the $HorV-representation `$horvrep`.
        """
        $lenp($horvrep::$HorVRep{N, T}) where {N, T} = length(Indices{N, T, $elemtype($horvrep)}($horvrep))

        """
            $isnotemptyp($horvrep::$HorVRep)

        Returns whether the $HorV-representation `$horvrep` has any $singular.
        """
        $isnotemptyp($horvrep::$HorVRep{N, T}) where {N, T} = !isempty(Indices{N, T, $elemtype($horvrep)}($horvrep))

        $elemtype(p::Polyhedron) = $elemtype($horvrep(p))
        if $singularstr == "point"
            $elemtype(p::$HorVRep) = $vectortype(typeof(p))
        else
            $elemtype(p::$HorVRep{N, T}) where {N, T} = $elt{N, T, $vectortype(typeof(p))}
        end

        function $plural(p::$HorVRep{N, T}...) where {N, T}
            ElemT = promote_type($elemtype.(p)...)
            RepIterator{N, T, ElemT}(p)
        end

        function $mapit(f::Function, d::FullDim{N}, ::Type{T}, p::$HorVRep...) where {N, T}
            ElemT = promote_type(similar_type.($elemtype.(p), d, T)...)
            MapRepIterator{N, T, ElemT}(p, f)
        end

        Base.length(it::$typename) where {N, T} = sum($lenp, it.ps)
        Base.isempty(it::$typename) where {N, T} = !any($isnotemptyp.(it.ps))
    end
end

# Combines an element type with its linear version.
# e.g. combines rays with the lines by splitting lines in two points.
struct AllRepIterator{N, T, ElemT, LinElemT, PT}
    itlin::RepIterator{N, T, LinElemT, PT}
    it::RepIterator{N, T, ElemT, PT}
end

Base.eltype(it::AllRepIterator{N, T, ElemT}) where {N, T, ElemT} = ElemT
Base.length(it::AllRepIterator) = 2length(it.itlin) + length(it.it)
Base.isempty(it::AllRepIterator) = isempty(it.itlin) && isempty(it.it)

function checknext(it::AllRepIterator, i, state)
    while i <= 3 && ((i <= 2 && done(it.itlin, state)) || (i == 3 && done(it.it, state)))
        i += 1
        if i <= 2
            state = start(it.itlin)
        elseif i == 3
            state = start(it.it)
        end # Otherwise we leave state as it is to be type stable
    end
    i, state
end

splitlin(h::HyperPlane, i) = (i == 1 ? HalfSpace(h.a, h.β) : HalfSpace(-h.a, -h.β))
#splitlin(s::SymPoint, i) = (i == 1 ? coord(s) : -coord(s))
splitlin(l::Line, i) = (i == 1 ? Ray(coord(l)) : Ray(-coord(l)))

Base.start(it::AllRepIterator) = checknext(it, 1, start(it.itlin))
Base.done(::AllRepIterator, state) = state[1] > 3
function Base.next(it::AllRepIterator, istate)
    i, state = istate
    if i <= 2
        @assert i >= 1
        itemlin, newstate = next(it.itlin, state)
        item = splitlin(itemlin, i)
    else
        @assert i == 3
        item, newstate = next(it.it, state)
    end
    newistate = checknext(it, i, newstate)
    (item, newistate)
end

for (isVrep, loop_singularlin,
     loop_singular, loop_repexem,
     loop_listexem) in [#(true, :sympoint, :point, "convexhull(SymPoint([1, 0]), [0, 1])", "[1, 0], [-1, 0], [0, 1]"),
                   (true, :line, :ray, "Line([1, 0]) + Ray([0, 1])",
                    "Ray([1, 0]), Ray([-1, 0]), Ray([0, 1])"),
                   (false, :hyperplane, :halfspace,
                    "HyperPlane([1, 0], 1) ∩ HalfSpace([0, 1], 1)",
                    "HalfSpace([1, 0]), HalfSpace([-1, 0]), HalfSpace([0, 1])")]
    global singularlin = loop_singularlin
    global singular = loop_singular
    global repexem = loop_repexem
    global listexem = loop_listexem
    if isVrep
        global HorV = :V
        global HorVRep = :VRep
        global horvrep = :vrep
    else
        global HorV = :H
        global HorVRep = :HRep
        global horvrep = :hrep
    end
    pluralstrlin = string(singularlin) * "s"
    global plurallin = Symbol(pluralstrlin)
    lenplin = Symbol("n" * pluralstrlin)
    isnotemptyplin = Symbol("has" * pluralstrlin)
    pluralstr = string(singular) * "s"
    global plural = Symbol(pluralstr)
    global lenp = Symbol("n" * pluralstr)
    global isnotemptyp = Symbol("has" * pluralstr)
    allpluralstr = "all" * pluralstr
    allplural = Symbol(allpluralstr)
    alllenp = Symbol("n" * allpluralstr)
    allisnotemptyp = Symbol("has" * allpluralstr)
    @eval begin
        export $allplural, $alllenp, $allisnotemptyp

        """
            all$plural($horvrep::$HorVRep)

        Returns an iterator over the $plural and $plurallin in the $HorV-representation `$horvrep` splitting $plurallin in two $plural.

        ### Examples

        ```julia
        $horvrep = $repexem
        collect(all$plural($horvrep)) # Returns [$listexem]
        ```
        """
        $allplural(p::$HorVRep...) = AllRepIterator($plurallin(p...), $plural(p...))

        """
            nall$plural($horvrep::$HorVRep)

        Returns the number of $plural plus twice the number of $plurallin in the $HorV-representation `$horvrep`, i.e. `length(all$plural($horvrep))`
        """
        $alllenp($horvrep::$HorVRep) = 2 * $lenplin($horvrep) + $lenp($horvrep)

        """
            hasall$plural($horvrep::$HorVRep)

        Returns whether the $HorV-representation `$horvrep` contains any $singular or $singularlin.
        """
        $allisnotemptyp($horvrep::$HorVRep) = $isnotemptyplin($horvrep) || $isnotemptyp($horvrep)
    end
end

const ElemIt{ElemT} = Union{AllRepIterator{<:Any, <:Any, ElemT}, AbstractRepIterator{<:Any, <:Any, ElemT}, AbstractVector{ElemT}}
const HyperPlaneIt{N, T} = ElemIt{<:HyperPlane{N, T}}
const HalfSpaceIt{N, T} = ElemIt{<:HalfSpace{N, T}}
const HIt{N, T} = Union{HyperPlaneIt{N, T}, HalfSpaceIt{N, T}}

#const SymPointIt{N, T} = ElemIt{<:SymPoint{N, T}}
const PointIt{N, T} = ElemIt{<:AbstractPoint{N, T}}
const PIt{N, T} = PointIt{N, T}
#const PIt{N, T} = Union{SymPointIt{N, T}, PointIt{N, T}}
const LineIt{N, T} = ElemIt{<:Line{N, T}}
const RayIt{N, T} = ElemIt{<:Ray{N, T}}
const RIt{N, T} = Union{LineIt{N, T}, RayIt{N, T}}
const VIt{N, T} = Union{PIt{N, T}, RIt{N, T}}

const It{N, T} = Union{HIt{N, T}, VIt{N, T}}

function fillvits(::FullDim{N}, points::ElemIt{AT}, lines::ElemIt{Line{N, T, AT}}=Line{N, T, AT}[], rays::ElemIt{Ray{N, T, AT}}=Ray{N, T, AT}[]) where {N, T, AT<:AbstractPoint{N, T}}
    if isempty(points) && !(isempty(lines) && isempty(rays))
        vconsistencyerror()
    end
    return points, lines, rays
end
function fillvits(::FullDim{N}, lines::ElemIt{Line{N, T, AT}}, rays::ElemIt{Ray{N, T, AT}}=Ray{N, T, AT}[]) where {N, T, AT}
    if isempty(lines) && isempty(rays)
        ps = AT[]
    else
        ps = [origin(AT, FullDim{N}())]
    end
    ps, lines, rays
end

hreps(p::HRep{N, T}...) where {N, T} = hyperplanes(p...), halfspaces(p...)
hreps(p::HAffineSpace{N, T}...) where {N, T} = tuple(hyperplanes(p...))

hmap(f, d::FullDim, ::Type{T}, p::HRep...) where T = maphyperplanes(f, d, T, p...), maphalfspaces(f, d, T, p...)
hmap(f, d::FullDim, ::Type{T}, p::HAffineSpace...) where T = tuple(maphyperplanes(f, d, T, p...))

hconvert(RepT::Type{<:HRep{N, T}}, p::HRep{N, T}) where {N, T} = constructpolyhedron(RepT, (p,), hreps(p)...)
hconvert(RepT::Type{<:HRep{N, T}}, p::HRep{N})    where {N, T} = constructpolyhedron(RepT, (p,), RepIterator{N, T}.(hreps(p))...)

vreps(p...) = preps(p...)..., rreps(p...)...
preps(p::VRep...) = tuple(points(p...))
preps(p::VCone...) = tuple()
rreps(p::VRep...) = lines(p...), rays(p...)
rreps(p::VLinearSpace...) = tuple(lines(p...))
rreps(p::VPolytope...) = tuple()

vmap(f, d::FullDim, ::Type{T}, p::VRep...) where T = pmap(f, d, T, p...)..., rmap(f, d, T, p...)...
pmap(f, d::FullDim, ::Type{T}, p::VRep...) where T = tuple(mappoints(f, d, T, p...))
pmap(f, d::FullDim, ::Type, p::VCone...) = tuple()
rmap(f, d::FullDim, ::Type{T}, p::VRep...) where T = maplines(f, d, T, p...), maprays(f, d, T, p...)
rmap(f, d::FullDim, ::Type{T}, p::VLinearSpace...) where T = tuple(maplines(f, d, T, p...))
rmap(f, d::FullDim, ::Type, p::VPolytope...) = tuple()

vconvert(RepT::Type{<:VRep{N, T}}, p::VRep{N, T}) where {N, T} = constructpolyhedron(RepT, (p,), vreps(p)...)
vconvert(RepT::Type{<:VRep{N, T}}, p::VRep{N})    where {N, T} = constructpolyhedron(RepT, (p,), RepIterator{N, T}.(vreps(p))...)


# Non-deterministic Finite Automaton
# ==================================

struct NFANode
    edges::Vector{Tuple{Edge,NFANode}}
end

function NFANode()
    return NFANode(Tuple{Edge,NFANode}[])
end

function Base.show(io::IO, node::NFANode)
    print(io, summary(node), "(<", length(node.edges), " edges", '@', objectid(node), ">)")
end

struct NFA
    start::NFANode
    final::NFANode

    function NFA(start::NFANode, final::NFANode)
        @assert start !== final
        return new(start, final)
    end
end

# epsilon transition
const eps = ByteSet()

function iseps(e::Edge)
    return isempty(e.labels)
end

function re2nfa(re::RegExp.RE, predefined_actions::Dict{Symbol,Action}=Dict{Symbol,Action}())
    actions = Dict{Tuple{RegExp.RE,Symbol},Action}()
    action_order = 1

    function make_action_list(re, names)
        list = ActionList()
        for name in names
            if haskey(predefined_actions, name)  # pick up a predefined action
                action = predefined_actions[name]
            elseif haskey(actions, (re, name))
                action = actions[(re, name)]
            else
                action = Action(name, action_order)
                actions[(re, name)] = action
                action_order += 1
            end
            push!(list, action)
        end
        return list
    end

    # Thompson's construction.
    function rec!(start, re)
        if haskey(re.actions, :enter)
            start_in = NFANode()
            push!(start.edges, (Edge(eps, make_action_list(re, re.actions[:enter])), start_in))
        else
            start_in = start
        end

        re′ = RegExp.shallow_desugar(re)
        head = re′.head
        args = re′.args

        if head == :set
            @assert length(args) == 1
            final_in = NFANode()
            push!(start_in.edges, (Edge(args[1]), final_in))
        elseif head == :cat
            f = start_in
            for arg in args
                f = rec!(f, arg)
            end
            if f == start_in
                final_in = NFANode()
                push!(start_in.edges, (Edge(eps), final_in))
            else
                final_in = f
            end
        elseif head == :alt
            @assert length(args) > 0
            final_in = NFANode()
            for arg in args
                s = NFANode()
                f = rec!(s, arg)
                push!(start_in.edges, (Edge(eps), s))
                push!(       f.edges, (Edge(eps), final_in))
            end
        elseif head == :rep
            @assert length(args) == 1
            s = NFANode()
            f = rec!(s, args[1])
            final_in = NFANode()
            push!(start_in.edges, (Edge(eps), s))
            push!(start_in.edges, (Edge(eps), final_in))
            push!(       f.edges, (Edge(eps), s))
            push!(       f.edges, (Edge(eps), final_in))
        elseif head == :isec || head == :diff
            @assert length(args) == 2
            final_in = NFANode()
            s1 = NFANode()
            f1 = rec!(s1, args[1])
            push!(start_in.edges, (Edge(eps), s1))
            push!(      f1.edges, (Edge(eps), final_in))
            s2 = NFANode()
            f2 = rec!(s2, args[2])
            push!(start_in.edges, (Edge(eps), s2))
            push!(      f2.edges, (Edge(eps), final_in))
            if head == :isec
                revoke = s -> f1 ∉ s.nfanodes || f2 ∉ s.nfanodes
            else  # re.head == :diff
                revoke = s -> f2 ∈ s.nfanodes
            end
            nfa = dfa2nfa(revoke_finals(revoke, nfa2dfa(NFA(start_in, final_in))))
            push!(start_in.edges, (Edge(eps), nfa.start))
            final_in = nfa.final
        else
            error("unsupported operation: $(head)")
        end

        if haskey(re.actions, :all)
            as = make_action_list(re, re.actions[:all])
            for s in traverse(start), (e, _) in s.edges
                if !iseps(e)
                    union!(e.actions, as)
                end
            end
        end

        if haskey(re.actions, :final)
            as = make_action_list(re, re.actions[:final])
            for s in traverse(start), (e, t) in s.edges
                if !iseps(e) && final_in ∈ epsilon_closure(t)
                    union!(e.actions, as)
                end
            end
        end

        if haskey(re.actions, :exit)
            final = NFANode()
            push!(final_in.edges, (Edge(eps, make_action_list(re, re.actions[:exit])), final))
        else
            final = final_in
        end

        if re.when != nothing
            name = re.when
            for s in traverse(start), (e, _) in s.edges
                if !iseps(e)
                    push!(e.precond, (name => TRUE))
                end
            end
        end

        return final
    end

    nfa_start = NFANode()
    nfa_final = rec!(nfa_start, re)
    return NFA(nfa_start, nfa_final)
end

function remove_dead_nodes(nfa::NFA)
    backrefs = Dict(nfa.start => Set{NFANode}())
    for s in traverse(nfa.start), (_, t) in s.edges
        push!(get!(() -> Set{NFANode}(), backrefs, t), s)
    end

    alive = Set{NFANode}()
    unvisited = [nfa.final]
    while !isempty(unvisited)
        s = pop!(unvisited)
        push!(alive, s)
        for t in backrefs[s]
            if t ∉ alive
                push!(unvisited, t)
            end
        end
    end
    @assert nfa.start ∈ alive
    @assert nfa.final ∈ alive

    newnodes = Dict{NFANode,NFANode}()
    new(s) = get!(() -> NFANode(), newnodes, s)
    isvisited(s) = haskey(newnodes, s)
    unvisited = [nfa.start]
    while !isempty(unvisited)
        s = pop!(unvisited)
        s′ = new(s)
        for (e, t) in s.edges
            if t ∈ alive
                if !isvisited(t)
                    push!(unvisited, t)
                end
                push!(s′.edges, (e, new(t)))
            end
        end
    end

    return NFA(new(nfa.start), new(nfa.final))
end


module TestingQuicjPriorityQueues

using Test

using Random

using Base: @propagate_inbounds, lt,
    ReverseOrdering, Reverse,
    IteratorEltype, HasEltype,
    IteratorSize, HasLength

using QuickHeaps
using QuickHeaps:
    AbstractPriorityQueue, PriorityQueue, FastPriorityQueue,
    AbstractNode, getkey, getval,
    isheap, index, nodes, ordering, in_range, heap_index

function test_queue!(A::AbstractPriorityQueue{K,V},
                     key_list::AbstractVector{K},
                     val_list::AbstractVector{V}) where {K,V}
    # Check arguments.
    axes(val_list) == axes(key_list) || error("incompatible indices")
    n = length(key_list)
    axes(key_list) == (1:n,) || error("non-standard indices")
    o = ordering(A)

    # Dummy private function used to "label" the tests.
    check(flag::Bool, comment) = flag

    # Checks keytype, etc.
    @test keytype(A) == K
    @test keytype(typeof(A)) == K
    @test IteratorSize(keys(A)) == HasLength()
    @test IteratorSize(typeof(keys(A))) == HasLength()
    @test IteratorEltype(keys(A)) == HasEltype()
    @test IteratorEltype(typeof(keys(A))) == HasEltype()
    @test eltype(keys(A)) == K
    @test eltype(typeof(keys(A))) == K

    # Checks valtype, etc.
    @test valtype(A) == V
    @test valtype(typeof(A)) == V
    @test IteratorSize(values(A)) == HasLength()
    @test IteratorSize(typeof(values(A))) == HasLength()
    @test IteratorEltype(values(A)) == HasEltype()
    @test IteratorEltype(typeof(values(A))) == HasEltype()
    @test eltype(values(A)) == V
    @test eltype(typeof(values(A))) == V

    # Check that `enqueue!` maintains the binary-heap structure.
    R = Dict{K,V}() # reference dictionary
    test_1 = true
    test_2 = true
    for (k, v) in zip(key_list, val_list)
        enqueue!(A, k, v)
        test_1 &= isheap(o, nodes(A))
        test_2 &= !haskey(R, k) # unique key?
        R[k] = v
    end
    @test check(test_1, "binary-heap structure after `enqueue!`")
    @test check(test_2, "unique keys after `enqueue!`")
    @test length(A) == n

    # Check `first`, `peek`, `keys`, and `values`.
    k, v = first(A)
    @test v == (isa(o, ReverseOrdering) ? maximum : minimum)(values(A))
    @test v == first(values(A))
    @test k == first(keys(A))
    x = peek(AbstractNode, A)
    @test v == getval(x)
    @test k == getkey(x)

    # Check `getindex` in random order.
    test_1 = true
    for i in randperm(n)
        k = key_list[i]
        test_1 &= (haskey(A,k) && A[k] == R[k])
    end
    @test check(test_1, "`getindex` in random order")

    # Test that `keys` and `values` yield all elements in the same order as
    # `iterate`.
    test_1 = true
    test_2 = true
    for (k,v,kv) in zip(keys(A), values(A), A)
        test_1 &= (k == kv.first)
        test_2 &= (v == kv.second)
    end
    @test check(test_1, "keys in heap order")
    @test check(test_2, "values in heap order")

    # Check `copy`.
    B = copy(A)
    @test typeof(B) === typeof(A)
    @test length(B) == length(A)
    @test nodes(A) == nodes(B)
    @test index(A) == index(B)
    S = Set(keys(B))
    @test length(S) == length(B) # keys are unique?
    test_1 = true
    for k in S
        test_1 &= (haskey(A, k) && haskey(B, k) && A[k] == B[k])
    end
    @test check(test_1, "`copy` yields same nodes")

    # Check `delete!` and result of addressing a non-existing key.
    k = key_list[rand(1:n)]
    delete!(B, k)
    @test !haskey(B, k)
    @test get(B, k, nothing) === nothing
    @test_throws ArgumentError B[k]

    # Check `isempty`, `empty!`, etc.
    empty!(B)
    @test isempty(B)
    @test length(B) == 0
    @test length(A) == n # no side effects on A

    # Check `setindex!` in heap order.
    length(B) > 1 && empty!(B)
    test_1 = true
    test_2 = true
    for i in randperm(n)
        k, v = key_list[i], val_list[i]
        B[k] = v
        test_1 &= isheap(o, nodes(B))
        test_2 &= (haskey(B, k) && B[k] == R[k])
    end
    @test check(test_1, "heap structure preserved by `setindex!`")
    @test check(test_2, "same value for key after `setindex!`")
    @test length(B) == length(A)
    test_1 = true
    test_2 = true
    for i in randperm(n)
        k = key_list[i]
        test_1 &= haskey(B, k)
        test_2 &= B[k] == R[k]
    end
    @test check(test_1, "no missing keys in random order")
    @test check(test_2, "same values in random order")

    # Check that `delete!` maintains the binary-heap structure.
    test_1 = true
    test_2 = true
    test_3 = true
    for i in randperm(n)
        k = key_list[i]
        test_1 &= haskey(B, k)
        delete!(B, k)
        test_2 &= isheap(o, nodes(B))
        test_3 &= !haskey(B, k)
    end
    @test check(test_1, "keys exist before `delete!`")
    @test check(test_2, "heap structure preserved by `delete!`")
    @test check(test_3, "keys do not exist after `delete!`")
    @test isempty(B)

    # Check that `pop!` extracts nodes in order and maintains the binary-heap
    # structure.
    B = copy(A)
    test_1 = true
    test_2 = true
    prev = (isa(o, ReverseOrdering) ? typemax : typemin)(V)
    for i in 1:n
        k, v = pop!(B)
        test_1 &= isheap(o, nodes(B))
        test_2 &= !lt(o, v, prev)
        prev = v
    end
    @test check(test_1, "heap structure preserved by `pop!`")
    @test check(test_2, "`pop!` yields keys in order")
    @test isempty(B)

    # Tests for fast priority queues.
    if isa(A, FastPriorityQueue)
        cartesian_index = CartesianIndices(index(A))
        linear_index = LinearIndices(index(A))
        test_1 = true
        test_2 = true
        test_3 = true
        test_4 = true
        test_5 = true
        test_6 = true
        test_7 = true
        test_8 = true
        for i in randperm(n)
            k = key_list[i]
            c = cartesian_index[k]
            inds = c.I
            test_1 &= haskey(A, c)
            test_2 &= haskey(A, inds)
            test_3 &= (A[k] == A[c])
            test_4 &= (A[k] == A[inds...])
            v = A[k]
            test_5 &= !haskey(delete!(A, c), k)
            A[c] = v
            test_6 &= (haskey(A, k) && A[k] == v)
            test_7 &= !haskey(delete!(A, inds), k)
            A[inds...] = v
            test_8 &= (haskey(A, k) && A[k] == v)
        end
        @test check(test_1, "`haskey` with Cartesian index")
        @test check(test_2, "`haskey` with multi-dimensional index")
        @test check(test_3, "`getindex` with Cartesian index")
        @test check(test_4, "`getindex` with multi-dimensional index")
        @test check(test_5, "`delete!` with Cartesian index")
        @test check(test_6, "`setindex!` with Cartesian index")
        @test check(test_7, "`delete!` with multi-dimensional index")
        @test check(test_8, "`setindex!` with multi-dimensional index")
    end
end

@testset "Priority queues       " begin
    K, V, n = Int, Float64, 20
    key_list = map(K, 1:n)
    val_list = rand(V, n)
    test_queue!(PriorityQueue{K,V}(), key_list, val_list)
    test_queue!(PriorityQueue{K,V}(Reverse), key_list, val_list)
end

@testset "Fast priority queues  " begin
    V, dims = Float32, (2,3,4)
    n = prod(dims)
    m = div(9n + 5, 10) # keep ~90% of indices
    key_list = randperm(n)[1:m]
    val_list = rand(V, m)
    test_queue!(FastPriorityQueue{V}(dims), key_list, val_list)
    test_queue!(FastPriorityQueue{V}(Reverse, dims), key_list, val_list)
end

end # module


# This file was generated, do not modify it. # hide
mach_basic = machine(pipe, X, y)
fit!(mach_basic, verbosity=0)

ŷ_basic = predict(mach_basic, Xtest);

print(
    "Basic model measurements on test set:\n",
    "  brier loss: ", brier_loss(ŷ_basic, ytest) |> mean, "\n",
    "  auc:        ", auc(ŷ_basic, ytest),                "\n",
    "  accuracy:   ", accuracy(mode.(ŷ_basic), ytest)
)

rm("tuned_iterated_pipe.jls") # hide