using Oceananigans
using Oceananigans.Architectures: arch_array
using Oceananigans: AbstractModel
using Oceananigans.Models.HydrostaticFreeSurfaceModels: HydrostaticFreeSurfaceModel

function get_parameter(filename, group, parameter_name, default=nothing)
    parameter = default

    jldopen(filename) do file
        if parameter_name ∈ keys(file["$group"])
            parameter = file["$group/$parameter_name"]
        end
    end

    return parameter
end

"""
    OneDimensionalEnsembleModel(observations::OneDimensionalTimeSeriesBatch; architecture = CPU(), ensemble_size = 50, kwargs...)

Build an Oceananigans `HydrostaticFreeSurfaceModel` with many independent
columns. The model grid is given by the data in `observations`, and the
dynamics in each column is attached to its own `CATKEVerticalDiffusivity` closure stored in
an `(Nx, Ny)` Matrix of closures. `ensemble_size = 50` is the
desired count of ensemble members for calibration with Ensemble Kalman Inversion (EKI), and the
remaining keyword arguments `kwargs` define the default closure across all columns.

In the "many columns" configuration, we run the model on a 3D grid with `(Flat, Flat, Bounded)` boundary
conditions so that many independent columns can be evolved at once with much of the computational overhead
split among the columns. The `Nx` rows of vertical columns are each reserved for an "ensemble" member
whose attached parameter value (updated at each iteration of EKI) sets the diffusivity closure
used to predict the model solution for the `Ny` physical scenarios described by the simulation-specific
`OneDimensionalTimeSeries` objects in `observations`.
"""
function OneDimensionalEnsembleModel(observations::OneDimensionalTimeSeriesBatch;
                       architecture = CPU(),
                       ensemble_size = 50,
                       closure = CATKEVerticalDiffusivity(Float64; warning=false, kwargs...)
                      )

    #
    # Harvest observation file metadata
    #

    datapath = observations.file_path

    α = get_parameter(datapath, "buoyancy", "equation_of_state/α", 2e-4)
    g = get_parameter(datapath, "buoyancy", "gravitational_acceleration", 9.81)
    αg = α * g

    f = get_parameter(datapath, "coriolis", "f", 0.0)

    # Surface fluxes
    Qᵘ = get_parameter(datapath, "parameters", "boundary_condition_u_top", 0.0)
    Qᵛ = get_parameter(datapath, "parameters", "boundary_conditions_v_top", 0.0)
    Qᶿ = get_parameter(datapath, "parameters", "boundary_condition_θ_top", 0.0)
    Qᵇ = Qᶿ * αg

    # Bottom gradients
    dudz_bottom = get_parameter(datapath, "parameters", "boundary_condition_u_bottom", 0.0)
    dθdz_bottom = get_parameter(datapath, "parameters", "boundary_condition_θ_bottom", 0.0)
    dbdz_bottom = dθdz_bottom * αg

    #
    # Build model using metadata
    #

    grid = OneDimensionalEnsembleGrid(observations.grid; size=(ensemble_size, length(observations), data_grid.Nz))

    closure = [closure for i=1:ensemble_size, j=1:length(observations)]

    coriolis = [FPlane(f=td.constants[:f]) for i=1:ensemble_size, td in observations]
    coriolis = arch_array(architecture, coriolis)

    bc_matrix(x) = [x for i = 1:ensemble_size, d in observations]
    Qᵇ = bc_matrix(Qᵇ)
    Qᵘ = bc_matrix(Qᵘ)
    Qᵛ = bc_matrix(Qᵛ)
    dbdz_bottom = bc_matrix(dbdz_bottom)
    dudz_bottom = bc_matrix(dudz_bottom)

    # Convert to CuArray if necessary
    closure = arch_array(architecture, closure)
    Qᵇ = arch_array(architecture, Qᵇ)
    Qᵘ = arch_array(architecture, Qᵘ)
    Qᵛ = arch_array(architecture, Qᵛ)
    dbdz_bottom = arch_array(architecture, dbdz_bottom)
    dudz_bottom = arch_array(architecture, dudz_bottom)

    u_bcs = FieldBoundaryConditions(top = FluxBoundaryCondition(Qᵘ),
                                    bottom = GradientBoundaryCondition(dudz_bottom))
    v_bcs = FieldBoundaryConditions(top = FluxBoundaryCondition(Qᵛ))
    b_bcs = FieldBoundaryConditions(top = FluxBoundaryCondition(Qᵇ),
                                    bottom = GradientBoundaryCondition(dbdz_bottom))

    model = HydrostaticFreeSurfaceModel(architecture = architecture,
                                         grid = grid,
                                         tracers = (:b, :e),
                                         buoyancy = BuoyancyTracer(),
                                         coriolis = coriolis,
                                         boundary_conditions = (b=b_bcs, u=u_bcs, v=v_bcs),
                                         closure = closure)

    return model
end


using Images, MXNet

### LOADING THE MODEL
const MODEL_NAME = "weights/inception-v3/Inception-7"

nnet = mx.load_checkpoint(MODEL_NAME, 1, mx.FeedForward);

layers = mx.get_internals(nnet.arch);
layers_flatten = nothing
layers_to_remove = Symbol[]

for i = 1:2000

    layer = layers[i];
    layer_name = mx.get_name(layer)

    if layers_flatten == nothing && layer_name == :flatten
        layers_flatten = layer
    elseif layers_flatten != nothing
        push!(layers_to_remove, layer_name)
        if layer_name in [:softmax, :label] break end
    end
end

nnet.arch = layers_flatten
map(x -> delete!(nnet.arg_params, x), layers_to_remove);
map(x -> delete!(nnet.aux_params, x), layers_to_remove);

mx.save_checkpoint(nnet, "weights/inception-v3/InceptionV3-FE", mx.OptimizationState(1, 0, 0, 0))


module ForcesWrapper

using JavaCall

const AbsoluteDate = @jimport org.orekit.time.AbsoluteDate
const AbstractGaussianContribution = @jimport org.orekit.propagation.semianalytical.dsst.forces.AbstractGaussianContribution
const AbstractGaussianContributionContext = @jimport org.orekit.propagation.semianalytical.dsst.forces.AbstractGaussianContributionContext
const Atmosphere = @jimport org.orekit.models.earth.atmosphere.Atmosphere
const AttitudeProvider = @jimport org.orekit.attitudes.AttitudeProvider
const AuxiliaryElements = @jimport org.orekit.propagation.semianalytical.dsst.utilities.AuxiliaryElements
const CelestialBody = @jimport org.orekit.bodies.CelestialBody
const Class = @jimport java.lang.Class
const DSSTAtmosphericDrag = @jimport org.orekit.propagation.semianalytical.dsst.forces.DSSTAtmosphericDrag
const DSSTForceModel = @jimport org.orekit.propagation.semianalytical.dsst.forces.DSSTForceModel
const DSSTNewtonianAttraction = @jimport org.orekit.propagation.semianalytical.dsst.forces.DSSTNewtonianAttraction
const DSSTNewtonianAttractionContext = @jimport org.orekit.propagation.semianalytical.dsst.forces.DSSTNewtonianAttractionContext
const DSSTSolarRadiationPressure = @jimport org.orekit.propagation.semianalytical.dsst.forces.DSSTSolarRadiationPressure
const DSSTTesseral = @jimport org.orekit.propagation.semianalytical.dsst.forces.DSSTTesseral
const DSSTTesseralContext = @jimport org.orekit.propagation.semianalytical.dsst.forces.DSSTTesseralContext
const DSSTThirdBody = @jimport org.orekit.propagation.semianalytical.dsst.forces.DSSTThirdBody
const DSSTThirdBodyContext = @jimport org.orekit.propagation.semianalytical.dsst.forces.DSSTThirdBodyContext
const DSSTZonal = @jimport org.orekit.propagation.semianalytical.dsst.forces.DSSTZonal
const DSSTZonalContext = @jimport org.orekit.propagation.semianalytical.dsst.forces.DSSTZonalContext
const DragForce = @jimport org.orekit.forces.drag.DragForce
const DragSensitive = @jimport org.orekit.forces.drag.DragSensitive
const EventDetector = @jimport org.orekit.propagation.events.EventDetector
const ExtendedPVCoordinatesProvider = @jimport org.orekit.utils.ExtendedPVCoordinatesProvider
const Field = @jimport org.hipparchus.Field
const FieldAbsoluteDate = @jimport org.orekit.time.FieldAbsoluteDate
const FieldAbstractGaussianContributionContext = @jimport org.orekit.propagation.semianalytical.dsst.forces.FieldAbstractGaussianContributionContext
const FieldAuxiliaryElements = @jimport org.orekit.propagation.semianalytical.dsst.utilities.FieldAuxiliaryElements
const FieldDSSTNewtonianAttractionContext = @jimport org.orekit.propagation.semianalytical.dsst.forces.FieldDSSTNewtonianAttractionContext
const FieldDSSTTesseralContext = @jimport org.orekit.propagation.semianalytical.dsst.forces.FieldDSSTTesseralContext
const FieldDSSTThirdBodyContext = @jimport org.orekit.propagation.semianalytical.dsst.forces.FieldDSSTThirdBodyContext
const FieldDSSTZonalContext = @jimport org.orekit.propagation.semianalytical.dsst.forces.FieldDSSTZonalContext
const FieldEventDetector = @jimport org.orekit.propagation.events.FieldEventDetector
const FieldForceModelContext = @jimport org.orekit.propagation.semianalytical.dsst.forces.FieldForceModelContext
const FieldOrbit = @jimport org.orekit.orbits.FieldOrbit
const FieldShortPeriodTerms = @jimport org.orekit.propagation.semianalytical.dsst.forces.FieldShortPeriodTerms
const FieldSpacecraftState = @jimport org.orekit.propagation.FieldSpacecraftState
const ForceModelContext = @jimport org.orekit.propagation.semianalytical.dsst.forces.ForceModelContext
const Frame = @jimport org.orekit.frames.Frame
const JString = @jimport java.lang.String
const List = @jimport java.util.List
const Map = @jimport java.util.Map
const Object = @jimport java.lang.Object
const Orbit = @jimport org.orekit.orbits.Orbit
const ParameterDriver = @jimport org.orekit.utils.ParameterDriver
const PropagationType = @jimport org.orekit.propagation.PropagationType
const RadiationSensitive = @jimport org.orekit.forces.radiation.RadiationSensitive
const RealFieldElement = @jimport org.hipparchus.RealFieldElement
const Set = @jimport java.util.Set
const ShortPeriodTerms = @jimport org.orekit.propagation.semianalytical.dsst.forces.ShortPeriodTerms
const SpacecraftState = @jimport org.orekit.propagation.SpacecraftState
const UnnormalizedSphericalHarmonicsProvider = @jimport org.orekit.forces.gravity.potential.UnnormalizedSphericalHarmonicsProvider

export AbstractGaussianContribution
export AbstractGaussianContributionContext
export DSSTAtmosphericDrag
export DSSTForceModel
export DSSTNewtonianAttraction
export DSSTNewtonianAttractionContext
export DSSTSolarRadiationPressure
export DSSTTesseral
export DSSTTesseralContext
export DSSTThirdBody
export DSSTThirdBodyContext
export DSSTZonal
export DSSTZonalContext
export FieldAbstractGaussianContributionContext
export FieldDSSTNewtonianAttractionContext
export FieldDSSTTesseralContext
export FieldDSSTThirdBodyContext
export FieldDSSTZonalContext
export FieldForceModelContext
export FieldShortPeriodTerms
export ForceModelContext
export ShortPeriodTerms
export equals
export get_a
export get_alpha
export get_ao_r3_pow
export get_atmosphere
export get_auxiliary_elements
export get_ax2o_a
export get_bb
export get_bbb
export get_beta
export get_bo_a
export get_bo_a_bpo
export get_body
export get_chi
export get_chi2
export get_class
export get_co2ab
export get_coefficients
export get_coefficients_key_prefix
export get_cxo2n2a2
export get_drag
export get_e2
export get_equatorial_radius
export get_events_detectors
export get_field_auxiliary_elements
export get_field_events_detectors
export get_gamma
export get_gm
export get_hk
export get_hxxx
export get_k2mh2
export get_k2mh2o2
export get_kxxx
export get_m2ao_a
export get_m_co2ab
export get_max_ar3_pow
export get_max_ecc_pow
export get_max_freq_f
export get_mean_element_rate
export get_mean_motion
export get_moa
export get_mu
export get_muo_r3
export get_muoa
export get_oo_ab
export get_oo_bpo
export get_oo_mu
export get_ooa
export get_ooab
export get_oon2a2
export get_orbit_period
export get_parameters
export get_parameters_drivers
export get_provider
export get_qns
export get_ratio
export get_rbar
export get_res_orders
export get_roa
export get_spacecraft
export get_theta
export get_ton2a
export get_x
export get_x2on2a2xp1
export get_x3on2a
export get_xon2a2
export get_xx
export get_xxx
export getb
export hash_code
export initialize
export notify
export notify_all
export register_attitude_provider
export to_string
export update_short_period_terms
export value
export wait

include("abstract_gaussian_contribution.jl")
include("abstract_gaussian_contribution_context.jl")
include("dsst_atmospheric_drag.jl")
include("dsst_force_model.jl")
include("dsst_newtonian_attraction.jl")
include("dsst_newtonian_attraction_context.jl")
include("dsst_solar_radiation_pressure.jl")
include("dsst_tesseral.jl")
include("dsst_tesseral_context.jl")
include("dsst_third_body.jl")
include("dsst_third_body_context.jl")
include("dsst_zonal.jl")
include("dsst_zonal_context.jl")
include("field_abstract_gaussian_contribution_context.jl")
include("field_dsst_newtonian_attraction_context.jl")
include("field_dsst_tesseral_context.jl")
include("field_dsst_third_body_context.jl")
include("field_dsst_zonal_context.jl")
include("field_force_model_context.jl")
include("field_short_period_terms.jl")
include("force_model_context.jl")
include("short_period_terms.jl")

end


# Заполняет словарь d индексами индивидуальных значений
#=
function indsdict!(d::Dict{T}, cdata::Tuple) where T
    @inbounds for (i, element) in enumerate(zip(cdata...))
        ind = ht_keyindex(d, element)
        if ind > 0
            push!(d.vals[ind], i)
        else
            v = Vector{Int}(undef, 1)
            v[1] = i
            d[element] = v
        end
    end
    d
end
=#
nonunique(v) = [k for (k, v) in StatsBase.countmap(v) if v > 1]

function floatparse(data)
    v = Vector{Float64}(undef, length(data))
    @inbounds for i = 1:length(data)
        if !isa(data[i], AbstractFloat) && !ismissing(data[i])
            if isa(data[i], AbstractString)
                try
                    v[i] = parse(Float64, data[i])
                catch
                    v[i] = NaN
                end
            else
                try
                    v[i] = float(data[i])
                catch
                    v[i] = NaN
                end
            end
        elseif ismissing(data[i])
            v[i] = NaN
        else
            v[i] = data[i]
        end
    end
    identity.(v)
end

function checkvalues(timevals_sp, concvals_sp)
    if !(eltype(timevals_sp) <: Number)
        timevals_sp = identity.(timevals_sp)
        eltype(timevals_sp) <: Number || error("Some time values not a number!")
    end
    if !(eltype(concvals_sp) <: Union{Number, Missing})
        concvals_sp = identity.(concvals_sp)
        if !(eltype(concvals_sp) <: Union{Number, Missing})
            @warn "Some concentration values not a number, try to fix."
            concvals_sp = floatparse(concvals_sp)
        end
    end
    timevals_sp, concvals_sp
end

"""
    pkimport(data, time, conc, sort; kelauto = true,  elimrange = ElimRange(), dosetime = DoseTime())

Import PK data from table `data`.

* `time` - time column;
* `conc` - concentration column;
* `sort` - subject sorting columns.

keywords:

* `kelauto` - if `true` auto range settings, if `false` used `kelstart`/`kelend` from `elimrange`;
* `elimrange` - set elimination range settings;
* `dosetime` - set dose and dose time, by default dosetime = 0, dose is `NaN`.

!!! note

    If time column have non-unique values - last pair time-concentration will be used.

"""
function pkimport(data, time, conc, sort; kelauto = true,  elimrange = ElimRange(), dosetime = DoseTime())
    if isa(sort, String) sort = [Symbol(sort)] end
    if isa(sort, Symbol) sort = [sort] end

    Tables.istable(data) || error("Data not a table!")

    cols   = Tables.columns(data)
    cdata  = Tuple(Tables.getcolumn(cols, y) for y in sort)
    d      = Dict{Tuple{eltype.(cdata)...}, Vector{Int}}()
    indsdict!(d, cdata)

    timec = Tables.getcolumn(data, time)
    concc = Tables.getcolumn(data, conc)

    any(isnanormissing, timec) && error("Some time values is NaN or Missing!")

    sdata = Vector{PKSubject}(undef, length(d))
    i = one(Int)
    @inbounds for (k, v) in d
        timevals = timec[v]
        concvals = concc[v]
        if !allunique(timevals)
            @warn "Not all time values is unique, last observation used! ($k)"
            nuv = nonunique(timevals)
            nuvinds = findall(x -> x == first(nuv), timevals)[1:end-1]
            if length(nuv) > 1
                for cnt = 2:length(nuv)
                    append!(nuvinds, findall(x -> x == nuv[cnt], timevals)[1:end-1])
                end
            end
            sort!(nuvinds)
            deleteat!(timevals, nuvinds)
            deleteat!(concvals, nuvinds)
        end
        sp = sortperm(timevals)
        timevals_sp = timevals[sp]
        concvals_sp = concvals[sp]
        timevals_sp, concvals_sp = checkvalues(timevals_sp, concvals_sp)
        sdata[i] = PKSubject(timevals_sp, concvals_sp, kelauto, elimrange,  dosetime, Dict(sort .=> k))
        i += one(Int)
    end
    return DataSet(identity.(sdata))
end
"""
    pkimport(data, time, conc; kelauto = true,  elimrange = ElimRange(), dosetime = DoseTime())

Import PK data from tabular data `data`, `time` - time column, `conc` - concentration column.
"""
function pkimport(data, time, conc; kelauto = true,  elimrange = ElimRange(), dosetime = DoseTime())
    timevals_sp, concvals_sp = checkvalues(copy(Tables.getcolumn(data, time)), copy(Tables.getcolumn(data, conc)))
    pkimport(timevals_sp, concvals_sp; kelauto = kelauto,  elimrange = elimrange, dosetime = dosetime)
end
"""
    pkimport(time, conc; kelauto = true,  elimrange = ElimRange(), dosetime = DoseTime(), id = Dict{Symbol, Any}())

Import PK data from time vector `time` and concentration vector `conc`.
"""
function pkimport(time, conc; kelauto = true,  elimrange = ElimRange(), dosetime = DoseTime(), id = Dict{Symbol, Any}())
    timevals_sp, concvals_sp = checkvalues(copy(time), copy(conc))
    PKSubject(timevals_sp, concvals_sp, kelauto, elimrange,  dosetime, id)
end

"""
    upkimport(data, stime, etime, conc, vol, sort; kelauto = true,  elimrange = ElimRange(), dosetime = DoseTime())

Import urine PK data from table `data`.

* `stime` - start time column;
* `etime` - end time column;
* `conc` - concentration column;
* `vol` - volume column;
* `sort` - subject sorting columns.
"""
function upkimport(data, stime, etime, conc, vol, sort; kelauto = true,  elimrange = ElimRange(), dosetime = DoseTime())
    if isa(sort, String) sort = [Symbol(sort)] end
    if isa(sort, Symbol) sort = [sort] end

    cols   = Tables.columns(data)
    cdata  = Tuple(Tables.getcolumn(cols, y) for y in sort)
    d      = Dict{Tuple{eltype.(cdata)...}, Vector{Int}}()
    indsdict!(d, cdata)

    tnames = Symbol.(names(data))
    isa(stime, Symbol) || stime in tnames || error("column Start Time ($stime) not found")
    isa(etime, Symbol) || etime in tnames || error("column End Time ($etime) not found")
    isa(conc, Symbol) || conc in tnames || error("column Concentration ($conc) not found")
    isa(vol, Symbol) || vol in tnames || error("column Volume ($vol) not found")

    stimec = Tables.getcolumn(data, stime)
    etimec = Tables.getcolumn(data, etime)
    concc = Tables.getcolumn(data, conc)
    volc = Tables.getcolumn(data, vol)

    any(isnanormissing, stimec) && error("Some Start Time values is NaN or Missing!")
    any(isnanormissing, etimec) && error("Some End Time values is NaN or Missing!")

    sdata = Vector{UPKSubject}(undef, length(d))
    i = one(Int)
    @inbounds for (k, v) in d
        stimevals = stimec[v]
        etimevals = etimec[v]
        concvals  = concc[v]
        volvals   = volc[v]
        #=
        timeranges = collect(zip(stimevals, etimevals))
        sp = sortperm(stimevals)

        timevals_sp = timeranges[sp]
        concvals_sp = concvals[sp]
        volvals_sp  = volvals[sp]

        if length(timevals_sp) > 1
            for c = 2:length(timevals_sp)
                timevals_sp[c][1] == timevals_sp[c-1][2] || error("Start time ($(timevals_sp[c][1])) for observation $c not equal End time ($(timevals_sp[c-1][2])) for observation $(c-1)!")
            end
        end
        sdata[i] = UPKSubject(timevals_sp, concvals_sp, volvals_sp, kelauto, elimrange,  dosetime, Dict(sort .=> k))
        =#
        sdata[i] = upkimport(stimevals, etimevals, concvals, volvals; kelauto = kelauto,  elimrange = elimrange, dosetime = dosetime, id = Dict(sort .=> k))
        i += one(Int)
    end
    return DataSet(identity.(sdata))
end
"""
    upkimport(data, stime, etime, conc, vol; kelauto = true,  elimrange = ElimRange(), dosetime = DoseTime())

Import single urine PK data from table `data`.

* `stime` - start time column;
* `etime` - end time column;
* `conc` - concentration column;
* `vol` - volume column.
"""
function upkimport(data, stime, etime, conc, vol; kelauto = true,  elimrange = ElimRange(), dosetime = DoseTime())
    upkimport(Tables.getcolumn(data, stime), Tables.getcolumn(data, etime), Tables.getcolumn(data, conc), Tables.getcolumn(data, vol); kelauto = kelauto,  elimrange = elimrange, dosetime = dosetime)
end
"""
    upkimport(stime, etime, conc, vol; kelauto = true,  elimrange = ElimRange(), dosetime = DoseTime())

Import urine PK data from time vectors:
* `stime` - start times;
* `etime` - end times;
* `conc` - concentrations;
* `vol` - volumes.
"""
function upkimport(stime, etime, conc, vol; kelauto = true,  elimrange = ElimRange(), dosetime = DoseTime(), id = Dict{Symbol, Any}())
    any(isnanormissing, stime) && error("Some Start Time values is NaN or Missing!")
    any(isnanormissing, etime) && error("Some End Time values is NaN or Missing!")
    timeranges = collect(zip(stime, etime))
    sp = sortperm(stime)
    timevals_sp = timeranges[sp]
    concvals_sp = conc[sp]
    volvals_sp  = vol[sp]
    if length(timevals_sp) > 1
        for c = 2:length(timevals_sp)
            timevals_sp[c][1] == timevals_sp[c-1][2] || error("Start time ($(timevals_sp[c][1])) for observation $c not equal End time ($(timevals_sp[c-1][2])) for observation $(c-1)!")
        end
    end
    UPKSubject(timevals_sp, concvals_sp, volvals_sp, kelauto, elimrange,  dosetime, id)
end

function pdimport(data, time, obs, sort; bl = 0, th = NaN)
    if isa(sort, String) sort = [Symbol(sort)] end
    if isa(sort, Symbol) sort = [sort] end

    Tables.istable(data) || error("Data not a table!")

    cols   = Tables.columns(data)
    cdata  = Tuple(Tables.getcolumn(cols, y) for y in sort)
    d      = Dict{Tuple{eltype.(cdata)...}, Vector{Int}}()
    indsdict!(d, cdata)

    timec = Tables.getcolumn(data, time)
    obsc  = Tables.getcolumn(data, obs)

    any(isnanormissing, timec) && error("Some time values is NaN or Missing!")

    sdata = Vector{PDSubject}(undef, length(d))
    i = one(Int)
    @inbounds for (k, v) in d
        timevals = timec[v]
        obsvals  = obsc[v]

        if !allunique(timevals)
            @warn "Not all time values is unique, last observation used! ($k)"
            nuv = nonunique(timevals)
            nuvinds = findall(x -> x == first(nuv), timevals)[1:end-1]
            if length(nuv) > 1
                for cnt = 2:length(nuv)
                    append!(nuvinds, findall(x -> x == nuv[cnt], timevals)[1:end-1])
                end
            end
            sort!(nuvinds)
            deleteat!(timevals, nuvinds)
            deleteat!(obsvals, nuvinds)
        end
        sp = sortperm(timevals)
        timevals_sp = timevals[sp]
        obsvals_sp = obsvals[sp]
        timevals_sp, obsvals_sp = checkvalues(timevals_sp, obsvals_sp)
        sdata[i] = PDSubject(timevals_sp, obsvals_sp, bl, th, Dict(sort .=> k))
        i += one(Int)
    end
    return DataSet(identity.(sdata))
end


function S1Point(arg0::jdouble)
    return S1Point((jdouble,), arg0)
end

function distance(::Type{S1Point}, arg0::S1Point, arg1::S1Point)
    return jcall(S1Point, "distance", jdouble, (S1Point, S1Point), arg0, arg1)
end

function distance(obj::S1Point, arg0::Point)
    return jcall(obj, "distance", jdouble, (Point,), arg0)
end

function equals(obj::S1Point, arg0::Object)
    return jcall(obj, "equals", jboolean, (Object,), arg0)
end

function get_alpha(obj::S1Point)
    return jcall(obj, "getAlpha", jdouble, ())
end

function get_space(obj::S1Point)
    return jcall(obj, "getSpace", Space, ())
end

function get_vector(obj::S1Point)
    return jcall(obj, "getVector", Vector2D, ())
end

function hash_code(obj::S1Point)
    return jcall(obj, "hashCode", jint, ())
end

function is_na_n(obj::S1Point)
    return jcall(obj, "isNaN", jboolean, ())
end


using Gen: run_first_pass, jacobian_correction, check_round_trip, run_transform

##################################
# ExtendingTraceTranslator #
##################################

"""
    translator = ExtendingTraceTranslator(;
        p_new_args::Tuple = (),
        p_argdiffs::Tuple = (),
        new_observations::ChoiceMap = EmptyChoiceMap(),
        q_forward::GenerativeFunction,
        q_forward_args::Tuple = (),
        transform::Union{TraceTransformDSLProgram,Nothing} = nothing)
Constructor for a extending trace translator.
Run the translator with:
    (output_trace, log_weight) = translator(input_trace)
"""
@with_kw mutable struct ExtendingTraceTranslator <: TraceTranslator
    p_new_args::Tuple = ()
    p_argdiffs::Tuple = ()
    new_observations::ChoiceMap = EmptyChoiceMap()
    q_forward::GenerativeFunction
    q_forward_args::Tuple = ()
    transform::Union{TraceTransformDSLProgram,Nothing} = nothing # a bijection
end

function (translator::ExtendingTraceTranslator)(prev_model_trace::Trace)

    # simulate from auxiliary program
    forward_proposal_trace = simulate(translator.q_forward,
        (prev_model_trace, translator.q_forward_args...,))
    forward_proposal_score = get_score(forward_proposal_trace)

    # transform forward proposal
    if translator.transform === nothing
        constraints = get_choices(forward_proposal_trace)
        log_abs_determinant = 0.0
    else
        first_pass_results =
            run_first_pass(translator.transform, forward_proposal_trace, nothing)
        log_abs_determinant =
            jacobian_correction(translator.transform, forward_proposal_trace,
                                nothing, first_pass_results, nothing)
        constraints = first_pass_results.constraints
    end

    # computing the new trace via update
    constraints = merge(constraints, translator.new_observations)
    (new_model_trace, log_model_weight, _, discard) = update(
        prev_model_trace, translator.p_new_args,
        translator.p_argdiffs, constraints)

    if !isempty(discard)
        @error("Can only extend with new choices, not remove existing choices")
        error("Invalid ExtendingTraceTranslator")
    end

    log_weight = log_model_weight - forward_proposal_score + log_abs_determinant
    return (new_model_trace, log_weight)
end

##################################
# UpdatingTraceTranslator #
##################################

"""
    translator = UpdatingTraceTranslator(;
        p_new_args::Tuple = (),
        p_argdiffs::Tuple = (),
        new_observations::ChoiceMap = EmptyChoiceMap(),
        q_forward::GenerativeFunction,
        q_forward_args::Tuple  = (),
        q_backward::GenerativeFunction,
        q_backward_args::Tuple  = (),
        f::TraceTransformDSLProgram)

Constructor for a updating trace translator, which updates the trace of model
given a forward kernel, backward kernel, new arguments for that model, and
a trace transform that specifies

Run the translator with:
```
    (output_trace, log_weight) =
        translator(input_trace; check=false, prev_observations=EmptyChoiceMap())
```
Use `check` to enable a bijection check (this requires that the transform `f`
has been paired with its inverse using `pair_bijections! or `is_involution!`).
If `check` is enabled, then `prev_observations` is a choice map containing
the observed random choices in the previous trace.
"""
@with_kw mutable struct UpdatingTraceTranslator <: TraceTranslator
    p_new_args::Tuple = ()
    p_argdiffs::Tuple = ()
    new_observations::ChoiceMap = EmptyChoiceMap()
    q_forward::GenerativeFunction
    q_forward_args::Tuple  = ()
    q_backward::GenerativeFunction
    q_backward_args::Tuple  = ()
    transform::TraceTransformDSLProgram
end

function Gen.inverse(translator::UpdatingTraceTranslator, prev_model_trace::Trace,
                     prev_observations::ChoiceMap=EmptyChoiceMap())
    return UpdatingTraceTranslator(
        get_args(prev_model_trace), map((_)->UnknownChange(), get_args(prev_model_trace)),
        prev_observations, translator.q_backward, translator.q_backward_args,
        translator.q_forward, translator.q_forward_args,
        inverse(translator.transform))
end

function Gen.run_transform(translator::UpdatingTraceTranslator,
                           prev_model_trace::Trace, forward_proposal_trace::Trace)
    @unpack transform, new_observations = translator
    @unpack p_new_args, p_argdiffs, q_backward, q_backward_args = translator
    first_pass_results = run_first_pass(
        transform, prev_model_trace, forward_proposal_trace)
    constraints = merge(first_pass_results.constraints, new_observations)
    new_model_trace, _, _, discard = update(
        prev_model_trace, p_new_args, p_argdiffs, constraints)
    log_abs_determinant = jacobian_correction(
        transform, prev_model_trace, forward_proposal_trace, first_pass_results, discard)
    backward_proposal_trace, _ = generate(
        q_backward, (new_model_trace, q_backward_args...), first_pass_results.u_back)
    return (new_model_trace, backward_proposal_trace, log_abs_determinant)
end

function (translator::UpdatingTraceTranslator)(
        prev_model_trace::Trace; check=false, prev_observations=EmptyChoiceMap())

    # sample auxiliary trace
    forward_proposal_trace = simulate(translator.q_forward,
        (prev_model_trace, translator.q_forward_args...,))

    # apply trace transform
    (new_model_trace, backward_proposal_trace, log_abs_determinant) =
        run_transform(translator, prev_model_trace, forward_proposal_trace)

    # compute log weight
    prev_model_score = get_score(prev_model_trace)
    new_model_score = get_score(new_model_trace)
    forward_proposal_score = get_score(forward_proposal_trace)
    backward_proposal_score = get_score(backward_proposal_trace)
    log_weight = new_model_score - prev_model_score +
        backward_proposal_score + forward_proposal_score + log_abs_determinant

    if check
        inverter = inverse(translator, prev_model_trace, prev_observations)
        argdiffs = map((_) -> UnknownChange(), get_args(prev_model_trace))
        (prev_model_trace_rt, forward_proposal_trace_rt, _) =
            run_transform(inverter, new_model_trace, backward_proposal_trace)
        check_round_trip(prev_model_trace, prev_model_trace_rt,
                         forward_proposal_trace, forward_proposal_trace_rt)
    end

    return (new_model_trace, log_weight)
end


"""
  Penalty for BR2
"""
struct InviscidPenalty{Tsol, Tres} <: AbstractDiffusionPenalty

  function InviscidPenalty{Tsol, Tres}(mesh, sbp, opts) where {Tsol, Tres}

    return new()
  end
end

function applyPenalty(penalty::InviscidPenalty{Tsol, Tres}, sbp, params::AbstractParamType,
                      sbpface,
                      diffusion::AbstractDiffusion, iface::Interface,
                      delta_w::AbstractMatrix{Tsol}, theta::AbstractMatrix{Tres},
                      qL_el::AbstractMatrix, qR_el::AbstractMatrix,
                      wL::AbstractMatrix, wR::AbstractMatrix,
                      coordsL::AbstractMatrix, coordsR::AbstractMatrix,
                      nrm_face::AbstractMatrix,
                      alphas::AbstractVector,
                      dxidxL::Abstract3DArray, dxidxR::Abstract3DArray,
                      jacL::AbstractVector, jacR::AbstractVector,
                      res1L::AbstractMatrix, res1R::AbstractMatrix,
                      res2L::AbstractMatrix, res2R::AbstractMatrix
                     ) where {Tsol, Tres}

  fill!(res1L, 0); fill!(res1R, 0)
  fill!(res2L, 0); fill!(res2R, 0)

  numDofPerNode, numNodesPerFace = size(delta_w)
  numNodesPerElement = length(jacL)
  dim = size(nrm_face, 1)

  for i=1:numNodesPerFace
    fac = calcLength(params, ro_sview(nrm_face, :, i))*sbpface.wface[i]
    for j=1:numDofPerNode
      res1L[j, i] =  fac*delta_w[j]
      res1R[j, i] = -fac*delta_w[j]
    end
  end

  return nothing
end

function applyPenalty_diff(penalty::InviscidPenalty{Tsol, Tres}, sbp,
                      params::AbstractParamType, sbpface,
                      diffusion::AbstractDiffusion, iface::Interface,
                      delta_w::AbstractMatrix{Tsol},
                      delta_w_dotL::Abstract4DArray, delta_w_dotR::Abstract4DArray,
                      theta::AbstractMatrix{Tres},
                      theta_dotL::Abstract4DArray, theta_dotR::Abstract4DArray,
                      qL_el::AbstractMatrix, qR_el::AbstractMatrix,
                      wL::AbstractMatrix, wR::AbstractMatrix,
                      coordsL::AbstractMatrix, coordsR::AbstractMatrix,
                      nrm_face::AbstractMatrix,
                      alphas::AbstractVector,
                      dxidxL::Abstract3DArray, dxidxR::Abstract3DArray,
                      jacL::AbstractVector, jacR::AbstractVector,
                      res1L_dotL::Abstract4DArray, res1L_dotR::Abstract4DArray,
                      res1R_dotL::Abstract4DArray, res1R_dotR::Abstract4DArray,
                      res2L::AbstractMatrix, res2R::AbstractMatrix,
                      res2L_dotL::Abstract4DArray, res2L_dotR::Abstract4DArray,
                      res2R_dotL::Abstract4DArray, res2R_dotR::Abstract4DArray,
                     ) where {Tsol, Tres}

  numDofPerNode, numNodesPerFace = size(delta_w)
  numNodesPerElement = length(jacL)
  dim = size(nrm_face, 1)

  for i=1:numNodesPerFace
    fac = calcLength(params, ro_sview(nrm_face, :, i))
    for j=1:numDofPerNode
      #res1L[j, i] =  fac*delta_w[j]
      #res1R[j, i] = -fac*delta_w[j]
    end
  end

  for q=1:numNodesPerElement
    for p=1:numNodesPerFace
      fac = calcLength(params, ro_sview(nrm_face, :, p))*sbpface.wface[p]
      for j=1:numDofPerNode
        for i=1:numDofPerNode
          res1L_dotL[i, j, p, q] =  fac*delta_w_dotL[i, j, p, q]
          res1L_dotR[i, j, p, q] =  fac*delta_w_dotR[i, j, p, q]
          res1R_dotL[i, j, p, q] = -fac*delta_w_dotL[i, j, p, q]
          res1R_dotL[i, j, p, q] = -fac*delta_w_dotR[i, j, p, q]
        end
      end
    end
  end

  fill!(res2L_dotL, 0); fill!(res2L_dotR, 0)
  fill!(res2R_dotL, 0); fill!(res2R_dotR, 0)

  return false
end


# This file is a part of BATBase.jl, licensed under the MIT License (MIT).

"""
    abstract type DifferentiationAlgorithm

*Experimental feature, not part of stable public API.*

Abstract type for integrated autocorrelation length estimation algorithms.
"""
abstract type DifferentiationAlgorithm end
export DifferentiationAlgorithm

"""
    bat_valgrad(f::Function, [algorithm::DifferentiationAlgorithm])

*Experimental feature, not part of stable public API.*

Generated a function that calculates both value and gradient of `f` at given
points.

The function `f` must support `ValueShapes.varshape(f)` and
`ValueShapes.unshaped(f)`.

Returns a NamedTuple of the shape

```julia
(result = valgrad_f, ...)
```

with

```julia
f_at_x, grad_of_f_at_x = valgrad_f(x)

grad_of_f_at_x = zero(x)
f_at_x = valgrad_f(!, grad_of_f_at_x, x)
```

Result properties not listed here are algorithm-specific and are not part
of the stable public API.

!!! note

    Do not add add methods to `bat_valgrad`, add methods to
    `bat_valgrad_impl` instead.
"""
function bat_valgrad end
export bat_valgrad

function bat_valgrad_impl end

function bat_valgrad(
    f::Function,
    algorithm = bat_default_withdebug(bat_valgrad, Val(:algorithm), f)
)
    r = bat_valgrad_impl(f, algorithm)
    result_with_args(r, (algorithm = algorithm,))
end

function argchoice_msg(::typeof(bat_valgrad), ::Val{:algorithm}, x::DifferentiationAlgorithm)
    "Using automiatic differentiation algorithm $x"
end

# ToDo, move to BAT.jl:
#=
gradient_shape(vs::AbstractValueShape) = replace_const_shapes(ValueShapes.const_zero_shape, vs)

abstract type GradientFunction end
=#


__precompile__()

module FastqSubsample

using Libz

export fastqsubsample

"""
    fastqsubsample(ifastq, ofastq, nSubsample [, seed = seed])

Subsample fixed number of reads from input FASTQ file.

- ifastq: Input FASTQ. File names ending with ".gz" will be treated as gzipped files.
- ofastq: Output FASTQ. File names ending with ".gz" will be treated as gzipped files.
- nSubsample: Number of reads after subsampling. Integer.
- seed: Seed for random number generation. The results of subsampling is the same when using the same seed. Integer.

"""

function fastqsubsample(ifastq::String, ofastq::String, nSubsample::Int; kwargs...)
    S = zeros(Int, nSubsample)
    n = countFastq(ifastq)

    println("Reads in $ifastq: $n")
    println("Reads will be subsampled to: $nSubsample")
    if n < nSubsample
        println("Cannot subsample more than the number of the original reads")
        exit()
    end

    isSeeded, seed = checkSeed(kwargs, :seed, 12345)
    rng = ifelse(isSeeded, MersenneTwister(seed), MersenneTwister())

    for i in 1:nSubsample
        S[i] = i
    end

    for i in (nSubsample+1):n
        j = rand(rng, 1:i)
        if j <= nSubsample
            S[j] = i
        end
    end

    # accepted = IntSet(shuffle(rng, 1:n)[1:nSubsample])
    accepted = IntSet(S)

    writeFastqSubsample(ifastq, ofastq, accepted)
    println("Subsampled FASTQ is written: $ofastq")
end

function checkSeed(kwargs::Array{Any,1}, s::Symbol, val::Int)
    for element in kwargs
        if element[1] == s
            return true, element[2]
        end
    end
    return false, val
end

function countFastq(ifastq::String)
    n = 0

    if ismatch(r".gz$", ifastq)
        f = ZlibInflateInputStream(open(ifastq))
    else
        f = open(ifastq)
    end

    try # For avoid EOFerror of GZip.open
        for line in eachline(f)
            n += 1
            readline(f)
            readline(f)
            readline(f)
        end
    end

    close(f)

    n
end

function writeFastqSubsample(ifastq::String, ofastq::String, accepted::IntSet)
    i = 0
    if ismatch(r".gz$", ifastq)
        f = ZlibInflateInputStream(open(ifastq))
    else
        f = open(ifastq)
    end

    if ismatch(r".gz$", ofastq)
        fw = ZlibDeflateOutputStream(open(ofastq, "w"))
    else
        fw = open(ofastq, "w")
    end

    try
        for line in eachline(f)
            i += 1
            if in(i, accepted)
                write(fw, line)
                write(fw, readline(f))
                write(fw, readline(f))
                write(fw, readline(f))
            else
                readline(f)
                readline(f)
                readline(f)
            end
        end
    end

    close(f)
    close(fw)
end

end # module


module TestMaterialize

include("preamble.jl")
using BangBang: air

@testset begin
    @test materialize!!(nothing, air.([0, 1] .+ 2))::Vector{Int} == [2, 3]
    @test materialize!!([NaN, NaN], air.([0, 1] .+ 2))::Vector{Real} == [2, 3]
    @test materialize!!(nothing, air.((0, 1) .+ 2))::Tuple === (2, 3)
    @test materialize!!(nothing, air.(SVector(0, 1) .+ 2))::SVector === SVector(2, 3)
end

@testset "widen" begin
    a0 = Real[1:3;]
    a = copy(a0)
    b = [0im] .+ (1:3)
    c = materialize!!(a, air.(a .+ b))
    @test c == a .+ b
    @test a == a0  # not modified
end

@testset "type-unstable materialize!!" begin
    unstabletrue = Ref{Any}(true)
    unstableone(unused) = unstabletrue[] ? unstabletrue[] : unused
    @test materialize!!(nothing, air.([1] .+ unstableone.([0]) ./ [1])) == [2]
end

end  # module


"""

```
coor(atoms; selection)
```

Returns the coordinates of the atoms. The input may be one atom (type `Atom`), a vector of atoms, or a `Residue`.
The coordinates are returned as a vector of static vectors (from `StaticArrays`), more specifically
as a `Vector{SVector{3,Float64}}`.

### Examples

```julia-repl
julia> using PDBTools, StaticArrays

julia> protein = wget("1LBD");

julia> coor(protein[1])
3-element SVector{3, Float64} with indices SOneTo(3):
 45.228
 84.358
 70.638

julia> coor(protein[1],as=SVector{3,Float32})
3-element SVector{3, Float32} with indices SOneTo(3):
 45.228
 84.358
 70.638

julia> coor(protein, "index <= 2")
2-element Vector{SVector{3, Float64}}:
 [45.228, 84.358, 70.638]
 [46.08, 83.165, 70.327]

julia> coor(protein, only = at -> at.resname == "ALA")
110-element Vector{SVector{3, Float64}}:
 [43.94, 81.982, 70.474]
 [43.02, 80.825, 70.455]
 [41.996, 80.878, 69.34]
 ⋮
 [-17.866, 84.088, 51.741]
 [-18.496, 83.942, 52.777]
 [-15.888, 82.583, 51.706]

julia> residues = collect(eachresidue(protein));

julia> coor(residues[1])
6-element Vector{SVector{3, Float64}}:
 [45.228, 84.358, 70.638]
 [46.08, 83.165, 70.327]
 [45.257, 81.872, 70.236]
 [45.823, 80.796, 69.974]
 [47.147, 82.98, 71.413]
 [46.541, 82.639, 72.662]

```

"""
coor(atom::Atom) = SVector{3,Float64}(atom.x, atom.y, atom.z)

function coor(atoms::AbstractVector{Atom}, selection::String)
    query = parse_query(selection)
    return coor(atoms, only = atom -> apply_query(query, atom))
end

function coor(atoms::AbstractVector{Atom}; only = all)
    n = 0
    for atom in atoms
        !only(atom) && continue
        n = n + 1
    end
    x = fill(zero(SVector{3,Float64}),n)
    i = 0
    for atom in atoms
        !only(atom) && continue
        i += 1
        x[i] = coor(atom)
    end
    return x
end

#
# Coordinates of the atoms of a residue/molecule
#
coor(residue::Residue; only = all) = coor(residue.atoms[residue.range], only = only)
coor(residue::Residue, selection::String) = coor(residue.atoms[residue.range], selection)


############################################################
## joAbstractDAparallelLinearOperator(s) - overloaded Base functions

# eltype(jo)
eltype(A::joAbstractDAparallelLinearOperator{DDT,RDT}) where {DDT,RDT} = promote_type(DDT,RDT)

# deltype(jo)
deltype(A::joAbstractDAparallelLinearOperator{DDT,RDT}) where {DDT,RDT} = DDT

# reltype(jo)
reltype(A::joAbstractDAparallelLinearOperator{DDT,RDT}) where {DDT,RDT} = RDT

# show(jo)
function show(A::joAbstractDAparallelLinearOperator)
    println("Type: $(typeof(A).name)")
    println("Name: $(A.name)")
    println("Size: $(size(A))")
    println(" NVC: $(A.nvc)")
    println(" DDT: $(deltype(A))")
    println(" RDT: $(reltype(A))")
    return nothing
end

# display(jo)
display(A::joAbstractDAparallelLinearOperator) = println((typeof(A),((A.m,A.n),A.nvc),A.name))

# size(jo)
size(A::joAbstractDAparallelLinearOperator) = A.m,A.n

# size(jo,1/2)
function size(A::joAbstractDAparallelLinearOperator,ind::Integer)
    if ind==1
		return A.m
	elseif ind==2
		return A.n
	elseif ind==3
		return A.nvc
	else
		throw(joAbstractDAparallelLinearOperatorException("invalid index"))
	end
end

# length(jo)
length(A::joAbstractDAparallelLinearOperator) = A.m*A.n

# jo_full(jo)

# norm(jo)

# real(jo)

# imag(jo)

# conj(jo)
conj(A::joDALinearOperator{DDT,RDT,N}) where {DDT,RDT,N} =
    joDALinearOperator{DDT,RDT,N}("conj("*A.name*")",A.m,A.n,A.nvc,
        get(A.fop_C), A.fop_A, A.fop_T, A.fop,
        A.iop_C, A.iop_A, A.iop_T, A.iop,
        A.PAs_in, A.PAs_out, A.fclean, A.rclean
        )
conj(A::joDAdistributedLinearOperator{DDT,RDT,N}) where {DDT,RDT,N} =
    joDAdistributedLinearOperator{DDT,RDT,N}("conj("*A.name*")",A.m,A.n,A.nvc,
        get(A.fop_C), A.fop_A, A.fop_T, A.fop,
        A.iop_C, A.iop_A, A.iop_T, A.iop,
        A.PAs_in, A.PAs_out, A.fclean, A.rclean
        )
conj(A::joDAdistributingLinearOperator{DDT,RDT,N}) where {DDT,RDT,N} =
    joDAdistributingLinearOperator{DDT,RDT,N}("conj("*A.name*")",A.m,A.n,A.nvc,
        get(A.fop_C), A.fop_A, A.fop_T, A.fop,
        A.iop_C, A.iop_A, A.iop_T, A.iop,
        A.PAs_out, A.gclean
        )
conj(A::joDAgatheringLinearOperator{DDT,RDT,N}) where {DDT,RDT,N} =
    joDAgatheringLinearOperator{DDT,RDT,N}("conj("*A.name*")",A.m,A.n,A.nvc,
        get(A.fop_C), A.fop_A, A.fop_T, A.fop,
        A.iop_C, A.iop_A, A.iop_T, A.iop,
        A.PAs_in, A.gclean
        )

# transpose(jo)
transpose(A::joDALinearOperator{DDT,RDT,N}) where {DDT,RDT,N} =
    joDALinearOperator{RDT,DDT,N}("transpose("*A.name*")",A.n,A.m,A.nvc,
        get(A.fop_T), A.fop, A.fop_C, A.fop_A,
        A.iop_T, A.iop, A.iop_C, A.iop_A,
        A.PAs_out, A.PAs_in, A.rclean, A.fclean
        )
transpose(A::joDAdistributedLinearOperator{DDT,RDT,N}) where {DDT,RDT,N} =
    joDAdistributedLinearOperator{RDT,DDT,N}("transpose("*A.name*")",A.n,A.m,A.nvc,
        get(A.fop_T), A.fop, A.fop_C, A.fop_A,
        A.iop_T, A.iop, A.iop_C, A.iop_A,
        A.PAs_out, A.PAs_in, A.rclean, A.fclean
        )
transpose(A::joDAdistributingLinearOperator{DDT,RDT,N}) where {DDT,RDT,N} =
    joDAgatheringLinearOperator{RDT,DDT,N}("transpose("*A.name*")",A.n,A.m,A.nvc,
        get(A.fop_T), A.fop, A.fop_C, A.fop_A,
        A.iop_T, A.iop, A.iop_C, A.iop_A,
        A.PAs_out, A.gclean
        )
transpose(A::joDAgatheringLinearOperator{DDT,RDT,N}) where {DDT,RDT,N} =
    joDAdistributingLinearOperator{RDT,DDT,N}("transpose("*A.name*")",A.n,A.m,A.nvc,
        get(A.fop_T), A.fop, A.fop_C, A.fop_A,
        A.iop_T, A.iop, A.iop_C, A.iop_A,
        A.PAs_in, A.gclean
        )

# adjoint(jo)
adjoint(A::joDALinearOperator{DDT,RDT,N}) where {DDT,RDT,N} =
    joDALinearOperator{RDT,DDT,N}("adjoint("*A.name*")",A.n,A.m,A.nvc,
        get(A.fop_A), A.fop_C, A.fop, A.fop_T,
        A.iop_A, A.iop_C, A.iop, A.iop_T,
        A.PAs_out, A.PAs_in, A.rclean, A.fclean
        )
adjoint(A::joDAdistributedLinearOperator{DDT,RDT,N}) where {DDT,RDT,N} =
    joDAdistributedLinearOperator{RDT,DDT,N}("adjoint("*A.name*")",A.n,A.m,A.nvc,
        get(A.fop_A), A.fop_C, A.fop, A.fop_T,
        A.iop_A, A.iop_C, A.iop, A.iop_T,
        A.PAs_out, A.PAs_in, A.rclean, A.fclean
        )
adjoint(A::joDAdistributingLinearOperator{DDT,RDT,N}) where {DDT,RDT,N} =
    joDAgatheringLinearOperator{RDT,DDT,N}("adjoint("*A.name*")",A.n,A.m,A.nvc,
        get(A.fop_A), A.fop_C, A.fop, A.fop_T,
        A.iop_A, A.iop_C, A.iop, A.iop_T,
        A.PAs_out, A.gclean
        )
adjoint(A::joDAgatheringLinearOperator{DDT,RDT,N}) where {DDT,RDT,N} =
    joDAdistributingLinearOperator{RDT,DDT,N}("adjoint("*A.name*")",A.n,A.m,A.nvc,
        get(A.fop_A), A.fop_C, A.fop, A.fop_T,
        A.iop_A, A.iop_C, A.iop, A.iop_T,
        A.PAs_in, A.gclean
        )

# isreal(jo)
isreal(A :: joAbstractDAparallelLinearOperator{DDT,RDT}) where {DDT,RDT} = (DDT<:Real && RDT<:Real)

# issymmetric(jo)

# ishermitian(jo)

# getindex(jo,...)

############################################################
## overloaded Base *(...jo...)

# *(jo,jo)
function *(A::joDALinOpsUnion{CDT,ARDT,2},B::joDALinOpsUnion{BDDT,CDT,2}) where {ARDT,BDDT,CDT}
    size(A,2) == size(B,1) || throw(joDALinearOperatorException("*($(A.name),$(B.name)): shape mismatch"))
    A.nvc==B.nvc || throw(joDALinearOperatorException("*($(A.name),$(B.name)): nvc mismatch"))
    isapprox(A.PAs_in,B.PAs_out) || throw(joDALinearOperatorException("*($(A.name),$(B.name)): distributor mismatch"))
    return joDALinearOperator{BDDT,ARDT,2}("($(A.name)*$(B.name))",size(A,1),size(B,2),A.nvc,
               v1->A*(B*v1),
               v2->transpose(B)*(transpose(A)*v2),
               v3->adjoint(B)*(adjoint(A)*v3),
               v4->conj(A)*(conj(B)*v4),
               @joNF, @joNF, @joNF, @joNF,
               B.PAs_in,A.PAs_out,B.fclean,A.rclean)
end
function *(A::joDALinOpsUnion{CDT,ARDT,2},B::joDAdistributeLinOpsUnion{BDDT,CDT,2}) where {ARDT,BDDT,CDT}
    size(A,2) == size(B,1) || throw(joDALinearOperatorException("*($(A.name),$(B.name)): shape mismatch"))
    A.nvc==B.nvc || throw(joDALinearOperatorException("*($(A.name),$(B.name)): nvc mismatch"))
    isapprox(A.PAs_in,B.PAs_out) || throw(joDALinearOperatorException("*($(A.name),$(B.name)): distributor mismatch"))
    return joDAdistributingLinearOperator{BDDT,ARDT,2}("($(A.name)*$(B.name))",size(A,1),size(B,2),A.nvc,
               v1->A*(B*v1),
               v2->transpose(B)*(transpose(A)*v2),
               v3->adjoint(B)*(adjoint(A)*v3),
               v4->conj(A)*(conj(B)*v4),
               @joNF, @joNF, @joNF, @joNF,
               A.PAs_out,B.gclean)
end
function *(A::joDAgatherLinOpsUnion{CDT,ARDT,2},B::joDALinOpsUnion{BDDT,CDT,2}) where {ARDT,BDDT,CDT}
    size(A,2) == size(B,1) || throw(joDAgatheringLinearOperatorException("*($(A.name),$(B.name)): shape mismatch"))
    A.nvc==B.nvc || throw(joDAgatheringLinearOperatorException("*($(A.name),$(B.name)): nvc mismatch"))
    isapprox(A.PAs_in,B.PAs_out) || throw(joDAgatheringLinearOperatorException("*($(A.name),$(B.name)): distributor mismatch"))
    return joDAgatheringLinearOperator{BDDT,ARDT,2}("($(A.name)*$(B.name))",size(A,1),size(B,2),A.nvc,
               v1->A*(B*v1),
               v2->transpose(B)*(transpose(A)*v2),
               v3->adjoint(B)*(adjoint(A)*v3),
               v4->conj(A)*(conj(B)*v4),
               @joNF, @joNF, @joNF, @joNF,
               B.PAs_in,B.fclean)
end
function *(A::joDAdistributeLinOpsUnion{CDT,ARDT,2},B::joDAgatherLinOpsUnion{BDDT,CDT,2}) where {ARDT,BDDT,CDT}
    size(A,2) == size(B,1) || throw(joDALinearOperatorException("*($(A.name),$(B.name)): shape mismatch"))
    A.nvc==B.nvc || throw(joDALinearOperatorException("*($(A.name),$(B.name)): nvc mismatch"))
    return joDALinearOperator{BDDT,ARDT,2}("($(A.name)*$(B.name))",size(A,1),size(B,2),A.nvc,
               v1->A*(B*v1),
               v2->transpose(B)*(transpose(A)*v2),
               v3->adjoint(B)*(adjoint(A)*v3),
               v4->conj(A)*(conj(B)*v4),
               @joNF, @joNF, @joNF, @joNF,
               B.PAs_in,A.PAs_out,B.gclean,A.gclean)
end
function *(A::joDAgatherLinOpsUnion{CDT,ARDT,2},B::joDAdistributeLinOpsUnion{BDDT,CDT,2}) where {ARDT,BDDT,CDT}
    size(A,2) == size(B,1) || throw(joLinearOperatorException("*($(A.name),$(B.name)): shape mismatch"))
    A.nvc==B.nvc || throw(joLinearOperatorException("*($(A.name),$(B.name)): nvc mismatch"))
    isapprox(A.PAs_in,B.PAs_out) || throw(joLinearOperatorException("*($(A.name),$(B.name)): distributor mismatch"))
    return joLinearOperator{BDDT,ARDT}("($(A.name)*$(B.name))",size(A,1),size(B,2),
               v1->A*(B*v1),
               v2->transpose(B)*(transpose(A)*v2),
               v3->adjoint(B)*(adjoint(A)*v3),
               v4->conj(A)*(conj(B)*v4),
               @joNF, @joNF, @joNF, @joNF)
end
# extras with warnings
function *(A::joDAdistribute{CDT,ARDT,2},B::joDAgather{BDDT,CDT,2}) where {ARDT,BDDT,CDT}
    size(A,2) == size(B,1) || throw(joDALinearOperatorException("*($(A.name),$(B.name)): shape mismatch"))
    A.PAs_out.dims[2]==B.PAs_in.dims[2] || throw(joDALinearOperatorException("*($(A.name),$(B.name)): nvc mismatch"))
    @warn "*($(typeof(A)),$(typeof(B))) is a senseless operation. Get rid of it!"
    return joDALinearOperator{BDDT,ARDT,2}("($(A.name)*$(B.name))",size(A,1),size(B,2),A.PAs_out.dims[2],
               v1->v1, v2->v2, v3->v3, v4->v4,
               @joNF, @joNF, @joNF, @joNF,
               B.PAs_in,A.PAs_out,B.gclean,A.gclean)
end
function *(A::joDAgather{CDT,ARDT,2},B::joDAdistribute{BDDT,CDT,2}) where {ARDT,BDDT,CDT}
    size(A,2) == size(B,1) || throw(joLinearOperatorException("*($(A.name),$(B.name)): shape mismatch"))
    A.PAs_in.dims[2]==B.PAs_out.dims[2] || throw(joLinearOperatorException("*($(A.name),$(B.name)): nvc mismatch"))
    isapprox(A.PAs_in,B.PAs_out) || throw(joLinearOperatorException("*($(A.name),$(B.name)): distributor mismatch"))
    @warn "*($(typeof(A)),$(typeof(B))) is a senseless operation. Get rid of it!"
    return joLinearOperator{BDDT,ARDT}("($(A.name)*$(B.name))",size(A,1),size(B,2),
               v1->v1, v2->v2, v3->v3, v4->v4,
               @joNF, @joNF, @joNF, @joNF)
end
# joDA with joAbstractLinearOperator
function *(A::joDAdistributeLinOpsUnion{CDT,ARDT,2},B::joAbstractLinearOperator{BDDT,CDT}) where {ARDT,BDDT,CDT}
    size(A,2) == size(B,1) || throw(joAbstractLinearOperatorException("*($(A.name),$(B.name)): shape mismatch"))
    return joDAdistributingLinearOperator{BDDT,ARDT,2}("($(A.name)*$(B.name))",size(A,1),size(B,2),A.nvc,
               v1->A*(B*v1),
               v2->transpose(B)*(transpose(A)*v2),
               v3->adjoint(B)*(adjoint(A)*v3),
               v4->conj(A)*(conj(B)*v4),
               @joNF, @joNF, @joNF, @joNF,
               A.PAs_out,A.gclean)
end
function *(A::joAbstractLinearOperator{CDT,ARDT},B::joDAgatherLinOpsUnion{BDDT,CDT,2}) where {ARDT,BDDT,CDT}
    size(A,2) == size(B,1) || throw(joAbstractLinearOperatorException("*($(A.name),$(B.name)): shape mismatch"))
    return joDAgatheringLinearOperator{BDDT,ARDT,2}("($(A.name)*$(B.name))",size(A,1),size(B,2),B.nvc,
               v1->A*(B*v1),
               v2->transpose(B)*(transpose(A)*v2),
               v3->adjoint(B)*(adjoint(A)*v3),
               v4->conj(A)*(conj(B)*v4),
               @joNF, @joNF, @joNF, @joNF,
               B.PAs_in,B.gclean)
end

# *(jo,mvec)
function *(A::joDALinearOperator{ADDT,ARDT,2},mv::DArray{mvDT,2}) where {ADDT,ARDT,mvDT<:Number}
    A.n == size(mv,1) || throw(joDALinearOperatorException("shape mismatch"))
    A.nvc == size(mv,2) || throw(joDALinearOperatorException("shape mismatch"))
    jo_check_type_match(ADDT,mvDT,join(["DDT for *(jo,mvec):",A.name,typeof(A),mvDT]," / "))
    isequiv(A.PAs_in,mv) || throw(joDALinearOperatorException("*($(A.name),mv::DArray): input distributor mismatch"))
    MV=A.fop(mv)
    jo_check_type_match(ARDT,eltype(MV),join(["RDT from *(jo,mvec):",A.name,typeof(A),eltype(MV)]," / "))
    return MV
end
function *(A::joDAdistributedLinearOperator{ADDT,ARDT,2},mv::DArray{mvDT,2}) where {ADDT,ARDT,mvDT<:Number}
    A.n == size(mv,1) || throw(joDAdistributedLinearOperatorException("shape mismatch"))
    A.nvc == size(mv,2) || throw(joDAdistributedLinearOperatorException("shape mismatch"))
    jo_check_type_match(ADDT,mvDT,join(["DDT for *(jo,mvec):",A.name,typeof(A),mvDT]," / "))
    isequiv(A.PAs_in,mv) || throw(joDAdistributedLinearOperatorException("*($(A.name),mv::DArray): input distributor mismatch"))
    MV=dalloc(A.PAs_out)
    spmd(joDAutils.jo_x_mv!,A.fop,mv,MV,pids=A.PAs_out.procs)
    jo_check_type_match(ARDT,eltype(MV),join(["RDT from *(jo,mvec):",A.name,typeof(A),eltype(MV)]," / "))
    return MV
end
function *(A::joDAdistributingLinearOperator{ADDT,ARDT,2},mv::LocalMatrix{mvDT}) where {ADDT,ARDT,mvDT<:Number}
    A.n == size(mv,1) || throw(joDAdistributingLinearOperatorException("shape mismatch in A$(size(A))*v$(size(mv))"))
    A.nvc == size(mv,2) || throw(joDAdistributingLinearOperatorException("nvc size mismatch"))
    jo_check_type_match(ADDT,mvDT,join(["DDT for *(jo,mvec):",A.name,typeof(A),mvDT]," / "))
    MV = A.fop(mv)
    jo_check_type_match(ARDT,eltype(MV),join(["RDT from *(jo,mvec):",A.name,typeof(A),eltype(MV)]," / "))
    return MV
end
function *(A::joDAgatheringLinearOperator{ADDT,ARDT,2},mv::DArray{mvDT,2}) where {ADDT,ARDT,mvDT<:Number}
    A.n == size(mv,1) || throw(joDAgatheringLinearOperatorException("shape mismatch in A$(size(A))*v$(size(mv))"))
    A.nvc == size(mv,2) || throw(joDAgatheringLinearOperatorException("nvc size mismatch"))
    jo_check_type_match(ADDT,mvDT,join(["DDT for *(jo,mvec):",A.name,typeof(A),mvDT]," / "))
    isequiv(A.PAs_in,mv) || throw(joDAgatheringLinearOperatorException("*($(A.name),mv::DArray): input distributor mismatch"))
    MV = A.fop(mv)
    jo_check_type_match(ARDT,eltype(MV),join(["RDT from *(jo,mvec):",A.name,typeof(A),eltype(MV)]," / "))
    return MV
end

# *(mvec,jo)

# *(jo,vec)

# *(vec,jo)

# *(num,jo)

# *(jo,num)

############################################################
## overloaded Base \(...jo...)

# \(jo,jo)

# \(jo,mvec)

# \(mvec,jo)

# \(jo,vec)

# \(vec,jo)

# \(num,jo)

# \(jo,num)
#\{ADDT,ARDT}(A::joAbstractDAparallelLinearOperator{ADDT,ARDT},a::Number) = inv(a)*A
#\{ADDT,ARDT}(A::joAbstractDAparallelLinearOperator{ADDT,ARDT},a::joNumber{ADDT,ARDT}) = inv(a)*A

############################################################
## overloaded Base +(...jo...)

# +(jo)
+(A::joAbstractDAparallelLinearOperator) = A

# +(jo,jo)

# +(jo,mvec)

# +(mvec,jo)

# +(jo,vec)

# +(vec,jo)

# +(jo,num)

# +(num,jo)
+(b::Number,A::joAbstractDAparallelLinearOperator{ADDT,ARDT}) where {ADDT,ARDT} = A+b
+(b::joNumber{ADDT,ARDT},A::joAbstractDAparallelLinearOperator{ADDT,ARDT}) where {ADDT,ARDT} = A+b

############################################################
## overloaded Base -(...jo...)

# -(jo)

# -(jo,jo)
-(A::joAbstractDAparallelLinearOperator,B::joAbstractDAparallelLinearOperator) = A+(-B)

# -(jo,mvec)

# -(mvec,jo)

# -(jo,vec)

# -(vec,jo)

# -(jo,num)
-(A::joAbstractDAparallelLinearOperator,b::Number) = A+(-b)
-(A::joAbstractDAparallelLinearOperator,b::joNumber) = A+(-b)

# -(num,jo)
-(b::Number,A::joAbstractDAparallelLinearOperator) = -A+b
-(b::joNumber,A::joAbstractDAparallelLinearOperator) = -A+b

############################################################
## overloaded Base .*(...jo...)

# .*(num,jo)
Base.Broadcast.broadcasted(::typeof(*),a::Number,A::joAbstractDAparallelLinearOperator) = a*A
Base.Broadcast.broadcasted(::typeof(*),a::joNumber,A::joAbstractDAparallelLinearOperator) = a*A

# .*(jo,num)
Base.Broadcast.broadcasted(::typeof(*),A::joAbstractDAparallelLinearOperator,a::Number) = a*A
Base.Broadcast.broadcasted(::typeof(*),A::joAbstractDAparallelLinearOperator,a::joNumber) = a*A

############################################################
## overloaded Base .\(...jo...)

############################################################
## overloaded Base .+(...jo...)

# .+(jo,num)
Base.Broadcast.broadcasted(::typeof(+),A::joAbstractDAparallelLinearOperator,b::Number) = A+b
Base.Broadcast.broadcasted(::typeof(+),A::joAbstractDAparallelLinearOperator,b::joNumber) = A+b

# .+(num,jo)
Base.Broadcast.broadcasted(::typeof(+),b::Number,A::joAbstractDAparallelLinearOperator) = A+b
Base.Broadcast.broadcasted(::typeof(+),b::joNumber,A::joAbstractDAparallelLinearOperator) = A+b

############################################################
## overloaded Base .-(...jo...)

# .-(jo,num)
Base.Broadcast.broadcasted(::typeof(-),A::joAbstractDAparallelLinearOperator,b::Number) = A+(-b)
Base.Broadcast.broadcasted(::typeof(-),A::joAbstractDAparallelLinearOperator,b::joNumber) = A+(-b)

# .-(num,jo)
Base.Broadcast.broadcasted(::typeof(-),b::Number,A::joAbstractDAparallelLinearOperator) = -A+b
Base.Broadcast.broadcasted(::typeof(-),b::joNumber,A::joAbstractDAparallelLinearOperator) = -A+b

############################################################
## overloaded Base block methods

# hcat(...jo...)

# vcat(...jo...)

# hvcat(...jo...)

############################################################
## overloaded LinearAlgebra functions

# mul!(...,jo,...)

# ldiv!(...,jo,...)


using HWfuncapp
using Test

@testset "HWfuncapp.jl" begin
	# test that q1 with 15 nodes has an error smaller than 1e-9

    # test that the integral of function h [-10,0] ≈ 1.46039789878568

    @test false  # by default fail
end


function getPawnAttack(attack::UInt64, b::UInt64, white::Bool)
    white ? lx_clear = CLEAR_FILE_A : lx_clear = CLEAR_FILE_H
    white ? lx_shift = 9 : lx_shift = -9
    attack |= (b & lx_clear) << lx_shift
    white ? rx_shift = 7 : rx_shift = -7
    white ? rx_clear = CLEAR_FILE_H : rx_clear = CLEAR_FILE_A
    return attack |= (b & rx_clear) << rx_shift
end

function getPawnMoves!(moves::Moves, bitboard::UInt64, taken::UInt64, friends::UInt64,
                       enemy::ChessSet, white::Bool, enpassant::UInt64)
    white ? shift = 8 : shift = -8
    white ? home = MASK_RANKS[2] : home = MASK_RANKS[7]
    white ? prom = MASK_RANKS[8] : prom = MASK_RANKS[1]

    one_step = (bitboard << shift) & ~taken
    if one_step != EMPTY
        b0, d = bitshift(one_step)
        for j = 0:d
            if b0<<j & one_step != EMPTY
                target = b0<<j
                if target & prom != EMPTY
                    src = target>>shift
                    for p in [:knight, :bishop, :rook, :queen]
                        push!(moves, Move(:pawn, src, target, NONE, EMPTY, p, NOCASTLING))
                    end
                else
                    push!(moves, Move(:pawn, target>>shift, target, NONE, EMPTY, :none, NOCASTLING))
                end
            end
        end
    end

    two_steps = ((((bitboard & home) << shift) & ~taken) << shift) & ~taken
    if two_steps != EMPTY
        b0, d = bitshift(two_steps)
        for j = 0:d
            if b0<<j & two_steps != EMPTY
                target = b0<<j
                push!(moves, Move(:pawn, target>>(shift*2), target, NONE, target>>shift, :none, NOCASTLING))
            end
        end
    end

    white ? lx_clear = CLEAR_FILE_A : lx_clear = CLEAR_FILE_H
    white ? lx_shift = 9 : lx_shift = -9
    x_lx = (((bitboard & lx_clear) << lx_shift) & taken) & ~friends
    if x_lx != EMPTY
        b0, d = bitshift(x_lx)
        for j = 0:d
            if b0<<j & x_lx != EMPTY
                target = b0<<j
                if target & enemy.friends != EMPTY
                    take = Piece(getTypeAt(enemy, target), target)
                    if target & prom != EMPTY
                        src = target>>lx_shift
                        for p in  [:knight, :bishop, :rook, :queen]
                            push!(moves, Move(:pawn, src, target, take, EMPTY, p, NOCASTLING))
                        end
                    else
                        push!(moves, Move(:pawn, target>>lx_shift, target, take, EMPTY, :none, NOCASTLING))
                    end
                end
            end
        end
    end

    en_lx = (((bitboard & lx_clear) << lx_shift) & ~taken) & enpassant
    if en_lx != EMPTY
        b0, d = bitshift(en_lx)
        take = Piece(:pawn, enpassant<<-shift)
        push!(moves, Move(:pawn, b0>>lx_shift, enpassant, take, EMPTY, :none, NOCASTLING))
    end

    white ? rx_shift = 7 : rx_shift = -7
    white ? rx_clear = CLEAR_FILE_H : rx_clear = CLEAR_FILE_A
    x_rx = (((bitboard & rx_clear) << rx_shift) & taken) & ~friends
    if x_rx != EMPTY
        b0, d = bitshift(x_rx)
        for j = 0:d
            if b0<<j & x_rx != EMPTY
                target = b0<<j
                if target & enemy.friends != EMPTY
                    take = Piece(getTypeAt(enemy, target), target)
                    if target & prom != EMPTY
                        src = target>>rx_shift
                        for  p in  [:knight, :bishop, :rook, :queen]
                            push!(moves, Move(:pawn, src, target, take, EMPTY, p, NOCASTLING))
                        end
                    else
                        push!(moves, Move(:pawn, target>>rx_shift, target, take, EMPTY, :none, NOCASTLING))
                    end
                end
            end
        end
    end

    en_rx = (((bitboard & rx_clear) << rx_shift) & ~taken) & enpassant
    if en_rx != EMPTY
        b0, d = bitshift(en_rx)
        take = Piece(:pawn, enpassant<<-shift)
        push!(moves, Move(:pawn, b0>>rx_shift, enpassant, take, EMPTY, :none, NOCASTLING))
    end
end

function pawnMovesGen(white::Bool)
    pawn_moves = Dict{UInt64,UInt64}()
    pawn_attacks = Dict{UInt64,UInt64}()

    white ? home = MASK_RANKS[2] : home = MASK_RANKS[7]
    white ? lx_clear = CLEAR_FILE_A : lx_clear = CLEAR_FILE_H
    white ? rx_clear = CLEAR_FILE_H : rx_clear = CLEAR_FILE_A

    for square in values(PGN2UINT)
        targets = EMPTY
        attacks = EMPTY
        white ? targets |= square << 8 : targets |= square >> 8
        if square & lx_clear != EMPTY
            attacks |= square << (white ? 9 : -9)
        end
        if square & rx_clear != EMPTY
            attacks |= square << (white ? 7 : -7)
        end
        if square & home != EMPTY
            white ? targets |= square << 16 : targets |= square >> 16
        end
        push!(pawn_moves, square=>targets)
        push!(pawn_attacks, square=>attacks)
    end

    return pawn_moves, pawn_attacks
end
const PAWN_WHITE, PAWN_X_WHITE = pawnMovesGen(true)
const PAWN_BLACK, PAWN_X_BLACK = pawnMovesGen(false)


function GaussIntegratorFactory()
    return GaussIntegratorFactory(())
end

function hermite(obj::GaussIntegratorFactory, arg0::jint)
    return jcall(obj, "hermite", SymmetricGaussIntegrator, (jint,), arg0)
end

function laguerre(obj::GaussIntegratorFactory, arg0::jint)
    return jcall(obj, "laguerre", GaussIntegrator, (jint,), arg0)
end

function legendre(obj::GaussIntegratorFactory, arg0::jint)
    return jcall(obj, "legendre", GaussIntegrator, (jint,), arg0)
end

function legendre(obj::GaussIntegratorFactory, arg0::jint, arg1::jdouble, arg2::jdouble)
    return jcall(obj, "legendre", GaussIntegrator, (jint, jdouble, jdouble), arg0, arg1, arg2)
end

function legendre_high_precision(obj::GaussIntegratorFactory, arg0::jint)
    return jcall(obj, "legendreHighPrecision", GaussIntegrator, (jint,), arg0)
end

function legendre_high_precision(obj::GaussIntegratorFactory, arg0::jint, arg1::jdouble, arg2::jdouble)
    return jcall(obj, "legendreHighPrecision", GaussIntegrator, (jint, jdouble, jdouble), arg0, arg1, arg2)
end


# cell.jl

# This is the main script, to run this script type: include("cell.jl") into Julia
# Set up the following scripts to define your problem and solution:
# (1) genetics.jl -> This is your system, feel free to design or evolve an ideal genetic system
# (2) input.jl -> This defines the problem you are trying to solve in terms of a flow of data into the cell
# (3) output.jl -> This defines the conditions by which you monitor success/failure/end conditions for the system

#Set a random seed to ensure consistent results
srand(0)

###### Initial Set Up ######
println("Creating Cell")

# Set Up Data Types

type Molecule
	# data stored in molecule
	data
end

type Gene
	# tree structure
	parent
	# protein function
	protein
	# length of protein life
	access
	# name of protein it creates
	lifetime
end

type ProteinAbundance
	# Gene for Protein
	gene
	# Raw Number (not normalized)
	abundance
end

include("output.jl")

include("genetics.jl")

include("input.jl")

# (1) Inputter
# This gets molecules for proteins on demand using a specified function from include.jl

pool = Molecule[]

function fromFactory()
	while true
		produce(getMolecule())
	end
end

function fromEnvironment(outside, permeability)
	while true
		r = rand()
		if r < permeability
			produce(consume(outside))
		else
			if length(fetch(pool)) > 0
				println("Processing Molecule from Pool")
				produce(shift!(fetch(pool)))
			else
				produce(consume(outside))
			end
		end
	end
end

outside = @task fromFactory()
environment = @task fromEnvironment(outside, 0.5)

# (2) Consumer

# This assigns molecules to proteins based on distribution
# Using Proteins to do (terrible) things to molecules

function consumer()
	println("Consume a Molecule")
	result = processMolecule(consume(environment), proteinDistribution)
	#println("")
	#println(pool)
	#println("")
	append!(pool, result[1])
	return (result[2], result[3])
end

function processMolecule(molecule, pd)

	# 1 - Normalised Protein Distribution
	normPD = ProteinAbundance[]
	sumPD = 0

	for p in pd
		sumPD = sumPD + p.abundance
	end

	for p in pd
		normPA = ProteinAbundance(p.gene, (p.abundance/sumPD))
		push!(normPD,normPA)
	end

	# 2 - Distribute
	handOfGod = rand()
	sumNPD = 0
	# Bad Default
	assignedProtein = root

	for p in normPD
		sumNPD = sumNPD + p.abundance
		if (sumNPD > handOfGod)
			assignedProtein = p.gene
			break
		end
	end

	# 3 - Run Process

	return (assignedProtein.protein(molecule))

end

###### Main Loop ######
println("Cell is Alive")

complete = false
#outputCount = 0
while complete == false
	# Run it like its hot
	result = consumer()
	outputCount = outputCount + result[2]
	backreg = result[1]
	complete = checkEndCon()
end

println(outputCount)
###### Final Steps ######
println("Cell is Dead")


# Wrapper around CounterValues so subtraction automatically handles wrapping
"""
    CounterValue(x::UInt)

A raw value returned from a performance counter.
This type will automatically detect and correct for counter roll-over.

```julia
julia> a = CounterTools.CounterValue(0x0)
CV(0)

julia> b = CounterTools.CounterValue(0x1)
CV(1)

julia> b - a
1

julia> UInt(a - b)
0x00007fffffffffff
```
"""
struct CounterValue
    value::UInt64
end
value(x::CounterValue) = x.value
Base.iszero(x::CounterValue) = iszero(x.value)

function Base.:-(x::CounterValue, y::CounterValue)
    # Test if overflow happend.
    start = (value(x) < value(y)) ? (UInt(1) << 48) : zero(UInt)
    if (value(x) < value(y))
        @show value(x)
        @show value(y)
    end
    return convert(Int, start + value(x) - value(y))
end

Base.show(io::IO, x::CounterValue) = print(io, "CV($(value(x)))")

#####
##### Bit maniuplation functions
#####
clearbit(x, i) = x & ~(one(x) << i)
setbit(x, i) = x | (one(x) << i)

isbitset(x, i) = !iszero(x & mask(i))

clearbits(x, i) = reduce(clearbit, i; init = x)
setbits(x, i) = reduce(setbit, i; init = x)

mask(lo::T, hi::T) where {T} = (one(lo) << (hi + 1)) - (1 << lo)
mask(i) = one(i) << i

hex(i::Integer) = string(i; base = 16)

#####
##### MSR Utilities
#####

msrpath(cpu::IndexZero) = "/dev/cpu/$(value(cpu))/msr"
msrpath(cpu::Integer) = msrpath(indexzero(cpu))
function readmsr(cpu, register)
    # Path to the kernel interface for MSRs
    path = msrpath(cpu)

    # We seek to the register and read a 64 bit int
    val = open(path) do f
        seek(f, value(indexzero(register)))
        return read(f, Int64)
    end
    return val
end

function writemsr(cpu::INDEX_TYPES, register, v)
    path = msrpath(cpu)
    open(path; write = true) do f
        seek(f, value(indexzero(register)))
        write(f, v)
    end
    return nothing
end

#####
##### Reading from Counters
#####

# Fixed counter values.
@enum FixedCounter::Int64 begin
    RetiredInstructions = 0
    ClockUnhaltedThread = 1
    ClockUnhaltedCore = 2
end

# This path goes through MSRs and is expected to be much much slower than the rdpmc
# based instructions.
readcounter(cpu, counter::INDEX_TYPES) = CounterValue(readmsr(cpu, PMC_MSRS[counter]))

"""
unsafe_rdpmc(i::Integer)

Read the contents of performance counter `i`.
If `i` is between 0 and 3, a programable counter is read.
If `i` is in 2^30, 2^30+1, or 2^30 + 2, a fixed function counter is read.

This function is unsafe since reading from an illegal value will recklessly segfault.
"""
function unsafe_rdpmc(i::Union{Integer,IndexZero,FixedCounter})
    high, low = unsafe_partial_rdpmc(i)
    return CounterValue((widen(high) << 32) | low)
end

unsafe_partial_rdpmc(i::Integer) = unsafe_partial_rdpmc(indexzero(i))
unsafe_partial_rdpmc(i::FixedCounter) = unsafe_partial_rdpmc(IndexZero(2^30 + Int64(i)))

function unsafe_partial_rdpmc(i::IndexZero)
    Base.@_inline_meta
    # This is reverse engineered from `ref.cpp` in the `ref/` directory and from Julia's
    # Tuple constructing syntax.
    #
    # The Tuple constructing syntax can be queried using
    # ```
    # f(a, b) = (a, b)
    # @code_llvm f(UInt32(0), UInt32(0))
    # ```
    val = Base.llvmcall(
        raw"""
        %val = call { i32, i32 } asm sideeffect "rdpmc", "={ax},={dx},{cx},~{dirflag},~{fpsr},~{flags}"(i32 %0)
        %low = extractvalue { i32, i32 } %val, 0
        %high = extractvalue { i32, i32 } %val, 1

        %.arr.0 = insertvalue [2 x i32] undef, i32 %high, 0
        %.arr.1 = insertvalue [2 x i32] %.arr.0, i32 %low, 1
        ret [2 x i32] %.arr.1
        """,
        Tuple{UInt32, UInt32},
        Tuple{Int32},
        convert(Int32, value(i))
    )
    return val
end


module Photopolarimetry

using Light, Images, ImageMagick, Colors, FixedPointNumbers

export path2stokes, getfnames, build_opt, getsz, stokes2mats, mats2images, stokes2profile, stokes2ellipse

const N = 1000
const N2 = round(Int,N/2)
const prop = Dict(:spatialorder => ["x","y"], :colorspace => "RGB", :pixelspacing => [1,1])

function getfnames(path::AbstractString)
	files = readdir(path)
	filter!(r"^[^\.].*\.(?:NEF|CR2)$",files)
	@assert length(files) == 4 || length(files) == 6
	return map(x -> joinpath(path,x), files)
end
getsz(fname) = map(x -> round(Int,parse(Int,x)/2),split(split(readall(`dcraw -i -v $fname`),'\n')[14])[[3,5]])
function build_opt(SZ,flip, flop, rotate, cropright, cropbottom, cropleft, croptop, scale)
	toflip = flip ? `-flip` : ``
	toflop = flop ? `-flop` : ``
	sz = SZ*scale/100
	w0 = sz[1]*cosd(rotate) +  sz[2]*cosd(90 - abs(rotate))
	w = round(Int, w0*cropright/100)
	h0 = sz[1]*sind(abs(rotate)) +  sz[2]*sind(90 - abs(rotate))
	h = round(Int, h0*cropbottom/100)
	x = round(Int, w0*cropleft/100)
	y = round(Int, h0*croptop/100)
	return ([w,h], `-scale $scale% -rotate $rotate +repage -crop $(w)x$h+$x+$y $toflip $toflop `)
end
#@everywhere
function myimread(fname::AbstractString,sz::Vector{Int},opt::Cmd)
	stream, _ = open(pipeline(`dcraw -w -H 0 -o 0 -h -4 -c $fname`,`convert - $opt -channel G -separate  Gray:-`))
	read(stream, UInt16, 1, sz...)
end
function loaddata{T <: AbstractString}(fnames::Vector{T},sz::Vector{Int},opt::Cmd)
	n = length(fnames)
	#x = Shared
	x = Array(UInt16,n,sz...)
	#@sync @parallel
	for i = 1:n
		x[i,:,:] = myimread(fnames[i],sz,opt)
	end
	return x
end
function data2stokes(x::Array{UInt16,3}, angleoffset::Real)
	n,w,h = size(x)
	S = Array(Stokes,w,h)
	m = zeros(n)
	for i = 1:w*h
		for j = 1:n
			m[j] = (x[j,i] + 1)/65536
		end
		tmp = Stokes(measurements2stokes(m...)...)
		S[i] = rotate(tmp,angleoffset)
	end
	return S
end
function path2stokes(path::AbstractString,opts...; angleoffset::Real = 0)
	fnames = getfnames(path)
	SZ = getsz(fnames[1])
	sz, opt = build_opt(SZ, opts...)
	x = loaddata(fnames,sz,opt)
	return data2stokes(x, angleoffset)
end
function path2stokes(path::AbstractString; angleoffset::Real = 0)
	fnames = getfnames(path)
	SZ = getsz(fnames[1])
	x = loaddata(fnames,SZ,` `)
	return data2stokes(x, angleoffset)
end

roundit(x::Float64) = round(Int, x*(N - 1) + 1)
function normalize(p::Polar)
	if isnan(p.I)
		return (1,1,rand(1:N),N2)
	else
		I = p.I > 1 ? N : roundit(p.I)
		dolp = p.dolp > 1 ? N : roundit(p.dolp)
		aop = I == 1 || dolp == 1 ? rand(1:N) : roundit(p.aop/pi + 0.5)
		tmp = (p.docp + 1)/2
		docp = tmp < 0 ? 1 : tmp > 1 ? N : roundit(tmp)
	end
	return (I,dolp,aop,docp)
end
function colorwheel(r::Int)
	row = Int[]
	col = Int[]
	ind = Int[]
	r2 = r*r
	for i in -r:r, j in -r:r
		if i*i + j*j <= r2
			push!(row,i + r + 1)
			push!(col,j + r + 1)
			tmp = atan2(j,i)/pi
			a = tmp > 0 ? roundit(tmp) : roundit(1 + tmp)
			push!(ind,a)
		end
	end
	return (row,col,ind)
end
function stokes2mats(s::Array{Stokes,2})
	sz = size(s)
	I = zeros(Int,sz)
	dolp = zeros(Int,sz)
	aop = zeros(Int,sz)
	docp = zeros(Int,sz)
	for i in eachindex(s)
		I[i], dolp[i], aop[i], docp[i] = normalize(Polar(s[i]))
	end
	return (I, dolp, aop, docp)
end
function mats2images(I::Array{Int,2},dolp::Array{Int,2},aop::Array{Int,2},docp::Array{Int,2})
	sz = size(I)
	row,col,ind = colorwheel(round(Int,0.1*mean(sz)))
	buff = round(Int,0.025*mean(sz))
	for (r,c,i) in zip(row,col,ind)
		aop[sz[1] - r - buff, c + buff] = i
	end
	Iimg = ImageCmap(I, linspace(RGB(0,0,0),RGB(1,1,1),N); prop...)
	colorbar = repeat(reshape(collect(round(Int,linspace(N,1,sz[2]))),1,sz[2]),outer = [round(Int,0.1sz[1]),1])
	dolpimg = ImageCmap(cat(1,dolp,colorbar),linspace(RGB(0,0,0),RGB(1,1,1),N); prop...)
	aopimg = convert(Image, ImageCmap(aop,convert(Array{RGB{FixedPointNumbers.UFixed{UInt8,8}},1}, linspace(HSV(0,1,1), HSV(360,1,1),N)); prop...))
	docpimg = ImageCmap(cat(1,docp,colorbar), [linspace(RGB(1,0,0),RGB(0,0,0),Int(N/2)); linspace(RGB(0,0,0), RGB(0,1,0),Int(N/2))]; prop...)
	return (Iimg, dolpimg, aopimg, docpimg)
end

profile(a::Stokes,ψ) = (a.s0 + ψ*a.s1)/(a.s0 - ψ*a.s1)
normalizeprofile(x,Sp) = (x - 1/Sp)/(Sp - 1/Sp)

function stokes2profile(s::Array{Stokes,2}, Sp::Number)
	sz = size(s)
	p = zeros(Int,sz)
	ψ = (Sp - 1)/(Sp + 1)
	for i in eachindex(s)
		p[i] = isnan(s[i].s0) ? rand(1:N) : roundit(normalizeprofile(profile(s[i],ψ),Sp))
	end
	colorbar = repeat(reshape(collect(round(Int,linspace(N,1,sz[2]))),1,sz[2]),outer = [round(Int,0.1sz[1]),1])
	ImageCmap(cat(1,p,colorbar),linspace(RGB(0,0,0),RGB(1,1,1),N); prop...)
end

import Base: /, *
immutable Centimeter
	x::Real
end
*(x::Centimeter,y::Real) = x.x*y
*(y::Real,x::Centimeter) = x*y
/(x::Centimeter,y::Real) = x.x/y
/(y::Real,x::Centimeter) = x/y
immutable Pixel
	x::Integer
end
*(x::Pixel,y::Real) = x.x*y
*(y::Real,x::Pixel) = x*y
/(x::Pixel,y::Real) = x.x/y
/(y::Real,x::Pixel) = x/y
immutable Length
	cm::Centimeter
	px::Pixel
end
Length(x::Centimeter,RATIO::Real) = Length(x,Pixel(round(Int,x/RATIO)))
Length(x::Pixel,RATIO::Real) = Length(Centimeter(x*RATIO),x)
findwindow(x::Real,RATIO::Real) = round(x/2/RATIO)*RATIO
function sense(el::PolEllipse,window::Length,maxpol::Real)
	p = round(el.polarization/maxpol,4)
	a = round(window.cm.x*el.a/maxpol,4)
	b = round(window.cm.x*el.b/maxpol,4)
	theta = round(rad2deg(el.angle),4)
	color = el.lefthand ? "dolcpcolor" : "dorcpcolor"
	return (color,p,theta,a,b)
end
function point2measurement(p1::Int,p2::Int,s::Array{Stokes,2},w::Int)
	m = Stokes(0.,0.,0.,0.)
	for j = -w:w, k = -w:w
		tmp = s[p1 + j, p2 + k]
		m += isnan(tmp.s0) ? Stokes(0.,0.,0.,0.) : tmp
	end
	return m/4*w*w
end
function grid(s::Array{Stokes,2},sz::Tuple{Length,Length},window::Length,maxpol::Real,RATIO::Real, minpol::Real, eccentricity::Real, name::AbstractString, linearcolor::RGB)
	open("ellipse.tikz","w") do o
		println(o,"""\\begin{tikzpicture}
			\\begin{axis}[xmin=0,xmax=1,ymin=0,ymax=1, ticks=none, axis lines=none, width = $(sz[1].cm.x) cm, height = $(sz[2].cm.x) cm, scale only axis]
				\\addplot graphics [xmin=0,xmax=1,ymin=0,ymax=1]{$name.jpg};
			\\end{axis}""")

		for i = (1 + window.px.x):2window.px.x:(sz[1].px.x - window.px.x), j = (1 + window.px.x):2window.px.x:(sz[2].px.x - window.px.x)
			ss = point2measurement(i,sz[2].px.x - j + 1,s,window.px.x)
			isnan(ss.s0) && continue
			el = PolEllipse(ss)
			el.polarization < minpol && continue
			#isnan(el.a) < minpol && continue
			color,p,theta,a,b = sense(el,window,maxpol)
			x = RATIO*(i - .5)
			y = RATIO*(j - .5)
			#y = RATIO*(sz[2].px.x - (j - 1.5))
			if a/b < eccentricity
				x1 = x + b*cosd(theta + 180 + 90)
				y1 = y + b*sind(theta + 180 + 90)
				x2 = x1 + 2b*cosd(theta + 90)
				y2 = y1 + 2b*sind(theta + 90)
				#println(o,"\\draw [$color,draw opacity = $p] ($x1 cm,$y1 cm) -- ($x2 cm, $y2 cm);")
				println(o,"\\draw [dolpcolor] ($x1 cm,$y1 cm) -- ($x2 cm, $y2 cm);")
			else
				#println(o,"\\draw [$color,draw opacity = $p] ($x cm,$y cm) circle [x radius=$a cm, y radius=$b cm, rotate=$theta];")
				println(o,"\\draw [$color] ($x cm,$y cm) circle [x radius=$a cm, y radius=$b cm, rotate=$theta];")
			end
		end
		println(o,"""\\draw[step = $(2window.cm.x) cm, gray, very thin, draw opacity=0.25] (0,0) grid ($(sz[1].cm.x) cm, $(sz[2].cm.x) cm);
	\\end{tikzpicture}""")
	end
end
function stokes2ellipse(s::Array{Stokes,2}; dpi::Int = 96, window_in_cm::Real = 0.15, maxpol::Real = 1, minpol::Real = 0, eccentricity::Real = 0.01, name = "RGB", linearcolor::RGB = RGB(0.0,1.0,0.0))
	SZ = size(s)
	const RATIO = 2.54/dpi
	sz = (Length(Pixel(SZ[1]),RATIO),Length(Pixel(SZ[2]),RATIO))
	window = Length(Centimeter(findwindow(window_in_cm,RATIO)),RATIO)
	grid(s,sz,window,maxpol,RATIO, minpol, eccentricity,name,linearcolor)
end

end


export flow, flowdθ, flowdσ, flowdψ, STARTING_σ_ψ, STARTING_log_ogip, STARTING_t

const STARTING_σ_ψ      = 0x1.baddbb87af68ap-2 # = 0.432
const STARTING_log_ogip = 0x1.670bf3d5b282dp-1 # = 0.701
const STARTING_t = 2*0.042/(2016-2003)

# functions in case we have months to expiration
@inline flow(  FF::Type, θ::AbstractVector{T}, σ::T, z::NTuple{N,T}, ψ::T,             d::Integer, d1::Integer, Dgt0::Bool, sgn_ext::Bool, τ::Real, geoid::Real, roy::T) where {N,T} = flow(  FF, θ, σ, z, ψ,    d, d1, Dgt0, sgn_ext, geoid, roy)
@inline flowdθ(FF::Type, θ::AbstractVector{T}, σ::T, z::NTuple{N,T}, ψ::T, k::Integer, d::Integer, d1::Integer, Dgt0::Bool, sgn_ext::Bool, τ::Real, geoid::Real, roy::T) where {N,T} = flowdθ(FF, θ, σ, z, ψ, k, d, d1, Dgt0, sgn_ext, geoid, roy)
@inline flowdσ(FF::Type, θ::AbstractVector{T}, σ::T, z::NTuple{N,T}, ψ::T,             d::Integer, d1::Integer, Dgt0::Bool, sgn_ext::Bool, τ::Real, geoid::Real, roy::T) where {N,T} = flowdσ(FF, θ, σ, z, ψ,    d,                    geoid, roy)
@inline flowdψ(FF::Type, θ::AbstractVector{T}, σ::T, z::NTuple{N,T}, ψ::T,             d::Integer,                          sgn_ext::Bool, τ::Real, geoid::Real, roy::T) where {N,T} = flowdψ(FF, θ, σ, z, ψ,    d,           sgn_ext, geoid, roy)
@inline flowdψ(FF::Type, θ::AbstractVector{T}, σ::T, z::NTuple{N,T}, ψ::T,             d::Integer, d1::Integer, Dgt0::Bool, sgn_ext::Bool, τ::Real, geoid::Real, roy::T) where {N,T} = flowdψ(FF, θ, σ, z, ψ,    d,           sgn_ext, geoid, roy)

@inline flowdσ(FF::Type, θ::AbstractVector{T}, σ::T, z::NTuple{N,T}, ψ::T,             d::Integer,                                                  geoid::Real, roy::T) where {N,T} = flowdσ(FF, θ, σ, z, ψ,    d,                    geoid, roy)

# functions in case we have months to expiration
@inline flow(  FF::Type, wp::AbstractUnitProblem, i::Integer, θ::AbstractVector{T}, σ::T, z::NTuple{N,T}, ψ::T,             d::Integer, geoid::Real, roy::T) where {N,T} = flow(  FF, θ, σ, z, ψ,    d, _d1(wp,i), _Dgt0(wp,i), _sgnext(wp,i), geoid, roy)
@inline flowdθ(FF::Type, wp::AbstractUnitProblem, i::Integer, θ::AbstractVector{T}, σ::T, z::NTuple{N,T}, ψ::T, k::Integer, d::Integer, geoid::Real, roy::T) where {N,T} = flowdθ(FF, θ, σ, z, ψ, k, d, _d1(wp,i), _Dgt0(wp,i), _sgnext(wp,i), geoid, roy)
@inline flowdσ(FF::Type, wp::AbstractUnitProblem, i::Integer, θ::AbstractVector{T}, σ::T, z::NTuple{N,T}, ψ::T,             d::Integer, geoid::Real, roy::T) where {N,T} = flowdσ(FF, θ, σ, z, ψ,    d,                                        geoid, roy)
@inline flowdψ(FF::Type, wp::AbstractUnitProblem, i::Integer, θ::AbstractVector{T}, σ::T, z::NTuple{N,T}, ψ::T,             d::Integer, geoid::Real, roy::T) where {N,T} = flowdψ(FF, θ, σ, z, ψ,    d,                         _sgnext(wp,i), geoid, roy)

# @inline function flowdθ(::Type, θ::AbstractVector{T}, σ::T, z::NTuple{N,T}, ψ::T, k::Integer, d::Integer, d1::Integer, Dgt0::Bool, sgn_ext::Bool, geoid::Real, roy::T)::T where {N,T}

# --------------------------- common revenue functions & derivatives  --------------------------------------

@inline    rev_exp_restricted(θ1::T, σ::T, logp::Real,          ψ::Real, Dgt0::Bool, geoid::Real, roy::Real) where {T} = rev_exp(   1, θ1, 1, STARTING_log_ogip, STARTING_σ_ψ,             σ, logp,    ψ, Dgt0, geoid, roy)
@inline drevdσ_exp_restricted(θ1::T, σ::T, logp::Real,          ψ::Real,             geoid::Real, roy::Real) where {T} = drevdσ_exp(1, θ1, 1, STARTING_log_ogip, STARTING_σ_ψ,             σ, logp,    ψ,       geoid, roy)
@inline drevdψ_exp_restricted(θ1::T, σ::T, logp::Real,          ψ::Real,             geoid::Real, roy::Real) where {T} = drevdψ_exp(1, θ1, 1, STARTING_log_ogip, STARTING_σ_ψ,             σ, logp,    ψ,       geoid, roy)

@inline    rev_exp_restricted(θ1::T, σ::T, logp::Real, t::Real, ψ::Real, Dgt0::Bool, geoid::Real, roy::Real) where {T} = rev_exp(   1, θ1, 1, STARTING_log_ogip, STARTING_σ_ψ, STARTING_t, σ, logp, t, ψ, Dgt0, geoid, roy)
@inline drevdσ_exp_restricted(θ1::T, σ::T, logp::Real, t::Real, ψ::Real,             geoid::Real, roy::Real) where {T} = drevdσ_exp(1, θ1, 1, STARTING_log_ogip, STARTING_σ_ψ, STARTING_t, σ, logp, t, ψ,       geoid, roy)
@inline drevdψ_exp_restricted(θ1::T, σ::T, logp::Real, t::Real, ψ::Real,             geoid::Real, roy::Real) where {T} = drevdψ_exp(1, θ1, 1, STARTING_log_ogip, STARTING_σ_ψ, STARTING_t, σ, logp, t, ψ,       geoid, roy)

# chebshev polynomials
# See http://www.aip.de/groups/soe/local/numres/bookcpdf/c5-8.pdf
@inline checkinterval(x::Real,min::Real,max::Real) =  min <= x <= max || throw(DomainError("x = $x must be in [$min,$max]"))
@inline checkinterval(x::Real) = checkinterval(x,-1,1)
@inline cheb0(x::Real) = (checkinterval(x); return one(Float64))
@inline cheb1(x::Real) = (checkinterval(x); return x)
@inline cheb2(x::Real) = (checkinterval(x); return 2*x^2 - 1)
@inline cheb3(x::Real) = (checkinterval(x); return 4*x^3 - 3*x)
@inline cheb4(x::Real) = (checkinterval(x); return 8*(x^4 - x^2) + 1)

# -----------------------------------------
# Flows
# -----------------------------------------

include("flow-payoffs-more.jl")

@inline function flow(::Type{Val{:cheb3_cost_tech}}, wp::AbstractUnitProblem, i::Integer, θ::AbstractVector{T}, σ::T, z::NTuple{N,T}, ψ::T, d::Integer, geoid::Real, roy::T) where {N,T}
# @inline function flow(::Type{Val{:cheb3_cost_tech}}, θ::AbstractVector{T}, σ::T, z::NTuple{N,T}, ψ::T, d::Integer, d1::Integer, Dgt0::Bool, sgn_ext::Bool, geoid::Real, roy::T) where {N,T}
    if d == 0
        _sgnext(wp,i) && return θ[11]
        return zero(T)
    end
    logp, logc, t = z
    u = rev_exp(1,θ[1],1,θ[2],θ[3],θ[4],σ, logp, t, ψ, _Dgt0(wp,i), geoid,roy) + (d==1 ? θ[5] : θ[6] )*cheb0(t) + θ[7]*cheb1(t) + θ[8]*cheb2(t) + θ[9]*cheb3(t) + θ[10]*exp(logc)
    d>1 && (u *= d)
    return u::T
end

@inline function flow(::Type{Val{:cheb3_cost_tech_restr}}, θ::AbstractVector{T}, σ::T, z::NTuple{N,T}, ψ::T, d::Integer, d1::Integer, Dgt0::Bool, sgn_ext::Bool, geoid::Real, roy::T) where {N,T}
    if d == 0
        sgn_ext && return θ[8]
        return zero(T)
    end
    logp, logc, t = z
    u = rev_exp_restricted(θ[1], σ, logp, t, ψ, Dgt0, geoid, roy) + (d==1 ? θ[2] : θ[3] )*cheb0(t) + θ[4]*cheb1(t) + θ[5]*cheb2(t) + θ[6]*cheb3(t) + θ[7]*exp(logc)
    d>1 && (u *= d)
    return u::T
end

# -----------------------------------------
# number of parms
# -----------------------------------------

function number_of_model_parms(FF::Symbol)::Int
    FF ∈ (:one_restr,)                                                           && return  3
    FF ∈ (:dgt1_restr,:Dgt0_restr,)                                              && return  4
    FF ∈ (:one,:dgt1_ext_restr, :dgt1_d1_restr,:dgt1_cost_restr,:cheb2_restr)   && return  5
    FF ∈ (:dgt1,:Dgt0,:dgt1_cost_Dgt0_restr,:dgt1_pricecost_restr,:cheb3_restr) && return  6
    FF ∈ (:dgt1_ext,:dgt1_d1,:dgt1_cost,:dgt1_pricebreak_restr,:cheb2, :cheb3_dgt1_restr,)  && return  7
    FF ∈ (:dgt1_cost_Dgt0,:dgt1_pricecost,:cheb3,:cheb3_cost_restr,:ttbuild_cost_restr,:cheb3_cost_tech_restr,)  && return  8
    FF ∈ (:dgt1_pricebreak,:cheb3_dgt1,)                                         && return  9
    FF ∈ (:cheb3_cost,:ttbuild_cost,)                                            && return 10
    FF ∈ (:cheb3_cost_tech,)                                                     && return 11
    throw(error("FF = $(FF) not recognized"))
end

# -----------------------------------------
# dθ
# -----------------------------------------

@inline function flowdθ(::Type{Val{:cheb3_cost_tech}}, wp::AbstractUnitProblem, i::Integer, θ::AbstractVector{T}, σ::T, z::NTuple{N,T}, ψ::T, k::Integer, d::Integer, geoid::Real, roy::T) where {N,T}
# @inline function flowdθ(::Type{Val{:cheb3_cost_tech}}, θ::AbstractVector{T}, σ::T,     z::NTuple{N,T}, ψ::T, k::Integer,d::Integer,           d1::Integer, Dgt0::Bool, sgn_ext::Bool, geoid::Real, roy::T)::T where {N,T}
    d == 0 && !_sgnext(wp,i) && return zero(T)
    Dgt0 = _Dgt0(wp,i)
    logp, logc, t = z

    # revenue
    k == 1  && return   d * rev_exp(1,θ[1],1,θ[2],θ[3],θ[4],σ,logp,t,ψ,Dgt0,geoid,roy)
    k == 2  && return   d * rev_exp(1,θ[1],1,θ[2],θ[3],θ[4],σ,logp,t,ψ,Dgt0,geoid,roy) * geoid
    k == 3  && return   d * rev_exp(1,θ[1],1,θ[2],θ[3],θ[4],σ,logp,t,ψ,Dgt0,geoid,roy) * ( Dgt0 ? ψ : ψ*_ρ(σ) + θ[k]*(1-_ρ2(σ)))
    k == 4  && return   d * rev_exp(1,θ[1],1,θ[2],θ[3],θ[4],σ,logp,t,ψ,Dgt0,geoid,roy) * t

    # drilling cost
    k == 5   && return d != 1 ? zero(T) : T(   cheb0(t) )
    k == 6   && return d <= 1 ? zero(T) : T( d*cheb0(t) )
    k == 7   && return d == 0 ? zero(T) : T( d*cheb1(t) )
    k == 8   && return d == 0 ? zero(T) : T( d*cheb2(t) )
    k == 9   && return d == 0 ? zero(T) : T( d*cheb3(t) )
    k == 10  && return d == 0 ? zero(T) : T( d*exp(logc) )

    # extension cost
    k == 11  && return d == 0 && _sgnext(wp,i) ? one(T) : zero(T)

    throw(error("$k out of bounds"))
end

@inline function flowdθ(::Type{Val{:cheb3_cost_tech_restr}}, θ::AbstractVector{T}, σ::T,     z::NTuple{N,T}, ψ::T, k::Integer,d::Integer,           d1::Integer, Dgt0::Bool, sgn_ext::Bool, geoid::Real, roy::T)::T where {N,T}
    d == 0 && !sgn_ext && return zero(T)

    logp, logc, t = z

    # revenue
    k == 1  && return   d * rev_exp_restricted(θ[1], σ, logp, t, ψ, Dgt0, geoid, roy)

    # drilling cost
    k == 2  && return d != 1 ? zero(T) : T(   cheb0(t) )
    k == 3  && return d <= 1 ? zero(T) : T( d*cheb0(t) )
    k == 4  && return d == 0 ? zero(T) : T( d*cheb1(t) )
    k == 5  && return d == 0 ? zero(T) : T( d*cheb2(t) )
    k == 6  && return d == 0 ? zero(T) : T( d*cheb3(t) )
    k == 7  && return d == 0 ? zero(T) : T( d*exp(logc) )
    # extension cost
    k == 8  && return d == 0 && sgn_ext ? one(T) : zero(T)

    throw(error("$k out of bounds"))
end

# -----------------------------------------
# dσ
# -----------------------------------------

@inline function flowdσ(::FF, θ::AbstractVector{T}, σ::T, z::NTuple{N,T}, ψ::T, d::Integer, geoid::Real, roy::T)::T where {FF <: Union{ Type{Val{:one}}, Type{Val{:dgt1}}, Type{Val{:Dgt0}}, Type{Val{:dgt1_ext}}, Type{Val{:dgt1_d1}}, Type{Val{:dgt1_cost}}, Type{Val{:dgt1_cost_Dgt0}}, Type{Val{:cheb2}}, Type{Val{:cheb3}}, Type{Val{:cheb3_dgt1}}, Type{Val{:cheb3_cost}}, Type{Val{:ttbuild_cost}} }, N, T}
    d == 0 && return zero(T)
    return d * drevdσ_exp(1,θ[1],1,θ[2],θ[3],σ,z[1],ψ,geoid,roy)
end

@inline function flowdσ(::FF, θ::AbstractVector{T}, σ::T, z::NTuple{N,T},  ψ::T, d::Integer, geoid::Real, roy::T)::T where {FF <: Union{ Type{Val{:one_restr}}, Type{Val{:dgt1_restr}}, Type{Val{:Dgt0_restr}}, Type{Val{:dgt1_ext_restr}}, Type{Val{:dgt1_d1_restr}}, Type{Val{:dgt1_cost_restr}}, Type{Val{:dgt1_cost_Dgt0_restr}}, Type{Val{:cheb2_restr}}, Type{Val{:cheb3_restr}}, Type{Val{:cheb3_dgt1_restr}}, Type{Val{:cheb3_cost_restr}}, Type{Val{:ttbuild_cost_restr}} },N, T}
    d == 0 && return zero(T)
    return d * drevdσ_exp_restricted(θ[1],σ,z[1],ψ,geoid,roy)
end

@inline function flowdσ(::FF, wp::AbstractUnitProblem, i::Integer, θ::AbstractVector{T}, σ::T, z::NTuple{N,T}, ψ::T, d::Integer, geoid::Real, roy::T)::T where {FF <: Union{ Type{Val{:cheb3_cost_tech}} }, N, T}
    d == 0 && return zero(T)
    return d * drevdσ_exp(1,θ[1],1,θ[2],θ[3],θ[4],σ,first(z),last(z),ψ,geoid,roy)
end

@inline function flowdσ(::FF, θ::AbstractVector{T}, σ::T, z::NTuple{N,T},  ψ::T, d::Integer, geoid::Real, roy::T)::T where {FF <: Union{ Type{Val{:cheb3_cost_tech_restr}}  },N, T}
    d == 0 && return zero(T)
    return d * drevdσ_exp_restricted(θ[1],σ,first(z),last(z),ψ,geoid,roy)
end

# -----------------------------------------
# dψ
# -----------------------------------------

@inline function flowdψ(::FF, θ::AbstractVector{T}, σ::T, z::NTuple{N,T}, ψ::T, d::Integer, sgn_ext::Bool, geoid::Real, roy::T) where {FF <: Union{ Type{Val{:one}}, Type{Val{:dgt1}}, Type{Val{:Dgt0}}, Type{Val{:dgt1_d1}}, Type{Val{:dgt1_cost}}, Type{Val{:dgt1_cost_Dgt0}}, Type{Val{:cheb2}}, Type{Val{:cheb3}}, Type{Val{:cheb3_dgt1}}, Type{Val{:cheb3_cost}}, Type{Val{:ttbuild_cost}} },N, T}
    d == 0  && return zero(T) # sgn_ext ? θ[10] : zero(T)
    return (d * drevdψ_exp(1,θ[1],1,θ[2],θ[3],σ,z[1],ψ,geoid,roy))::T
end

@inline function flowdψ(::FF, θ::AbstractVector{T}, σ::T, z::NTuple{N,T}, ψ::T, d::Integer, sgn_ext::Bool, geoid::Real, roy::T) where {FF <: Union{ Type{Val{:one_restr}}, Type{Val{:dgt1_restr}}, Type{Val{:Dgt0_restr}}, Type{Val{:dgt1_d1_restr}}, Type{Val{:dgt1_cost_restr}}, Type{Val{:dgt1_cost_Dgt0_restr}}, Type{Val{:cheb2_restr}}, Type{Val{:cheb3_restr}}, Type{Val{:cheb3_dgt1_restr}}, Type{Val{:cheb3_cost_restr}}, Type{Val{:ttbuild_cost_restr}} },N, T}
    d == 0  && return zero(T) # sgn_ext ? θ[10] : zero(T)
    return (d * drevdψ_exp_restricted(θ[1],σ,z[1],ψ,geoid,roy))::T
end

# @inline function flow(  FF::Type, wp::AbstractUnitProblem, i::Integer, θ::AbstractVector{T}, σ::T, z::NTuple{N,T}, ψ::T,             d::Integer, geoid::Real, roy::T)
# @inline function flowdθ(FF::Type, wp::AbstractUnitProblem, i::Integer, θ::AbstractVector{T}, σ::T, z::NTuple{N,T}, ψ::T, k::Integer, d::Integer, geoid::Real, roy::T)
# @inline function flowdσ(FF::Type, wp::AbstractUnitProblem, i::Integer, θ::AbstractVector{T}, σ::T, z::NTuple{N,T}, ψ::T,             d::Integer, geoid::Real, roy::T)
# @inline function flowdψ(FF::Type, wp::AbstractUnitProblem, i::Integer, θ::AbstractVector{T}, σ::T, z::NTuple{N,T}, ψ::T,             d::Integer, geoid::Real, roy::T)

@inline function flowdψ(::FF, wp::AbstractUnitProblem, i::Integer, θ::AbstractVector{T}, σ::T, z::NTuple{N,T}, ψ::T, d::Integer, geoid::Real, roy::T) where {FF <: Union{ Type{Val{:cheb3_cost_tech}} }, N, T}
    d == 0  && return zero(T) # _sgnext(wp,i) ? θ[10] : zero(T)
    return (d * drevdψ_exp(1,θ[1],1,θ[2],θ[3],θ[4],σ,first(z),last(z),ψ,geoid,roy))::T
end

@inline function flowdψ(::FF, θ::AbstractVector{T}, σ::T, z::NTuple{N,T}, ψ::T, d::Integer, sgn_ext::Bool, geoid::Real, roy::T) where {FF <: Union{ Type{Val{:cheb3_cost_tech_restr}} }, N, T}
    d == 0  && return zero(T) # sgn_ext ? θ[10] : zero(T)
    return (d * drevdψ_exp_restricted(θ[1],σ,first(z),last(z),ψ,geoid,roy))::T
end