module Datasets
import ..Cubes.Axes: axsym, axname, CubeAxis, findAxis, CategoricalAxis, RangeAxis, caxes
import ..Cubes: YAXArray, concatenatecubes, CleanMe
using ...YAXArrays: YAXArrays, YAXDefaults
using DataStructures: OrderedDict, counter
using Dates: Day,Hour,Minute,Second,Month,Year, Date, DateTime, TimeType
using IntervalSets: Interval, (..)
using CFTime: timedecode, timeencode, DateTimeNoLeap, DateTime360Day, DateTimeAllLeap
using YAXArrayBase
using YAXArrayBase: iscontdimval
using DiskArrayTools: CFDiskArray, ConcatDiskArray
using Glob: glob

struct Dataset
    cubes::OrderedDict{Symbol,YAXArray}
    axes::Dict{Symbol,CubeAxis}
end
function Dataset(;cubesnew...)
    axesall = Set{CubeAxis}()
    foreach(values(cubesnew)) do c
        ax = caxes(c)
        foreach(a->push!(axesall,a),ax)
    end
    axesall = collect(axesall)
    axnameall = axsym.(axesall)
    axesnew = Dict{Symbol,CubeAxis}(axnameall[i]=>axesall[i] for i in 1:length(axesall))
    Dataset(OrderedDict(cubesnew), axesnew)
end

function Base.show(io::IO,ds::Dataset)
    println(io,"YAXArray Dataset")
    println(io,"Dimensions: ")
    foreach(a->println(io,"   ", a),values(ds.axes))
    print(io,"Variables: ")
    foreach(i->print(io,i," "),keys(ds.cubes))
end
function Base.propertynames(x::Dataset, private::Bool=false)
    if private
        Symbol[:cubes; :axes; collect(keys(x.cubes)); collect(keys(x.axes))]
    else
        Symbol[collect(keys(x.cubes)); collect(keys(x.axes))]
    end
end
function Base.getproperty(x::Dataset,k::Symbol)
    if k === :cubes
        return getfield(x,:cubes)
    elseif k === :axes
        return getfield(x,:axes)
    else
        x[k]
    end
end
Base.getindex(x::Dataset,i::Symbol) = haskey(x.cubes, i) ? x.cubes[i] : haskey(x.axes,i) ? x.axes[i] : throw(ArgumentError("$i not found in Dataset"))
function Base.getindex(x::Dataset,i::Vector{Symbol})
  cubesnew = [j=>x.cubes[j] for j=i]
    Dataset(;cubesnew...)
end

function fuzzyfind(s::String,comp::Vector{String})
    sl = lowercase(s)
    f = findall(i->startswith(lowercase(i),sl),comp)
    if length(f) != 1
        throw(KeyError("Name $s not found"))
    else
        f[1]
    end
end
function Base.getindex(x::Dataset,i::Vector{String})
    istr = string.(keys(x.cubes))
    ids  = map(name->fuzzyfind(name,istr),i)
    syms   = map(j->Symbol(istr[j]),ids)
    cubesnew = [Symbol(i[j])=>x.cubes[syms[j]] for j=1:length(ids)]
    Dataset(;cubesnew...)
end
Base.getindex(x::Dataset,i::String)=getproperty(x,Symbol(i))
function subsetcube(x::Dataset; var=nothing, kwargs...)
    if var ===nothing
        cc = x.cubes
        Dataset(;map(ds->ds=>subsetcube(cc[ds];kwargs...),collect(keys(cc)))...)
    elseif isa(var,String) || isa(var, Symbol)
        subsetcube(getproperty(x,Symbol(var));kwargs...)
    else
        cc = x[var].cubes
        Dataset(;map(ds->ds=>subsetcube(cc[ds];kwargs...),collect(keys(cc)))...)
    end
end
function collectdims(g)
  dlist = Set{Tuple{String,Int,Int}}()
  varnames = get_varnames(g)
  foreach(varnames) do k
    d = get_var_dims(g,k)
    v = get_var_handle(g,k)
    for (len,dname) in zip(size(v),d)
      if !occursin("bnd",dname) && !occursin("bounds",dname)
        datts = get_var_attrs(g,dname)
        offs = get(datts,"_ARRAY_OFFSET",0)
        push!(dlist,(dname,offs,len))
      end
    end
  end
  outd = Dict(d[1] => (ax = toaxis(d[1],g,d[2],d[3]), offs = d[2]) for d in dlist)
  length(outd)==length(dlist) || throw(ArgumentError("All Arrays must have the same offset"))
  outd
end

function toaxis(dimname,g,offs,len)
    axname = dimname
    if !haskey(g,dimname)
      return RangeAxis(dimname, 1:len)
    end
    ar = g[dimname]
    aratts = get_var_attrs(g,dimname)
    if uppercase(axname)=="TIME" && haskey(aratts,"units")
        tsteps = timedecode(ar[:],aratts["units"],get(aratts,"calendar","standard"))
        RangeAxis(dimname,tsteps[offs+1:end])
    elseif haskey(aratts,"_ARRAYVALUES")
      vals = identity.(aratts["_ARRAYVALUES"])
      CategoricalAxis(axname,vals)
    else
      axdata = testrange(ar[offs+1:end])
      if eltype(axdata)<:AbstractString || (!issorted(axdata) && !issorted(axdata,rev=true))
        CategoricalAxis(axname,axdata)
      else
        RangeAxis(axname,axdata)
      end
    end
end
propfromattr(attr) = Dict{String,Any}(filter(i->i[1]!="_ARRAY_DIMENSIONS",attr))

"Test if data in x can be approximated by a step range"
function testrange(x)
  r = range(first(x),last(x),length=length(x))
  all(i->isapprox(i...),zip(x,r)) ? r : x
end

testrange(x::AbstractArray{<:AbstractString}) = x

_glob(x) = startswith(x,"/") ? glob(x[2:end], "/") : glob(x)

open_mfdataset(g::AbstractString; kwargs...) = open_mfdataset(_glob(g); kwargs...)
open_mfdataset(g::Vector{<:AbstractString};kwargs...) =
  merge_datasets(map(i->open_dataset(i;kwargs...), g))

function open_dataset(g; driver = :all)
  g = to_dataset(g, driver=driver)
  isempty(get_varnames(g)) && throw(ArgumentError("Group does not contain datasets."))
  dimlist = collectdims(g)
  dnames  = string.(keys(dimlist))
  varlist = filter(get_varnames(g)) do vn
    upname = uppercase(vn)
    !occursin("BNDS",upname) && !occursin("BOUNDS",upname) && !any(i->isequal(upname,uppercase(i)),dnames)
  end
  allcubes = OrderedDict{Symbol,YAXArray}()
  for vname in  varlist
    vardims = get_var_dims(g,vname)
    iax = [dimlist[vd].ax for vd in vardims]
    offs = [dimlist[vd].offs for vd in vardims]
    subs = if all(iszero,offs)
      nothing
    else
      ntuple(i->(offs[i]+1):(offs[i]+length(iax[i])),length(offs))
    end
    ar = get_var_handle(g,vname)
    att = get_var_attrs(g,vname)
    if subs !== nothing
      ar = view(ar,subs...)
    end
    allcubes[Symbol(vname)] = YAXArray(iax,ar,propfromattr(att), cleaner=CleanMe[])
  end
  sdimlist = Dict(Symbol(k)=>v.ax for (k,v) in dimlist)
  Dataset(allcubes,sdimlist)
end
Base.getindex(x::Dataset;kwargs...) = subsetcube(x;kwargs...)
YAXDataset(;kwargs...) = Dataset(YAXArrays.YAXDefaults.cubedir[];kwargs...)

function Cube(ds::Dataset; joinname="Variable")

  dl = collect(keys(ds.axes))
  dls = string.(dl)
  length(ds.cubes)==1 && return first(values(ds.cubes))
  #TODO This is an ugly workaround to merge cubes with different element types,
  # There should bde a more generic solution
  eltypes = map(eltype,values(ds.cubes))
  majtype = findmax(counter(eltypes))[2]
  newkeys = Symbol[]
  for k in keys(ds.cubes)
    c = ds.cubes[k]
    if all(axn->findAxis(axn,c)!==nothing,dls) && eltype(c)==majtype
      push!(newkeys,k)
    end
  end
  if length(newkeys)==1
    return ds.cubes[first(newkeys)]
  else
    varax = CategoricalAxis(joinname, string.(newkeys))
    return concatenatecubes([ds.cubes[k] for k in newkeys], varax)
  end
end

"""
    function createdataset(DS::Type,axlist; kwargs...)

Creates a new datacube with axes specified in `axlist`. Each axis must be a subtype
of `CubeAxis`. A new empty Zarr array will be created and can serve as a sink for
`mapCube` operations.

### Keyword arguments

* `folder=tempname()` location where the new cube is stored
* `T=Union{Float32,Missing}` data type of the target cube
* `chunksize = ntuple(i->length(axlist[i]),length(axlist))` chunk sizes of the array
* `chunkoffset = ntuple(i->0,length(axlist))` offsets of the chunks
* `persist::Bool=true` shall the disk data be garbage-collected when the cube goes out of scope?
* `overwrite::Bool=false` overwrite cube if it already exists
* `properties=Dict{String,Any}()` additional cube properties
* `fillvalue= T>:Missing ? defaultfillval(Base.nonmissingtype(T)) : nothing` fill value
* `datasetaxis="Variable"` special treatment of a categorical axis that gets written into separate zarr arrays
"""
function createdataset(DS, axlist;
  path="",
  persist = nothing,
  T=Union{Float32,Missing},
  chunksize = ntuple(i->length(axlist[i]),length(axlist)),
  chunkoffset = ntuple(i->0,length(axlist)),
  overwrite::Bool=false,
  properties=Dict{String,Any}(),
  datasetaxis = "Variable",
  kwargs...
  )
  if persist === nothing
    persist = !isempty(path)
  end
  path = getsavefolder(path, persist)
  check_overwrite(path, overwrite)
  splice_generic(x::AbstractArray,i) = [x[1:(i-1)];x[(i+1:end)]]
  splice_generic(x::Tuple,i)         = (x[1:(i-1)]...,x[(i+1:end)]...)
  myar = create_empty(DS, path)
  finalperm = nothing
  idatasetax = datasetaxis === nothing ? nothing : findAxis(datasetaxis,axlist)
  if idatasetax !== nothing
    groupaxis = axlist[idatasetax]
    axlist = splice_generic(axlist,idatasetax)
    chunksize = splice_generic(chunksize,idatasetax)
    chunkoffset = splice_generic(chunkoffset,idatasetax)
    finalperm = ((1:idatasetax-1)..., length(axlist)+1, (idatasetax:length(axlist))...)
  else
    groupaxis = nothing
  end
  foreach(axlist,chunkoffset) do ax,co
    arrayfromaxis(myar,ax,co)
  end
  attr = properties
  s = map(length,axlist) .+ chunkoffset
  if all(iszero,chunkoffset)
    subs = nothing
  else
    subs = ntuple(length(axlist)) do i
      (chunkoffset[i]+1):(length(axlist[i].values)+chunkoffset[i])
    end
  end
  if groupaxis===nothing
    cubenames = ["layer"]
  else
    cubenames = groupaxis.values
  end
  cleaner = CleanMe[]
  persist || push!(cleaner,CleanMe(path,false))
  allcubes = map(cubenames) do cn
    axnames = map(axname,axlist)
    axlengths = map(i->length(get_var_handle(myar,i)),axnames)
    v = if allow_missings(myar) || !(T>:Missing)
      add_var(myar, T, cn, axlengths, axnames, attr; chunksize = chunksize, kwargs...)
    else
      S = Base.nonmissingtype(T)
      if !haskey(attr,"missing_value")
        attr["missing_value"] = YAXArrayBase.defaultfillval(S)
      end
      v = add_var(myar, S, cn, axlengths, axnames, attr; chunksize = chunksize, kwargs...)
      CFDiskArray(v,attr)
    end
    if subs !== nothing
      v = view(v,subs...)
    end
    YAXArray(axlist,v,propfromattr(attr),cleaner=cleaner)
  end
  if groupaxis===nothing
    return allcubes[1],allcubes[1]
  else
    cube = concatenatecubes(allcubes,groupaxis)
    return permutedims(cube, finalperm), cube
  end
end

function getsavefolder(name,persist)
  if isempty(name)
    name = persist ? split(tempname(),"/")[end] : tempname()[2:end]
    joinpath(YAXDefaults.workdir[],name)
  else
    occursin("/",name) ? name : joinpath(YAXDefaults.workdir[],name)
  end
end

function check_overwrite(newfolder, overwrite)
  if isdir(newfolder) || isfile(newfolder)
    if overwrite
      rm(newfolder, recursive=true)
    else
      error("$(newfolder) already exists, please pick another name or use `overwrite=true`")
    end
  end
end

function arrayfromaxis(p,ax::CubeAxis,offs)
    data, attr = dataattfromaxis(ax,offs)
    attr["_ARRAY_OFFSET"]=offs
    za = add_var(p, data, axname(ax), (axname(ax),), attr)
    za
end

prependrange(r::AbstractRange,n) = n==0 ? r : range(first(r)-n*step(r),last(r),length=n+length(r))
function prependrange(r::AbstractVector,n)
  if n==0
    return r
  else
    step = r[2]-r[1]
    first = r[1] - step*n
    last = r[1] - step
    radd = range(first,last,length=n)
    return [radd;r]
  end
end

defaultcal(::Type{<:TimeType}) = "standard"
defaultcal(::Type{<:DateTimeNoLeap}) = "noleap"
defaultcal(::Type{<:DateTimeAllLeap}) = "allleap"
defaultcal(::Type{<:DateTime360Day}) = "360_day"

datetodatetime(vals::AbstractArray{<:Date}) = DateTime.(vals)
datetodatetime(vals) = vals

function dataattfromaxis(ax::CubeAxis,n)
    prependrange(ax.values,n), Dict{String,Any}()
end
# function dataattfromaxis(ax::CubeAxis,n)
#     prependrange(1:length(ax.values),n), Dict{String,Any}("_ARRAYVALUES"=>collect(ax.values))
# end
function dataattfromaxis(ax::CubeAxis{T},n) where T<:TimeType
    data = timeencode(datetodatetime(ax.values),"days since 1980-01-01",defaultcal(T))
    prependrange(data,n), Dict{String,Any}("units"=>"days since 1980-01-01","calendar"=>defaultcal(T))
end

#The good old Cube function:
Cube(s::String;kwargs...) = Cube(open_dataset(s);kwargs...)
function Cube(;kwargs...)
  if !isempty(YAXArrays.YAXDefaults.cubedir[])
    Cube(YAXArrays.YAXDefaults.cubedir[];kwargs...)
  else
    error("A path should be specified")
  end
end

#Defining joins of Datasets
abstract type AxisJoin end
struct AllEqual <: AxisJoin
     ax
end
struct SortedRanges <: AxisJoin
     axlist
     perm
end
blocksize(x::AllEqual) = 1
blocksize(x::SortedRanges) = length(x.axlist)
getperminds(x::AllEqual) = 1:1
getperminds(x::SortedRanges) = x.perm
wholeax(x::AllEqual) = x.ax
wholeax(x::SortedRanges) = reduce(vcat,x.axlist[x.perm])
struct NewDim <: AxisJoin
     newax
end
#Test for a range of categorical axes how to concatenate them
function analyse_axjoin_ranges(dimvallist)
    firstax = first(dimvallist)
    if all(isequal(firstax), dimvallist)
        return AllEqual(firstax)
    end
    revorder = if all(issorted,dimvallist)
        false
    elseif all(i->issorted(i,rev=true),dimvallist)
        true
    else
        error("Dimension values are not sorted")
    end
    function ltfunc(ax1,ax2)
        min1,max1 = extrema(ax1)
        min2,max2 = extrema(ax2)
        if max1 < min2
            return true
        elseif min1 > max2
            return false
        else
            error("Dimension ranges overlap")
        end
    end
    sp = sortperm(dimvallist, rev = revorder, lt = ltfunc)
    SortedRanges(dimvallist, sp)
end
using YAXArrayBase: YAXArrayBase, getdata, getattributes, yaxcreate
function create_mergedict(dimvallist)
    allmerges = Dict{Symbol,Any}()

    for (axn,dimvals) in dimvallist
        iscont = iscontdimval.(dimvals)
        if all(iscont)
            allmerges[axn] = analyse_axjoin_ranges(dimvals)
        elseif any(iscont)
            error("Mix of continous and non-continous values")
        else
            allmerges[axn] = analyse_axjoin_categorical(dimvals)
        end
    end
    allmerges
end
function merge_datasets(dslist)
    allaxnames = counter(Symbol)
    for ds in dslist, k in keys(ds.axes)
        push!(allaxnames,k)
    end
    dimvallist = Dict(ax=>map(i->i.axes[ax].values,dslist) for ax in keys(allaxnames))
    allmerges = create_mergedict(dimvallist)
    repvars = counter(Symbol)
    for ds in dslist, v in keys(ds.cubes)
        push!(repvars,v)
    end
    tomergevars = filter(i->i[2]==length(dslist), repvars)
    mergedvars=Dict{Symbol,Any}()
    for v in keys(tomergevars)
        dn = YAXArrayBase.dimnames(first(dslist)[v])
        howmerge = getindex.(Ref(allmerges), dn)
        sizeblockar = map(blocksize,howmerge)
        perminds = map(getperminds,howmerge)
        @assert length(dslist) == prod(sizeblockar)
        vcol = map(i->getdata(i[v]), dslist)
        allatts = mapreduce(i->getattributes(i[v]),merge,dslist, init=Dict{String,Any}())
        aa = [vcol[i] for (i,_) in enumerate(Iterators.product(perminds...))]
        dvals = map(wholeax, howmerge)
        mergedvars[v] = yaxcreate(YAXArray, ConcatDiskArray(aa), dn, dvals, allatts)
    end
    Dataset(;mergedvars...)
end

end


############################################################################################
"""
$(TYPEDEF)

Default arguments for SMC constructor.

# Fields
$(TYPEDFIELDS)
"""
struct SMCDefault{
    F<:Function,
    T<:BaytesCore.ResamplingMethod,
    B<:BaytesCore.UpdateBool,
    U<:BaytesCore.UpdateBool
}
    "Number of tuning steps used when constructing sampler."
    Ntuning::Int64
    "Threshold for resampling chains."
    resamplingmethod::T
    "Threshold for resampling chains. Set to 1.0 if resampling should always be applied."
    resamplingthreshold::Float64
    "Function that is applied to correlation vector of parameter to determine if jittering can be stopped."
    jitterfun::F
    "Boolean if fixed number of jittersteps are applied or jittering is based on parameter correlation."
    jitteradaption::B
    "Stopping threshold against mean correlation of jittered model parameter."
    jitterthreshold::Float64
    "Minimum number of jittering steps."
    jittermin::Int64
    "Maximum number of jittering steps."
    jittermax::Int64
    "Boolean if generate(_rng, objective) for corresponding model is stored in PF Diagnostics."
    generated::U
    function SMCDefault(;
        Ntuning=10,
        resamplingmethod::T = BaytesFilters.Systematic(),
        resamplingthreshold=0.9,
        jitterfun::F = maximum,
        jitteradaption::B = BaytesCore.UpdateTrue(),
        jitterthreshold=0.9,
        jittermin=1,
        jittermax=10,
        generated=BaytesCore.UpdateFalse(),
    ) where {
    F<:Function,
    T<:BaytesCore.ResamplingMethod,
    B<:BaytesCore.UpdateBool
    }
        ArgCheck.@argcheck 0 < Ntuning
        ArgCheck.@argcheck 0.0 <= resamplingthreshold <= 1.0
        ArgCheck.@argcheck 0.0 <= jitterthreshold <= 1.0
        ArgCheck.@argcheck 0 < jittermin <= jittermax
        return new{F, T, B, typeof(generated)}(
            Ntuning,
            resamplingmethod,
            resamplingthreshold,
            jitterfun,
            jitteradaption,
            jitterthreshold,
            jittermin,
            jittermax,
            generated
            )
    end
end

################################################################################
function update!(particles::SMCParticles, model::ModelWrapper, tagged::Tagged, upd::BaytesCore.UpdateFalse)
    return nothing
end
function update!(particles::SMCParticles, model::ModelWrapper, tagged::Tagged, upd::BaytesCore.UpdateTrue)
    # Update parameter particles that are not tagged
    for iter in eachindex(particles.model)
        ModelWrappers.fill!(particles.model[iter],
            merge(model.val, ModelWrappers.subset(particles.model[iter].val, tagged.parameter))
        )
    end
    return nothing
end

############################################################################################
"""
$(TYPEDEF)

SMC Algorithm.

# Fields
$(TYPEDFIELDS)
"""
struct SMC{A<:SMCParticles,B<:SMCTune} <: AbstractAlgorithm
    particles::A
    tune::B
    function SMC(particles::A, tune::B) where {A<:SMCParticles,B<:SMCTune}
        return new{A,B}(particles, tune)
    end
end

function SMC(
    _rng::Random.AbstractRNG,
    Jitterkernel::J,
    objective::Objective,
    default::SMCDefault=SMCDefault(),
    info::BaytesCore.SampleDefault = BaytesCore.SampleDefault()
) where {
    J<:AbstractConstructor
}
    ## Print assumptions to user
    @unpack chains = info
    @unpack Ntuning, resamplingmethod, resamplingthreshold, jitterfun, jitteradaption, jitterthreshold, jittermin, jittermax, generated = default
    Ndata = maximum(size(objective.data))
    #!NOTE: Check if steps after first jitter call can be captured
    capture = Jitterkernel isa MCMCConstructor ? BaytesCore.UpdateFalse() : BaytesCore.UpdateTrue()
    ## Initialize tune
    tagged = ModelWrappers.Tagged(objective.model, BaytesCore.get_sym(Jitterkernel))
    tune = SMCTune(
        tagged,
        chains,
        Ndata,
        Ntuning,
        resamplingthreshold,
        resamplingmethod,
        capture,
        jitterfun,
        jitteradaption,
        jitterthreshold,
        jittermin,
        jittermax,
        generated
    )
    ## Initialize SMCParticles
    particles = SMCParticles(_rng, Jitterkernel, objective, info, tune)
    ## Return SMC container
    return SMC(particles, tune)
end

############################################################################################
"""
$(SIGNATURES)
Propose new parameter with smc sampler. If update=true, objective function will be updated with input model and data.

# Examples
```julia
```

"""
function propose!(
    _rng::Random.AbstractRNG,
    smc::SMC,
    model::ModelWrapper,
    data::D,
    temperature::F = model.info.reconstruct.default.output(1.0),
    update::U=BaytesCore.UpdateTrue()
) where {D,F<:AbstractFloat,U<:BaytesCore.UpdateBool}
    ## Update kernel parameter values with non-tagged parameter from other sampler
    update!(smc.particles, model, smc.tune.tagged, update)
    ## Set back tune.jitter and update buffer to store correct information for diagnostics
    update!(smc.tune)
    update!(smc.particles.buffer)
    ## If latent θ trajectory increasing - propagate forward
    propagate!(_rng, smc.particles, smc.tune, data, temperature)
    ## Predict new data given current particles
    predict!(_rng, smc.particles, smc.tune, data, temperature)
    ## Adjust weights with log likelihood INCREMENT at time t
    weight!(_rng, smc.particles, smc.tune, data, temperature)
    ## Resample and jitter particles if criterion fulfilled
    #!NOTE: Cannot use resample before propagate! (as in pf) as previous temperature otherwise unknown
    ESS, resampled = resample!(_rng, smc.particles, smc.tune, data, temperature)
    ## Choose proposal model parameter to update model.val
    path = BaytesFilters.draw!(_rng, smc.particles.weights)
    ModelWrappers.fill!(model,
        merge(model.val, ModelWrappers.subset(smc.particles.model[path].val, smc.tune.tagged.parameter))
    )
    ## Pack and return Models and particles
    return model.val,
    SMCDiagnostics(
        smc.particles,
        temperature,
        ESS,
        resampled,
        smc.tune.jitter.Nsteps.current,
        smc.tune.iter.current,
        ModelWrappers.generate(_rng, Objective(model, data, smc.tune.tagged, temperature), smc.tune.generated)
    )
end

############################################################################################
#export
export SMC, SMCDefault, propose!


function eval_cooling(data_fill,cols)
    i_nonmissing = ismissing.(data_fill[!,cols["temp_col"]]).==false
    df = data_fill[i_nonmissing,[:datetime,Symbol(cols["temp_col"])]]

    # Eval cooling windows
    cooling_ranges = ECHOsched.cooling_onoff_segmentation(
        df.datetime,
        df.temperature_deg_c,
        θ_grad_onset=0.05,
    )

    # Map cooling ranges to data_fill dataframe (which includes missing timestamps)
    cooling_ranges_fill = ECHOsched.map_data_to_datafill.((df,),(data_fill,),cooling_ranges)
    data_fill[!,Symbol("$(cols["temp_col"])_neg_grad")] = fill!(Array{Bool}(undef,nrow(data_fill)),false)
    data_fill[!,Symbol("$(cols["temp_col"])_neg_grad")][vcat(collect.(cooling_ranges_fill)...)] .= true
    # Include "ac_on" bool in data_fill
    data_fill[!,:ac_on] = fill!(Array{Bool}(undef,nrow(data_fill)),false)
    data_fill[!,:ac_on][vcat(collect.(cooling_ranges_fill)...)] .= true

    # Find standby ranges
    standby_ranges_fill = ECHOsched.find_standby_segments(cooling_ranges_fill,nrow(data_fill))
    # Include "ac_standby" bool in data_fill
    data_fill[!,:ac_standby] = fill!(Array{Bool}(undef,nrow(data_fill)),false)
    data_fill.ac_standby[vcat(collect.(standby_ranges_fill)...)] .= true

    # Find gaps longer than 5 minutes
    gap_ranges_fill = ECHOsched.find_gaps(df,data_fill;gap_treshold_minutes=5)
    # Include gap bool in data_fill
    data_fill[!,:prolonged_gap] = fill!(Array{Bool}(undef,nrow(data_fill)),false)
    data_fill.prolonged_gap[vcat(collect.(gap_ranges_fill)...)] .= true

    # Impose gaps in cooling/standby segments
    data_fill.ac_on[data_fill.prolonged_gap] .= false
    data_fill.ac_standby[data_fill.prolonged_gap] .= false

    # Check for mischaracterised standby periods (high loads, even where temperature increases)
    standby_ranges_fill = ECHOsched.ranges_from_vec(findall(data_fill.ac_standby))
    i_cooling_corr = ECHOsched.check_standby_segment.((data_fill[!,cols["power_col"]],),standby_ranges_fill;power_threshold=50)
    standby_ranges_fill = standby_ranges_fill[Not(i_cooling_corr)]
    data_fill.ac_standby[Not(vcat(collect.(standby_ranges_fill)...))] .= false
    data_fill.ac_on[setdiff(findall(data_fill.prolonged_gap.==false),findall(data_fill.ac_standby))] .= true

    # Ground truth against power
    i_nonmissing = findall(ismissing.(data_fill[!,cols["power_col"]]).==false)
    i_stby = i_nonmissing[findall(data_fill[i_nonmissing,cols["power_col"]].<50)]
    i_ac_on = i_nonmissing[findall(data_fill[i_nonmissing,cols["power_col"]].>=50)]
    data_fill.ac_on[i_stby] .= false
    data_fill.ac_on[i_ac_on] .= true
    data_fill.ac_standby[i_stby] .= true
    data_fill.ac_standby[i_ac_on] .= false

    return data_fill, cooling_ranges_fill, standby_ranges_fill
end


mutable struct AlphaEdge
    vec::Vector{Float64}
    id::UInt64
end
Base.hash(a::AlphaEdge, h::UInt64=zero(UInt64)) = hash(a.vec, hash(a.id, h))

mutable struct MCVINode
    id::UInt64
    act::Any
    states::Any
    alpha_edges::Vector{AlphaEdge}
    MCVINode() = new()
    MCVINode(id, act, states, aes) = new(id, act, states, aes)
end

hasnext(n::MCVINode) = !isempty(n.alpha_edges)

copy(n::MCVINode) = MCVINode(n.id, n.act, n.states, n.alpha_edges)

"""
Returns the next node in the policygraph
"""
mutable struct MCVIUpdater{P<:POMDP} <: POMDPs.Updater
    problem::P
    root
    root_belief
    nodes::Dict{UInt64, MCVINode}
    nodes_queue::Vector{MCVINode}
end

MCVIUpdater(problem) = MCVIUpdater(problem, nothing, nothing, Dict{UInt64, MCVINode}(), Vector{MCVINode}())

Base.length(ps::MCVIUpdater) = length(ps.nodes)

hasroot(ps::MCVIUpdater) = ps.root != nothing

hasnode(ps::MCVIUpdater, n::MCVINode) = haskey(ps.nodes, n.id)

create_belief(ps::MCVIUpdater) = MCVINode()

function initialize_belief(up::MCVIUpdater, b::Any)
    if up.root_belief != b
        if @implemented initialstate_distribution(::typeof(up.problem))
            is = initialstate_distribution(up.problem)
        else
            is = "<initialstate_distribution(::$(typeof(up.problem))) not implemented>"
        end
        @warn("""
             The belief used to start MCVI policy execution was (potentially) different from the initial belief used in the MCVI solution.

             updater root belief: $(up.root_belief)
             b0 for policy execution: $b
             initialstate_distribution(pomdp): $is
             """)
    end
    return up.root
end

function create_node(ps::MCVIUpdater, a::A, states::Any, alpha_edges::Vector{AlphaEdge}) where {A}
    if states == nothing
        st_hash = Base.hash(states)
    else
        st_hash = hash(states) # TODO: recursive hash of array?
    end
    id = hash(a, hash(alpha_edges, st_hash))

    return MCVINode(id, a, states, alpha_edges)
end

mutable struct MCVIPolicy <: POMDPs.Policy
    problem::POMDPs.POMDP
    updater::Union{Nothing, POMDPs.Updater}
    MCVIPolicy() = new()
    MCVIPolicy(p) = new(p, nothing)
    MCVIPolicy(p, up) = new(p, up)
    MCVIPolicy(p, up, root) = new(p, up)
end

function init_node(ps::MCVIUpdater, problem::POMDPs.POMDP)
    return create_node(ps, init_lower_action(problem), nothing, Vector{AlphaEdge}())
end

function init_nodes(policy::MCVIPolicy, ps::MCVIUpdater)
    ns = Vector{MCVINode}()
    for a in actions(policy.problem)
        if isterminal(policy.problem, a) # FIXME: Can use init_lower_action instead?
            push!(ns, create_node(ps, a, nothing, Vector{AlphaEdge}()))
        end
    end
    return ns
end

function addnode!(ps::MCVIUpdater, n::MCVINode)
    ps.nodes[n.id] = n
    push!(ps.nodes_queue, n)
end

updater(policy::MCVIPolicy) = policy.updater

"""
Initialize and return the MCVIUpdater
"""
function initialize_updater!(policy::MCVIPolicy)
    ps = MCVIUpdater(policy.problem)
    ns = init_nodes(policy, ps)
    for n in ns
        addnode!(ps, n)
        ps.root = n
    end
    policy.updater = ps
end

"""
Move to the next policy state given observation
"""
function update(ps::MCVIUpdater, n::MCVINode, act::A, obs::O, np::MCVINode=create_belief(ps)) where {A,O}
    @assert hasnext(n) "No next policy state exists"

    local nid::UInt64
    if length(n.alpha_edges) == 1
        nid = n.alpha_edges[1].id
    else
        @assert n.states != nothing "No states in the current node"
        obs_wt = zeros(length(n.states))
        for (i,s) in enumerate(n.states)
            obs_wt[i] = obs_weight(ps.problem, act, s, obs)
        end
        maxv = -Inf
        for ae in n.alpha_edges
            v = dot(obs_wt, ae.vec)
            if v > maxv
                maxv = v
                nid = ae.id
            end
        end
    end
    @assert ps.nodes[nid] != nothing
    np = ps.nodes[nid]
    return np
end

"""
Return all next policy states for the given node
"""
function next_nodes(ps::MCVIUpdater, n::MCVINode)
    nodes = Vector{MCVINode}(length(n.alpha_edges))
    for i in 1:length(nodes)
        nid = n.alpha_edges[i].id
        nodes[i] = ps.nodes[nid]
    end
    return nodes
end

function to_dict(n::MCVINode)
    return Dict("id"          => n.id,
                "act"         => n.act,
                "states"      => n.states,
                "alpha_edges" => n.alpha_edges)
end

"""
Dump policy graph for inspection
"""
function dump_json(policy::MCVIPolicy, filename::AbstractString)
    root = policy.updater.root
    nodes = policy.updater.nodes
    open(filename, "w") do f
        write(f, JSON.json(Dict(root.id => to_dict(root))))
        for k in keys(nodes)
            write(f, JSON.json(Dict(k => to_dict(nodes[k]))))
        end
    end
end


using NCTiles,NCDatasets,NetCDF,Dates,Printf

examplesdir = joinpath("data","ex3")
indir = joinpath(examplesdir,"data","cs510","diags_interp")
availdiagsfname = joinpath(examplesdir,"data","available_diagnostics.log")
savedir = joinpath(examplesdir,"data","cs510","interp_nctiles")

if ~ispath(savedir); mkpath(savedir); end
README = readlines(joinpath(examplesdir,"README"))

exdir = "../../../run/run_cs510/"
fnameprefix = "darwin_v0.2_cs510"
groups = ["Nutrients","Bulk_Ecosystem_Characteristcs","Phytoplankton_Functional_Types","Ocean_Color"]

rename = Dict(["TRAC05" => "PO4",
"TRAC06" => "SiO2",
"TRAC07" => "FeT",
"TRAC19" => "O2"])

unitsrename = Dict(["mmol C/" => "mmol N/m^3",
                    "mmol O/" => "mmol O/m^3",
                    "mmol P/" => "mmol P/m^3",
                    "mmol Si" => "mmol Si/m^3",
                    "mmol Fe" => "mmol Fe/m^3"])

# Hard coding dims for now
lon=-179.75:0.5:179.75; lat=-89.75:0.5:89.75;

timeinterval = 3
tstart = 732; tend = 8766;
#tstart = 6; tend = 6;
timesteps = (tstart - timeinterval/2):timeinterval:(tend - timeinterval/2)
nsteps = length(timesteps)
timeinfname = lpad.(string.(Integer.(collect((timesteps.+timeinterval/2).*24))),10,'0')
timeunits = "days since 1992-1-1 0:0:0"
timestart = DateTime(1992,01,01)
dep = -1 .* collect([-5 -15 -25 -35 -45 -55 -65 -75.0049972534180 -85.0250015258789 -95.0950012207031 -105.309997558594 -115.870002746582 -127.150001525879 -139.740005493164 -154.470001220703 -172.399993896484 -194.735000610352 -222.710006713867 -257.470001220703 -299.929992675781 -350.679992675781 -409.929992675781 -477.470001220703 -552.710021972656 -634.734985351563 -722.400024414063 -814.469970703125 -909.739990234375 -1007.15502929688 -1105.90502929688 -1205.53503417969 -1306.20495605469 -1409.15002441406  -1517.09497070313 -1634.17504882813 -1765.13500976563 -1914.15002441406 -2084.03491210938 -2276.22509765625 -2491.25000000000 -2729.25000000000 -2990.25000000000 -3274.25000000000 -3581.25000000000 -3911.25000000000 -4264.25000000000 -4640.25000000000 -5039.25000000000 -5461.25000000000 -5906.25000000000]')
n1,n2,n3 = (length(lon),length(lat),length(dep))

dimdict = Dict{AbstractString,NCvar}([
    "lon_c" => NCvar("lon_c","degrees_east",size(lon),lon,Dict("long_name" => "longitude"),NCDatasets),
    "lat_c" => NCvar("lat_c","degrees_north",size(lat),lat,Dict("long_name" => "latitude"),NCDatasets),
    "dep_c" => NCvar("dep_c","m",size(dep),dep,Dict("long_name" => "depth","standard_name" => "depth","positive" => "down"),NCDatasets),
    "time" => NCvar("time",timeunits,Inf,timesteps,Dict(("long_name" => "time","standard_name" => "time")),NCDatasets)
])

dims = [NCvar("lon_c","degrees_east",size(lon),lon,Dict("long_name" => "longitude"),NCDatasets),
        NCvar("lat_c","degrees_north",size(lat),lat,Dict("long_name" => "latitude"),NCDatasets),
        NCvar("dep_c","m",size(dep),dep,Dict("long_name" => "depth","standard_name" => "depth","positive" => "down"),NCDatasets),
        NCvar("time",timeunits,Inf,timesteps,Dict(("long_name" => "time","standard_name" => "time")),NCDatasets)]

function createTimestep(timestep,idx,group,flds,timestart,fnameprefix,README,dimdict)
    thistime = timestart + Dates.Day(floor(timestep))

    yyyyddd = @sprintf("%i%03i",year(thistime),dayofyear(thistime))

    thisfilename = join((fnameprefix,lowercase(group),yyyyddd),"_")*".nc"
    if isfile(thisfilename)
        try
            ds = Dataset(thisfilename)
            ds["time"]
            println(thisfilename*" already done")
            return
        catch
            rm(thisfilename)
        end
    end
    ds,fieldvars,dimlist = createfile(thisfilename,flds,README)

    # Add field and dimension data and close the file
    for k in keys(flds)
        addData(ds[k],flds[k][idx])
    end

    if haskey(dimdict,"time")
        hastimedim = true
        timedim = pop!(dimdict,"time")
    else
        hastimedim = false
    end

    addDimData.(Ref(ds),collect(values(dimdict)))
    if hastimedim
        addDimData(ds,timedim[idx])
        dimdict["time"] = timedim
    end

    close(ds)
    println("finished "*thisfilename)
end

for group in groups
    println(group)
    groupindir = joinpath(indir,group)
    groupsavedir = joinpath(savedir,group)
    if !isdir(groupsavedir); mkpath(groupsavedir); end

    selectfields = filter(x -> isdir(joinpath(groupindir,x)),readdir(groupindir))

    flds = Dict{AbstractString,NCvar}()

    timestepidx = []

    for fldname in selectfields
        fullpath = joinpath(groupindir,fldname)
        fnames_timerange = fldname*'.'.*timeinfname.*".data"
        fnames_avail = filter(x -> occursin(".data",x), readdir(fullpath))
        fnames = joinpath.(Ref(fullpath),intersect(fnames_timerange,fnames_avail))
        timestepidx = [findfirst(occursin.(f,fnames_timerange)) for f in fnames_avail]
        timestepidx = timestepidx[.!isa.(timestepidx,Nothing)]
        metafile = joinpath.(Ref(fullpath),fldname*'.'.*timeinfname.*".meta")
        #filter(x -> occursin(".meta",x), readdir(fullpath))[1]
        metafile = parsemeta(metafile[1])
        diaginfo = readAvailDiagnosticsLog(availdiagsfname,fldname)

        #timesteps = parse.(Int,getindex.(split.(fnames,'.'),2))./24 .- timeinterval/2

        # Rename if needed
        if any(occursin.(fldname,collect(keys(rename))))
            fldname = rename[fldname]
        end

        prec = eval(Meta.parse(metafile["dataprec"]))
        units = get(unitsrename,diaginfo["units"],diaginfo["units"])
        longname = diaginfo["title"]

        mydims = Array{NCvar,1}()
        for i in 1:metafile["nDims"]
            dimidx = getindex.(getfield.(dims,:dims),1) .== metafile["dimList"][i,1]
            mydims = cat(mydims,dims[dimidx],dims=1)
        end

        iosize = Tuple([d.dims[1] for d in mydims])

        mydims = cat(mydims,dimdict["time"],dims=1)

        fielddata = BinData(fnames,prec,iosize)
        flds[fldname] = NCvar(fldname,units,mydims,fielddata,
            Dict("long_name" => longname),NCDatasets)

    end

    # Set up NCvars
    filename = joinpath(savedir,join([fnameprefix,lowercase(group)],"_")*".nc")
    println(filename)
    if isfile(filename); rm(filename); end

    # Create the NetCDF file and populate with dimension and field info
    ds,fieldvars,dimlist = createfile(filename,flds,README)

    # Add field and dimension data and close the file
    for k in keys(flds)
        addData(ds[k],flds[k])
    end

    addDimData.(Ref(ds),dims)
    close(ds)

    createTimestep.(timesteps[timestepidx],timestepidx,Ref(group),Ref(flds),Ref(timestart),Ref(joinpath(groupsavedir,fnameprefix)),Ref(README),Ref(dimdict))
    #createTimestep.(dtsteps,didx,Ref(group),Ref(flds),Ref(timestart),Ref(ncdir*group*"/"*fnameprefix),Ref(README))

end


struct PlusMinusBridge{T, F <: MOI.AbstractVectorFunction,
                       G <: MOI.AbstractVectorFunction,
                       ST <: MOI.AbstractVectorSet} <: MOIB.Constraint.AbstractBridge
    plus::MOI.ConstraintIndex{F, ST}
    minus::MOI.ConstraintIndex{G, ST}
end

function MOIB.Constraint.bridge_constraint(
    ::Type{PlusMinusBridge{T, F, G, ST}},
    model::MOI.ModelLike,
    f::MOI.AbstractVectorFunction,
    set::PlusMinusSet{ST}) where {T, F, G, ST}

    plus = MOI.add_constraint(model, f, set.set)
    minus = MOI.add_constraint(model, MOIU.operate(-, T, f), set.set)
    return PlusMinusBridge{T, F, G, ST}(plus, minus)
end

function MOI.supports_constraint(::Type{PlusMinusBridge{T}},
                                 ::Type{<:MOI.AbstractVectorFunction},
                                 ::Type{<:PlusMinusSet}) where T
    return true
end
function MOIB.added_constrained_variable_types(::Type{<:PlusMinusBridge})
    return Tuple{DataType}[]
end
function MOIB.added_constraint_types(::Type{<:PlusMinusBridge{T, F, F, ST}}) where {T, F, ST}
    return [(F, ST)]
end
function MOIB.added_constraint_types(::Type{<:PlusMinusBridge{T, F, G, ST}}) where {T, F, G, ST}
    return [(F, ST), (G, ST)]
end
function MOIB.Constraint.concrete_bridge_type(
    ::Type{<:PlusMinusBridge{T}},
    F::Type{<:MOI.AbstractVectorFunction},
    ::Type{<:PlusMinusSet{ST}}) where {T, ST}

    G = MOI.Utilities.promote_operation(-, T, F)
    return PlusMinusBridge{T, F, G, ST}
end

# Attributes, Bridge acting as an model
function MOI.get(::PlusMinusBridge{T, F, F, ST},
                 ::MOI.NumberOfConstraints{F, ST}) where {T, F, ST}
    return 2
end
function MOI.get(::PlusMinusBridge{T, F, G, ST},
                 ::MOI.NumberOfConstraints{F, ST}) where {T, F, G, ST}
    return 1
end
function MOI.get(::PlusMinusBridge{T, F, G, ST},
                 ::MOI.NumberOfConstraints{G, ST}) where {T, F, G, ST}
    return 1
end
function MOI.get(bridge::PlusMinusBridge{T, F, F, ST},
                 ::MOI.ListOfConstraintIndices{F, ST}) where {T, F, ST}
    return [bridge.plus, bridge.minus]
end
function MOI.get(bridge::PlusMinusBridge{T, F, G, ST},
                 ::MOI.ListOfConstraintIndices{F, ST}) where {T, F, G, ST}
    return [bridge.plus]
end
function MOI.get(bridge::PlusMinusBridge{T, F, G, ST},
                 ::MOI.ListOfConstraintIndices{G, ST}) where {T, F, G, ST}
    return [bridge.minus]
end

# Indices
function MOI.delete(model::MOI.ModelLike, bridge::PlusMinusBridge)
    MOI.delete(model, bridge.plus)
    MOI.delete(model, bridge.minus)
end


#!/usr/bin/env julia

f1 = ARGS[1]
f2 = ARGS[2]

"""
Expanding the dashed intervals into single nucleotide positions
example:
Y:2688071-2688075 =>
                    Y:2688071
                    Y:2688072
                    Y:2688073
                    Y:2688074
                    Y:2688075
"""
function expand(locci)
    list = []
    pos = split(locci, ':')
        chr = pos[1]
        region = split(pos[2], '-')
        n = parse(Int64, region[1])
        while n <= parse(Int64, region[end])
             out = "$chr" * ":" * "$n"
            push!(list, out)
            n +=  1
        end
    return list
end

"""
Reads EnsEMBL Variant Effect Predictor (VEP) Results
and returning mutation locci as a list
"""
function readVEP(fh)
    locci = []
    file  = open(fh)
    lines = readlines(file)
    for line in lines
        if ismatch(r"^#", line)
            continue
        else
            cols = split(line, '\t')
            push!(locci, cols[2])
        end
    end

    list = []
    for g in locci
        if ismatch(r"-", g)
            test = expand(g)
            for i in test
                push!(list, i)
            end
        else
            push!(list, g)
        end
    end
    return list
end

gl = readVEP(f1)
sm = readVEP(f2)

glOnly = setdiff(Set(gl), Set(sm))

"""
smOnly contains only variants excluding the common variants found in germline
"""
smOnly = setdiff(Set(sm), Set(gl))

common = intersect(Set(gl), Set(sm))
smf = open(f2)
lines = readlines(smf)
"""
fetching the variant position based annoation for the filtered
somatic variants
"""
for g in smOnly
    for line in lines
        if ismatch(r"^#", line)
            continue
        else
            cols = split(line, '\t')
            if g == cols[2]
                println(chomp(line))
            elseif ismatch(r"-", cols[2])
                pos = expand(cols[2])
                for item in pos
                    if g == item
                        println(chomp(line))
                    end
                end
            end
        end
    end
end


using MPIMeasurements
using Base.Test
using Unitful
using Compat
using HDF5

# define Grid
shape = [10,10,1]
fov = [200.0,200.0,1.0]Unitful.mm
center = [0.0,0.0,0.0]Unitful.mm
positions = MeanderingGridPositions( RegularGridPositions(shape,fov,center) )

# create Scanner
scanner = MPIScanner("HeadScanner.toml")
#scanner = MPIScanner("DummyScanner.toml")
robot = getRobot(scanner)
safety = getSafety(scanner)
gauss = getGaussMeter(scanner)
rp = RedPitaya("192.168.1.100")
waitTime = 4.0
currRange = 2.0:2.0:10.0
currents = hcat(hcat(currRange,zeros(length(currRange)))',
                hcat(zeros(length(currRange)),currRange)',
                hcat(currRange,currRange)')
voltToCurrent = 0.08547008547008547

#TODO mT sollte man hier nicht angeben müssen. Das sollte im Gaussmeter gekapselt sein
mfMeasObj = MagneticFieldSweepCurrentsMeas(rp, gauss, Unitful.mT, positions,
                                           currents, waitTime, voltToCurrent)

res = performTour!(robot, safety, positions, mfMeasObj)

filenameField = joinpath(homedir(),"MagneticField$(string(now())).h5")
saveMagneticFieldAsHDF5(mfMeasObj, filenameField)

pos, field = loadMagneticField(filenameField)

MPISimulations.plotMagneticField(field, pos, 3, 1)


function king_in_check(board::Bitboard, color::String="white")
    if color == "white"
        if board.K & board.black_attacks == EMPTY
            return false
        end
    else
        if board.k & board.white_attacks == EMPTY
            return false
        end
    end
    return kingtrace(board, color)
end

function square_in_check(board::Bitboard, ui::UInt64, color::String="white")
    if color == "white"
        if ui & board.a[1] != EMPTY
            return true
        end
        if ui & board.a[2] != EMPTY
            return true
        end
        if ui & board.a[4] != EMPTY || ui & board.a[3] != EMPTY
            squares, edges = orthogonal_attack(board.taken, ui)
            for rook in board.r
                if rook in edges
                    return true
                end
            end
            for queen in board.q
                if queen in edges
                    return true
                end
            end
        end
        if ui & board.a[5] != EMPTY || ui & board.a[3] != EMPTY
            squares, edges = cross_attack(board.taken, ui)
            for bishop in board.b
                if bishop in edges
                    return true
                end
            end
            for queen in board.q
                if queen in edges
                    return true
                end
            end
        end
        return false
    else
        if ui & board.A[1] != EMPTY
            return true
        end
        if ui & board.A[2] != EMPTY
            return true
        end
        if ui & board.A[4] != EMPTY || ui & board.A[3] != EMPTY
            squares, edges = orthogonal_attack(board.taken, ui)
            for rook in board.R
                if rook in edges
                    return true
                end
            end
            for queen in board.Q
                if queen in edges
                    return true
                end
            end
        end
        if ui & board.A[5] != EMPTY || ui & board.A[3] != EMPTY
            squares, edges = cross_attack(board.taken, ui)
            for bishop in board.B
                if bishop in edges
                    return true
                end
            end
            for queen in board.Q
                if queen in edges
                    return true
                end
            end
        end
        return false
    end
end

function kingtrace(board::Bitboard, color::String="white")
    if color == "white"
        return square_in_check(board, board.K, color)
    else
        return square_in_check(board, board.k, color)
    end
end

function check_mate(board::Bitboard, color::String="white")
    if color == "white"
        king = board.K
    else
        king = board.k
    end

    if length(get_current_king_valid(board, color)) != 0
        return false
    else
        if length(get_all_valid_moves(board, color)) == 0
            return true
        else
            return false
        end
    end
end


################################################################################
# Use the Julia-PerpleX interface to run the same isobaric Perplex calculation
# with different solution models to compare results

## --- Import some useful packages
    using StatGeochem
    using Plots; gr();
    using Statistics, DelimitedFiles, SpecialFunctions

## --- Configure

    # Absolute paths to perplex resources
    perplexdir = joinpath(resourcepath,"perplex-stable")
    scratchdir = "./scratch/" # Location of directory to store output files

    # Attempt to install perplex, if not already extant
    if !isfile(joinpath(perplexdir,"vertex"))
        # Make sure resourcepath exists
        run(`mkdir -p $resourcepath`)

        # Download Perplex v6.8.7 -- known to work with interface used here
        file = download("https://storage.googleapis.com/statgeochem/perplex-stable-6.8.7.zip", joinpath(resourcepath,"perplex-stable.zip"))

        # # For a more updated perplex version, also try
        # file = download("https://petrol.natur.cuni.cz/~ondro/perplex-sources-stable.zip", joinpath(resourcepath,"perplex-stable.zip"))

        run(`unzip -u $file -d $resourcepath`) # Extract
        system("cd $perplexdir; make") # Compile
    end

## --- # # # # # # # # # # # # # Initial composition # # # # # # # # # # # # # #
    ## McDonough Pyrolite
    #elements =    [ "SIO2", "TIO2", "AL2O3",  "FEO",  "MNO",  "MGO",  "CAO", "NA2O",  "K2O",  "H2O",  "CO2",]
    #composition = [45.1242, 0.2005, 4.4623, 8.0723, 0.1354, 37.9043, 3.5598, 0.3610, 0.0291, 0.1511, 0.0440,]

    ## Kelemen (2014) primitive continental basalt. H2O and CO2 are guesses
    #elements =    [ "SIO2", "TIO2", "AL2O3",  "FEO",  "MNO",  "MGO",  "CAO", "NA2O",  "K2O",  "H2O",  "CO2",]
    #composition = [50.0956, 0.9564, 15.3224, 8.5103, 0.1659, 9.2520, 9.6912, 2.5472, 0.8588, 2.0000, 0.6000,]

    # Kelemen (2014) primitive continental basalt excluding Mn and Ti since most melt models can"t handle them..
    elements =    [ "SIO2", "AL2O3",  "FEO",  "MGO",  "CAO", "NA2O",  "K2O",  "H2O",  "CO2",]
    composition = [50.0956, 15.3224, 8.5103, 9.2520, 9.6912, 2.5472, 0.8588, 2.0000, 0.6000,]

    ## Average Archean basalt (EarthChem data)
    #elements =    [ "SIO2", "TIO2", "AL2O3",   "FEO",  "MNO",   "MGO",  "CAO", "NA2O",  "K2O",  "H2O",  "CO2",]
    #composition = [49.2054, 0.8401, 12.0551, 11.4018, 0.2198, 12.3997, 9.3113, 1.6549, 0.4630, 1.8935, 0.5555,]

## --- # # # # # # # # # # # Some solution model options # # # # # # # # # # # #
    # Emphasis on phases from Green (2016) -- developed for metabasites, includes what is probably the best (and most expensive) amphibole model. Use with hp11ver.dat
    G_solution_phases = "Augite(G)\nOpx(JH)\ncAmph(G)\noAmph(DP)\nO(JH)\nSp(JH)\nGrt(JH)\nfeldspar_B\nMica(W)\nBio(TCC)\nChl(W)\nCtd(W)\nCrd(W)\nSa(WP)\nSt(W)\nIlm(WPH)\nAtg(PN)\nT\nB\nF\nDo(HP)\nScap\nChum\nNeph(FB)\n"
    G_excludes ="ged\nfanth\ngl\n"

    # Emphasis on phases from White (2014) -- developed for metapelites. Use with hp11ver.dat
    W_solution_phases = "Omph(HP)\nOpx(W)\ncAmph(DP)\noAmph(DP)\nO(JH)\nSp(JH)\nGt(W)\nfeldspar_B\nMica(W)\nBi(W)\nChl(W)\nCtd(W)\nCrd(W)\nSa(WP)\nSt(W) \nIlm(WPH)\nAtg(PN)\nT\nB\nF\nDo(HP)\nScap\nChum\nPu(M)\n"
    W_excludes = "andr\nts\nparg\ngl\nged\nfanth\n"

    # Emphasis on phases from Jennings and Holland (2015) -- developed for mantle melting. Use with hp11ver.dat
    JH_solution_phases = "Cpx(JH)\nOpx(JH)\ncAmph(DP)\noAmph(DP)\nO(JH)\nSp(JH)\nGrt(JH)\nfeldspar_B\nMica(W)\nBio(TCC)\nChl(W)\nCtd(W)\nCrd(W)\nSa(WP)\nSt(W)\nIlm(WPH)\nAtg(PN)\nT\nB\nF\nDo(HP)\nScap\nChum\nNeph(FB)\n"
    JH_excludes = "ts\nparg\ngl\nged\nfanth\n"

    # Emphasis on phases from Holland and Powell -- all phases can be used with hp02ver.dat.
    HP_solution_phases = "Omph(HP)\nOpx(HP)\nGlTrTsPg\nAnth\nO(HP)\nSp(HP)\nGt(HP)\nfeldspar_B\nMica(CF)\nBio(TCC)\nChl(HP)\nCtd(HP)\nSapp(HP)\nSt(HP)\nIlHm(A)\nDo(HP)\nT\nB\nF\n"
    HP_excludes = "";

## --- Prepare for plotting

    h0 = plot(xlabel="T (C)", ylabel="Melt percent")

## --- # # # # # # # # # # melt(G) + G_solution_phases # # # # # # # # # # # # #

    # Input parameters
    P = 10000 # bar
    T_range = [500+273.15, 1500+273.15]
    idx = 1
    print("\nmelt(G) + G_solution_phases\n")
    @time perplex_configure_isobar(perplexdir, scratchdir, composition, elements, P, T_range, dataset="hp11ver.dat", solution_phases="melt(G)\n"*G_solution_phases, excludes=G_excludes, index=idx)

    # Query the full isobar -- results returned as elementified dictionary
    T_range_inc = [floor(Int,T_range[1])+1, ceil(Int,T_range[2])-1]
    npoints = T_range_inc[2] - T_range_inc[1] + 1
    bulk = perplex_query_system(perplexdir, scratchdir, index=idx)            # Get system data for all temperatures. Set include_fluid = "n" to get solid+melt only
    modes = perplex_query_modes(perplexdir, scratchdir, index=idx)            # || phase modes
    melt = perplex_query_phase(perplexdir, scratchdir, "melt(G)", index=idx)  # || melt data

    # Create dictionary to hold solid composition and fill it using what we know from system and melt
    solid = Dict()
    solid["wt_pct"] = 100 .- melt["wt_pct"]
    for e in ["SIO2","AL2O3","FEO","MGO","CAO","NA2O","K2O"]
        solid[e] = (bulk[e] .- (melt[e] .* melt["wt_pct"]/100)) ./ (solid["wt_pct"]/100)
    end

    # Add results to melt % vs temperature figure
    plot!(h0, melt["T(K)"] .- 273.15, melt["wt_pct"], label="melt(G) + G")

    # Plot melt composition as a function of melt percent
    h = plot(xlabel="Percent melt", ylabel="Wt. % in melt", title="melt(G) + G_solution_phases, $P bar")
    for e in ["SIO2","AL2O3","FEO","MGO","CAO","NA2O","K2O"]
        plot!(h, melt["wt_pct"], melt[e], label=e)
    end
    plot!(h,fg_color_legend=:white, framestyle=:box)
    savefig(h,"Perplex_MeltTest_G.pdf")

    # Plot solid composition as a function of melt percent
    h = plot(xlabel="Percent melt", ylabel="Wt. % in solid", title="melt(G) + G_solution_phases, $P bar")
    for e in ["SIO2","AL2O3","FEO","MGO","CAO","NA2O","K2O"]
        plot!(h, melt["wt_pct"], solid[e], label=e)
    end
    plot!(h,fg_color_legend=:white, framestyle=:box, legend=:topleft)
    savefig(h,"Perplex_SolidTest_G.pdf")

## --- # # # # # # # # # # melt(G) + W_solution_phases # # # # # # # # # # # # #

    # Input parameters
    P = 10000 # bar
    T_range = [500+273.15, 1500+273.15]
    idx = 2
    print("\nmelt(G) + W_solution_phases\n")
    @time perplex_configure_isobar(perplexdir, scratchdir, composition, elements, P, T_range, dataset="hp11ver.dat", solution_phases="melt(G)\n"*W_solution_phases, excludes=W_excludes, index=idx)

    # Query the full isobar -- results returned as elementified dictionary
    T_range_inc = [floor(Int,T_range[1])+1, ceil(Int,T_range[2])-1]
    npoints = T_range_inc[2] - T_range_inc[1] + 1
    bulk = perplex_query_system(perplexdir, scratchdir, index=idx)            # Get system data for all temperatures. Set include_fluid = "n" to get solid+melt only
    modes = perplex_query_system(perplexdir, scratchdir, index=idx)            # || phase modes
    melt = perplex_query_phase(perplexdir, scratchdir, "melt(G)", index=idx)  # || melt data

    # Create dictionary to hold solid composition and fill it using what we know from system and melt
    solid = Dict()
    solid["wt_pct"] = 100 .- melt["wt_pct"]
    for e in ["SIO2","AL2O3","FEO","MGO","CAO","NA2O","K2O"]
        solid[e] = (bulk[e] - (melt[e] .* melt["wt_pct"]/100)) ./ (solid["wt_pct"]/100)
    end

    # Add results to melt % vs temperature figure
    plot!(h0, melt["T(K)"] .- 273.15, melt["wt_pct"], label="melt(G) + W")

    # Plot melt composition as a function of melt percent
    h = plot(xlabel="Percent melt", ylabel="Wt. % in melt", title="melt(G) + W_solution_phases, $P bar")
    for e in ["SIO2","AL2O3","FEO","MGO","CAO","NA2O","K2O"]
        plot!(h, melt["wt_pct"], melt[e], label=e)
    end
    plot!(h,fg_color_legend=:white, framestyle=:box)
    savefig(h,"Perplex_MeltTest_G_W.pdf")

    # Plot solid composition as a function of melt percent
    h = plot(xlabel="Percent melt", ylabel="Wt. % in solid", title="melt(G) + W_solution_phases, $P bar")
    for e in ["SIO2","AL2O3","FEO","MGO","CAO","NA2O","K2O"]
        plot!(h, melt["wt_pct"], solid[e], label=e)
    end
    plot!(h,fg_color_legend=:white, framestyle=:box, legend=:topleft)
    savefig(h,"Perplex_SolidTest_G_W.pdf")

## --- # # # # # # # # # # melt(G) +JH_solution_phases # # # # # # # # # # # # #

    # Input parameters
    P = 10000 # bar
    T_range = [500+273.15, 1500+273.15]
    idx = 3
    print("\nmelt(G) + JH_solution_phases\n")
    @time perplex_configure_isobar(perplexdir, scratchdir, composition, elements, P, T_range, dataset="hp11ver.dat", solution_phases="melt(G)\n"*JH_solution_phases, excludes=JH_excludes, index=idx)

    # Query the full isobar -- results returned as elementified dictionary
    T_range_inc = [floor(Int,T_range[1])+1, ceil(Int,T_range[2])-1]
    npoints = T_range_inc[2] - T_range_inc[1] + 1
    bulk = perplex_query_system(perplexdir, scratchdir, index=idx)            # Get system data for all temperatures. Set include_fluid = "n" to get solid+melt only
    modes = perplex_query_modes(perplexdir, scratchdir, index=idx)            # || phase modes
    melt = perplex_query_phase(perplexdir, scratchdir, "melt(G)", index=idx)  # || melt data

    # Create dictionary to hold solid composition and fill it using what we know from system and melt
    solid = Dict()
    solid["wt_pct"] = 100 .- melt["wt_pct"]
    for e in ["SIO2","AL2O3","FEO","MGO","CAO","NA2O","K2O"]
        solid[e] = (bulk[e] - (melt[e] .* melt["wt_pct"]/100)) ./ (solid["wt_pct"]/100)
    end

    # Add results to melt % vs temperature figure
    plot!(h0, melt["T(K)"] .- 273.15, melt["wt_pct"], label="melt(G) + JH")

    # Plot melt composition as a function of melt percent
    h = plot(xlabel="Percent melt", ylabel="Wt. % in melt", title="melt(G) + JH_solution_phases, $P bar")
    for e in ["SIO2","AL2O3","FEO","MGO","CAO","NA2O","K2O"]
        plot!(h, melt["wt_pct"], melt[e], label=e)
    end
    plot!(h,fg_color_legend=:white, framestyle=:box)
    savefig(h,"Perplex_MeltTest_G_JH.pdf")

    # Plot solid composition as a function of melt percent
    h = plot(xlabel="Percent melt", ylabel="Wt. % in solid", title="melt(G) + JH_solution_phases, $P bar")
    for e in ["SIO2","AL2O3","FEO","MGO","CAO","NA2O","K2O"]
        plot!(h, melt["wt_pct"], solid[e], label=e)
    end
    plot!(h,fg_color_legend=:white, framestyle=:box, legend=:topleft)
    savefig(h,"Perplex_SolidTest_G_JH.pdf")

## --- # # # # # # # # # # pMELTS(G) +HP_solution_phases # # # # # # # # # # # #

    # Input parameters
    P = 10000 # bar
    T_range = [500+273.15, 1500+273.15]
    idx = 4
    print("\npMELTS(G) + JH_solution_phases\n")
    @time perplex_configure_isobar(perplexdir, scratchdir, composition, elements, P, T_range, dataset="hp02ver.dat", solution_phases="pMELTS(G)\n"*JH_solution_phases, excludes=JH_excludes, index=idx)

    # Query the full isobar -- results returned as elementified dictionary
    T_range_inc = [floor(Int,T_range[1])+1, ceil(Int,T_range[2])-1]
    npoints = T_range_inc[2] - T_range_inc[1] + 1
    bulk = perplex_query_system(perplexdir, scratchdir, index=idx)            # Get system data for all temperatures. Set include_fluid = "n" to get solid+melt only
    modes = perplex_query_modes(perplexdir, scratchdir, index=idx)            # || phase modes
    melt = perplex_query_phase(perplexdir, scratchdir, "pMELTS(G)", index=idx)  # || melt data

    # Create dictionary to hold solid composition and fill it using what we know from system and melt
    solid = Dict()
    solid["wt_pct"] = 100 .- melt["wt_pct"]
    for e in ["SIO2","AL2O3","FEO","MGO","CAO","NA2O","K2O"]
        solid[e] = (bulk[e] - (melt[e] .* melt["wt_pct"]/100)) ./ (solid["wt_pct"]/100)
    end

    # Add results to melt % vs temperature figure
    plot!(h0, melt["T(K)"] .- 273.15, melt["wt_pct"], label="pMELTS(G) + JH")

    # Plot melt composition as a function of melt percent
    h = plot(xlabel="Percent melt", ylabel="Wt. % in melt", title="pMELTS(G) + JH_solution_phases, $P bar")
    for e in ["SIO2","AL2O3","FEO","MGO","CAO","NA2O","K2O"]
        plot!(h, melt["wt_pct"], melt[e], label=e)
    end
    plot!(h,fg_color_legend=:white, framestyle=:box)
    savefig(h,"Perplex_MeltTest_pMELTS_JH.pdf")

    # Plot solid composition as a function of melt percent
    h = plot(xlabel="Percent melt", ylabel="Wt. % in solid", title="pMELTS(G) + JH_solution_phases, $P bar")
    for e in ["SIO2","AL2O3","FEO","MGO","CAO","NA2O","K2O"]
        plot!(h, melt["wt_pct"], solid[e], label=e)
    end
    plot!(h,fg_color_legend=:white, framestyle=:box, legend=:topleft)
    savefig(h,"Perplex_SolidTest_pMELTS_JH.pdf")

## --- # # # # # # # # # # # melt(W) + W_solution_phases # # # # # # # # # # # #

    # Input parameters
    P = 10000 # bar
    T_range = [500+273.15, 1500+273.15]
    idx = 5
    print("\nmelt(W) + W_solution_phases\n")
    @time perplex_configure_isobar(perplexdir, scratchdir, composition, elements, P, T_range, dataset="hp11ver.dat", solution_phases="melt(W)\n"*W_solution_phases, excludes=W_excludes, index=idx)

    # Query the full isobar -- results returned as elementified dictionary
    T_range_inc = [floor(Int,T_range[1])+1, ceil(Int,T_range[2])-1]
    npoints = T_range_inc[2] - T_range_inc[1] + 1
    bulk = perplex_query_system(perplexdir, scratchdir, index=idx)            # Get system data for all temperatures. Set include_fluid = "n" to get solid+melt only
    modes = perplex_query_modes(perplexdir, scratchdir, index=idx)            # || phase modes
    melt = perplex_query_phase(perplexdir, scratchdir, "melt(W)", index=idx)  # || melt data

    # Create dictionary to hold solid composition and fill it using what we know from system and melt
    solid = Dict()
    solid["wt_pct"] = 100 .- melt["wt_pct"]
    for e in ["SIO2","AL2O3","FEO","MGO","CAO","NA2O","K2O"]
        solid[e] = (bulk[e] - (melt[e] .* melt["wt_pct"]/100)) ./ (solid["wt_pct"]/100)
    end

    # Add results to melt % vs temperature figure
    plot!(h0, melt["T(K)"] .- 273.15, melt["wt_pct"], label="melt(W) + W")

    # Plot melt composition as a function of melt percent
    h = plot(xlabel="Percent melt", ylabel="Wt. % in melt", title="melt(W) + W_solution_phases, $P bar")
    for e in ["SIO2","AL2O3","FEO","MGO","CAO","NA2O","K2O"]
        plot!(h, melt["wt_pct"], melt[e], label=e)
    end
    plot!(h,fg_color_legend=:white, framestyle=:box)
    savefig(h,"Perplex_MeltTest_W.pdf")

    # Plot solid composition as a function of melt percent
    h = plot(xlabel="Percent melt", ylabel="Wt. % in solid", title="melt(W) + W_solution_phases, $P bar")
    for e in ["SIO2","AL2O3","FEO","MGO","CAO","NA2O","K2O"]
        plot!(h, melt["wt_pct"], solid[e], label=e)
    end
    plot!(h,fg_color_legend=:white, framestyle=:box, legend=:topleft)
    savefig(h,"Perplex_SolidTest_W.pdf")

## --- # # # # # # # # # # melt(W) + G_solution_phases # # # # # # # # # # # # #

    # Input parameters
    P = 10000 # bar
    T_range = [500+273.15, 1500+273.15]
    idx = 6
    print("\nmelt(W) + G_solution_phases\n")
    @time perplex_configure_isobar(perplexdir, scratchdir, composition, elements, P, T_range, dataset="hp11ver.dat", solution_phases="melt(W)\n"*G_solution_phases, excludes=G_excludes, index=idx)

    # Query the full isobar -- results returned as elementified dictionary
    T_range_inc = [floor(Int,T_range[1])+1, ceil(Int,T_range[2])-1]
    npoints = T_range_inc[2] - T_range_inc[1] + 1
    bulk = perplex_query_system(perplexdir, scratchdir, index=idx)            # Get system data for all temperatures. Set include_fluid = "n" to get solid+melt only
    modes = perplex_query_modes(perplexdir, scratchdir, index=idx)            # || phase modes
    melt = perplex_query_phase(perplexdir, scratchdir, "melt(W)", index=idx)  # || melt data

    # Create dictionary to hold solid composition and fill it using what we know from system and melt
    solid = Dict()
    solid["wt_pct"] = 100 .- melt["wt_pct"]
    for e in ["SIO2","AL2O3","FEO","MGO","CAO","NA2O","K2O"]
        solid[e] = (bulk[e] - (melt[e] .* melt["wt_pct"]/100)) ./ (solid["wt_pct"]/100)
    end

    # Add results to melt % vs temperature figure
    plot!(h0, melt["T(K)"] .- 273.15, melt["wt_pct"], label="melt(W) + G")

    # Plot melt composition as a function of melt percent
    h = plot(xlabel="Percent melt", ylabel="Wt. % in melt", title="melt(W) + G_solution_phases, $P bar")
    for e in ["SIO2","AL2O3","FEO","MGO","CAO","NA2O","K2O"]
        plot!(h, melt["wt_pct"], melt[e], label=e)
    end
    plot!(h,fg_color_legend=:white, framestyle=:box)
    savefig(h,"Perplex_MeltTest_W_G.pdf")

    # Plot solid composition as a function of melt percent
    h = plot(xlabel="Percent melt", ylabel="Wt. % in solid", title="melt(W) + G_solution_phases, $P bar")
    for e in ["SIO2","AL2O3","FEO","MGO","CAO","NA2O","K2O"]
        plot!(h, melt["wt_pct"], solid[e], label=e)
    end
    plot!(h,fg_color_legend=:white, framestyle=:box, legend=:topleft)
    savefig(h,"Perplex_SolidTest_W_G.pdf")

## --- # # # # # # # # # # # melt(W) +JH_solution_phases # # # # # # # # # # # #

    # Input parameters
    P = 10000 # bar
    T_range = [500+273.15, 1500+273.15]
    idx = 7
    print("\nmelt(W) + JH_solution_phases\n")
    @time perplex_configure_isobar(perplexdir, scratchdir, composition, elements, P, T_range, dataset="hp11ver.dat", solution_phases="melt(W)\n"*JH_solution_phases, excludes=JH_excludes, index=idx)

    # Query the full isobar -- results returned as elementified dictionary
    T_range_inc = [floor(Int,T_range[1])+1, ceil(Int,T_range[2])-1]
    npoints = T_range_inc[2] - T_range_inc[1] + 1
    bulk = perplex_query_system(perplexdir, scratchdir, index=idx)            # Get system data for all temperatures. Set include_fluid = "n" to get solid+melt only
    modes = perplex_query_modes(perplexdir, scratchdir, index=idx)            # || phase modes
    melt = perplex_query_phase(perplexdir, scratchdir, "melt(W)", index=idx)  # || melt data

    # Create dictionary to hold solid composition and fill it using what we know from system and melt
    solid = Dict()
    solid["wt_pct"] = 100 .- melt["wt_pct"]
    for e in ["SIO2","AL2O3","FEO","MGO","CAO","NA2O","K2O"]
        solid[e] = (bulk[e] - (melt[e] .* melt["wt_pct"]/100)) ./ (solid["wt_pct"]/100)
    end

    # Add results to melt % vs temperature figure
    plot!(h0, melt["T(K)"] .- 273.15, melt["wt_pct"], label="melt(W) + JH")

    # Plot melt composition as a function of melt percent
    h = plot(xlabel="Percent melt", ylabel="Wt. % in melt", title="melt(W) + JH_solution_phases, $P bar")
    for e in ["SIO2","AL2O3","FEO","MGO","CAO","NA2O","K2O"]
        plot!(h, melt["wt_pct"], melt[e], label=e)
    end
    plot!(h,fg_color_legend=:white, framestyle=:box)
    savefig(h,"Perplex_MeltTest_W_JH.pdf")

    # Plot solid composition as a function of melt percent
    h = plot(xlabel="Percent melt", ylabel="Wt. % in solid", title="melt(W) + JH_solution_phases, $P bar")
    for e in ["SIO2","AL2O3","FEO","MGO","CAO","NA2O","K2O"]
        plot!(h, melt["wt_pct"], solid[e], label=e)
    end
    plot!(h,fg_color_legend=:white, framestyle=:box, legend=:topleft)
    savefig(h,"Perplex_SolidTest_W_JH.pdf")

## --- Format melt comparison

    plot!(h0,fg_color_legend=:white,legend=:topleft)
    display(h0)
    savefig(h0,"Perplex_T-F_comparison.pdf")

## --- End of File


function nextline(pf::IO, c::Char)
    eof(pf) && return "-1"
    s = "\0"
    while s[1] != c
        eof(pf) && return "-1"
        s = chomp(readline(pf))
    end
    return s
end

@doc """
    H, R = uwpf(pf[, v])

Read UW-format seismic pick file `pf` into SeisHdr object `H`, with seismic
source description (focal mechanism) returned in SeisSrc object `R`.

    uwpf!(W, pf[, v::Integer=KW.v])

Read UW-format seismic pick info from pickfile `f` into SeisEvent object `W`.
Overwrites W.source and W.hdr with pickfile information. Keyword `v` controls
verbosity.

!!! caution

    Reader has no safety check to guarantee that `pf` is from the same event.

""" uwpf
function uwpf(pickfile::String; v::Integer=KW.v)
  # Initialize variables that will fill SeisHdr structure
  D   = Dict{String, Any}()
  MAG = -5.0f0
  ID  = ""
  OT  = zero(Float64)
  loc = zeros(Float64, 12)
  sig = ""
  locflags = Array{Char, 1}(undef,8)
  R = SeisSrc()
  fill!(locflags, '0')

  # Read begins
  pf = open(pickfile, "r")

  # ========================================================================
  # Acard line
  A = nextline(pf, 'A')
  (v > 1)  && println(stdout, A)
  c = 0
  if length(A) == 75 || length(A) == 12
    y = zero(Int8)
    c = 1900
  else
    y = Int8(2)
  end
  D["type"] = getindex(A, 2)

  # ACard indices
  #
  # Crosson's notes:
  # Type, Year, Month,  Day,  Hour, Min,  Sec,  LatDeg, NS, Latmin*100, LongDeg,  EW, Lonmin*100, Depth, Fix, Magnitude, Numsta, numphase, Gap, Mindelta, RMS, ERR, Q1, Q2, Velmodel
  # ATYYYYMMDDHHMM SS.SS LLNMMMM LLLWMMMM DD.DD* M.M NN/0NN GGG DD R.RR EE.EQQ VV
  # AF200206291436  4.79 45N2009 121W4118  6.20* 4.5 41/041  37  9 0.27  0.1BB O0
  #
  #        Sec,Lat,Lon,Dep,Fix,Mag,Nst,Nph,Gap, d0,RMS,ERR, Q, mod
  si = Int8[13, 19, 27, 36, 42, 43, 47, 51, 54, 58, 61, 66, 71, 74] .+ y
  ei = Int8[18, 26, 35, 41, 42, 46, 49, 53, 57, 60, 65, 70, 72, 75] .+ y
  L = length(si)

  # Parse reset of Acard line
  ah = Array{String,1}(undef, L)
  for i = 1:L
    setindex!(ah, getindex(A, getindex(si, i):getindex(ei, i)), i)
  end
  v > 2 && println("ah = ", ah)

  # origin time, event depth, and magnitude
  OT            = d2u(DateTime(string(parse(Int64, A[3:4+y]) + c)*A[5+y:12+y],
                      "yyyymmddHHMM")) + parse(Float64, getindex(ah, 1))
  evla          = getindex(ah, 2)
  evlo          = getindex(ah, 3)
  loc[3]        = parse(Float64, getindex(ah, 4))       # depth           :dep
  locflags[3]   = (getindex(ah, 5) == "F") ? '1' : '0'
  MAG           = parse(Float32, getindex(ah, 6))
  nst           = parse(Int64, getindex(ah, 7))
  D["numpha"]   = parse(Int64, getindex(ah, 8))
  loc[10]       = parse(Float64, getindex(ah, 9))       # gap             :gap
  loc[11]       = parse(Float64, getindex(ah, 10))      # min distance    :dmin
  loc[9]        = parse(Float64, getindex(ah, 11))      # rms pick error  :rms
  loc[8]        = parse(Float64, getindex(ah, 12))      # standard error  :se
  D["qual"]     = getindex(ah, 13)
  D["vmod"]     = getindex(ah, 14)

  # Convert lat and lon to decimal degrees
  loc[1] = (parse(Float64, evla[1:3]) +
            parse(Float64, evla[5:6])/60.0 +
            parse(Float64, evla[7:8])/6000.0) * (evla[4] == 'S' ? -1.0 : 1.0)
  loc[2] = (parse(Float64, evlo[1:4]) +
            parse(Float64, evlo[6:7])/60.0 +
            parse(Float64, evlo[8:9])/6000.0) * (evlo[5] == 'W' ? -1.0 : 1.0)

  # ========================================================================
  # Error line
  seekstart(pf)
  eline = nextline(pf, 'E')
  if eline != "-1"
    # E O0  0.27 0.022 0.281 0.281  141.30  38   Z  0.05 0.05 0.11 0.01 4.50 0.000.03
    # Effectively: 10x   MeanRMS    SDabout0    SDaboutMean    SSWRES    NDFR    FIXXYZT    SDx    SDy    SDz    SDt    Mag  5x MeanUncert
    #              10x   f6.3       f6.3        f6.3           f8.2      i4      a4        f5.2   f5.2   f5.2   f5.2   f5.2  5x f4.2
    eline_keys = String["MeanRMS", "SDabout0", "SDaboutMean", "SSWRES", "NDFR", "FIXXYZT", "SDx", "SDy", "SDz", "SDt", "Mag", "MeanUncert"]
    si =           Int8[       11,         17,            23,       29,     37,        42,    46,     51,    56,   61,    66,           76]
    ei =           Int8[       16,         22,            28,       36,     40,        45,    50,     55,    60,   65,    70,           79]
    j = 0

    while j < length(eline_keys)
      j = j + 1
      a = getindex(si, j)
      b = getindex(ei, j)
      s = getindex(eline, a:b)
      if j == 6
        (s[1] == 'X') && (locflags[1] = '1')
        (s[2] == 'Y') && (locflags[2] = '1')
        (s[3] == 'Z') && (locflags[3] = '1')
        (s[4] == 'T') && (locflags[4] = '1')
      elseif j == 7
        loc[4] = parse(Float64, s)
      elseif j == 8
        loc[5] = parse(Float64, s)
      elseif j == 9
        loc[6] = parse(Float64, s)
      elseif j == 10
        if s != "*****"
          loc[7] = parse(Float64, s)
        end
      elseif !isempty(s)
        k = getindex(eline_keys, j)
        D[k] = parse(j == 5 ? Int32 : Float32, s)
      end
    end
    sig = "1σ"
  end
  LOC = EQLoc(loc..., nst, parse(UInt8, join(locflags), base=2), "", "hypocenter", "", "SPONG")

  # ========================================================================
  # Focal mechanism line(s)
  #= Note: planes F and G azimuth and an incidence are NOT in N°E. They're
    measured clockwise from the (N°E) azimuth of the dip vector, because R.
    Crosson wanted to be a unique special snowflake.

    The axes copied to Hdr.axes are therefore P, T, with the last field of
    the 3Tuple set to 0.0.
  =#
  seekstart(pf)
  mline = nextline(pf,'M')
  m = 0
  PAX = Array{Float64,2}(undef, 2 ,2)
  NP = Array{Float64,2}(undef, 2, 2)
  if mline != "-1"
    # Convert first mechanism line
    M = split(mline)

    setindex!(NP, parse(Float64, getindex(M, 3)), 1)
    setindex!(NP, parse(Float64, getindex(M, 4)), 2)
    setindex!(NP, parse(Float64, getindex(M, 6)), 3)
    setindex!(NP, parse(Float64, getindex(M, 7)), 4)

    # Order here is T, P
    setindex!(PAX, parse(Float64, getindex(M, 18)), 1)
    setindex!(PAX, parse(Float64, getindex(M, 19)), 2)
    setindex!(PAX, parse(Float64, getindex(M, 15)), 3)
    setindex!(PAX, parse(Float64, getindex(M, 16)), 4)

    setfield!(R, :pax, PAX)
    setfield!(R, :planes, NP)
    pf_str = abspath(pickfile)
    setfield!(R, :src, pf_str)
    note!(R, "+source ¦ " * pf_str)

    # Add rest to a dictionary
    mech_lines = Array{String, 1}(undef, 0)
    mline = nextline(pf,'M')
    while mline != "-1"
      m += 1
      push!(mech_lines, mline)
      mline = nextline(pf,'M')
    end
    R.gap = getindex(loc, 10)
    if !isempty(mech_lines)
      R.misc["mech_lines"] = mech_lines
    end
    v>0 && println(stdout, "Processed ", m, " focal mechanism lines.")
    note!(R, string("planes are arranged [θ₁ θ₂; ϕ₁ ϕ₂] but θ₁, θ₂ are NOT oriented N°E. ",
                    "They're measured clockwise from the N°E azimuth of the dip vector."))
    note!(R, string("gap is copied from the A-card line; this likely ",
                    "under-represents the true focal mechanism gap."))
  end

  # ========================================================================
  # Comment lines
  seekstart(pf)
  m = 0
  cline = nextline(pf,'C')
  if cline != "-1"
    D["comment"] = Array{String, 1}(undef, 0)
    while cline != "-1"
      m = m + 1
      L = lastindex(cline)
      if occursin("NEAR", cline)
        D["loc_name"] = getindex(cline, 8:L)
      elseif occursin("EVENT ID", cline)
        ID = getindex(cline, 13:L)
      elseif occursin("LOCATED BY", cline)
        setfield!(LOC, :src, getfield(LOC, :src) * "; " * getindex(cline, 3:L))
      else
        push!(D["comment"], getindex(cline, 3:L))
      end
      cline = nextline(pf,'C')
    end
    v > 0 && println(stdout, "Processed ", m, " comment lines.")
  end

  # ========================================================================
  # Done reading
  close(pf)
  H = SeisHdr()
  src_str = abspath(pickfile)
  setfield!(H, :loc, LOC)
  setfield!(H, :src, src_str)
  note!(H, "+source ¦ " * src_str)
  if MAG != -5.0f0
    setfield!(H, :mag, EQMag(val = MAG, scale = "Md", src = "SPONG"))
  end
  if OT != zero(Float64)
    setfield!(H, :ot, u2d(OT))
  end
  if isempty(ID) == false
    setfield!(H, :id, string(ID))
    setfield!(R, :eid, string(ID))
  end
  if isempty(D) == false
    setfield!(H, :misc, D)
  end

  return H, R
end

@doc (@doc uwpf)
function uwpf!(S::SeisEvent, pickfile::String; v::Integer=KW.v)
  (H, R) = uwpf(pickfile, v=v)

  N   = getfield(getfield(S, :data), :n)
  ID  = getfield(getfield(S, :data), :id)
  PHA = getfield(getfield(S, :data), :pha)
  cha = Array{String,1}(undef, N)
  for i = 1:N
    id = split(getindex(ID, i), ".", keepempty=true)
    setindex!(cha, string(".", getindex(id, 2), ".", getindex(id, 4)), i)
  end
  v > 2 && println("cha = ", cha)

  # Pick lines (requires reopening/rereading file)
  m = 0
  ndur = 0
  npol = 0
  pf = open(pickfile, "r")
  pick_line = nextline(pf, '.')

  while pick_line != "-1"
    v>1 && println(stdout, "pick_line = ", pick_line)
    m += 1
    pdat = split(pick_line, "(")
    pick_cha = getindex(pdat, 1)
    pcat = PhaseCat()
    dur = 0.0

    # Parse picks
    for j = 2:length(pdat)
      pha = split(getindex(pdat, j))
      if getindex(pha, 1) == "P"
        v > 2 && println(pha)
        pol = getindex(pha, 3)[1]
        if pol != '_'
          npol = npol + 1
        end
        pcat[getindex(pha,2)] = SeisPha(
          0.0,                              # amp
          0.0,                              # d
          0.0,                              # inc
          parse(Float64, pha[7][1:end-1]),  # res
          0.0,                              # rp
          0.0,                              # ta
          parse(Float64, pha[4]),           # tt --> relative to file begin; fixed below
          parse(Float64, pha[6]),           # unc
          pol,                              # polarity
          pha[5][1]                         # quality
          )
      elseif getindex(pha, 1) == "D"
        dur = parse(Float64, getindex(pha,2)[1:end-1])
      end
    end

    # Assign to the correct channel
    for i = 1:N
      if startswith(pick_cha, getindex(cha, i))
        for p in keys(pcat)
          PHA[i][p] = pcat[p]
        end
        if dur > 0.0
          ndur = ndur + 1
          S.data.misc[i]["dur"] = dur
        end
        break
      end
    end
    pick_line = nextline(pf, '.')
  end
  v>0 && println(stdout, "Processed ", m, " pick lines.")
  close(pf)

  # Set ndur, npol
  setfield!(getfield(H, :mag), :nst, ndur)
  setfield!(R, :npol, npol)

  # Place hdr, source
  setfield!(S, :hdr, H)
  setfield!(S, :source, R)

  # Done
  return S
end


export ChunkedVector

immutable ChunkedVector{T,A<:AVector,N} <: AVector{T}
  data::Vector{A}
end

typealias ChunkedVectorT{T,A,N} Type{ChunkedVector{T,A,N}}

(::ChunkedVectorT{T,A,N}){T,A,N}(data = A[A()]) =
  ChunkedVector{T,A,N}(data)

(::ChunkedVectorT{T,A}){T,A}(data = A[A()]) =
  ChunkedVector{T,A,128}(data)

(::ChunkedVectorT{T}){T}(a...) =
  ChunkedVector{T,Vector{T}}(a...)

chunksize{T,A,N}(::Type{ChunkedVector{T,A,N}}) = N
chunksize(xs::ChunkedVector) = chunksize(typeof(xs))

chunks(xs::ChunkedVector) = xs.data

function makeroom!(xs::ChunkedVector)
  push!(xs.data, similar(xs.data[end], 0))
end

function Base.push!(xs::ChunkedVector, y)
  length(xs.data[end]) == chunksize(xs) && makeroom!(xs)
  push!(xs.data[end], y)
  return xs
end

function ChunkedVector(xs, a...)
  v = ChunkedVector{eltype(xs)}(a...)
  for x in xs
    push!(v, x)
  end
  return v
end

function index_slow(xs::ChunkedVector, i)
  j, i′ = divrem(i-1, chunksize(xs))
  j + 1, i′ + 1
end

@generated function index_fast(xs::ChunkedVector, i)
  pow = round(Int, log2(chunksize(xs)))
  :((i-1) >> $pow + 1, (i-1) & $(2^pow-1) + 1)
end

@generated function index(xs::ChunkedVector, i)
  ispow2(chunksize(xs)) ?
    :(index_fast(xs, i)) :
    :(index_slow(xs, i))
end

@inline function Base.getindex(xs::ChunkedVector, i::Integer)
  j, i′ = index(xs, i)
  @inbounds return xs.data[j][i′]
end

function Base.setindex!(xs::ChunkedVector, v, i::Integer)
  j, i′ = index(xs, i)
  @inbounds xs.data[j][i′] = v
end

Base.length(xs::ChunkedVector) = chunksize(xs)*(length(xs.data)-1)+length(xs.data[end])

Base.size(xs::ChunkedVector) = (length(xs),)

# Iteration utils

immutable ChunkIter{N}
  max::Tuple{Int,Int}
end

chunksize{N}(::Type{ChunkIter{N}}) = N
chunksize(it::ChunkIter) = chunksize(typeof(it))

Base.start(::ChunkIter) = (1, 0)

Base.done(it::ChunkIter, state) =
  state[2] == it.max[2] && state[1] == it.max[1]

@inline function _next(it::ChunkIter, last)
  i, j = last
  j == chunksize(it) ? (i+1, 1) : (i, j+1)
end

function Base.next(it::ChunkIter, last)
  next = _next(it, last)
  next, next
end

Base.length(it::ChunkIter) = (it.max[1]-1)*chunksize(it)+it.max[2]

# ChunkVector iteration

ChunkIter(xs::ChunkedVector) =
  ChunkIter{chunksize(xs)}((length(xs.data), length(xs.data[end])))

@iter xs::ChunkedVector -> ChunkIter(xs)

@inline function Base.next(xs::ChunkedVector, sub::SubIter)
  i, j = _next(sub.iter, sub.state)
  @inbounds return xs.data[i][j], SubIter(sub.iter, (i, j))
end


export
    ConstraintSet,
	add_constraint!,
	findmax_violation

############################################################################################
#  								CONSTRAINT SETS 										   #
############################################################################################

""" $(TYPEDEF) Set of all constraints for a trajectory optimization problem
Holds a vector of [`ConstraintVals`](@ref) that specify where in the trajectory each constraint
is applied. The `ConstraintSet` efficiently dispatches functions to all of the constraints.

# Constructors:
```julia
ConstraintSet(n,m,N)
ConstraintSet(n,m,Vector{<:ConstraintVals},N)
```

"""
struct ConstraintSet{T}
	n::Int
	m::Int
	constraints::Vector{ConstraintVals}
	p::Vector{Int}
	c_max::Vector{T}
end

function ConstraintSet(n::Int,m::Int,constraints, N)
	@assert !isempty(constraints)
	p = zeros(Int,N)
	c_max = zeros(length(constraints))
	for con in constraints
		for k = 1:N
			p[k] += length(con, k)
		end
	end
	cons = ConstraintVals[]
	append!(cons, constraints)
	ConstraintSet(n,m, cons, p, c_max)
end

function ConstraintSet(n,m,N)
	constraints = Vector{ConstraintVals}()
	p = zeros(Int,N)
	c_max = zeros(0)
	ConstraintSet(n,m,constraints,p,c_max)
end

"Get size of state and control dimensions"
Base.size(conSet::ConstraintSet) = conSet.n, conSet.m, length(conSet.constraints)
"Get number of separate constraints (i.e. ConstraintVals) in the set"
Base.length(conSet::ConstraintSet) = length(conSet.constraints)
Base.copy(conSet::ConstraintSet) = ConstraintSet(conSet.n, conSet.m,
	copy(conSet.constraints), copy(conSet.p), copy(conSet.c_max))

"Re-calculate the number of constraints in the constraint set"
function num_constraints!(conSet::ConstraintSet)
	p = conSet.p
	p .*= 0
	for con in conSet.constraints
		for k = 1:length(p)
			p[k] += length(con, k)
		end
	end
end

"""```julia
num_constraints(::ConstraintSet)
num_constraints(::AbstractSolver)
num_constraints(::Problem)
```
Get the total number of constraints at each time step
"""
@inline num_constraints(conSet::ConstraintSet) = conSet.p

"""```julia
add_constraint!(conSet, conVal::ConstraintVals, idx=-1)
add_constraint!(conSet, con::AbstractConstraint, inds::UnitRange, idx=-1)
```
Add a constraint to the constraint set. You can directly add a `ConstraintVals` type,
but when adding a normal `AbstractConstraint` you must specify the range of knotpoints
to which the constraint applies.

The optional `idx` argument allows you to specify exactly where in the vector of constraints
you'd like the constraint to be added. The ordering will effect the order in which the constraint
appears in concatenated vectors of constraint values (that show up in direct methods), potentially
effecting the band structure of the resultant Jacobian. See solvers for more details.
"""
function add_constraint!(conSet::ConstraintSet, conVal::ConstraintVals, idx=-1)
	if idx == -1
		push!(conSet.constraints, conVal)
		push!(conSet.c_max, 0)
	else
		insert!(conSet.constraints, idx, conVal)
		insert!(conSet.c_max, idx, 0)
	end
	num_constraints!(conSet)
end

function add_constraint!(conSet::ConstraintSet, con::AbstractConstraint,
		inds::UnitRange, idx=-1)
	conVal = ConstraintVals(con, inds)
	add_constraint!(conSet, conVal, idx)
end

"""```julia
max_violation(conSet::ConstraintSet)
max_violation(conSet::ConstraintSet, Z::Traj)
max_violation(prob::Problem, Z=prob.Z)
max_violation(solver::AbstractSolver)
max_violation(solver::AbstractSolver, Z)
```
Calculate the maximum constraint violation for the entire constraint set.
	If the a trajectory is not passed in, the violation is computed from the currently
	stored constraint values; otherwise, the constraints are re-computed using the
	trajectory passed in.
"""
function max_violation(conSet::ConstraintSet)
	max_violation!(conSet)
	maximum(conSet.c_max)
end

function max_violation(conSet::ConstraintSet, Z::Traj)
	evaluate!(conSet, Z)
	max_violation(conSet)
end

function max_violation!(conSet::ConstraintSet{T}) where T
	for i in eachindex(conSet.constraints)
		con = conSet.constraints[i]
		max_violation!(con)
		conSet.c_max[i] = maximum(con.c_max::Vector{T})
	end
end

function findmax_violation(conSet::ConstraintSet)
	max_violation!(conSet)
	c_max0, j_con = findmax(conSet.c_max) # which constraint
	if c_max0 < 1e-16
		return "No constraints violated"
	end
	con = conSet.constraints[j_con]
	i_con = findmax(con.c_max)[2]  # whicn index
	k_con = con.inds[i_con] # time step
	con_sense = sense(con)
	viol = violation(con_sense, con.vals[i_con])
	c_max, i_max = findmax(viol)  # index into constraint
	@assert c_max == c_max0
	con_name = string(typeof(con.con).name)
	return con_name * " at time step $k_con at " * con_label(con.con, i_max)
end

"Calculate the maximum penalty parameter across all constraints"
function max_penalty(conSet::ConstraintSet)
	max_penalty!(conSet)
	maximum(conSet.c_max)
end

function max_penalty!(conSet::ConstraintSet{T}) where T
	for (i,con) in enumerate(conSet.constraints)
		max_penalty!(con)
		conSet.c_max[i] = maximum(con.c_max::Vector{T})
	end
end

function shift_fill!(conSet::ConstraintSet)
	for con in conSet.constraints
		shift_fill!(con)
	end
end

"""```julia
evaluate!(conSet::ConstraintSet, Z::Traj)
```
Compute constraint values for all constraints for the entire trajectory
"""
function evaluate!(conSet::ConstraintSet, Z::Traj)
	for con in conSet.constraints
		evaluate!(con, Z)
	end
end

"""```julia
jacobian!(conSet::ConstraintSet, Z::Traj)
```
Compute constraint Jacobians for all constraints for the entire trajectory
"""
function jacobian!(conSet::ConstraintSet, Z::Traj)
	for con in conSet.constraints
		jacobian!(con, Z)
	end
end

"""```julia
update_active_set!(conSet::ConstraintSet, Z::Traj, ::Val{tol})
```
Compute the active set for the current constraint values, with tolerance tol.
	Uses a value type to avoid an allocation down the line.
"""
function update_active_set!(conSet::ConstraintSet, Z::Traj, val::Val{tol}=Val(0.0)) where tol
	for con in conSet.constraints
		update_active_set!(con, val)
	end
end

Base.iterate(conSet::ConstraintSet) = @inbounds (conSet.constraints[1], 1)
Base.iterate(conSet::ConstraintSet, i) = @inbounds i >= length(conSet.constraints) ? nothing : (conSet.constraints[i+1], i+1)
@inline Base.getindex(conSet::ConstraintSet, i) = conSet.constraints[i]

"Reset all the Lagrange multipliers and constraint values to zero and
	penalties the their initial value"
function reset!(conSet::ConstraintSet)
	for con in conSet.constraints
		reset!(con)
	end
end

function reset_duals!(conSet::ConstraintSet)
	for con in conSet.constraints
		reset_duals!(con)
	end
end

function reset_penalties!(conSet::ConstraintSet)
	for con in conSet.constraints
		reset_penalties!(con)
	end
end

""" ```julia
change_dimension(conSet::ConstraintSet, n, m)
```
Change the dimensionality of the constraint set to one that is strictly larger
than the current dimensions. Useful for state and control augmentation. If the dimension
change affects a particular constraint, it will be wrapped in an [`IndexedConstraint`](@ref)
that simply passes the original sizes to the original constraints. Right now, this assumes
that the original state and controls are first (i.e. the new states or controls are appended
to the end of the state and control vectors).
"""
function change_dimension(conSet::ConstraintSet, n, m)
	cons = map(conSet.constraints) do con

		# Check if the new dimensions match the old ones
		if check_dims(con,n,m)
			return con
		end

		con_idx = IndexedConstraint(n,m,con.con)
		con_val = ConstraintVals(con_idx, con.inds)
	end
	ConstraintSet(n,m,cons, length(conSet.p))
end


@reexport module FirstPoincareInvariants

end  # module FirstPoincareInvariants


#ДАНО: Робот находится в произвольной клетке ограниченного прямоугольного поля с внутренними перегородками.
#РЕЗУЛЬТАТ: Робот — в исходном положении в центре прямого креста из маркеров, расставленных вплоть до внешней рамки.
include("library.jl")
function krest_with_obstacles(r, dx::Integer = 0, dy::Integer = 0)

    R = CrossRobot(r, Coords(0, 0), true, dx, dy)
    back_path = BackPath(R)
    marks!(R)
    BackPath(r)
    back!(R, back_path)
end


export uniform_mutate, goldman_mutate, profiling_mutate

"create a child by randomly mutating genes"
function uniform_mutate(cfg::NamedTuple, ind::CGPInd; kwargs...)::CGPInd
    chromosome = copy(ind.chromosome)
    chance = rand(length(ind.chromosome))
    non_output = length(ind.chromosome) - length(ind.outputs)
    change = [chance[1:non_output] .<= cfg.m_rate;
              chance[(non_output+1):end] .<= cfg.out_m_rate]
    chromosome[change] = rand(sum(change))
    kwargs_dict = Dict(kwargs)
    if haskey(kwargs_dict, :init_function)
        return kwargs_dict[:init_function](cfg, chromosome)
    else
        return CGPInd(cfg, chromosome)
    end
end

"create a child that is structurally different from the parent"
function goldman_mutate(cfg::NamedTuple, ind::CGPInd; kwargs...)::CGPInd
    child = uniform_mutate(cfg, ind; kwargs...)
    while true
        if any(ind.outputs != child.outputs)
            return child
        else
            for i in eachindex(ind.nodes)
                if ind.nodes[i].active
                    if child.nodes[i].active
                        if (ind.nodes[i].f != child.nodes[i].f
                            || ind.nodes[i].x != child.nodes[i].x
                            || ind.nodes[i].y != child.nodes[i].y)
                            return child
                        end
                    else
                        return child
                    end
                end
            end
        end
        child = uniform_mutate(cfg, ind; kwargs...)
    end
    nothing
end

"create a child that gives different outputs based on the provided inputs (a 2D matrix of (n_in, n_samples))"
function profiling_mutate(cfg::NamedTuple, ind::CGPInd, inputs::AbstractArray; kwargs...)::CGPInd
    child = uniform_mutate(cfg, ind; kwargs...)
    while true
        for i in 1:size(inputs, 2)
            out_ind = process(ind, inputs[:, i])
            out_child = process(child, inputs[:, i])
            if any(out_ind .!= out_child)
                return child
            end
        end
        child = uniform_mutate(cfg, ind; kwargs...)
    end
    nothing
end


include("../src/geom_module.jl")

module linetest
using ..geom
using PyPlot
PyPlot.pygui(true)

function test_line()

end

end


### Functions for loading/anntoting mutations/variants

using FASTX

### load mutations

function loadmutations(; file="hk1mutations.tsv", projdir=getprojectdir())
    filepath = joinpath(projdir, "data", file)
    df = CSV.read(filepath, DataFrame)

    df.Ref = [ismissing(s) ? dna"" : LongDNASeq(s) for s in df.Ref]
    df.Alt = [ismissing(s) ? dna"" : LongDNASeq(s) for s in df.Alt]

    df.PatientGroup = Int.(floor.(df.Patient))
    df[!, :ID] = string.(df.start, "_", df.Ref, "_", df.Alt)

    df
end

function parsechromloc(s)
    fields = split(replace(s, "," => ""), r"[:-]")
    fields[1], parse(Int, fields[2]):parse(Int, fields[3])
end

function loadregregion(;file="hk1_reg_region.fa",  projdir=getprojectdir())
    filepath = joinpath(projdir, "data", file)
    reader = open(FASTA.Reader, filepath)
    record = first(collect(reader))
    seq = sequence(record)
    chromloc = replace(first(split(description(record))), "range=" => "")
    chrom, loc = parsechromloc(chromloc)
    seq
    Interval(chrom, loc, '+', seq)
end

function annotateregion!(mutations, region)

    regc = leftposition(region):rightposition(region)
    mutreg = [s:e for (s, e) in zip(mutations.start, mutations.stop)]
    relreg = [mr .- leftposition(region) .+ 1 for mr in mutreg]
    mutations[!, :RelReg] = relreg
    mutations[!, :Contained] = length.(intersect.(Ref(regc), mutreg)) .== length.(mutreg)

    ### assert that ref alleles match sequence
    refseqs = [region.metadata[r] for r in mutations[mutations.Contained, :RelReg]]
    @assert refseqs == mutations.Ref[mutations.Contained]
    mutations

end

buildmutation(refreg, kind, ref, alt, contained) = (coords=refreg, kind=kind, ref=ref, alt=alt, contained=contained)

function buildaltseq(seq, mutation)
    if !mutation.contained
        return missing, missing, missing, missing
    end
    leftind = 1:(first(mutation.coords) - 1)
    refrightind = (last(mutation.coords) + 1):length(seq)
    if mutation.kind == "var"
        altind  = mutation.coords
        altrightind = refrightind
    elseif mutation.kind == "del"
        altind = first(mutation.coords) .+ (0:-1)
        altrightind = first(mutation.coords):(length(seq) - length(mutation.coords))
    else
        error("Mutation type: $mutation.kind not recognised")
    end

    altseq = seq[leftind]*mutation.alt*seq[refrightind]
    leftind, altind, refrightind, altrightind, altseq
end

function distinctmuts(mutations, region)
    distinctmutations = combine(groupby(@subset(mutations, :Contained), names(mutations, Not([:Patient, :Mutation, :Inheritance]))), nrow=>:count)

    muts = buildmutation.(distinctmutations.RelReg, distinctmutations.type, distinctmutations.Ref, distinctmutations.Alt, distinctmutations.Contained);

    refseq = region.metadata
    altseqindex = buildaltseq.(Ref(refseq), muts)
    distinctmutations.LeftInd = first.(altseqindex)
    distinctmutations.AltInd = getindex.(altseqindex, 2)
    distinctmutations.RefRightInd = getindex.(altseqindex, 3)
    distinctmutations.AltRightInd = getindex.(altseqindex, 4)

    altseqs = last.(altseqindex)

    distinctmutations, refseq, altseqs
end


using NonconvexMultistart
function convert_common_kwargs(opt::NonconvexMultistart.HyperoptAlg, opt_kwargs;
    callback=nothing,
    maxiters=nothing,
    maxtime=nothing,
    abstol=nothing,
    reltol=nothing)

    conv_opt_kwargs = (; opt_kwargs...)

    if !isnothing(callback)
        @warn "common callback argument is currently not used by $(opt)"
    end

    if !isnothing(maxiters)
        @info "common maxiters argument refers to how many of the potential starting points will be evaluated by $(opt)"
        conv_opt_kwargs = (; conv_opt_kwargs..., iters=maxiters)
    end

    if !isnothing(maxtime)
        @warn "common maxtime argument is currently not used by $(opt)"
    end

    if !isnothing(abstol)
        @warn "common abstol argument is currently not used by $(opt)"
    end

    if !isnothing(reltol)
        @warn "common reltol argument is currently not used by $(opt)"
    end

    return conv_opt_kwargs
end

function __create_options(opt::NonconvexMultistart.HyperoptAlg;
    opt_kwargs=nothing)

    options = !isnothing(opt_kwargs) ? NonconvexMultistart.HyperoptOptions(;opt_kwargs...) : NonconvexMultistart.HyperoptOptions()

    if isa(options.sampler, NonconvexMultistart.Hyperopt.Hyperband)
        error("$(options.sampler) is currently not support by GalacticOptim")
    end

    return options
end

function _create_options(opt::NonconvexMultistart.HyperoptAlg;
    opt_kwargs=nothing,
    sub_options=nothing,
    convergence_criteria=nothing)

    options = (; options = !isnothing(opt_kwargs) ? NonconvexMultistart.HyperoptOptions(;sub_options= __create_options(opt.sub_alg, opt_kwargs=sub_options) ,opt_kwargs...) : NonconvexMultistart.HyperoptOptions(;sub_options= __create_options(opt.sub_alg, opt_kwargs=sub_options)))

    if isa(options.options.sampler, NonconvexMultistart.Hyperopt.Hyperband)
        error("$(options.options.sampler) is currently not support by GalacticOptim")
    end

    return options
end

check_optimizer_backend(opt::NonconvexMultistart.HyperoptAlg) = false


"""
A mutable collection of `Monetary` values of various currencies.

Like regular `Monetary` values, `Basket` values support basic arithmetic
operations, with both other baskets and with raw monetary values. The
constructor for `Basket` accepts either a monetary value or a vector of
monetary values.

```julia
basket = Basket([1USD, 5EUR])        # \$1 + 5€
basket += 2EUR                       # \$1 + 7€
basket *= 2                          # \$2 + 14€
```

`Basket` values also support iteration. The iteration order of currencies is
undefined. For instance, the following will print `1.00 USD` and `5.00 EUR`, in
some order. Iteration will skip over any currencies with zero weight in the
basket.

```julia
for amt in Basket([1USD, 5EUR])
    println(amt)
end
```

One can also mutate the basket with index notation, or with `push!`.

```julia
basket = Basket([1USD, 2EUR])
basket[:USD] = 3USD  # Basket([3USD, 2EUR])
push!(basket, 10GBP) # Basket([3USD, 2EUR, 10GBP])
```
"""
struct Basket <: AbstractMonetary
    table::SMDict
    Basket(ms::Union{AbstractArray,Tuple}) = new(buildtable(ms))
end

# basket outer constructor and convert-constructor
Basket() = Basket(())
(::Type{T})(x) where T <: Basket = convert(T, x)

# access methods
Base.length(b::Basket) = length(b.table)
Base.haskey(b::Basket, k) = haskey(b.table, k) && !iszero(b.table[k])
Base.getindex(b::Basket, T::Symbol) = get(b.table, T, zero(Monetary{T}))
Base.iterate(b::Basket, s...) = iterate(values(b.table), s...)

# arithmetic methods (for static & dynamic baskets)
- b::Basket = Basket([-x for x in b])
b::Basket + c::Basket = Basket((b, c))
b::Basket * k::Real   = Basket([x * k for x in b])
k::Real   * b::Basket = Basket([k * x for x in b])
b::Basket / k::Real   = Basket([x / k for x in b])

# methods for dynamic baskets
function Base.setindex!(b::Basket, m::Monetary, k::Symbol)
    if currency(m) ≠ k
        throw(ArgumentError("Monetary value type does not match currency"))
    end
    b.table[k] = m
    deleteifzero!(b.table, k)
    b
end

# somewhat strange because Baskets behave like collections and like numbers...
# maybe add! is a better name
Base.push!(b::Basket, m::Monetary) = (b[currency(m)] += m; b)
function Base.push!(b::Basket, c::Basket)
    for m in c
        push!(b, m)
    end
    b
end

# other methods (eltype, iszero, zero, ==)
          iszero(b::Basket) = isempty(b)
  Base.zero(::Type{Basket}) = Basket()
   Base.one(::Type{Basket}) = 1
Base.eltype(::Type{Basket}) = Monetary


using FlexJuMP
@static if VERSION < v"0.7.0-DEV.2005"
    using Base.Test
else
    using Test
end
using Pavito, GLPKMathProgInterface, Ipopt
using JuMP
using LinearAlgebra

# Run basic model definition tests

@time @testset "Model Definition Testing" begin
    @test include("initialize_model.jl")
    @test include("set_single_variables.jl")
    @test include("set_vector_variables.jl")
    @test include("define_constraints.jl")
    @test include("define_real_model.jl")
end
print("\n")

# Check basic data manipulation functions
@time @testset "Data Access Testing" begin
    @testset "Data Extraction" begin include("get_flexdata.jl") end
    @testset "Get/Set Covariance Matrix" begin include("manage_covariance.jl") end
    @testset "Get/Set Mean" begin include("manage_mean.jl") end
end
print("\n")

# Check functions that can be used before solving the model
@time @testset "Pre-Solve Method Testing" begin
    @testset "Set Uncertainty Set" begin include("define_uncertainty_set.jl") end
    @testset "Check Mean" begin include("check_mean.jl") end
    @testset "Find Centered Mean" begin include("find_center.jl") end
    @testset "SF Index" begin include("sf_index.jl") end
end
print("\n")

# Extensively test the solvehook
@time @testset "Solvehook Testing" begin include("solvehook_checks.jl") end
print("\n")

# Test function that can be called after the solution of the solvehook
@time @testset "Post-Solve Method Testing" begin
    @testset "Confidence Level" begin include("confidence_level.jl") end
    @testset "Critical Point" begin include("retrieve_critical_pt.jl") end
    @testset "Rank Constraints" begin include("constraint_ranker.jl") end
    @testset "SF Index" begin include("sf_index2.jl") end
end