function DfpField(arg0::jint)
    return DfpField((jint,), arg0)
end

function clear_ieee_flags(obj::DfpField)
    return jcall(obj, "clearIEEEFlags", void, ())
end

function compute_exp(::Type{DfpField}, arg0::Dfp, arg1::Dfp)
    return jcall(DfpField, "computeExp", Dfp, (Dfp, Dfp), arg0, arg1)
end

function compute_ln(::Type{DfpField}, arg0::Dfp, arg1::Dfp, arg2::Dfp)
    return jcall(DfpField, "computeLn", Dfp, (Dfp, Dfp, Dfp), arg0, arg1, arg2)
end

function equals(obj::DfpField, arg0::Object)
    return jcall(obj, "equals", jboolean, (Object,), arg0)
end

function get_deg_to_rad(obj::DfpField)
    return jcall(obj, "getDegToRad", Dfp, ())
end

function get_e(obj::DfpField)
    return jcall(obj, "getE", Dfp, ())
end

function get_e_split(obj::DfpField)
    return jcall(obj, "getESplit", Vector{Dfp}, ())
end

function get_extended_field(obj::DfpField, arg0::jint, arg1::jboolean)
    return jcall(obj, "getExtendedField", DfpField, (jint, jboolean), arg0, arg1)
end

function get_ieee_flags(obj::DfpField)
    return jcall(obj, "getIEEEFlags", jint, ())
end

function get_ln10(obj::DfpField)
    return jcall(obj, "getLn10", Dfp, ())
end

function get_ln2(obj::DfpField)
    return jcall(obj, "getLn2", Dfp, ())
end

function get_ln2_split(obj::DfpField)
    return jcall(obj, "getLn2Split", Vector{Dfp}, ())
end

function get_ln5(obj::DfpField)
    return jcall(obj, "getLn5", Dfp, ())
end

function get_ln5_split(obj::DfpField)
    return jcall(obj, "getLn5Split", Vector{Dfp}, ())
end

function get_one(obj::DfpField)
    return jcall(obj, "getOne", Dfp, ())
end

function get_pi(obj::DfpField)
    return jcall(obj, "getPi", Dfp, ())
end

function get_pi_split(obj::DfpField)
    return jcall(obj, "getPiSplit", Vector{Dfp}, ())
end

function get_rad_to_deg(obj::DfpField)
    return jcall(obj, "getRadToDeg", Dfp, ())
end

function get_radix_digits(obj::DfpField)
    return jcall(obj, "getRadixDigits", jint, ())
end

function get_rounding_mode(obj::DfpField)
    return jcall(obj, "getRoundingMode", DfpField_RoundingMode, ())
end

function get_runtime_class(obj::DfpField)
    return jcall(obj, "getRuntimeClass", Class, ())
end

function get_sqr2(obj::DfpField)
    return jcall(obj, "getSqr2", Dfp, ())
end

function get_sqr2_reciprocal(obj::DfpField)
    return jcall(obj, "getSqr2Reciprocal", Dfp, ())
end

function get_sqr2_split(obj::DfpField)
    return jcall(obj, "getSqr2Split", Vector{Dfp}, ())
end

function get_sqr3(obj::DfpField)
    return jcall(obj, "getSqr3", Dfp, ())
end

function get_sqr3_reciprocal(obj::DfpField)
    return jcall(obj, "getSqr3Reciprocal", Dfp, ())
end

function get_two(obj::DfpField)
    return jcall(obj, "getTwo", Dfp, ())
end

function get_zero(obj::DfpField)
    return jcall(obj, "getZero", Dfp, ())
end

function hash_code(obj::DfpField)
    return jcall(obj, "hashCode", jint, ())
end

function new_dfp(obj::DfpField)
    return jcall(obj, "newDfp", Dfp, ())
end

function new_dfp(obj::DfpField, arg0::Dfp)
    return jcall(obj, "newDfp", Dfp, (Dfp,), arg0)
end

function new_dfp(obj::DfpField, arg0::JString)
    return jcall(obj, "newDfp", Dfp, (JString,), arg0)
end

function new_dfp(obj::DfpField, arg0::jbyte)
    return jcall(obj, "newDfp", Dfp, (jbyte,), arg0)
end

function new_dfp(obj::DfpField, arg0::jbyte, arg1::jbyte)
    return jcall(obj, "newDfp", Dfp, (jbyte, jbyte), arg0, arg1)
end

function new_dfp(obj::DfpField, arg0::jdouble)
    return jcall(obj, "newDfp", Dfp, (jdouble,), arg0)
end

function new_dfp(obj::DfpField, arg0::jint)
    return jcall(obj, "newDfp", Dfp, (jint,), arg0)
end

function new_dfp(obj::DfpField, arg0::jlong)
    return jcall(obj, "newDfp", Dfp, (jlong,), arg0)
end

function set_ieee_flags(obj::DfpField, arg0::jint)
    return jcall(obj, "setIEEEFlags", void, (jint,), arg0)
end

function set_ieee_flags_bits(obj::DfpField, arg0::jint)
    return jcall(obj, "setIEEEFlagsBits", void, (jint,), arg0)
end

function set_rounding_mode(obj::DfpField, arg0::DfpField_RoundingMode)
    return jcall(obj, "setRoundingMode", void, (DfpField_RoundingMode,), arg0)
end

function to_string(obj::Object)
    return jcall(obj, "toString", JString, ())
end


# This file is a part of Julia. License is MIT: https://julialang.org/license

module QueryTests

using Test
using Dates

Jan = DatesPlus.DateTime(2013, 1, 1) # Tuesday
Feb = DatesPlus.DateTime(2013, 2, 2) # Saturday
Mar = DatesPlus.DateTime(2013, 3, 3) # Sunday
Apr = DatesPlus.DateTime(2013, 4, 4) # Thursday
May = DatesPlus.DateTime(2013, 5, 5) # Sunday
Jun = DatesPlus.DateTime(2013, 6, 7) # Friday
Jul = DatesPlus.DateTime(2013, 7, 7) # Sunday
Aug = DatesPlus.DateTime(2013, 8, 8) # Thursday
Sep = DatesPlus.DateTime(2013, 9, 9) # Monday
Oct = DatesPlus.DateTime(2013, 10, 10) # Thursday
Nov = DatesPlus.DateTime(2013, 11, 11) # Monday
Dec = DatesPlus.DateTime(2013, 12, 11) # Wednesday
monthnames = ["January", "February", "March", "April",
                "May", "June", "July", "August", "September",
                "October", "November", "December"]
daysofweek = [DatesPlus.Tue, DatesPlus.Sat, DatesPlus.Sun, DatesPlus.Thu, DatesPlus.Sun, DatesPlus.Fri,
              DatesPlus.Sun, DatesPlus.Thu, DatesPlus.Mon, DatesPlus.Thu, DatesPlus.Mon, DatesPlus.Wed]
dows = ["Tuesday", "Saturday", "Sunday", "Thursday", "Sunday", "Friday",
        "Sunday", "Thursday", "Monday", "Thursday", "Monday", "Wednesday"]
daysinmonth = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
@testset "Name functions" begin
    for (i, dt) in enumerate([Jan, Feb, Mar, Apr, May, Jun, Jul, Aug, Sep, Oct, Nov, Dec])
        @test DatesPlus.month(dt) == i
        @test DatesPlus.monthname(dt) == monthnames[i]
        @test DatesPlus.monthname(i) == monthnames[i]
        @test DatesPlus.monthabbr(dt) == monthnames[i][1:3]
        @test DatesPlus.monthabbr(i) == monthnames[i][1:3]
        @test DatesPlus.dayofweek(dt) == daysofweek[i]
        @test DatesPlus.dayname(dt) == dows[i]
        @test DatesPlus.dayname(DatesPlus.dayofweek(dt)) == dows[i]
        @test DatesPlus.dayabbr(dt) == dows[i][1:3]
        @test DatesPlus.dayabbr(DatesPlus.dayofweek(dt)) == dows[i][1:3]
        @test DatesPlus.daysinmonth(dt) == daysinmonth[i]
    end
end
@testset "Customizing locale" begin
    DatesPlus.LOCALES["french"] = DatesPlus.DateLocale(
        ["janvier", "février", "mars", "avril", "mai", "juin",
         "juillet", "août", "septembre", "octobre", "novembre", "décembre"],
        ["janv", "févr", "mars", "avril", "mai", "juin",
         "juil", "août", "sept", "oct", "nov", "déc"],
        ["lundi", "mardi", "mercredi", "jeudi", "vendredi", "samedi", "dimanche"],
        [""],
    )
    @test DatesPlus.dayname(Nov; locale="french") == "lundi"
    @test DatesPlus.dayname(Jan; locale="french") == "mardi"
    @test DatesPlus.dayname(Dec; locale="french") == "mercredi"
    @test DatesPlus.dayname(Apr; locale="french") == "jeudi"
    @test DatesPlus.dayname(Jun; locale="french") == "vendredi"
    @test DatesPlus.dayname(Feb; locale="french") == "samedi"
    @test DatesPlus.dayname(May; locale="french") == "dimanche"

    @test DatesPlus.monthname(Jan; locale="french") == "janvier"
    @test DatesPlus.monthname(Feb; locale="french") == "février"
    @test DatesPlus.monthname(Mar; locale="french") == "mars"
    @test DatesPlus.monthname(Apr; locale="french") == "avril"
    @test DatesPlus.monthname(May; locale="french") == "mai"
    @test DatesPlus.monthname(Jun; locale="french") == "juin"
    @test DatesPlus.monthname(Jul; locale="french") == "juillet"
    @test DatesPlus.monthname(Aug; locale="french") == "août"
    @test DatesPlus.monthname(Sep; locale="french") == "septembre"
    @test DatesPlus.monthname(Oct; locale="french") == "octobre"
    @test DatesPlus.monthname(Nov; locale="french") == "novembre"
    @test DatesPlus.monthname(Dec; locale="french") == "décembre"
end
@testset "leap years" begin
    @test DatesPlus.isleapyear(DatesPlus.DateTime(1900)) == false
    @test DatesPlus.isleapyear(DatesPlus.DateTime(2000)) == true
    @test DatesPlus.isleapyear(DatesPlus.DateTime(2004)) == true
    @test DatesPlus.isleapyear(DatesPlus.DateTime(2008)) == true
    @test DatesPlus.isleapyear(DatesPlus.DateTime(0)) == true
    @test DatesPlus.isleapyear(DatesPlus.DateTime(1)) == false
    @test DatesPlus.isleapyear(DatesPlus.DateTime(-1)) == false
    @test DatesPlus.isleapyear(DatesPlus.DateTime(4)) == true
    @test DatesPlus.isleapyear(DatesPlus.DateTime(-4)) == true

    @test DatesPlus.daysinyear(2000) == 366
    @test DatesPlus.daysinyear(2001) == 365
    @test DatesPlus.daysinyear(2000) == 366
    @test DatesPlus.daysinyear(2001) == 365

    @test DatesPlus.daysinyear(DatesPlus.Date(2000)) == 366
    @test DatesPlus.daysinyear(DatesPlus.Date(2001)) == 365
    @test DatesPlus.daysinyear(DatesPlus.DateTime(2000)) == 366
    @test DatesPlus.daysinyear(DatesPlus.DateTime(2001)) == 365
end
@testset "dayofweek" begin
    # Days of week from Monday = 1 to Sunday = 7
    @test DatesPlus.dayofweek(DatesPlus.DateTime(2013, 12, 22)) == 7
    @test DatesPlus.dayofweek(DatesPlus.DateTime(2013, 12, 23)) == 1
    @test DatesPlus.dayofweek(DatesPlus.DateTime(2013, 12, 24)) == 2
    @test DatesPlus.dayofweek(DatesPlus.DateTime(2013, 12, 25)) == 3
    @test DatesPlus.dayofweek(DatesPlus.DateTime(2013, 12, 26)) == 4
    @test DatesPlus.dayofweek(DatesPlus.DateTime(2013, 12, 27)) == 5
    @test DatesPlus.dayofweek(DatesPlus.DateTime(2013, 12, 28)) == 6
    @test DatesPlus.dayofweek(DatesPlus.DateTime(2013, 12, 29)) == 7
end
@testset "daysofweekinmonth" begin
    # There are 5 Sundays in December, 2013
    @test DatesPlus.daysofweekinmonth(DatesPlus.DateTime(2013, 12, 1)) == 5
    # There are 4 Sundays in November, 2013
    @test DatesPlus.daysofweekinmonth(DatesPlus.DateTime(2013, 11, 24)) == 4

    @test DatesPlus.dayofweekofmonth(DatesPlus.DateTime(2013, 12, 1)) == 1
    @test DatesPlus.dayofweekofmonth(DatesPlus.DateTime(2013, 12, 8)) == 2
    @test DatesPlus.dayofweekofmonth(DatesPlus.DateTime(2013, 12, 15)) == 3
    @test DatesPlus.dayofweekofmonth(DatesPlus.DateTime(2013, 12, 22)) == 4
    @test DatesPlus.dayofweekofmonth(DatesPlus.DateTime(2013, 12, 29)) == 5
end
@testset "dayofyear" begin
    @test DatesPlus.dayofyear(2000, 1, 1) == 1
    @test DatesPlus.dayofyear(2004, 1, 1) == 1
    @test DatesPlus.dayofyear(20013, 1, 1) == 1
    # Leap year
    @test DatesPlus.dayofyear(2000, 12, 31) == 366
    # Non-leap year
    @test DatesPlus.dayofyear(2001, 12, 31) == 365

    @test DatesPlus.dayofyear(DatesPlus.DateTime(2000, 1, 1)) == 1
    @test DatesPlus.dayofyear(DatesPlus.DateTime(2004, 1, 1)) == 1
    @test DatesPlus.dayofyear(DatesPlus.DateTime(20013, 1, 1)) == 1
    # Leap year
    @test DatesPlus.dayofyear(DatesPlus.DateTime(2000, 12, 31)) == 366
    # Non-leap year
    @test DatesPlus.dayofyear(DatesPlus.DateTime(2001, 12, 31)) == 365
    # Test every day of a year
    dt = DatesPlus.DateTime(2000, 1, 1)
    for i = 1:366
        @test DatesPlus.dayofyear(dt) == i
        dt += DatesPlus.Day(1)
    end
    dt = DatesPlus.DateTime(2001, 1, 1)
    for i = 1:365
        @test DatesPlus.dayofyear(dt) == i
        dt += DatesPlus.Day(1)
    end
end
@testset "quarterofyear" begin
    @test DatesPlus.quarterofyear(DatesPlus.Date(2000, 1, 1))  == 1
    @test DatesPlus.quarterofyear(DatesPlus.Date(2000, 1, 31))  == 1
    @test DatesPlus.quarterofyear(DatesPlus.Date(2000, 2, 1))  == 1
    @test DatesPlus.quarterofyear(DatesPlus.Date(2000, 2, 29))  == 1
    @test DatesPlus.quarterofyear(DatesPlus.Date(2000, 3, 1))  == 1
    @test DatesPlus.quarterofyear(DatesPlus.Date(2000, 3, 31))  == 1
    @test DatesPlus.quarterofyear(DatesPlus.Date(2000, 4, 1)) == 2
    @test DatesPlus.quarterofyear(DatesPlus.Date(2000, 4, 30)) == 2
    @test DatesPlus.quarterofyear(DatesPlus.Date(2000, 5, 1)) == 2
    @test DatesPlus.quarterofyear(DatesPlus.Date(2000, 5, 31)) == 2
    @test DatesPlus.quarterofyear(DatesPlus.Date(2000, 6, 1)) == 2
    @test DatesPlus.quarterofyear(DatesPlus.Date(2000, 6, 30)) == 2
    @test DatesPlus.quarterofyear(DatesPlus.Date(2000, 7, 1)) == 3
    @test DatesPlus.quarterofyear(DatesPlus.Date(2000, 7, 31)) == 3
    @test DatesPlus.quarterofyear(DatesPlus.Date(2000, 8, 1)) == 3
    @test DatesPlus.quarterofyear(DatesPlus.Date(2000, 8, 31)) == 3
    @test DatesPlus.quarterofyear(DatesPlus.Date(2000, 9, 1)) == 3
    @test DatesPlus.quarterofyear(DatesPlus.Date(2000, 9, 30)) == 3
    @test DatesPlus.quarterofyear(DatesPlus.Date(2000, 10, 1)) == 4
    @test DatesPlus.quarterofyear(DatesPlus.Date(2000, 10, 31)) == 4
    @test DatesPlus.quarterofyear(DatesPlus.Date(2000, 11, 1)) == 4
    @test DatesPlus.quarterofyear(DatesPlus.Date(2000, 11, 30)) == 4
    @test DatesPlus.quarterofyear(DatesPlus.Date(2000, 12, 1)) == 4
    @test DatesPlus.quarterofyear(DatesPlus.Date(2000, 12, 31)) == 4

    @test DatesPlus.quarterofyear(DatesPlus.DateTime(2000, 1, 1))  == 1
    @test DatesPlus.quarterofyear(DatesPlus.DateTime(2000, 1, 31))  == 1
    @test DatesPlus.quarterofyear(DatesPlus.DateTime(2000, 2, 1))  == 1
    @test DatesPlus.quarterofyear(DatesPlus.DateTime(2000, 2, 29))  == 1
    @test DatesPlus.quarterofyear(DatesPlus.DateTime(2000, 3, 1))  == 1
    @test DatesPlus.quarterofyear(DatesPlus.DateTime(2000, 3, 31))  == 1
    @test DatesPlus.quarterofyear(DatesPlus.DateTime(2000, 4, 1)) == 2
    @test DatesPlus.quarterofyear(DatesPlus.DateTime(2000, 4, 30)) == 2
    @test DatesPlus.quarterofyear(DatesPlus.DateTime(2000, 5, 1)) == 2
    @test DatesPlus.quarterofyear(DatesPlus.DateTime(2000, 5, 31)) == 2
    @test DatesPlus.quarterofyear(DatesPlus.DateTime(2000, 6, 1)) == 2
    @test DatesPlus.quarterofyear(DatesPlus.DateTime(2000, 6, 30)) == 2
    @test DatesPlus.quarterofyear(DatesPlus.DateTime(2000, 7, 1)) == 3
    @test DatesPlus.quarterofyear(DatesPlus.DateTime(2000, 7, 31)) == 3
    @test DatesPlus.quarterofyear(DatesPlus.DateTime(2000, 8, 1)) == 3
    @test DatesPlus.quarterofyear(DatesPlus.DateTime(2000, 8, 31)) == 3
    @test DatesPlus.quarterofyear(DatesPlus.DateTime(2000, 9, 1)) == 3
    @test DatesPlus.quarterofyear(DatesPlus.DateTime(2000, 9, 30)) == 3
    @test DatesPlus.quarterofyear(DatesPlus.DateTime(2000, 10, 1)) == 4
    @test DatesPlus.quarterofyear(DatesPlus.DateTime(2000, 10, 31)) == 4
    @test DatesPlus.quarterofyear(DatesPlus.DateTime(2000, 11, 1)) == 4
    @test DatesPlus.quarterofyear(DatesPlus.DateTime(2000, 11, 30)) == 4
    @test DatesPlus.quarterofyear(DatesPlus.DateTime(2000, 12, 1)) == 4
    @test DatesPlus.quarterofyear(DatesPlus.DateTime(2000, 12, 31)) == 4
end
@testset "dayofquarter" begin
    # Non-leap and leap year
    for y in (2014, 2020)
        @test DatesPlus.dayofquarter(DatesPlus.Date(y, 1, 1)) == 1
        @test DatesPlus.dayofquarter(DatesPlus.Date(y, 4, 1)) == 1
        @test DatesPlus.dayofquarter(DatesPlus.Date(y, 7, 1)) == 1
        @test DatesPlus.dayofquarter(DatesPlus.Date(y, 10, 1)) == 1
        @test DatesPlus.dayofquarter(DatesPlus.Date(y, 3, 31)) == 90 + isleapyear(y)
        @test DatesPlus.dayofquarter(DatesPlus.Date(y, 6, 30)) == 91
        @test DatesPlus.dayofquarter(DatesPlus.Date(y, 9, 30)) == 92
        @test DatesPlus.dayofquarter(DatesPlus.Date(y, 12, 31)) == 92
        @test DatesPlus.dayofquarter(DatesPlus.DateTime(y, 1, 1)) == 1
        @test DatesPlus.dayofquarter(DatesPlus.DateTime(y, 4, 1)) == 1
        @test DatesPlus.dayofquarter(DatesPlus.DateTime(y, 7, 1)) == 1
        @test DatesPlus.dayofquarter(DatesPlus.DateTime(y, 10, 1)) == 1
        @test DatesPlus.dayofquarter(DatesPlus.DateTime(y, 3, 31)) == 90 + isleapyear(y)
        @test DatesPlus.dayofquarter(DatesPlus.DateTime(y, 6, 30)) == 91
        @test DatesPlus.dayofquarter(DatesPlus.DateTime(y, 9, 30)) == 92
        @test DatesPlus.dayofquarter(DatesPlus.DateTime(y, 12, 31)) == 92
    end
end

end


# MIT License
#
# Copyright (c) 2018 Martin Biel
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

"""
    AbstractSampler

Abstract supertype for sampler objects.
"""
abstract type AbstractSampler{S <: AbstractScenario} end
"""
    sample(sampler::AbstractSampler{S})

Sample a scenario of type `S` using `sampler`.
"""
function sample(sampler::AbstractSampler)
    return sampler()
end
function Base.show(io::IO, sampler::AbstractSampler{S}) where S <: AbstractScenario
    print(io, "$(S.name.name) sampler")
    return io
end
function Base.show(io::IO, sampler::AbstractSampler{S}) where S <: Scenario
    print(io, "Scenario sampler")
    return io
end
"""
    sample(sampler::AbstractSampler{S}, π::AbstractSampler)

Sample a scenario of type `S` using `sampler` and set the probability of the sampled scenario to `π`.
"""
function sample(sampler::AbstractSampler, π::AbstractFloat)
    scenario = sampler()
    set_probability!(scenario, π)
    return scenario
end
"""
    Sampler

General purpose sampler object that samples Scenario.

See also: [`Scenario`](@ref), [`@scenario`](@ref), [`@sampler`](@ref)
"""
struct Sampler <: AbstractSampler{Scenario}
    sampler::Function
end

function (sampler::Sampler)()
    return sampler.sampler()
end

function sample!(scenarios::Scenarios{S}, sampler::AbstractSampler{S}, n::Integer) where S <: AbstractScenario
    _sample!(scenarios, sampler, n, length(scenarios), 1/n)
    return nothing
end
function sample!(scenarios::Scenarios{S}, sampler::AbstractSampler{Scenario}, n::Integer) where S <: Scenario
    _sample!(scenarios, sampler, n, length(scenarios), 1/n)
    return nothing
end

function _sample!(scenarios::Scenarios, sampler::AbstractSampler, n::Integer, m::Integer, π::AbstractFloat)
    if m > 0
        # Rescale probabilities of existing scenarios
        for scenario in scenarios
            p = probability(scenario) * m / (m+n)
            set_probability!(scenario, p)
        end
        π *= n/(m+n)
    end
    for i = 1:n
        push!(scenarios, sample(sampler, π))
    end
    return nothing
end


using Distributed

using Test

@everywhere begin
    using MLJBase
    using MLJTuning
    using ..Models
    import ComputationalResources: CPU1, CPUProcesses, CPUThreads
end

using Random
Random.seed!(1234*myid())
using .TestUtilities

N = 30
x1 = rand(N);
x2 = rand(N);
x3 = rand(N);
X = (x1=x1, x2=x2, x3=x3);
y = 2*x1 .+ 5*x2 .- 3*x3 .+ 0.4*rand(N);

m(K) = KNNRegressor(K=K)
r = [m(K) for K in 2:13]

# TODO: replace the above with the line below and post an issue on
# the failure (a bug in Distributed, I reckon):
# r = (m(K) for K in 2:13)

@testset "constructor" begin
    @test_throws ErrorException TunedModel(model=first(r), tuning=Explicit(),
                                           measure=rms)
    @test_throws ErrorException TunedModel(tuning=Explicit(),
                                           range=r, measure=rms)
    @test_throws ErrorException TunedModel(model=42, tuning=Explicit(),
                                           range=r, measure=rms)
    @test_logs((:info, r"No measure"),
               TunedModel(model=first(r), tuning=Explicit(), range=r))
end

# @testset "duplicate models warning" begin
#     s = [m(K) for K in 2:13]
#     push!(s, m(13))
#     tm = TunedModel(model=first(s), tuning=Explicit(),
#                     range=s, resampling=CV(nfolds=2),
#                     measures=[rms, l1])
#     @test_logs((:info, r"Attempting"),
#                (:warn, r"A model already"),
#                fitresult, meta_state, report = fit(tm, 1, X, y))
#     history, _, state = meta_state;
#     @test length(history) == length(2:13) + 1
# end

results = [(evaluate(model, X, y,
                     resampling=CV(nfolds=2),
                     measure=rms,
                     verbosity=0,)).measurement[1] for model in r]

@testset "measure compatibility check" begin
    tm = TunedModel(model=first(r), tuning=Explicit(),
                    range=r, resampling=CV(nfolds=2),
                    measures=cross_entropy)
    @test_throws ArgumentError fit(tm, 0, X, y)
end

@testset_accelerated "basic fit (CPU1)" accel begin
    printstyled("\n Testing progressmeter basic fit with $(accel) and CPU1 resampling \n", color=:bold)
    best_index = argmin(results)
    tm = TunedModel(model=first(r), tuning=Explicit(),
                    range=r, resampling=CV(nfolds=2),
                    measures=[rms, l1], acceleration=accel)
    fitresult, meta_state, report = fit(tm, 1, X, y);
    history, _, state = meta_state;
    results2 = map(event -> event.measurement[1], history)
    @test results2 ≈ results
    @test fitresult.model == collect(r)[best_index]
    @test report.best_model == collect(r)[best_index]
    @test report.history[5] == MLJTuning.delete(history[5], :metadata)
end

@static if VERSION >= v"1.3.0-DEV.573"
@testset_accelerated "Basic fit (CPUThreads)" accel begin
    printstyled("\n Testing progressmeter basic fit with $(accel) and CPUThreads resampling \n", color=:bold)
    tm = TunedModel(model=first(r), tuning=Explicit(),
                    range=r, resampling=CV(nfolds=2),
                    measures=[rms, l1], acceleration= CPUThreads(),
                    acceleration_resampling=accel)
    fitresult, meta_state, report = fit(tm, 1, X, y);
    history, _, state = meta_state;
    results3 = map(event -> event.measurement[1], history)
    @test results3 ≈ results
end
end
@testset_accelerated "Basic fit (CPUProcesses)" accel begin
    printstyled("\n Testing progressmeter basic fit with $(accel) and CPUProcesses resampling \n", color=:bold)
    best_index = argmin(results)
    tm = TunedModel(model=first(r), tuning=Explicit(),
                    range=r, resampling=CV(nfolds=2),
                    measures=[rms, l1], acceleration=CPUProcesses(),
                    acceleration_resampling=accel)
    fitresult, meta_state, report = fit(tm, 1, X, y);
    history, _, state = meta_state;
    results4 = map(event -> event.measurement[1], history)
    @test results4 ≈ results
end

@testset_accelerated("under/over supply of models", accel, begin
                     tm = TunedModel(model=first(r), tuning=Explicit(),
                                     range=r, measures=[rms, l1],
                                     acceleration=accel,
                                     resampling=CV(nfolds=2),
                                     n=4)
    mach = machine(tm, X, y)
    fit!(mach, verbosity=1)
    history = MLJBase.report(mach).history
    @test map(event -> event.measurement[1], history) ≈ results[1:4]

    tm.n=100
    @test_logs (:info, r"Only 12") fit!(mach, verbosity=0)
    history = MLJBase.report(mach).history
    @test map(event -> event.measurement[1], history) ≈ results
end)

@everywhere begin

    # variation of the Explicit strategy that annotates the models
    # with metadata
    mutable struct MockExplicit <: MLJTuning.TuningStrategy end

    annotate(model) = (model, params(model)[1])

    _length(x) = length(x)
    _length(::Nothing) = 0
    MLJTuning.models(tuning::MockExplicit,
                     model,
                     history,
                     state,
                     n_remaining,
                     verbosity) =
                         annotate.(state)[_length(history) + 1:end], state

    function default_n(tuning::Explicit, range)
        try
            length(range)
        catch MethodError
            DEFAULT_N
        end

    end

end

@testset_accelerated("passing of model metadata", accel,
                  begin
                     tm = TunedModel(model=first(r), tuning=MockExplicit(),
                                     range=r, resampling=CV(nfolds=2),
                                     measures=[rms, l1], acceleration=accel)
                     fitresult, meta_state, report = fit(tm, 0, X, y);
                     history, _, state = meta_state;
                     @test all(history) do event
                         event.metadata == event.model.K
                     end
end)

true


module Densratio

export
    densratio,
    compute_density_ratio,
    search_sigma_and_lambda,
    compute_kernel_Gaussian,
    kernel_Gaussian

using PyCall
const densratio_py = PyNULL()

function __init__()
    copy!(densratio_py, pyimport_conda("densratio", "densratio"))
end

function densratio(x, y)
    densratio.densratio(x, y)
end

end # module


################################################################################
# CircularWall
################################################################################

mutable struct CircularWall{T}
    x::T
    y::T
    r::T
end

function CircularWall()
    x = 0.0
    y = 0.0
    r = 0.0
    return CircularWall{typeof(x)}(x,y,r)
end

################################################################################
# Lane
################################################################################

mutable struct Lane{T}
    id::Int
    name::Symbol
    wall::Vector{Wall}
    circle::Vector{CircularWall{T}}
    start::StartingArea{T}
    centerline::Function
end

function Lane()
    name = :lane_0
    w = Wall([0., 0.], [0., 0.], [0., 0.])
    c = CircularWall()
    start = StartingArea()
    return Lane{typeof(c.x)}(0, name, [w], [c], start, x -> x)
end

function Lane(id::Int, name::Symbol, wall::Vector{Wall}, circle::Vector{CircularWall}, start::StartingArea{T}, centerline) where {T}
    return Lane{T}(id, name, wall, circle, start, centerline)
end


"""
Adaptive Stress Testing for the POMDPs.jl ecosystem.
"""
module POMDPStressTesting

include("AST.jl")
include(joinpath("solvers", "drl", "PolicyOptimization.jl"))

using .AST
using .PolicyOptimization
using CrossEntropyMethod
using Distributions
using Markdown
using MCTS
using Parameters
using POMDPs
using ProgressMeter
using Random
using Statistics

# Visualization specific.
using D3Trees
using Requires

function __init__()
    @require PyPlot="d330b81b-6aea-500a-939a-2ce795aea3ee" begin
        @require Seaborn="d2ef9438-c967-53ab-8060-373fdd9e13eb" include(joinpath("visualization", "figures.jl"))
    end
end

export AST,
       ASTMDP,
       ASTParams,
       ASTState,
       ASTAction,
       ASTSeedAction,
       ASTSampleAction,
       actiontype,
       BlackBox,
       GrayBox,

       search!,
       playback,
       online_path,
       logpdf,
       get_top_path,
       hash_uint32,

       visualize,
       full_width_notebook,
       episodic_figures,
       distribution_figures,
       distribution_figures!,
       FailureMetrics,
       print_metrics,
       failure_metrics,
       combine_ast_metrics,
       latex_metrics,
       reset_metrics!,
       highest_loglikelihood_of_failure,
       most_likely_failure,

       MCTSPWSolver,
       CEMSolver,
       CEMPlanner,
       RandomSearchSolver,
       RandomSearchPlanner,
       TRPOSolver,
       TRPOPlanner,
       PPOSolver,
       PPOPlanner,
       get_action,
       sample_action,

       # For MCTS and RandomSearch
       solve,
       action

include(joinpath("utils", "metrics.jl"))
include(joinpath("visualization", "tree_visualization.jl"))

include(joinpath("solvers", "mcts.jl"))
include(joinpath("solvers", "cem.jl"))
include(joinpath("solvers", "random_search.jl"))

end # module POMDPStressTesting


# Read TOML file

"""
    read_home_config(toml_file::String) -> Dict

Read home configuration from a TOML file.
"""
function read_home_config(toml_file::String)::Dict
    h = TOML.parsefile(toml_file)
    # confirm unique ID
    reserved = Set(["load"])
    ids = Set{String}()
    for id in get_all_load_ids(h)
        if id in ids
            error("The ID '$id' is not unique!")
        end
        if id in reserved
            error("The ID '$id' is reserved and should not be used!")
        end
        push!(ids, id)
    end
    return h
end

"""
    get_all_load_ids(h::Dict) -> Vector{String}

Get the ID of all loads in the home `h`.
"""
function get_all_load_ids(h::Dict)::Vector{String}
    return [l["id"] for l in Iterators.flatten((h["CR"], h["AD"], h["SI"], h["SU"]))]
end

"""
    validate_solution(s::AbstractDataFrame, h::AbstractDict)

Given a solution in `s` along with the scenario data, validate whether it is feasible.
If it is feasible, return the total cost of this solution on `s`.
"""
function validate_solution(s::AbstractDataFrame, h::AbstractDict)
    C_SI = C_SU = C_AD = C_CR = 0.0
    # shiftable constraints
    for l in h["SU"]
        id = l["id"]
        if !isapprox(sum(s[!, Symbol("z_", id)]), 1)    # z_ is stored as Float in `s`
            @show sum(s[!, Symbol("z_", id)])
            error("Infeasible solution in SU load $id")
        end
        C_SU += sum(s[!, Symbol("C_", id)])
    end
    for l in h["SI"]
        id = l["id"]
        if !isapprox(sum(s[!, Symbol("z_", id)]), l["L"])
            error("Infeasible solution in SI load $id")
        end
        C_SI += sum(s[!, Symbol("C_", id)])
    end
    # power balance
    consumption = sum(s[!, Symbol("P_", l["id"])] for l in Iterators.Flatten((h["AD"], h["SI"], h["SU"]))) +
                  s.Ps .+ sum(l["P"] for l in h["CR"])
    generation = s.Pb .+ s.PPV
    if !all(consumption .≈ generation)
        @show consumption generation
        error("Power not balanced!")
    end
    for l in h["AD"]
        C_AD += sum(s[!, Symbol("C_", l["id"])])
    end
    for l in h["CR"]
        C_CR += sum(l["P"] .* s.ρ)
    end
    ρ = s.ρ
    ρs = h["UtilityGrid"]["β"] .* ρ
    return C_SI + C_SU + C_AD + C_CR - sum(ρ .* s.PPV) + sum((ρ .- ρs) .* s.Ps)
end

"""
    compute_cost!(df::DataFrame, h::AbstractDict)->Float64

For home `h`, `df` contains price, PV, and the power of each load. Then, the net power exchange is
computed and stored into `df` as `Pb` and `Ps` columns. Finally, the stepwise electricity cost `Cele` is
calculated, and the total cost `C` is also computed, both stored into `df` in place.

The total cost of all steps is returned.
"""
function compute_cost!(df::DataFrame, h::AbstractDict)::Float64
    T = nrow(df)
    c = 0.0
    for l in h["CR"]
        df[!, "P_$(l["id"])"] = fill(l["P"], T)
    end
    ids = [l["id"] for l in Iterators.Flatten((h["CR"], h["AD"], h["SU"], h["SI"]))]
    Pnet = sum(df[!, "P_$id"] for id in ids) .- df.PPV
    df.Pnet = Pnet
    Pb = max.(Pnet, 0.0)
    Ps = -min.(Pnet, 0.0)
    ρb = df.ρ
    ρs = h["UtilityGrid"]["β"] .* ρb
    Cele = @. ρb * Pb - ρs * Ps
    df.Pb = Pb
    df.Ps = Ps
    df.Cele = Cele
    # discomfort cost
    Cdc = zeros(T)
    for l in h["AD"]
        id = l["id"]
        Pmax = l["Pmax"]
        α = l["α"]
        df[!, "C_$(id)_dc"] = @. α * (Pmax - df[!, "P_$id"])^2
        Cdc .+= df[!, "C_$(id)_dc"]
    end
    df.Cdc = Cdc
    df.C = df.Cdc .+ df.Cele
    return sum(df.C)
end


immutable InputLayerState <: AbstractScatteredLayerState
    blobs::Vector{Mocha.Blob}
    layer::InputLayer
end

function InputLayerState(backend::Mocha.Backend, layer::InputLayer)
    ranges = ntuple(k -> kthrange(layer, k), ndims(layer.data))
    blob = ScatteredBlob(
        DataStructures.SortedDict((Path() => Node(layer.data, ranges),)))
    blobs = Mocha.Blob[blob]
    return InputLayerState(blobs, layer)
end

function Mocha.setup(
        backend::Mocha.Backend,
        layer::InputLayer,
        inputs::Vector{Mocha.Blob},
        diffs::Vector{Mocha.Blob})
    @assert length(diffs) == 0
    @assert length(inputs) == 0
    return InputLayerState(backend, layer)
end

kthrange(layer::InputLayer, k::Int) =
    (PathKey(layer.symbols[k]) => (1:1:size(layer.data, k)))


###############################################################################
# TASK ########################################################################
###### The `task` field should not be accessed outside this section. ##########

## Affine Constraints #########################################################
##################### lc ≤ Ax ≤ uc ############################################

function allocateconstraints(m::MosekModel, N::Int)
    numcon = getnumcon(m.task)
    alloced = ensurefree(m.c_block,N)
    id = newblock(m.c_block, N)

    if alloced > 0
        appendcons(m.task, alloced)
    end
    return id
end

function getconboundlist(t::Mosek.Task, subj::Vector{Int32})
    n = length(subj)
    bk = Vector{Boundkey}(undef,n)
    bl = Vector{Float64}(undef,n)
    bu = Vector{Float64}(undef,n)
    for i in 1:n
        bki,bli,bui = getconbound(t,subj[i])
        bk[i] = bki
        bl[i] = bli
        bu[i] = bui
    end
    bk,bl,bu
end

# `putbaraij` and `putbarcj` need the whole matrix as a sum of sparse mat at once
function split_scalar_matrix(m::MosekModel, terms::Vector{MOI.ScalarAffineTerm{Float64}},
                             set_sd::Function)
    cols = Int32[]
    values = Float64[]
    sd_row  = Vector{Int32}[Int32[] for i in 1:length(m.sd_dim)]
    sd_col  = Vector{Int32}[Int32[] for i in 1:length(m.sd_dim)]
    sd_coef = Vector{Float64}[Float64[] for i in 1:length(m.sd_dim)]
	function add(col::ColumnIndex, coefficient::Float64)
		push!(cols, col.value)
		push!(values, coefficient)
	end
	function add(mat::MatrixIndex, coefficient::Float64)
        coef = mat.row == mat.column ? coefficient : coefficient / 2
        push!(sd_row[mat.matrix], mat.row)
        push!(sd_col[mat.matrix], mat.column)
        push!(sd_coef[mat.matrix], coef)
    end
	for term in terms
        add(mosek_index(m, term.variable_index), term.coefficient)
    end
    for j in 1:length(m.sd_dim)
        if !isempty(sd_row[j])
            id = appendsparsesymmat(m.task, m.sd_dim[j], sd_row[j],
                                    sd_col[j], sd_coef[j])
            set_sd(j, [id], [1.0])
        end
    end
    return cols, values
end

function set_row(task::Mosek.MSKtask, row::Int32, cols::ColumnIndices,
                 values::Vector{Float64})
    putarow(task, row, cols.values, values)
end
function set_row(m::MosekModel, row::Int32,
                 terms::Vector{MOI.ScalarAffineTerm{Float64}})
    cols, values = split_scalar_matrix(m, terms,
        (j, ids, coefs) -> putbaraij(m.task, row, j, ids, coefs))
    set_row(m.task, row, ColumnIndices(cols), values)
end

function set_coefficients(task::Mosek.MSKtask, rows::Vector{Int32},
                          cols::ColumnIndices, values::Vector{Float64})
    putaijlist(task, rows, cols.values, values)
end

function set_coefficients(task::Mosek.MSKtask, rows::Vector{Int32},
                          col::ColumnIndex, values::Vector{Float64})
    n = length(rows)
    @assert n == length(values)
    set_coefficient(task, rows, ColumnIndices(fill(col.value, n)), values)
end
function set_coefficients(m::MosekModel, rows::Vector{Int32},
                          vi::MOI.VariableIndex, values::Vector{Float64})
    set_coefficient(m.task, rows, mosek_index(m, vi), values)
end

function set_coefficient(task::Mosek.MSKtask, row::Int32, col::ColumnIndex,
                         value::Float64)
    putaij(task, row, col.value, value)
end
function set_coefficient(m::MosekModel, row::Int32, vi::MOI.VariableIndex,
                         value::Float64)
    set_coefficient(m.task, row, mosek_index(m, vi), value)
end

bound_key(::Type{MOI.GreaterThan{Float64}}) = MSK_BK_LO
bound_key(::Type{MOI.LessThan{Float64}})    = MSK_BK_UP
bound_key(::Type{MOI.EqualTo{Float64}})     = MSK_BK_FX
bound_key(::Type{MOI.Interval{Float64}})    = MSK_BK_RA
add_bound(m::MosekModel, row::Int32, dom::MOI.GreaterThan{Float64}) = putconbound(m.task, row, bound_key(typeof(dom)), dom.lower, dom.lower)
add_bound(m::MosekModel, row::Int32, dom::MOI.LessThan{Float64})    = putconbound(m.task, row, bound_key(typeof(dom)), dom.upper, dom.upper)
add_bound(m::MosekModel, row::Int32, dom::MOI.EqualTo{Float64})     = putconbound(m.task, row, bound_key(typeof(dom)), dom.value, dom.value)
add_bound(m::MosekModel, row::Int32, dom::MOI.Interval{Float64})    = putconbound(m.task, row, bound_key(typeof(dom)), dom.lower, dom.upper)
function bounds_to_set(::Type{S}, bk, bl, bu) where S
    if S == MOI.GreaterThan{Float64}
        return S(bl)
    elseif S == MOI.LessThan{Float64}
        return S(bu)
    elseif S == MOI.EqualTo{Float64}
        @assert bl == bu
        return S(bu)
    else
        @assert S == MOI.Interval{Float64}
        return S(bl, bu)
    end
end
function get_bound(m::MosekModel,
                   ci::MOI.ConstraintIndex{MOI.ScalarAffineFunction{Float64}, S}) where {S}
    bounds_to_set(S, getconbound(m.task, row(m, ci))...)
end

## Variable Constraints #######################################################
####################### lx ≤ x ≤ u ############################################
#######################      x ∈ K ############################################

function getvarboundlist(t::Mosek.Task, subj::Vector{Int32})
    n = length(subj)
    bk = Vector{Boundkey}(undef,n)
    bl = Vector{Float64}(undef,n)
    bu = Vector{Float64}(undef,n)
    for i in 1:n
        bki,bli,bui = getvarbound(t, subj[i])
        bk[i] = bki
        bl[i] = bli
        bu[i] = bui
    end
    bk,bl,bu
end

function delete_variable_constraint(m::MosekModel, col::ColumnIndex,
                                    ::Type{<:Union{MOI.Interval, MOI.EqualTo}})
    putvarbound(m.task, col.value, MSK_BK_FR, 0.0, 0.0)
end
function delete_variable_constraint(m::MosekModel, col::ColumnIndex,
                                    ::Type{MOI.Integer})
    putvartype(m.task, col.value, MSK_VAR_TYPE_CONT)
end
function delete_variable_constraint(m::MosekModel, col::ColumnIndex,
                                    ::Type{MOI.ZeroOne})
    putvartype(m.task, col.value, MSK_VAR_TYPE_CONT)
    putvarbound(m.task, col.value, MSK_BK_FR, 0.0, 0.0)
end
function delete_variable_constraint(m::MosekModel, col::ColumnIndex,
                                    ::Type{MOI.LessThan{Float64}})
    bk, lo, up = getvarbound(m.task, col.value)
    if bk == MSK_BK_UP
        bk = MSK_BK_FR
    else
        @assert bk == MSK_BK_RA
        bk = MSK_BK_LO
    end
    putvarbound(m.task, col.value, bk, lo, 0.0)
end
function delete_variable_constraint(m::MosekModel, col::ColumnIndex,
                                    ::Type{MOI.GreaterThan{Float64}})
    bk, lo, up = getvarbound(m.task, col.value)
    if bk == MSK_BK_LO
        bk = MSK_BK_FR
    else
        @assert bk == MSK_BK_RA
        bk = MSK_BK_UP
    end
    putvarbound(m.task, col.value, bk, 0.0, up)
end
function add_variable_constraint(m::MosekModel, col::ColumnIndex, dom::MOI.Interval)
    putvarbound(m.task, col.value, MSK_BK_RA, dom.lower, dom.upper)
end
function add_variable_constraint(m::MosekModel, col::ColumnIndex, dom::MOI.EqualTo)
    putvarbound(m.task, col.value, MSK_BK_FX, dom.value, dom.value)
end
function add_variable_constraint(m::MosekModel, col::ColumnIndex, ::MOI.Integer)
    putvartype(m.task, col.value, MSK_VAR_TYPE_INT)
end
function add_variable_constraint(m::MosekModel, col::ColumnIndex, ::MOI.ZeroOne)
    putvartype(m.task, col.value, MSK_VAR_TYPE_INT)
    putvarbound(m.task, col.value, MSK_BK_RA, 0.0, 1.0)
end
function add_variable_constraint(m::MosekModel, col::ColumnIndex, dom::MOI.LessThan)
    bk, lo, up = getvarbound(m.task, col.value)
    if bk == MSK_BK_FR
        bk = MSK_BK_UP
    else
        @assert bk == MSK_BK_LO
        bk = MSK_BK_RA
    end
    putvarbound(m.task, col.value, bk, lo, dom.upper)
end
function add_variable_constraint(m::MosekModel, col::ColumnIndex,
                                 dom::MOI.GreaterThan)
    bk, lo, up = getvarbound(m.task, col.value)
    if bk == MSK_BK_FR
        bk = MSK_BK_LO
    else
        @assert bk == MSK_BK_UP
        bk = MSK_BK_RA
    end
    putvarbound(m.task, col.value, bk, dom.lower, up)
end
function get_variable_constraint(m::MosekModel,
                                 col::ColumnIndex,
                                 ci::MOI.ConstraintIndex{MOI.SingleVariable, S}) where S
    return bounds_to_set(S, getvarbound(m.task, col.value)...)
end
function get_variable_constraint(m::MosekModel, vi::MOI.VariableIndex,
                                 ci::MOI.ConstraintIndex)
    return get_variable_constraint(m, mosek_index(m, vi), ci)
end

cone_parameter(dom :: MOI.PowerCone{Float64})     = dom.exponent
cone_parameter(dom :: MOI.DualPowerCone{Float64}) = dom.exponent
cone_parameter(dom :: C) where C <: MOI.AbstractSet = 0.0

function add_cone(m::MosekModel, cols::ColumnIndices, set)
    appendcone(m.task, cone_type(typeof(set)), cone_parameter(set), cols.values)
    id = getnumcone(m.task)
    if DEBUG
        putconename(m.task, id, "$id")
    end
    return id
end

## Name #######################################################################
###############################################################################

function set_row_name(task::Mosek.MSKtask, row::Int32, name::String)
    putconname(task, row, name)
end

function set_row_name(m::MosekModel,
                      c::MOI.ConstraintIndex{MOI.ScalarAffineFunction{Float64}},
                      name::AbstractString)
    set_row_name(m.task, row(m, c), name)
end
function set_row_name(m::MosekModel, c::MOI.ConstraintIndex,
                      name::AbstractString)
    # Fallback for `SingleVariable` and `VectorOfVariables`.
    m.con_to_name[c] = name
end

function delete_name(m::MosekModel, ci::MOI.ConstraintIndex)
    name = MOI.get(m, MOI.ConstraintName(), ci)
    if !isempty(name)
        cis = m.constrnames[name]
        deleteat!(cis, findfirst(isequal(ci), cis))
    end
end

###############################################################################
# INDEXING ####################################################################
###############################################################################

function row(m::MosekModel,
             c::MOI.ConstraintIndex{MOI.ScalarAffineFunction{Float64}})::Int32
    return getindex(m.c_block, c.value)
end
function columns(m::MosekModel, ci::MOI.ConstraintIndex{MOI.VectorOfVariables})
    return ColumnIndices(getcone(m.task, ci.value)[4])
end

const VectorCone = Union{MOI.SecondOrderCone,
                         MOI.RotatedSecondOrderCone,
                         MOI.PowerCone,
                         MOI.DualPowerCone,
                         MOI.ExponentialCone,
                         MOI.DualExponentialCone}

# Two `SingleVariable`-in-`S` cannot be set to the same variable if
# the two constraints
# * both set a lower bound, or
# * both set an upper bound, or
# * both set it to integer.
# The `incompatible_mask` are computed according to these rules.
flag(::Type{MOI.EqualTo{Float64}}) = 0x1
incompatible_mask(::Type{MOI.EqualTo{Float64}}) = 0x2f
flag(::Type{MOI.GreaterThan{Float64}}) = 0x2
incompatible_mask(::Type{MOI.GreaterThan{Float64}}) = 0x2b
flag(::Type{MOI.LessThan{Float64}}) = 0x4
incompatible_mask(::Type{MOI.LessThan{Float64}}) = 0x2d
flag(::Type{MOI.Interval{Float64}}) = 0x8
incompatible_mask(::Type{MOI.Interval{Float64}}) = 0x2f
flag(::Type{MOI.Integer}) = 0x10
incompatible_mask(::Type{MOI.Integer}) = 0x30
flag(::Type{MOI.ZeroOne}) = 0x20
incompatible_mask(::Type{MOI.ZeroOne}) = 0x3f
flag(::Type{<:VectorCone}) = 0x40
incompatible_mask(::Type{<:VectorCone}) = 0x40

function set_flag(model::MosekModel, vi::MOI.VariableIndex, S::Type)
    model.x_constraints[vi.value] |= flag(S)
end
function unset_flag(model::MosekModel, vi::MOI.VariableIndex, S::Type)
    model.x_constraints[vi.value] &= ~flag(S)
end
function has_flag(model::MosekModel, vi::MOI.VariableIndex, S::Type)
    return !iszero(model.x_constraints[vi.value] & flag(S))
end

###############################################################################
# MOI #########################################################################
###############################################################################

const ScalarLinearDomain = Union{MOI.LessThan{Float64},
                                 MOI.GreaterThan{Float64},
                                 MOI.EqualTo{Float64},
                                 MOI.Interval{Float64}}
const ScalarIntegerDomain = Union{MOI.ZeroOne, MOI.Integer}

## Add ########################################################################
###############################################################################

MOI.supports_constraint(m::MosekModel, ::Type{<:Union{MOI.SingleVariable, MOI.ScalarAffineFunction}}, ::Type{<:ScalarLinearDomain}) = true
MOI.supports_constraint(m::MosekModel, ::Type{MOI.VectorOfVariables}, ::Type{<:Union{VectorCone, MOI.PositiveSemidefiniteConeTriangle}}) = true
MOI.supports_constraint(m::MosekModel, ::Type{MOI.SingleVariable}, ::Type{<:ScalarIntegerDomain}) = true

## Affine Constraints #########################################################
##################### lc ≤ Ax ≤ uc ############################################

function MOI.add_constraint(m  ::MosekModel,
                            axb::MOI.ScalarAffineFunction{Float64},
                            dom::D) where {D <: MOI.AbstractScalarSet}

    if !iszero(axb.constant)
        throw(MOI.ScalarFunctionConstantNotZero{Float64, typeof(axb), D}(axb.constant))
    end

    # Duplicate indices not supported
    axb = MOIU.canonical(axb)

    N = 1
    conid = allocateconstraints(m, N)
    ci = MOI.ConstraintIndex{MOI.ScalarAffineFunction{Float64}, D}(conid)
    r = row(m, ci)
    set_row(m, r, axb.terms)

    add_bound(m, r, dom)

    return ci
end

## Variable Constraints #######################################################
####################### lx ≤ x ≤ u ############################################
#######################      x ∈ K ############################################

# We allow following. Each variable can have
# - at most most upper and one lower bound
# - belong to at most one non-semidefinite cone
# - any number of semidefinite cones, which are implemented as ordinary constraints
# This is when things get a bit funky; By default a variable has no
# bounds, i.e. "free". Adding a `GreaterThan`
# constraint causes it to have a defined lower bound but no upper
# bound, allowing a `LessThan` constraint to be
# added later. Adding a `Interval` constraint defines both upper and
# lower bounds.

cone_type(::Type{MOI.ExponentialCone})        = MSK_CT_PEXP
cone_type(::Type{MOI.DualExponentialCone})    = MSK_CT_DEXP
cone_type(::Type{MOI.PowerCone{Float64}})     = MSK_CT_PPOW
cone_type(::Type{MOI.DualPowerCone{Float64}}) = MSK_CT_DPOW
cone_type(::Type{MOI.SecondOrderCone})        = MSK_CT_QUAD
cone_type(::Type{MOI.RotatedSecondOrderCone}) = MSK_CT_RQUAD

function MOI.add_constraint(
    m   :: MosekModel,
    xs  :: MOI.SingleVariable,
    dom :: D) where {D <: MOI.AbstractScalarSet}

    msk_idx = mosek_index(m, xs.variable)
    if !(msk_idx isa ColumnIndex)
        error("Cannot add $D constraint on a matrix variable")
    end

    if !iszero(incompatible_mask(D) & m.x_constraints[xs.variable.value])
        error("Cannot put multiple bound sets of the same type on a variable")
    end

    set_flag(m, xs.variable, D)

    add_variable_constraint(m, msk_idx, dom)

    return MOI.ConstraintIndex{MOI.SingleVariable, D}(xs.variable.value)
end

function MOI.add_constraint(m::MosekModel, xs::MOI.VectorOfVariables,
                            dom::D) where {D<:VectorCone}
    if any(vi -> is_matrix(m, vi), xs.variables)
        error("Cannot add $D constraint on a matrix variable")
    end
    cols = ColumnIndices(reorder(columns(m, xs.variables).values, D))

    if !all(vi -> iszero(incompatible_mask(D) & m.x_constraints[vi.value]), xs.variables)
        error("Cannot multiple bound sets of the same type to a variable")
    end

    N = MOI.dimension(dom)
    id = add_cone(m, cols, dom)

    return MOI.ConstraintIndex{MOI.VectorOfVariables, D}(id)
end

################################################################################
################################################################################

function MOI.add_constraint(m  ::MosekModel,
                            xs ::MOI.VectorOfVariables,
                            dom::MOI.PositiveSemidefiniteConeTriangle)
    N = dom.side_dimension
    if N < 2
        error("Invalid dimension for semidefinite constraint, got $N which is ",
              "smaller than the minimum dimension 2.")
    end
    appendbarvars(m.task, [Int32(N)])
    push!(m.sd_dim, N)
    id = length(m.sd_dim)
    sd_row  = Dict{Int, Vector{Int32}}()
    sd_col  = Dict{Int, Vector{Int32}}()
    sd_coef = Dict{Int, Vector{Float64}}()
    obj_row = Int32[]
    obj_col = Int32[]
    obj_coef = Float64[]
    k = 0
    for i in 1:N
        for j in 1:i
            k += 1
            vi = xs.variables[k]
            m.x_sd[vi.value] = MatrixIndex(id, i, j)

            if variable_type(m, vi) == MatrixVariable
                error("Variable $vi cannot be part of two matrix variables.")
            elseif variable_type(m, vi) == ScalarVariable
                # The variable has already been used, we need replace the
                # coefficients in the `A` matrix by matrix coefficients
                nnz, rows, vals = getacol(m.task, column(m, vi).value)
                @assert nnz == length(rows) == length(vals)
                for ii in 1:nnz
                    if !haskey(sd_row, rows[ii])
                        sd_row[rows[ii]] = Int32[]
                        sd_col[rows[ii]] = Int32[]
                        sd_coef[rows[ii]] = Float64[]
                    end
                    push!(sd_row[rows[ii]], i)
                    push!(sd_col[rows[ii]], j)
                    push!(sd_coef[rows[ii]], i == j ? vals[ii] : vals[ii] / 2)
                end
                cj = getcj(m.task, column(m, vi).value)
                if !iszero(cj)
                    push!(obj_row, i)
                    push!(obj_col, j)
                    push!(obj_coef, i == j ? cj : cj / 2)
                end
                MOI.delete(m, vi)
            end
            m.x_type[vi.value] = MatrixVariable
        end
    end
    for row in eachindex(sd_row)
        sid = appendsparsesymmat(m.task, N, sd_row[row], sd_col[row],
                                sd_coef[row])
        putbaraij(m.task, row, id, [sid], [1.0])
    end
    if !isempty(obj_row)
        sid = appendsparsesymmat(m.task, N, obj_row, obj_col, obj_coef)
        putbarcj(m.task, id, [sid], [1.0])
    end
    @assert k == length(xs.variables)

    conref = MOI.ConstraintIndex{MOI.VectorOfVariables, MOI.PositiveSemidefiniteConeTriangle}(id)

    return conref
end

## Get ########################################################################
###############################################################################

_variable(ci::MOI.ConstraintIndex{MOI.SingleVariable}) = MOI.VariableIndex(ci.value)
function MOI.get(m::MosekModel, ::MOI.ConstraintFunction,
                 ci::MOI.ConstraintIndex{MOI.SingleVariable}) where S <: ScalarLinearDomain
    return MOI.SingleVariable(_variable(ci))
end
function MOI.get(m::MosekModel, ::MOI.ConstraintSet,
                 ci::MOI.ConstraintIndex{MOI.SingleVariable, S}) where S <: ScalarIntegerDomain
    return S()
end
function MOI.get(m::MosekModel, ::MOI.ConstraintSet,
                 ci::MOI.ConstraintIndex{MOI.SingleVariable, S}) where S <: ScalarLinearDomain
    sv = MOI.get(m, MOI.ConstraintFunction(), ci)
    return get_variable_constraint(m, sv.variable, ci)
end

function MOI.get(m::MosekModel, ::MOI.ConstraintFunction,
                 ci::MOI.ConstraintIndex{MOI.ScalarAffineFunction{Float64},
                                         <:ScalarLinearDomain})
    nnz, cols, vals = getarow(m.task, row(m, ci))
    @assert nnz == length(cols) == length(vals)
    terms = MOI.ScalarAffineTerm{Float64}[
        MOI.ScalarAffineTerm(vals[i], index_of_column(m, cols[i])) for i in 1:nnz]
    # TODO add matrix terms
    return MOI.ScalarAffineFunction(terms, 0.0)
end
function MOI.get(m::MosekModel, ::MOI.ConstraintSet,
                 ci::MOI.ConstraintIndex{MOI.ScalarAffineFunction{Float64},
                                         S}) where S
    return get_bound(m, ci)
end

## Modify #####################################################################
###############################################################################

### SET
chgbound(bl::Float64,bu::Float64,k::Float64,dom :: MOI.LessThan{Float64})    = bl,dom.upper-k
chgbound(bl::Float64,bu::Float64,k::Float64,dom :: MOI.GreaterThan{Float64}) = dom.lower-k,bu
chgbound(bl::Float64,bu::Float64,k::Float64,dom :: MOI.EqualTo{Float64})     = dom.value-k,dom.value-k
chgbound(bl::Float64,bu::Float64,k::Float64,dom :: MOI.Interval{Float64})    = dom.lower-k,dom.upper-k

function MOI.set(m::MosekModel,
                 ::MOI.ConstraintSet,
                 ci::MOI.ConstraintIndex{F,D},
                 dom::D) where {F<:MOI.SingleVariable,
                                D<:ScalarLinearDomain}
    col = column(m, _variable(ci))
    bk, bl, bu = getvarbound(m.task, col.value)
    bl, bu = chgbound(bl, bu, 0.0, dom)
    putvarbound(m.task, col.value, bk, bl, bu)
end

function MOI.set(m::MosekModel,
                 ::MOI.ConstraintSet,
                 cref::MOI.ConstraintIndex{F,D},
                 dom::D) where { F    <: MOI.ScalarAffineFunction,
                                 D    <: ScalarLinearDomain }
    cid = ref2id(cref)
    i = getindex(m.c_block,cid) # since we are in a scalar domain
    bk, bl, bu = getconbound(m.task,i)
    bl, bu = chgbound(bl,bu,0.0,dom)
    putconbound(m.task,i,bk,bl,bu)
end

### MODIFY
function MOI.modify(m   ::MosekModel,
                    c   ::MOI.ConstraintIndex{MOI.ScalarAffineFunction{Float64},D},
                    func::MOI.ScalarConstantChange{Float64}) where {D <: MOI.AbstractSet}
    if !iszero(func.new_constant)
        throw(MOI.ScalarFunctionConstantNotZero{Float64, MOI.ScalarAffineFunction{Float64}, D}(func.new_constant))
    end
end

function MOI.modify(m   ::MosekModel,
                    c   ::MOI.ConstraintIndex{MOI.ScalarAffineFunction{Float64}},
                    func::MOI.ScalarCoefficientChange{Float64})
    set_coefficient(m, row(m, c), func.variable, func.new_coefficient)
end

### TRANSFORM
function MOI.transform(m::MosekModel,
                       cref::MOI.ConstraintIndex{F,D},
                       newdom::D) where {F <: MOI.AbstractFunction,
                                         D <: MOI.AbstractSet}
    MOI.modify(m,cref,newdom)
    return cref
end

function MOI.transform(m::MosekModel,
                       cref::MOI.ConstraintIndex{MOI.ScalarAffineFunction{Float64},D1},
                       newdom::D2) where {D1 <: ScalarLinearDomain,
                                          D2 <: ScalarLinearDomain}
    F = MOI.ScalarAffineFunction{Float64}

    cid = ref2id(cref)

    r = row(m, cref)
    add_bound(m, r, newdom)

    newcref = MOI.ConstraintIndex{F, D2}(cid)
    return newcref
end

## Delete #####################################################################
###############################################################################

function MOI.is_valid(model::MosekModel,
                      ci::MOI.ConstraintIndex{<:MOI.ScalarAffineFunction{Float64},
                                              S}) where S<:ScalarLinearDomain
    return allocated(model.c_block, ci.value) && getconbound(model.task, row(model, ci))[1] == bound_key(S)
end
function MOI.delete(
    m::MosekModel,
    cref::MOI.ConstraintIndex{F,D}) where {F <: MOI.ScalarAffineFunction{Float64},
                                           D <: ScalarLinearDomain}
    if !MOI.is_valid(m, cref)
        throw(MOI.InvalidIndex(cref))
    end

    delete_name(m, cref)

    subi = getindexes(m.c_block, cref.value)

    n = length(subi)
    subi_i32 = convert(Vector{Int32}, subi)
    ptr = fill(Int64(0), n)
    putarowlist(m.task,subi_i32,ptr,ptr,Int32[],Float64[])
    b = fill(0.0,n)
    putconboundlist(m.task,subi_i32,fill(MSK_BK_FX,n),b,b)

    deleteblock(m.c_block, cref.value)
end

function MOI.is_valid(model::MosekModel,
                      ci::MOI.ConstraintIndex{MOI.SingleVariable,
                                              S}) where S<:Union{ScalarLinearDomain,
                                                                 ScalarIntegerDomain}
    return allocated(model.x_block, ci.value) && has_flag(model, _variable(ci), S)
end
function MOI.delete(
    m::MosekModel,
    ci::MOI.ConstraintIndex{MOI.SingleVariable, S}) where S<:Union{ScalarLinearDomain,
                                                                   ScalarIntegerDomain}
    if !MOI.is_valid(m, ci)
        throw(MOI.InvalidIndex(ci))
    end
    delete_name(m, ci)
    vi = _variable(ci)
    unset_flag(m, vi, S)
    delete_variable_constraint(m, column(m, vi), S)
end
function MOI.is_valid(model::MosekModel,
                      ci::MOI.ConstraintIndex{MOI.VectorOfVariables,
                                              S}) where S<:VectorCone
    return 1 ≤ ci.value ≤ getnumcone(model.task) &&
        getconeinfo(model.task, ci.value)[1] == cone_type(S)
end
function MOI.is_valid(model::MosekModel,
                      ci::MOI.ConstraintIndex{MOI.VectorOfVariables,
                                              MOI.PositiveSemidefiniteConeTriangle})
    # TODO add supports for deletion
    return 1 ≤ ci.value ≤ length(model.sd_dim)
end

## List #######################################################################
###############################################################################
function MOI.get(m::MosekModel, ::MOI.ListOfConstraintAttributesSet)
    set = MOI.AbstractConstraintAttribute[]
    if !isempty(m.constrnames)
        push!(set, MOI.ConstraintName())
    end
    # TODO add VariablePrimalStart when get is implemented on it
    return set
end

## Name #######################################################################
###############################################################################
function MOI.supports(::MosekModel, ::MOI.ConstraintName,
                      ::Type{<:MOI.ConstraintIndex})
    return true
end
function MOI.set(m::MosekModel, ::MOI.ConstraintName, ci::MOI.ConstraintIndex,
                 name ::AbstractString)
    delete_name(m, ci)
    if !haskey(m.constrnames, name)
        m.constrnames[name] = MOI.ConstraintIndex[]
    end
    push!(m.constrnames[name], ci)
    set_row_name(m, ci, name)
end
function MOI.get(m::MosekModel, ::MOI.ConstraintName,
                 ci::MOI.ConstraintIndex{MOI.ScalarAffineFunction{Float64}})
    # All rows should have same name so we take the first one
    return getconname(m.task, getindexes(m.c_block, ref2id(ci))[1])
end
function MOI.get(m::MosekModel, ::MOI.ConstraintName, ci::MOI.ConstraintIndex)
    return get(m.con_to_name, ci, "")
end
function MOI.get(m::MosekModel, CI::Type{<:MOI.ConstraintIndex}, name::String)
    #asgn, row = getconnameindex(m.task, name)
    #if iszero(asgn)
    #    return nothing
    #else
    #    # TODO how to recover function and constraint type ?
    #end
    if !haskey(m.constrnames, name)
        return nothing
    end
    cis = m.constrnames[name]
    if isempty(cis)
        return nothing
    end
    if !isone(length(cis))
        error("Multiple constraints have the name $name.")
    end
    ci = first(cis)
    if !(ci isa CI)
        return nothing
    end
    return ci
end


module TestSort

using Random: shuffle
using Test
using ThreadsX

@testset for basesize in 1:8
    @testset for alg in [ThreadsX.MergeSort, ThreadsX.QuickSort, ThreadsX.StableQuickSort]
        @test ThreadsX.sort!(shuffle([1:1000;]); alg = alg, basesize = basesize) == 1:1000
        @test ThreadsX.sort!(
            shuffle([1:1000;]);
            alg = alg,
            basesize = basesize,
            by = inv,
        ) == 1000:-1:1
        @test ThreadsX.sort!([1:1000;]; alg = alg, basesize = basesize) == 1:1000
    end
end

@testset "stable sort" begin
    @testset for alg in [ThreadsX.MergeSort, ThreadsX.StableQuickSort]
        @test ThreadsX.sort(1:45; alg = alg, basesize = 25, by = _ -> 1) == 1:45
        @test ThreadsX.sort(1:1000; alg = alg, basesize = 200, by = _ -> 1) == 1:1000
    end
end

@testset "inplace" begin
    @testset "default" begin
        xs = [1:10;]
        @test ThreadsX.sort!(xs) === xs
        @test issorted(xs)
    end
    @testset for alg in [
        ThreadsX.MergeSort,
        ThreadsX.QuickSort,
        ThreadsX.StableQuickSort,
        MergeSort,
        QuickSort,
    ]
        xs = [1:10;]
        @test ThreadsX.sort!(xs; alg = alg) === xs
        @test issorted(xs)
    end
    @testset for alg in [ThreadsX.MergeSort, ThreadsX.QuickSort, ThreadsX.StableQuickSort]
        xs = [1:10;]
        @test sort!(xs; alg = alg) === xs
        @test issorted(xs)
    end
end

randnans(n) = reinterpret(Float64, [rand(UInt64) | 0x7ff8000000000000 for i in 1:n])

function randn_with_nans(n, p)
    v = randn(n)
    x = findall(rand(n) .< p)
    v[x] = randnans(length(x))
    return v
end

# Taken from "advanced sorting" testset.  This was the only test that
# failed when `fpsort!` was replaced with plain `sort!`.
@testset "NaN" begin
    @testset for n in [0:10; 100; 101; 1000; 1001]
        v = randn_with_nans(n, 0.1)
        @testset for alg in [
                ThreadsX.MergeSort,
                ThreadsX.QuickSort,
                ThreadsX.StableQuickSort,
            ],
            rev in [false, true],
            basesize in 1:8
            # test float sorting with NaNs
            s = ThreadsX.sort(v, alg = alg, rev = rev, basesize = basesize)
            @test issorted(s, rev = rev)
            @test reinterpret(UInt64, v[isnan.(v)]) == reinterpret(UInt64, s[isnan.(s)])
        end
    end
end

@testset "UI" begin
    @test ThreadsX.sort(1:10) == 1:10
    @test ThreadsX.sort(1:10; alg = MergeSort) == 1:10
    @test ThreadsX.sort(1:10; alg = QuickSort) == 1:10
end

end  # module


using RecipesBase
export SeqPlot
export TestPlot
export rectangle_corners

function rectangle_corners(x::Real,y::Real,w,h; anchor=:bottomright)
    if anchor == :botttomright
        [x,x+w,x+w,x], [y,y,y+h,y+h]
    elseif anchor == :center
        [x-w/2, x+w/2, x+w/2, x-w/2], [y-h/2, y-h/2, y+h/2, y+h/2]
    else
        error("Anchor not recognised.")
    end
end

function rectangle_corners(
    x_vec::Vector{T},y_vec::Vector{T},
    w,h; anchor=:bottomleft
    ) where {T<:Real}

    x_out = Vector{Real}[]
    y_out = Vector{Real}[]
    if anchor == :bottomleft
        for (x,y) in zip(x_vec, y_vec)
            push!(x_out, [x,x+w,x+w,x])
            push!(y_out, [y,y,y+h,y+h])
        end
    elseif anchor == :center
        for (x,y) in zip(x_vec, y_vec)
            push!(x_out, [x-w/2, x+w/2, x+w/2, x-w/2])
            push!(y_out, [y-h/2, y-h/2, y+h/2, y+h/2])
        end
    else
        error("Anchor not recognised.")
    end
    x_out, y_out
end

@userplot SeqPlot
@recipe function f(
    h::SeqPlot;
    entrymargin=0.1,
    entryfontsize=20
    )
    seq = h.args[1]
    w,h = (1,1)
    y = fill(0.0, length(seq))
    x = [0.0 + i*(w+entrymargin) for i in 0:(length(seq)-1)]
    annotations := [(x,y,string(i)) for (x,y,i) in zip(x,y,seq)]
    annotationfontsize := entryfontsize
    x_cords, y_cords = rectangle_corners(x,y, w, h; anchor=:center)
    showaxis --> false
    axis --> nothing
    aspect_ratio --> 1
    legend --> false
    @series begin
        seriestype := :shape
        x_cords,y_cords
    end
end

@userplot TestPlot
@recipe function f(
    h::TestPlot
    )
    aspect_ratio --> 1
    showaxis --> false
    @series begin
        seriestype := :shape
        1:4,1:4
    end
    @series begin
        seriestype := :line
        1:4, 1:4
        annotations := [[];[(i,i,string(i)) for i in 1:4]]
        annotationhalign --> :right
    end
end


"""
    mutable struct Structure_VMD  <: Str

This contains information for `converter_vmd()` to assign all potential parameters.

"""
mutable struct Structure_VMD <: Str
    atom_name
    num_atom_types::Int64
    num_bond_types::Int64
    num_angle_types::Int64
    num_dihedral_types::Int64
    num_improper_types::Int64
    para_mass
    para_pair
    para_bond
    para_angle
    para_dihedral
    para_improper
end


# The glaciers for ESA:

println("Time to load BITEModel.jl")
@time using BITEModel
const BM=BITEModel
using Parameters
import PyPlot
using Base.Test

"""
    get_glaciers(region)

Return the glacier numbers to be run.  Sorted with
large glaciers first.

Return:
- glnrs1 -- with radar
- glnrs2 -- without radar
- glnrs_all -- both combined.
"""
function get_glaciers(region)

    glnrs, lon, lat, subregion, area  = BM.read_attrib_file(region)
    # purge some
    inds = 1:length(glnrs)
    # if region==7
    #     inds = 1:length(glnrs)
    #     # inds = (.!( (lon.>18.5) .& (lat.>79.7) ) .& # rm Westfonna
    #     #         .!( (lon.>20) .& (lat.>79.2) )) # rm Austfonna
    # elseif region==11
    #     # Alps only
    #     inds = subregion.==1  # only Alps: 3892 glaciers
    if region==14
        inds = subregion.==2  # only Karakoram: 13757 glaciers
    end
    # else
    #     inds = 1:length(glnrs)
    # end;
    glnrs_all = glnrs[inds]

    # now get radar vs non-radar
    glnrs1 = BM.get_glaciers_with_radar(region)
    # purge glnrs1 which are not in glnrs_all
    glnrs1 = [n for n in glnrs1 if (n in glnrs_all)]
    glnrs2 = [n for n in glnrs_all if !(n in glnrs1)]

    # sort with big glaciers first
    area = BM.read_attrib_file(region, glnrs1)[5]
    glnrs1 = glnrs1[reverse(sortperm(area))]

    area = BM.read_attrib_file(region, glnrs2)[5]
    glnrs2 = glnrs2[reverse(sortperm(area))]

    area = BM.read_attrib_file(region, glnrs_all)[5]
    glnrs_all = glnrs_all[reverse(sortperm(area))]

    return glnrs1, glnrs2, glnrs_all
end

function fit_it(nr, region,
                sigma_of_model, fit_vars, fit_target, runtyp, posterior_2D_or_1D::Val{I},
                expect=nothing, stdev=nothing;
                store_thetas=false,
                update_cache=true, prior_only_resamples=300,
                retsol=false,
                dir_output=error("Provide dir_output as kwarg"),
                save_geotiff=true,
                save_sol=false,
                flroot="",
                fit_sigma=false,
                debugging=true,
                plkws...
                ) where I
    (gid, gl,gb,pp,pm,pn,pl,th0d,
     theta0, logposterior, logposterior1d, logprior, logprior_debug,
     loglikelihood, loglikelihood1d, fwdm_fn, fwdm1d_fn,
     pmcmc_defaults, fit_target) = BM.init(nr, region, fit_vars, fit_target,
                                           sigma_of_model, runtyp;
                                           fit_sigma=fit_sigma,
                                           update_cache=update_cache,
                                           plkws...)

    # Update prior (if needed) to take values of expect and stdev into account.
    # This assumes that the previous posteriors had independent normal distributions.
    if expect!=nothing
        @assert length(expect)==length(theta0.th0)
        logprior_ = logprior
        # update priors to take expect and stdev into account
        logprior = theta -> logprior_(theta) +
            sum( - (theta[i]-expect[i])^2/(2*stdev[i]^2) for i=1:length(expect))
        # update the logposterior and logposterior1d to take the new logprior into accout:
        logposterior, logposterior1d = make_logposterior(loglikelihood, loglikelihood1d, logprior)
    end
    if I==2 && !(runtyp=="test" || runtyp=="testmid")
        merge!(pmcmc_defaults, Dict(:niter=>2*10^4,
                                    :nthin =>10,
                                    :nburnin=>5*10^3,
                                    :nchains=>50))
    end

    tosample = (logposterior, logposterior1d, logprior)[I]
    pmcmc = BM.MCMCNum(;pmcmc_defaults...)

        # some tests:
    if debugging
        fwdm_fn(theta0.th0);
        print("    Time to run forward model:")
        @time fwdsol = fwdm_fn(theta0.th0);
        print("    Posterior value:")
        @show logposterior(theta0.th0)[1]
        print("    Time to calculate posterior:")
        @time logposterior(theta0.th0)[1]
        print("    Starting MCMC:")
    end

    if logprior(theta0.th0)==-Inf
        logprior_debug(theta0.th0)
        error("Zero density of prior of $gl at $(theta0.th0)")
    end

    sol = BM.mcmc(tosample, theta0, pmcmc;
                  verbose=debugging,
                  use_progress_meter=progress_bar,
                  misc=Dict(:logposterior=>logposterior,
                            :logposterior1d=>logposterior1d,
                            :logprior=>logprior,
                            :logprior_debug=>logprior_debug,
                            :loglikelihood=>loglikelihood,
                            :loglikelihood1d=>loglikelihood1d,
                            :fwdm_fn=>fwdm_fn,
                            :fwdm1d_fn=>fwdm1d_fn,
                            :fit_target=>fit_target,
                            :runtyp=>runtyp,
                            :posterior_or_prior=>I,
                            :expect=>expect,
                            :stdev=>stdev,
                            # pp, pm, pn, etc are part of FWDSol
                            ))

    # if we sampled the logprior or logposterior1d, then run the forward model for
    # prior_only_resamples to get ice thickness samples
    if I==2
        tosample = logposterior
        sol.misc[:fit_target] = BM.FitTarget.prior
        sol = BM.retrace_samples(tosample, sol, pmcmc, nsamples=prior_only_resamples, misc=sol.misc)
    end

    # postproc

    debugging && println("MCMC of experiment $exp of $(gl.gid) done in $(signif(sol.solverstats[:wall_time]/60,4)) minutes")
    debugging && println("   sampling $(["log-posterior", "log-prior"][I])")
    #debugging && I==1 && println("   Rhat=$(mean(sol.solverstats[:Rhat])), $(maximum(sol.solverstats[:Rhat]))")
    debugging && I==1 && println("   eff_sample_size=$(sol.solverstats[:eff_sample_size]), $(maximum(sol.solverstats[:eff_sample_sizes]))")
    debugging && println("\n")

    if dir_output!="" && save_geotiff
        BM.make_geotiff(sol, dir_output*"/h-$flroot-$(BM.getrgi(gid)).tif")
    end
    if dir_output!="" && save_sol
        open(dir_output*"/sol-$flroot-$(BM.getrgi(gid)).jls", "w") do io
            serialize(io, sol)
        end
    end
    if retsol
        return BM.condense_MCMCSol(sol, store_thetas=store_thetas), sol
    else
        return BM.condense_MCMCSol(sol, store_thetas=store_thetas)
    end
end

var2filename(gid, sigma_h_model, sigma_iv_model) =
    dir_output*"$(BM.getrgi(gid))_sh=$(sigma_h_model)_siv=$(sigma_iv_model)"

import OnlineStats
function calculate_prior_update(out1::Vector, drop_sigma)
    out1_ = filter(o-> o isa Dict, out1)
    # if there is no out1 data, return nothing:
    length(out1_)==0 && return nothing, nothing, nothing, nothing

    # find out length without sigma
    if drop_sigma
        d = Dict(out1_[1][:theta0][:names]...)
        # remove all sigmas
        haskey(d, :sigma_iv_model) && delete!(d, :sigma_iv_model)
        haskey(d, :sigma_h_model) && delete!(d, :sigma_h_model)
        # and see what's the last index
        len = maximum(map(maximum, values(d)))
    else
        len = length(out1_[1][:thetas_expect])
        @assert all([length(o[:thetas_expect]) for o in out1_].==len) "Not all o[:thetas_expect] the same length"
    end

    # calculate expectation.
    expect_mean = mean([o[:thetas_expect][1:len] for o in out1_])
    expect_median = median(hcat([o[:thetas_expect][1:len] for o in out1_]...),2)[:] # might be more robust because of outliers
    expect = expect_median

    # use OnlineStats to get the total variance (and quantiles (todo))
    v = OnlineStats.Group([OnlineStats.Variance() for i=1:len])
    q = OnlineStats.Group([OnlineStats.Quantile() for i=1:len])
    for o in out1_
        ns = o[:solverstats][:nsamples]
        v1 = OnlineStats.Group([OnlineStats.Variance(o[:thetas_std][i]^2, o[:thetas_expect][i], OnlineStats.EqualWeight, ns)
                                for i=1:len])
        merge!(v, v1)
    end
    stdev = sqrt.(map(x->OnlineStats.value(x), v))

    return expect, expect_median, expect_mean, stdev
end


using Minerva

function run_episodes()
    env = CartPole()
    # render(env)
    scores = []
    for i = 1:1000
        _ = reset!(env)
        total_reward = 0
        while true
            a = rand(env.action_space)
            s′, r, done, info = step!(env, a)
            total_reward += r
            # @info step, s′, r, done, info
            done && break
        end
        push!(scores, total_reward)
    end
    return scores
end

# scores = run_episodes()

function sample_run()
    env = CartPole()
    _ = reset!(env)
    render(env)
    total_reward = 0
    steps = 0
    while true
        a = rand(env.action_space)
        s′, r, done, info = step!(env, a)
        total_reward += r
        steps += 1
        # @info step, s′, r, done, info
        done && break
    end
    @info steps
    return total_reward, steps
end

_ = sample_run()


"""
    struct IntVar <: AbstractIntVar

A "simple" integer variable, whose domain can be any set of integers.
The constraints that affect this variable are stored in the `onDomainChange` array.
"""
struct IntVar <: AbstractIntVar
    onDomainChange      ::Array{Constraint}
    domain              ::SeaPearl.IntDomain
    id                  ::String
end

"""
    function IntVar(min::Int, max::Int, id::String, trailer::Trailer)

Create an `IntVar` with a domain being the integer range [`min`, `max`] with the `id` string identifier
and that will be backtracked by `trailer`.
"""
function IntVar(min::Int, max::Int, id::String, trailer::Trailer)
    offset = min - 1

    dom = IntDomain(trailer, max - min + 1, offset)

    return IntVar(Constraint[], dom, id)
end

function Base.show(io::IO, var::IntVar)
    print(io, var.id, " = ", var.domain)
end

function Base.show(io::IO, ::MIME"text/plain", var::IntVar)
    print(io, typeof(var), ": ", var.id, " = ", var.domain)
end

"""
    isbound(x::AbstractIntVar)

Check whether x has an assigned value.
"""
isbound(x::AbstractIntVar) = length(x.domain) == 1

"""
    assign!(x::AbstractIntVar, value::Int)

Remove everything from the domain of `x` but `value`.
"""
assign!(x::AbstractIntVar, value::Int) = assign!(x.domain, value)

"""
    assignedValue(x::AbstractIntVar)

Return the assigened value of `x`. Throw an error if `x` is not bound.
"""
function assignedValue(x::AbstractIntVar)
    @assert isbound(x)

    return minimum(x.domain)
end

"""
    rootVariable(x::IntVar)

Return the "true" variable behind `x`. For a `IntVar`, it simply returns `x`.
"""
rootVariable(x::IntVar) = x


module Crypto

using MbedTLS

hmacsha256_digest(s::String, k::Union{String, Vector{UInt8}}) = digest(MD_SHA256, s, k)
export hmacsha256_digest

hmacsha1_digest(s::String, k::Union{String, Vector{UInt8}}) = digest(MD_SHA1, s, k)
export hmacsha1_digest

function md5_file(s::String)
    f = open(s)
    md = nothing
    try
        md = md5(f)
    finally
        close(f)
    end
    md
end
export md5_file

md5(s::String) = digest(MD_MD5, s)

function md5(s::IO)
    md = MbedTLS.MD(MD_MD5)
    success = false
    try
        while !eof(s)
            b = read(s, UInt8, min(nb_available(s), 65536))    # Read in 64 K chunks....
            MbedTLS.write(md, b)
        end
        d = MbedTLS.finish!(md)
        success = true
        return d
    finally
        success || MbedTLS.finish!(md)
    end
end
export md5

sha256(s::String) = digest(MD_SHA256, s)

end # Module end


################################################################################
#
#   GENERAL INTERFACE FOR CREATING LATTICES INSIDE SHAPES
#   (USING ABSOLUTE COORDINATES)
#
################################################################################

# FALLBACK function
function getLatticeInShape(
            :: Type{L},
            unitcell :: U,
            shape    :: Function,
            origin   :: Integer = 1
        ) :: L where {
            D,N,LS,LB,S<:AbstractSite{LS,D},B<:AbstractBond{LB,N},
            U<:AbstractUnitcell{S,B},
            DL,LLS,LLB,SL<:AbstractSite{LLS,DL},BL<:AbstractBond{LLB,0},
            L<:AbstractLattice{SL,BL,U}
        }

    # throw an error as this is not implemented yet
    error("Either function 'getLatticeInShape' is not yet implemented for unitcells of type " * string(U) * ", i.e. N=" *
        string(N) * " / D=" * string(D) * " or you passed the wrong lattice type L = " * string(L))
end

# generate a lattice in 2D in the absolute shape given by the function shape (open boundary conditions)
function getLatticeInShape(
            :: Type{L},
            unitcell :: U,
            shape    :: Function,
            origin   :: Integer = 1
        ) :: L where {
            D,LS,LB,S<:AbstractSite{LS,D},B<:AbstractBond{LB,2},
            U<:AbstractUnitcell{S,B},
            DL,LLS,LLB,SL<:AbstractSite{LLS,DL},BL<:AbstractBond{LLB,0},
            L<:AbstractLattice{SL,BL,U}
        }

    # Format of checklist item
    #   Tuple{S, NTuple, Int64}
    # denoting
    #   Site - UC Indices (i,j,k,...) - Site Index (in UC)

    # checklist for all sites that are checked if added etc
    checklist = Tuple{S, NTuple{2,Int64}, Int64}[]
    # list for all accepted sites that are checked
    accepted  = Tuple{S, NTuple{2,Int64}, Int64}[]

    # List of all accepted bonds
    bond_list = BL[]

    # push the origin site to the checklist
    push!(
        checklist, (deepcopy(site(unitcell,origin)), (0,0), origin)
    )

    # iterate while the checklist is not empty
    while size(checklist,1) > 0

        # get the item that was on the checklist for longest
        item_to_handle = popfirst!(checklist)

        # check if the item is already in the accepted list
        found = false
        for acc in accepted
            # check if a site with index acc[3] in unitcell acc[2] is already found
            if item_to_handle[2] == acc[2] && item_to_handle[3] == acc[3]
                found = true
                break
            end
        end
        # if the element is found, continue with the next checklist item
        if found
            continue
        end

        # Item is NOT in the accepted list, push it into the list
        push!(accepted, item_to_handle)
        # set the current item index to the length of the list
        index_from = length(accepted)

        # search through all connections of the unitcell to find the ones relevant persuing
        for b in bonds(unitcell)

            # check if the bond actually describes a bond emerging from the site
            if from(b) != item_to_handle[3]
                continue
            end

            # build up the element that the bond connects to

            # find out in which copy of unitcell the site is located
            uc_to = (
                item_to_handle[2][1] + wrap(b)[1],
                item_to_handle[2][2] + wrap(b)[2]
            )
            # find out which site index it is
            alpha_to = to(b)

            # find out if the item already exists within the accepted list
            index_to = -1
            for (index,item_handled) in enumerate(accepted)
                # check if the item is the particular item
                if item_handled[2] == uc_to && item_handled[3] == alpha_to
                    # save the list index
                    index_to = index
                    # break the loop
                    break
                end
            end
            # determine whether the element is already inside the list (index was set to > 0)
            if index_to > 0
                # create a new bonds
                bond_new_1 = newBond(
                    BL,
                    index_from,
                    index_to,
                    label(b),
                    NTuple{0,Int64}()
                )
                bond_new_2 = newBond(
                    BL,
                    index_to,
                    index_from,
                    label(b),
                    NTuple{0,Int64}()
                )
                # check if the bonds are already added
                if !(bond_new_1 in bond_list)
                    push!(bond_list, bond_new_1)
                end
                if !(bond_new_2 in bond_list)
                    push!(bond_list, bond_new_2)
                end
            else
                # the respective site is not in list, create it to check the position, create a new site object
                site_new = deepcopy(site(unitcell, to(b)))
                # set the new position
                point!(site_new, (uc_to[1] .* a1(unitcell)) .+ (uc_to[2] .* a2(unitcell)) .+ point(site_new))
                # if still inside the shape, add it to the list
                if shape(point(site_new))
                    # format for sites on checklist: ([pos_x, pos_y, pos_z], [uc_i, uc_j, uc_k], index_in_UC)
                    push!(checklist, (
                        site_new,
                        uc_to,
                        to(b)
                    ))
                end
            end
        end
    end

    # insert missing connections (if (i to j) is present, insert (j to i))
    # TODO

    # save everything to a Lattice object and return it
    return newLattice(
            L,
        	Vector{Float64}[],
        	SL[newSite(SL, point(s[1]), label(s[1])) for s in accepted],
        	bond_list,
            unitcell
        )
end

# generate a lattice in 3D in the absolute shape given by the function shape (open boundary conditions)
function getLatticeInShape(
            :: Type{L},
            unitcell :: U,
            shape    :: Function,
            origin   :: Integer = 1
        ) :: L where {
            D,LS,LB,S<:AbstractSite{LS,D},B<:AbstractBond{LB,3},
            U<:AbstractUnitcell{S,B},
            DL,LLS,LLB,SL<:AbstractSite{LLS,DL},BL<:AbstractBond{LLB,0},
            L<:AbstractLattice{SL,BL,U}
        }

    # Format of checklist item
    #   Tuple{S, NTuple, Int64}
    # denoting
    #   Site - UC Indices (i,j,k,...) - Site Index (in UC)

    # checklist for all sites that are checked if added etc
    checklist = Tuple{S, NTuple{3,Int64}, Int64}[]
    # list for all accepted sites that are checked
    accepted  = Tuple{S, NTuple{3,Int64}, Int64}[]

    # List of all accepted bonds
    bond_list = BL[]

    # push the origin site to the checklist
    push!(
        checklist, (deepcopy(site(unitcell,origin)), (0,0,0), origin)
    )

    # iterate while the checklist is not empty
    while size(checklist,1) > 0

        # get the item that was on the checklist for longest
        item_to_handle = popfirst!(checklist)

        # check if the item is already in the accepted list
        found = false
        for acc in accepted
            # check if a site with index acc[3] in unitcell acc[2] is already found
            if item_to_handle[2] == acc[2] && item_to_handle[3] == acc[3]
                found = true
                break
            end
        end
        # if the element is found, continue with the next checklist item
        if found
            continue
        end

        # Item is NOT in the accepted list, push it into the list
        push!(accepted, item_to_handle)
        # set the current item index to the length of the list
        index_from = length(accepted)

        # search through all connections of the unitcell to find the ones relevant persuing
        for b in bonds(unitcell)

            # check if the bond actually describes a bond emerging from the site
            if from(b) != item_to_handle[3]
                continue
            end

            # build up the element that the bond connects to

            # find out in which copy of unitcell the site is located
            uc_to = (
                item_to_handle[2][1] + wrap(b)[1],
                item_to_handle[2][2] + wrap(b)[2],
                item_to_handle[2][3] + wrap(b)[3]
            )
            # find out which site index it is
            alpha_to = to(b)

            # find out if the item already exists within the accepted list
            index_to = -1
            for (index,item_handled) in enumerate(accepted)
                # check if the item is the particular item
                if item_handled[2] == uc_to && item_handled[3] == alpha_to
                    # save the list index
                    index_to = index
                    # break the loop
                    break
                end
            end
            # determine whether the element is already inside the list (index was set to > 0)
            if index_to > 0
                # create a new bonds
                bond_new_1 = newBond(
                    BL,
                    index_from,
                    index_to,
                    label(b),
                    NTuple{0,Int64}()
                )
                bond_new_2 = newBond(
                    BL,
                    index_to,
                    index_from,
                    label(b),
                    NTuple{0,Int64}()
                )
                # check if the bonds are already added
                if !(bond_new_1 in bond_list)
                    push!(bond_list, bond_new_1)
                end
                if !(bond_new_2 in bond_list)
                    push!(bond_list, bond_new_2)
                end
            else
                # the respective site is not in list, create it to check the position, create a new site object
                site_new = deepcopy(site(unitcell, to(b)))
                # set the new position
                point!(site_new, (uc_to[1] .* a1(unitcell)) .+ (uc_to[2] .* a2(unitcell)) .+ (uc_to[3] .* a3(unitcell)) .+ point(site_new))
                # if still inside the shape, add it to the list
                if shape(point(site_new))
                    # format for sites on checklist: ([pos_x, pos_y, pos_z], [uc_i, uc_j, uc_k], index_in_UC)
                    push!(checklist, (
                        site_new,
                        uc_to,
                        to(b)
                    ))
                end
            end
        end
    end

    # insert missing connections (if (i to j) is present, insert (j to i))
    # TODO

    # save everything to a Lattice object and return it
    return newLattice(
            L,
        	Vector{Float64}[],
        	SL[newSite(SL, point(s[1]), label(s[1])) for s in accepted],
        	bond_list,
            unitcell
        )
end

# wrapper
function getLatticeInShape(
            unitcell :: U,
            shape    :: Function,
            origin   :: Integer = 1
        ) :: Lattice{S,Bond{LB,0},U} where {
            D,N,LS,LB,S<:AbstractSite{LS,D},B<:AbstractBond{LB,N},
            U<:AbstractUnitcell{S,B}
        }

    # call the general function
    return getLatticeInShape(Lattice{S,Bond{LB,0},U}, unitcell, shape, origin)
end

# export the function
export getLatticeInShape

################################################################################
#
#   SPECIFIC FUNCTIONS FOR CREATING LATTICES INSIDE CERTAIN SHAPES
#   (USING ABSOLUTE COORDINATES & POSSIBILITY OF RELATIVE COORDINATES)
#
#   SUPPORTED SHAPES
#   - box
#   - sphere
#
################################################################################

#-------------
#   BOX
#-------------

# MOST GENERIC INTERFACE (FALLBACK)
function getLatticeInBox(
            :: Type{L},
            unitcell   :: U,
            dimensions :: Vector{<:Real},
            center     :: Vector{<:Real},
            origin     :: Integer = 1
        ) :: L where {
            D,N,LS,LB,S<:AbstractSite{LS,D},B<:AbstractBond{LB,N},
            U<:AbstractUnitcell{S,B},
            DL,LLS,LLB,SL<:AbstractSite{LLS,DL},BL<:AbstractBond{LLB,0},
            L<:AbstractLattice{SL,BL,U}
        }

    # throw an error as this is not implemented yet
    error("Either function 'getLatticeInBox' is not yet implemented for unitcells of type " * string(U) * ", i.e. N=" *
        string(N) * " / D=" * string(D) * " or you passed the wrong lattice type L = " * string(L))
end

# MOST GENERIC INTERFACE (2D)
function getLatticeInBox(
            :: Type{L},
            unitcell   :: U,
            dimensions :: Vector{<:Real},
            center     :: Vector{<:Real},
            origin     :: Integer = 1
        ) :: L where {
            N,LS,LB,S<:AbstractSite{LS,2},B<:AbstractBond{LB,N},
            U<:AbstractUnitcell{S,B},
            LLS,LLB,SL<:AbstractSite{LLS,2},BL<:AbstractBond{LLB,0},
            L<:AbstractLattice{SL,BL,U}
        }

    # create the shape function
    shape_function(p) = (center[1] - dimensions[1]/2 <= p[1] <= center[1] + dimensions[1]/2) &&
                        (center[2] - dimensions[2]/2 <= p[2] <= center[2] + dimensions[2]/2)

    # call the interface with the shape function
    return getLatticeInShape(L, unitcell, shape_function, origin)
end

# MOST GENERIC INTERFACE (3D)
function getLatticeInBox(
            :: Type{L},
            unitcell   :: U,
            dimensions :: Vector{<:Real},
            center     :: Vector{<:Real},
            origin     :: Integer = 1
        ) :: L where {
            N,LS,LB,S<:AbstractSite{LS,3},B<:AbstractBond{LB,N},
            U<:AbstractUnitcell{S,B},
            LLS,LLB,SL<:AbstractSite{LLS,3},BL<:AbstractBond{LLB,0},
            L<:AbstractLattice{SL,BL,U}
        }

    # create the shape function
    shape_function(p) = (center[1] - dimensions[1]/2 <= p[1] <= center[1] + dimensions[1]/2) &&
                        (center[2] - dimensions[2]/2 <= p[2] <= center[2] + dimensions[2]/2) &&
                        (center[3] - dimensions[3]/2 <= p[3] <= center[3] + dimensions[3]/2)

    # call the interface with the shape function
    return getLatticeInShape(L, unitcell, shape_function, origin)
end

# export the function
export getLatticeInBox

# Wrapper function (2D)
function getLatticeInBox(
            unitcell   :: U,
            dimensions :: Vector{<:Real},
            center     :: Vector{<:Real} = [0.0, 0.0],
            origin     :: Integer = 1
        ) :: Lattice{S,Bond{LB,0},U} where {
            N,LS,LB,S<:AbstractSite{LS,2},B<:AbstractBond{LB,N},
            U<:AbstractUnitcell{S,B}
        }

    # call the interface
    return getLatticeInBox(Lattice{S,Bond{LB,0},U}, unitcell, dimensions, center, origin)
end

# Wrapper function (3D)
function getLatticeInBox(
            unitcell   :: U,
            dimensions :: Vector{<:Real},
            center     :: Vector{<:Real} = [0.0, 0.0, 0.0],
            origin     :: Integer = 1
        ) :: Lattice{S,Bond{LB,0},U} where {
            N,LS,LB,S<:AbstractSite{LS,3},B<:AbstractBond{LB,N},
            U<:AbstractUnitcell{S,B}
        }

    # call the interface
    return getLatticeInBox(Lattice{S,Bond{LB,0},U}, unitcell, dimensions, center, origin)
end

#-------------
#   SPHERE
#-------------

# MOST GENERIC INTERFACE (FALLBACK)
function getLatticeInSphere(
            :: Type{L},
            unitcell   :: U,
            radius     :: Real,
            center     :: Vector{<:Real},
            origin     :: Integer = 1
        ) :: L where {
            D,N,LS,LB,S<:AbstractSite{LS,D},B<:AbstractBond{LB,N},
            U<:AbstractUnitcell{S,B},
            DL,LLS,LLB,SL<:AbstractSite{LLS,DL},BL<:AbstractBond{LLB,0},
            L<:AbstractLattice{SL,BL,U}
        }

    # throw an error as this is not implemented yet
    error("Either function 'getLatticeInSphere' is not yet implemented for unitcells of type " * string(U) * ", i.e. N=" *
        string(N) * " / D=" * string(D) * " or you passed the wrong lattice type L = " * string(L))
end

# MOST GENERIC INTERFACE (2D)
function getLatticeInSphere(
            :: Type{L},
            unitcell   :: U,
            radius     :: Real,
            center     :: Vector{<:Real},
            origin     :: Integer = 1
        ) :: L where {
            N,LS,LB,S<:AbstractSite{LS,2},B<:AbstractBond{LB,N},
            U<:AbstractUnitcell{S,B},
            LLS,LLB,SL<:AbstractSite{LLS,2},BL<:AbstractBond{LLB,0},
            L<:AbstractLattice{SL,BL,U}
        }

    # create the shape function
    shape_function(p) = (center[1]-p[1])^ 2 + (center[2]-p[2])^ 2  < radius^2

    # call the interface with the shape function
    return getLatticeInShape(L, unitcell, shape_function, origin)
end

# MOST GENERIC INTERFACE (3D)
function getLatticeInSphere(
            :: Type{L},
            unitcell   :: U,
            radius     :: Real,
            center     :: Vector{<:Real},
            origin     :: Integer = 1
        ) :: L where {
            N,LS,LB,S<:AbstractSite{LS,3},B<:AbstractBond{LB,N},
            U<:AbstractUnitcell{S,B},
            LLS,LLB,SL<:AbstractSite{LLS,3},BL<:AbstractBond{LLB,0},
            L<:AbstractLattice{SL,BL,U}
        }

    # create the shape function
    shape_function(p) = (center[1]-p[1])^ 2 + (center[2]-p[2])^ 2 + (center[3]-p[3])^ 2  < radius^2

    # call the interface with the shape function
    return getLatticeInShape(L, unitcell, shape_function, origin)
end

# export the function
export getLatticeInSphere

# Wrapper function (2D)
function getLatticeInSphere(
            unitcell   :: U,
            radius     :: Real,
            center     :: Vector{<:Real} = [0.0, 0.0],
            origin     :: Integer = 1
        ) :: Lattice{S,Bond{LB,0},U} where {
            N,LS,LB,S<:AbstractSite{LS,2},B<:AbstractBond{LB,N},
            U<:AbstractUnitcell{S,B}
        }

    # call the interface
    return getLatticeInSphere(Lattice{S,Bond{LB,0},U}, unitcell, radius, center, origin)
end

# Wrapper function (3D)
function getLatticeInSphere(
            unitcell   :: U,
            radius     :: Real,
            center     :: Vector{<:Real} = [0.0, 0.0, 0.0],
            origin     :: Integer = 1
        ) :: Lattice{S,Bond{LB,0},U} where {
            N,LS,LB,S<:AbstractSite{LS,3},B<:AbstractBond{LB,N},
            U<:AbstractUnitcell{S,B}
        }

    # call the interface
    return getLatticeInSphere(Lattice{S,Bond{LB,0},U}, unitcell, radius, center, origin)
end


module IteratedProbabilisticClassification

using
	KernelDensityEstimate,
	Distributions,
	StatsBase,
	Random,
	Gadfly,
	KernelDensityEstimatePlotting,
    Statistics

export
	# Major types and functions used for classification
	ClassificationSystem,
	TuningParameters,
	ClassificationStats,
	defaultClassificationStats,
	classifyOneStep,
	classifyConverge,
	plotClassification1D,
	plotClassificationStats,
	plotEMStatus,
	packDebugResults!,
	EMClassificationRun!,
	dispersePoints,

	# functions under development
	sdc2,
	plotUtil1D,
	plotUtil2D,
	# batchClassifyConverge

	# previous code
	simdata01_true,
	simdata02_true,
	DebugResults,
	defaultDebugResults,
	DataGroundTruth,
	SampleData,
	PALETTE

PALETTE = ["deepskyblue";"red";"magenta";"black";"green";"blue"]

const NothingUnion{T} = Union{Nothing, T}

include("ClassificationUtilities.jl")
include("PlottingUtilities.jl")
include("SimulationData.jl")

function plotMarginalClassificationBelief()
	pl_init = plotKDE([c1_expert,c2_expert],c=PALETTE)
	lyr_smpls = layer(x=samples, y=zeros(size(samples)), Geom.point)
	push!(pl_init.layers, lyr_smpls[1] )
	push!(pl_init.layers, lyr_smpls_trueA[1])
	push!(pl_init.layers, lyr_smpls_trueB[1])

	#Gadfly.draw(PDF("InitialClusterDist.pdf",20cm,12cm),pl_init)
	error("Incomplete function")
end

function plotClassification1D(
			;groundtruth=nothing,
			samples::Array{Float64,2}=Array{Float64,2}(),
			belief::BallTreeDensity=nothing)
		dims = size(samples, 1)

		belief == nothing ? nothing : (dims == Ndim(belief) ? nothing : error("belief and sample dimensions don't match") )

		plr = ceil(Int, sqrt(dims))
		plc = ceil(Int, dims/plr)

		PL = Gadfly.Context[]
		for i in dims
			# pl =
			push!(PL, pl)
		end

		error("Incomplete function")

		nothing
end

end # module


# helpers to make addition/multiplication of distance and regularizer possible

import Base.+
function +(a::Function, b::SparseMatrixCSC)
	f(x) = a(x) + b*x
	return f
end


mutable struct PPHMCTune <: SamplerTune
    logfgrad::Union{Function,Missing}
    epsilon::Float64
    delta::Float64
    nleap::Int64
    m::Int64
    randomization::Bool
    adapter::Float64
    moves::Vector{Int}

    PPHMCTune() = new()

    function PPHMCTune(x::Vector{T}, logfgrad::Union{Function,Missing}, epsilon::Float64, nleap::Int64, delta::Float64=0.003) where T <: GeneralNode
        new(logfgrad, epsilon, delta, nleap, 0, true, 0.4,Int[])
    end
end

#PPHMCTune(x::Vector{T}, logfgrad::Function, epsilon::Float64, nleap::Int64, delta::Float64=0.003) where T <: GeneralNode =
#    PPHMCTune(x, logfgrad, epsilon, nleap, delta)

const PPHMCVariate = Sampler{PPHMCTune, T} where T<:GeneralNode

#################### Sampler Constructor ####################

"""
    PPHMC(params::ElementOrVector{Symbol}; args...)

Construct a `Sampler` object for PPHMC sampling. The Parameter is assumed to be
a tree.

Returns a `Sampler{PPHMCTune}` type object.

* params: stochastic node to be updated with the sampler.

* args...: additional keyword arguments to be passed to the PNUTSVariate constructor.
"""
function PPHMC(params::ElementOrVector{Symbol}, epsilon::Float64, nleap::Int64, delta::Float64)
    samplerfx = function (model::Model, block::Integer)
        block = SamplingBlock(model, block, true)

        f = let block = block
            (x, sz, ll, gr) -> mlogpdfgrad!(block, x, sz, ll, gr)
        end
        v = Sampler(block, f, epsilon, nleap, delta)

        sample!(v::PPHMCVariate, f, model.iter <= model.burnin, model.burnin)

        relist(block, v)
    end
    Sampler(params, samplerfx, PPHMCTune())
end

sample!(v::PPHMCVariate; args...) = sample!(v, v.tune.logfgrad, bi)

function sample!(v::PPHMCVariate, logfgrad::Function, adapt::Bool, bi::Int64)
    mt = v.value[1]
    nl = size(mt)[1] - 1
    delta = v.tune.delta
    v.tune.m += 1
    r = randn(nl)
    blv = get_branchlength_vector(mt)
    set_branchlength_vector!(mt, molifier.(blv, delta))
    lf, grad = logfgrad(mt, nl, true, true)
    s = Tree_HMC_State(deepcopy(mt), r, grad, lf)

    currH = hamiltonian(s)
    ovnni = 0

    fac = adapt ? (1-v.tune.adapter)^(1-v.tune.m*1/bi) : 1
    epsilon = v.tune.epsilon * fac
    nl = v.tune.randomization ? rand(1:v.tune.nleap) : v.tune.nleap

    for i in 1:nl
        nni = refraction!(s, epsilon, logfgrad, delta, nl)
        @show hamiltonian(s)
        ovnni += nni
    end

    push!(v.tune.moves, ovnni)
    propH = hamiltonian(s)
    @show propH, currH
    if log(rand()) < propH - currH
        v.value[1] = s.x
    end
    v
end