#
# Cache routines for Business Days precalculated days
#

"""
Data structure for calendar cache.
"""
mutable struct HolidayCalendarCache
    hc::HolidayCalendar
    isbday_array::Vector{Bool}
    bdayscounter_array::Vector{UInt32}
    dtmin::Dates.Date
    dtmax::Dates.Date
    is_initialized::Bool # indicated wether isbday_array and bdayscounter_array is empty for this cache
end

"""
    HolidayCalendarCache()

creates an empty instance of HolidayCalendarCache
"""
HolidayCalendarCache() = HolidayCalendarCache(NullHolidayCalendar(), Vector{Bool}(), Vector{UInt32}(), Dates.Date(1900,1,1), Dates.Date(1900,1,1), false)

"""
Holds caches for Holiday Calendars.

* Key = `HolidayCalendar` instance
* Value = instance of `HolidayCalendarCache`
"""
const CACHE_DICT = Dict{HolidayCalendar, HolidayCalendarCache}()
const DEFAULT_CACHE_D0 = Dates.Date(1980, 01, 01)
const DEFAULT_CACHE_D1 = Dates.Date(2150, 12, 20)

@inline _getcachestate(hcc::HolidayCalendarCache) = hcc.is_initialized
@inline _getcachestate(hc::HolidayCalendar) = haskey(CACHE_DICT, hc) && _getcachestate(CACHE_DICT[hc])
@inline _getholidaycalendarcache(hc::HolidayCalendar) = CACHE_DICT[hc]
@inline checkbounds(hcc::HolidayCalendarCache, dt::Dates.Date) = @assert (hcc.dtmin <= dt) && (dt <= hcc.dtmax) "Date out of cache bounds. Use initcache function with a wider time spread. Provided date: $(dt)."
@inline _linenumber(hcc::HolidayCalendarCache, dt::Dates.Date) = Dates.days(dt) - Dates.days(hcc.dtmin) + 1

@inline function isbday(hcc::HolidayCalendarCache, dt::Dates.Date) :: Bool
    checkbounds(hcc, dt)
    return hcc.isbday_array[ _linenumber(hcc, dt) ]
end

function bdayscount(hcc::HolidayCalendarCache, dt0::Dates.Date, dt1::Dates.Date) :: Int
    # Computation is always based on next Business Days if given dates are not Business Days, inspired by Banking Account convention.
    dt0_tobday = tobday(hcc.hc, dt0) # cache bounds are checked inside tobday -> isbday
    dt1_tobday = tobday(hcc.hc, dt1) # cache bounds are checked inside tobday -> isbday

    return Int(hcc.bdayscounter_array[_linenumber(hcc, dt1_tobday)]) - Int(hcc.bdayscounter_array[_linenumber(hcc, dt0_tobday)])
end

@inline bdays(hcc::HolidayCalendarCache, dt0::Dates.Date, dt1::Dates.Date) :: Dates.Day = Dates.Day(bdayscount(hcc, dt0, dt1))

# Returns tuple
# tuple[1] = Array of Bool (isBday) , tuple[2] = Array of UInt32 (bdaycounter)
function _create_bdays_cache_arrays(hc::HolidayCalendar, d0::Dates.Date, d1::Dates.Date)

    d0_rata = Dates.days(d0)
    d1_rata = Dates.days(d1)

    # length of the cache arrays
    len::Int = d1_rata - d0_rata + 1

    # This function uses UInt32 to store bdayscounter array
    # We need to check if we'll exceed typemax(UInt32)
    @assert len <= typemax(UInt32) "Maximum size allowed for bdays cache array is $(typemax(UInt32)). The required lenght was $(len)."

    isbday_array = Vector{Bool}(undef, len)
    bdayscounter_array = Vector{UInt32}(undef, len)

    @inbounds isbday_array[1] = isbday(hc, d0)
    @inbounds bdayscounter_array[1] = 0

    for i in 2:len
        @inbounds isbday_array[i] = isbday(hc, d0 + Dates.Day(i-1))
        @inbounds bdayscounter_array[i] = bdayscounter_array[i-1] + isbday_array[i]
    end

    return isbday_array, bdayscounter_array
end

@inline needs_cache_update(cache::HolidayCalendarCache, d0::Dates.Date, d1::Dates.Date) :: Bool = _getcachestate(cache) && cache.dtmin == d0 && cache.dtmax == d1
@inline needs_cache_update(hc::HolidayCalendar, d0::Dates.Date, d1::Dates.Date) :: Bool = _getcachestate(hc) && CACHE_DICT[hc].dtmin == d0 && CACHE_DICT[hc].dtmax == d1

# Be sure to use this function on a syncronized code (not multithreaded).
"""
    initcache(calendar, [d0], [d1])

Creates cache for a given Holiday Calendar. After calling this function, any call to `isbday`
function, or any function that uses `isbday`, will be optimized to use this cache.

You can pass `calendar` as an instance of `HolidayCalendar`, `Symbol` or `AbstractString`.
You can also pass `calendar` as an `AbstractArray` of those types.
"""
function initcache(hc::HolidayCalendar, d0::Dates.Date=DEFAULT_CACHE_D0, d1::Dates.Date=DEFAULT_CACHE_D1)
    @assert d0 <= d1 "d1 < d0 not allowed."
    if needs_cache_update(hc, d0, d1)
        # will not repeat initcache for this already initialized cache
        return
    else
        isbday_array , bdayscounter_array = _create_bdays_cache_arrays(hc, d0, d1)
        CACHE_DICT[hc] = HolidayCalendarCache(hc, isbday_array, bdayscounter_array, d0, d1, true)
    end
    nothing
end

function initcache(hc_vec::Vector{HolidayCalendar}, d0::Dates.Date=DEFAULT_CACHE_D0, d1::Dates.Date=DEFAULT_CACHE_D1)
    for hc in hc_vec
        initcache(hc, d0, d1)
    end
end

initcache(calendars::A, d0::Dates.Date=DEFAULT_CACHE_D0, d1::Dates.Date=DEFAULT_CACHE_D1) where {A<:AbstractArray} = initcache(convert(Vector{HolidayCalendar}, calendars), d0, d1)
initcache(calendar, d0::Dates.Date=DEFAULT_CACHE_D0, d1::Dates.Date=DEFAULT_CACHE_D1) = initcache(convert(HolidayCalendar, calendar), d0, d1)

function initcache!(cache::HolidayCalendarCache, hc::HolidayCalendar, d0::Dates.Date=DEFAULT_CACHE_D0, d1::Dates.Date=DEFAULT_CACHE_D1)
    if needs_cache_update(cache, d0, d1)
        # will not repeat initcache for this already initialized cache
        return
    else
        cache.dtmin = d0
        cache.dtmax = d1
        isbday_array , bdayscounter_array = _create_bdays_cache_arrays(hc, d0, d1)
        cache.isbday_array = isbday_array
        cache.bdayscounter_array = bdayscounter_array
        cache.is_initialized = true
    end
    nothing
end

function cleancache!(cache::HolidayCalendarCache)
    if cache.is_initialized
        empty!(cache.isbday_array)
        empty!(cache.bdayscounter_array)
        cache.is_initialized = false
    end
    nothing
end

# remove all elements from cache
function cleancache()
    for k in keys(CACHE_DICT)
        delete!(CACHE_DICT, k)
    end
    nothing
end

"""
    cleancache([calendar])

Cleans cache for a given instance or list of `HolidayCalendar`, `Symbol` or `AbstractString`.
"""
function cleancache(hc::HolidayCalendar)
    if haskey(CACHE_DICT, hc)
        delete!(CACHE_DICT, hc)
    end
    nothing
end

function cleancache(hc_vec::Vector{HolidayCalendar})
    for k in hc_vec
        if k in keys(CACHE_DICT)
            delete!(CACHE_DICT, k)
        end
    end
    nothing
end

cleancache(calendar) = cleancache(convert(HolidayCalendar, calendar))
cleancache(calendars::A) where {A<:AbstractArray} = cleancache(convert(Vector{HolidayCalendar}, calendars))


using Caesar
using IncrementalInference
using RoME
using CloudGraphs

@show dbaddress = ARGS[1]
configuration = CloudGraphs.CloudGraphConfiguration(dbaddress, 7474, "", "", dbaddress, 27017, false, "", "");
cloudGraph = connect(configuration, encodePackedType, getpackedtype, decodePackedType);

# register types of interest in CloudGraphs
registerGeneralVariableTypes!(cloudGraph)
IncrementalInference.setCloudDataLayerAPI!()

nprocs() > 1 ? thxl = 2 : nothing

type SLAMWrapper
  fg::FactorGraph
  tree
  lndmidx::Int
end

USEREADY=0
include("SlamConvenienceFunctions.jl")

function tcpStringSLAMServer(;slamdata=Union{},port::Int=60001)
  println("Empty slam object created")
  if slamdata == Union{}
    # this is being replaced by cloudGraph, added here for development period
    fg = emptyFactorGraph()
    fg.cg = cloudGraph
    slamdata = SLAMWrapper(fg, Union{}, 0)
  end

  println("Listenting on $(port)")
  server = listen(port)
  loop = true
  while loop
   sock = accept(server)
     while isopen(sock)
         loop, retstr = parseTCP!(slamdata, readline(sock)[1:(end-1)])
         loop ? (isopen(sock) ? println(sock, retstr) : nothing) : close(sock)
     end
     println("connection lost")
  end
  !loop ? close(server) : nothing
  return slamdata
end

tcpStringSLAMServer()


# constructors

# values array + index array + fields singleton
TS(v::V, t::T, f::F) where {V<:VALARR,T<:IDXARR,F<:FLDTYPE} = TS(collect(hcat(v)), collect(t), [Symbol(f)])

# values array + index array + fields array
TS(v::V, t::T, f::F) where {V<:VALARR,T<:IDXARR,F<:FLDARR} = TS(collect(hcat(v)), collect(t), Symbol.(f))

# values array + index array
TS(v::V, t::T) where {V<:VALARR,T<:IDXARR} = TS(collect(hcat(v)), collect(t), autocol(1:size(v,2)))

# values array + index singleton + fields any
TS(v::V, t::T, f) where {V<:VALARR,T<:IDXTYPE} = TS(collect(hcat(v)), [t], f)

# values singleton + index array + fields any
TS(v::V, t::T, f) where {V<:VALTYPE,T<:IDXARR} = TS([v], collect(t), f)

# values singleton + index singelton + fields any
TS(v::V, t::T, f) where {V<:VALTYPE,T<:IDXTYPE} = TS([v][:,:], [t], f)

# values singleton + index array
TS(v::V, t::T) where {V<:VALTYPE, T<:IDXARR} = TS([v], collect(t), [:A])

# values singleton + index singleton
TS(v::V, t::T) where {V<:VALTYPE, T<:IDXTYPE} = TS([v], [t], [:A])

# values array
TS(v::V) where {V<:VALARR} = TS(collect(v), autoidx(size(v,1)), autocol(1:size(v,2)))

# default constructor
TS() = TS(zeros((0,0)), Date[], Symbol[])

# copy constructor
TS(X::TS) = copy(X)

# named tuple constructor
function TS(named_tuple::NamedTuple, index_element::Int=0)::TS
    function find_index_col(fields)::Int
        regex = r"date|time|index"i
        @inbounds for (j, field) in enumerate(fields)
            if occursin(regex, String(field))
                return j
            end
        end
        return 0
    end
    if index_element == 0
        index_element = find_index_col(keys(named_tuple))
    end
    index = named_tuple[index_element]
    array = zeros(Float64, (length(index), length(named_tuple)-1))
    value_elements = setdiff(1:length(named_tuple), index_element)
    fields = [f for f in keys(named_tuple)[value_elements]]
    @inbounds for (array_col, tuple_element) in enumerate(value_elements)
        array[:,array_col] = VALTYPE.(named_tuple[tuple_element])
    end
    return TS(array, index, fields)
end

# conversions

# boolean -> numeric
convert(::Type{TS{Bool}}, x::TS{V}) where {V<:VALTYPE} = TS{Bool}(map(V, x.values), x.index, x.fields)

## boolean -> float
#convert(::Type{TS{Float64}}, x::TS{Bool}) = TS{Float64}(map(Float64, x.values), x.index, x.fields)
#
## boolean -> integer
#convert(::Type{TS{Int}}, x::TS{Bool}) = TS{Int}(map(Int, x.values), x.index, x.fields)


"""
    _parse_grr(gpa::SBML.GeneProductAssociation)::GeneAssociation

Parse `SBML.GeneProductAssociation` structure to the simpler GeneAssociation.
The input must be (implicitly) in a positive DNF.
"""
function _parse_grr(gpa::SBML.GeneProductAssociation)::GeneAssociation
    parse_ref(x) =
        typeof(x) == SBML.GPARef ? [x.gene_product] :
        begin
            @_models_log @warn "Could not parse a part of gene association, ignoring: $x"
            String[]
        end
    parse_and(x) =
        typeof(x) == SBML.GPAAnd ? vcat([parse_and(i) for i in x.terms]...) : parse_ref(x)
    parse_or(x) =
        typeof(x) == SBML.GPAOr ? vcat([parse_or(i) for i in x.terms]...) : [parse_and(x)]
    return parse_or(gpa)
end

"""
    _unparse_grr(
        ::Type{SBML.GeneProductAssociation},
        x::GeneAssociation,
    )::SBML.GeneAssociation

Convert a GeneAssociation to the corresponding `SBML.jl` structure.
"""
function _unparse_grr(
    ::Type{SBML.GeneProductAssociation},
    x::GeneAssociation,
)::SBML.GeneAssociation
    SBML.GPAOr([SBML.GPAAnd([SBML.GPARef(j) for j in i]) for i in x])
end

"""
    _parse_grr(s::String)::GeneAssociation

Parse a DNF gene association rule in format `(YIL010W and YLR043C) or (YIL010W
and YGR209C)` to `GeneAssociation. Also accepts `OR`, `|`, `||`, `AND`, `&`,
and `&&`.

# Example
```
julia> _parse_grr("(YIL010W and YLR043C) or (YIL010W and YGR209C)")
2-element Array{Array{String,1},1}:
 ["YIL010W", "YLR043C"]
 ["YIL010W", "YGR209C"]
```
"""
_parse_grr(s::String)::Maybe{GeneAssociation} = _maybemap(_parse_grr, _parse_grr_to_sbml(s))

"""
    _parse_grr_to_sbml(str::String)::Maybe{SBML.GeneProductAssociation}

Internal helper for parsing the string GRRs into SBML data structures. More
general than [`_parse_grr`](@ref).
"""
function _parse_grr_to_sbml(str::String)::Maybe{SBML.GeneProductAssociation}
    s = str
    toks = String[]
    m = Nothing
    while !isnothing(begin
        m = match(r"( +|[a-zA-Z0-9]+|[^ a-zA-Z0-9()]+|[(]|[)])(.*)", s)
    end)
        tok = strip(m.captures[1])
        !isempty(tok) && push!(toks, tok)
        s = m.captures[2]
    end

    fail() = throw(DomainError(str, "Could not parse GRR"))

    # shunting yard
    ops = Symbol[]
    vals = SBML.GeneProductAssociation[]
    fold(sym, op) =
        while !isempty(ops) && last(ops) == sym
            r = pop!(vals)
            l = pop!(vals)
            pop!(ops)
            push!(vals, op([l, r]))
        end
    for tok in toks
        if tok in ["and", "AND", "&", "&&"]
            push!(ops, :and)
        elseif tok in ["or", "OR", "|", "||"]
            fold(:and, SBML.GPAAnd)
            push!(ops, :or)
        elseif tok == "("
            push!(ops, :paren)
        elseif tok == ")"
            fold(:and, SBML.GPAAnd)
            fold(:or, SBML.GPAOr)
            if isempty(ops) || last(ops) != :paren
                fail()
            else
                pop!(ops)
            end
        else
            push!(vals, SBML.GPARef(tok))
        end
    end

    fold(:and, SBML.GPAAnd)
    fold(:or, SBML.GPAOr)

    if !isempty(ops) || length(vals) > 1
        fail()
    end

    if isempty(vals)
        nothing
    else
        first(vals)
    end
end

"""
    unparse_grr(grr::Vector{Vector{Gene}}

Converts a nested string gene reaction array  back into a gene reaction rule
string.

# Example
```
julia> _unparse_grr(String, [["YIL010W", "YLR043C"], ["YIL010W", "YGR209C"]])
"(YIL010W and YLR043C) or (YIL010W and YGR209C)"
```
"""
function _unparse_grr(::Type{String}, grr::GeneAssociation)::String
    grr_strings = String[]
    for gr in grr
        push!(grr_strings, "(" * join([g for g in gr], " and ") * ")")
    end
    grr_string = join(grr_strings, " or ")
    return grr_string
end


export TransformedSource

struct TransformedSource
  f::Any
  src::Any
  fs::Float32
  blksize::Int
  nchannels::Int
end

function TransformedSource(f, src)
  fs = inputframerate(src)
  blksize = inputblocksize(src)
  ch = inputchannels(src)
  x = f(zeros(blksize, ch))
  fs *= size(x, 1) / blksize
  if x isa AbstractMatrix
    blksize, ch = size(x)
  else
    ch = 1
    blksize = length(x)
  end
  TransformedSource(f, src, fs, blksize, ch)
end

connect(src::TransformedSource) = connect(src.src)
Base.close(src::TransformedSource) = close(src.src)
stopinputstream(src::TransformedSource) = stopinputstream(src.src)

inputframerate(src::TransformedSource) = src.fs
inputblocksize(src::TransformedSource) = src.blksize
inputchannels(src::TransformedSource) = src.nchannels

function startinputstream(src::TransformedSource, callback)
  startinputstream(src.src, (t, data) -> callback(t, src.f(data)))
end


using Revise
using DataBench, RCall
using FastGroupBy
using IterableTables
using DataFrames
using CSV

function bench_gen_string_vec_var_len(n, strlen, skipbase = n >= 100_000_000)
    x = DataBench.gen_string_vec_var_len(n, strlen);
    bench_string_vec(x, n, strlen, "variable", skipbase)
end

function bench_gen_string_vec_fixed_len(n, strlen, skipbase = n >= 100_000_000)
    x = DataBench.gen_string_vec_fixed_len(n, strlen);
    bench_string_vec(x, n, strlen, "fixed", skipbase)
end

function bench_gen_string_vec_id_len(n, strlen, grps, skipbase = n >= 100_000_000)
    x = DataBench.gen_string_vec_id_fixed_len(n, strlen, grps)
    bench_string_vec(x, n, strlen, "id", skipbase)
end

function bench_string_vec(x, n, strlen, strlentype, skipbase = n >= 100_000_000)
    if !skipbase
        copyx = copy(x);
        julia_sort_elapsed = @elapsed sort!(copyx);
        @assert issorted(copyx)
    end

    copyx = copy(x);
    julia_lsd_radixsort_elapsed = @elapsed radixsort_lsd!(copyx);
    @assert issorted(copyx)

    copyx = copy(x);
    R"""
    memory.limit(2^32-1)
    rcopyx = $copyx
    summary(rcopyx)
    pt = proc.time()
    sort(rcopyx, method="radix")
    pt2 = proc.time()
    """;

    @rget pt;
    @rget pt2;

    r_radixsort_elapsed = pt2[3] - pt[3]

    c_qsort_strcmp_elapsed = @elapsed str_qsort!(copyx)
    @assert issorted(copyx)

    copyx = copy(x)
    three_way_radix_qsort_elapsed = @elapsed three_way_radix_qsort!(copyx)
    @assert issorted(copyx)

    if skipbase
        DataFrame(
        test = ["julia_lsd_radixsort_elapsed", "r_radixsort_elapsed", "c_qsort_strcmp_elapsed", "three_way_radix_qsort_elapsed"]
        ,elapsed = [julia_lsd_radixsort_elapsed, r_radixsort_elapsed, c_qsort_strcmp_elapsed, three_way_radix_qsort_elapsed]
        ,n = [n for i = 1:4]
        ,strlen = [strlen for i = 1:4]
        ,strlentype = [strlentype for i = 1:4]
        ,grps = [n÷100 for i=1:4])
    else
        DataFrame(
            test = ["julia_sort_elapsed", "julia_lsd_radixsort_elapsed", "r_radixsort_elapsed", "c_qsort_strcmp_elapsed", "three_way_radix_qsort_elapsed"]
            ,elapsed = [julia_sort_elapsed, julia_lsd_radixsort_elapsed, r_radixsort_elapsed, c_qsort_strcmp_elapsed, three_way_radix_qsort_elapsed]
            ,n = [n for i = 1:5]
            ,strlen = [strlen for i = 1:5]
            ,strlentype = [strlentype for i = 1:5]
            ,grps = [n÷100 for i=1:5])
    end
end

# bench_gen_string_vec_var_len(1_000_000, 8)
# bench_gen_string_vec_var_len(1_000_000, 16)
# bench_gen_string_vec_var_len(1_000_000, 24)
# bench_gen_string_vec_var_len(1_000_000, 32)
# bench_gen_string_vec_var_len(1_000_000, 64)

# bench_gen_string_vec_fixed_len(1_000_000, 8)
# bench_gen_string_vec_fixed_len(1_000_000, 16)
# bench_gen_string_vec_fixed_len(1_000_000, 24)
# bench_gen_string_vec_fixed_len(1_000_000, 32)
function withrepeat(x,y)
    [repeat(x, inner=[size(y,1)]) repeat(y, outer=[size(x,1)])]
end

ns = Int.(10.^(6:8))
strlens = [3, 10]
nsx = withrepeat(ns, strlens)

for i in 1:size(nsx,1)
    n = nsx[i,1]
    strlen = nsx[i,2]
    df = bench_gen_string_vec_id_len(n, 3, 100)
    println(df)
    CSV.write("benchmark/bench_string_results/string_sort_perf $(replace(string(now()),":"," ")).csv", df);
    df = bench_gen_string_vec_id_len(n, 10, n÷100)
    println(df)
    CSV.write("benchmark/bench_string_results/string_sort_perf $(replace(string(now()),":"," ")).csv", df);
end

ns = Int.(10.^(8:-1:6))
strlens = 8.*(8:-1:1)
nsx = withrepeat(ns, strlens)
df = DataFrame()
for i in 1:size(nsx,1)
    n = nsx[i,1]
    strlen = nsx[i,2]
    df = bench_gen_string_vec_var_len(n, strlen)
    CSV.write("benchmark/bench_string_results/string_sort_perf $(replace(string(now()),":"," ")).csv", df);
    println(df)
    df = bench_gen_string_vec_fixed_len(n, strlen)
    println(df)

    CSV.write("benchmark/bench_string_results/string_sort_perf $(replace(string(now()),":"," ")).csv", df);
end


using Immerse, Colors

hfig = figure()
x = range(0, stop=4pi, length=101)
display(plot(x=x, y=sin.(x), Geom.line(tag=:line)))

val = Immerse.setproperty!((hfig,:line), rand(1:5), :linewidth)
@test Immerse.getproperty((hfig,:line), :linewidth) == [val*mm]
val = Immerse.setproperty!((hfig,:line), RGB(rand(),rand(),rand()), :stroke)
@test Immerse.getproperty((hfig,:line), :stroke) == [coloralpha(val)]
Immerse.setproperty!((hfig,:line), false, :visible)
@test Immerse.getproperty((hfig,:line), :visible) == [false]
Immerse.setproperty!((hfig,:line), true, :visible)
@test Immerse.getproperty((hfig,:line), :visible) == [true]


# Addition
#
# commutativity
# S1 + S2 == S2 + S1
# S + C == C + S
# C1 + C2 == C2 + C1
# S + U - U == S (for sorted S)
#
# associativity
# (S1 + S2) + S3 == S1 + (S2 + S3)
# (S1 + S2) + C == S1 + (S2 + C)
function +(S::T, U::T) where {T<:GphysData}
  Ω = deepcopy(S)
  append!(Ω, U)
  merge!(Ω, purge_only=true)
  return Ω
end
+(S::SeisData, C::SeisChannel) = +(S, SeisData(C))
+(C::SeisChannel, S::SeisData) = +(S, SeisData(C))
+(C::SeisChannel, D::SeisChannel) = +(SeisData(C), SeisData(D))

# Subtraction
-(S::GphysData, i::Int)          = (U = deepcopy(S); deleteat!(U,i); return U)  # By channel #
-(S::GphysData, J::Array{Int,1}) = (U = deepcopy(S); deleteat!(U,J); return U)  # By array of channel #s

# Multiplication
# distributivity: (S1+S2)*S3) == (S1*S3 + S2*S3)
*(S::SeisData, U::SeisData) = merge(Array{SeisData,1}([S,U]))
*(S::SeisData, C::SeisChannel) = merge(S, SeisData(C))
function *(C::SeisChannel, D::SeisChannel)
  s1 = deepcopy(C)
  s2 = deepcopy(D)
  S = merge(SeisData(s1),SeisData(s2))
  return S
end

# Division will happen eventually; for S/U to be logical, we need to extract
# time ranges of data from S that are not in U. This will work like unix `diff`


using Test

# FIXME this is a hack to get the test to work.
if VERSION < v"1.9.0-DEV"
  include("cas_infer.jl")

  ex1 = :(cos(1 + 3.0) + 4 + (4 - 4im))
  ex2 = :("ciao" * 2)
  ex3 = :("ciao" * " mondo")

  @test ComplexF64 == infer(ex1)
  @test_throws MethodError infer(ex2)
  @test String == infer(ex3)
end


push!(LOAD_PATH, pwd())

using OffsetArrays
using DataStructures
using Circular

function Base.insert!(a::OffsetArray{T, 1, Array{T, 1}}, i::Integer, v::T) where T
    j = 1 + i - firstindex(a)
    insert!(a.parent, j, v)
end

function Base.splice!(a::OffsetArray{T, 1, Array{T, 1}}, i::Integer) where T
    j = 1 + i - firstindex(a)
    splice!(a.parent, j)
end

function solve011(filename)
    println(filename)
    basefreq = 0
    visited = Set([basefreq])
    open(filename) do file
        for line in eachline(file)
            change = parse(Int, line)
            basefreq += change
        end
    end

    return basefreq
end

function solve012(filename)
    nstep = 0
    basefreq = 0
    visited = Set([basefreq])
    changes = readlines(filename)
    while true
        for change_str in changes
            change = parse(Int, change_str)
            basefreq += change
            if in(basefreq, visited)
                return basefreq
            end
            union!(visited, Set([basefreq]))
        end
    end
end

function solve021(filename)
    count2, count3 = open(filename) do input
        count2 = 0
        count3 = 0
        counts = zeros(Int, 26)
        offset = Int('a')
        for id in eachline(input)
            # println(id)
            for char in id
                loc = 1 + Int(char) - offset
                counts[loc] += 1
            end
            # println(counts)
            n2 = any(counts .== 2)
            n3 = any(counts .== 3)
            # println("$n2 $n3")
            count2 += Int(n2)
            count3 += Int(n3)
            # println("$count2 $count3")
            # Reset and check next id.
            counts .= 0
        end
        return count2, count3
    end
    return count2 * count3
end

function solve022(filename)
    ids = readlines(filename)
    dist = id_dist.(reshape(ids, :, 1), reshape(ids, 1, :))
    # println(dist)

    id_indx = findfirst(dist) do d
        d == 1
    end
    correct_id1 = ids[id_indx[1]]
    correct_id2 = ids[id_indx[2]]
    println(correct_id1)
    println(correct_id2)
    matching = Array{Char}(undef, length(correct_id1) - 1)
    i = 1
    for cc in zip(correct_id1, correct_id2)
        c1, c2 = cc
        if c1 == c2
            matching[i] = c1
            i += 1
        end
    end
    return string(matching...)
end

function id_dist(id1, id2)
    diff = 0
    # println(id1)
    # println(id2)
    for cc in zip(split(id1, ""), split(id2, ""))
        c1, c2 = cc
        diff += Int(c1 != c2)
    end
    # println("dist $diff")
    return diff
end

function solve031(filename)
    claim_count = count_claims(readlines(filename))
    # display(claim_count[1:9, 1:9]); println()
    return sum(claim_count .>= 2)
end

function count_claims(claims)
    claim_count = zeros(Int, 1000, 1000)
    for claim in claims
        id, left, top, width, height = parse_claim(claim)
        claim_count[1+top:top+height, 1+left:left+width] .+= 1
    end
    return claim_count
end

function parse_claim(claim_str)
    claim_regex = r"#([0-9]*) @ ([0-9]*),([0-9]*): ([0-9]*)x([0-9]*)"
    id, left, top, width, height = match(claim_regex, claim_str).captures
    left, top, width, height = parse.(Int, [left, top, width, height])
    return id, left, top, width, height
end

function solve032(filename)
    claims = readlines(filename)
    claim_count = count_claims(claims)
    # display(claim_count[1:9, 1:9]); println()
    for claim in claims
        id, left, top, width, height = parse_claim(claim)
        claimed_section = claim_count[1+top:top+height, 1+left:left+width]
        # display(claimed_section); println()
        if all(claimed_section .== 1)
            return id
        end
    end
end

function solve041(filename)
    records = readlines(filename)
    minutes_asleep = create_sleep_table(records)
    most_sleep = first(sort(
        collect(minutes_asleep), by=entry->sum(entry[2]), rev=true))
    most_sleep_id = first(most_sleep)
    id_number = parse(Int, most_sleep_id[2:lastindex(most_sleep_id)])
    asleep = last(most_sleep)
    most_sleep_minute = argmax(asleep) - 1
    return id_number * most_sleep_minute
end

function create_sleep_table(records)
    sort!(records)
    n = length(records)
    minutes_asleep = Dict{String, Array{Int}}()
    record_regex = r"\[.*:([0-9]*)\].*(#[0-9]*|falls|wakes).*"
    asleep = Array{Int}(undef, 60)
    sleep_start::Int = 0
    sleep_end::Int = 0
    for record in records
        println(record)
        m = match(record_regex, record)
        if m[2][1] == '#'
            id = m[2]
            if haskey(minutes_asleep, id)
                asleep = minutes_asleep[id]
            else
                asleep = zeros(Int, 60)
                minutes_asleep[id] = asleep
            end
        elseif m[2] == "falls"
            sleep_start = parse(Int, m[1])
        else
            sleep_end = parse(Int, m[1])
            # Array is 1 to 60, minutes 0 to 59, so offset by 1.
            asleep[1+sleep_start:sleep_end] .+= 1
        end
    end
    return minutes_asleep
end

function print_sleep_table(sleep_table)
    print("    ")
    for i in 1:60
        print(div(i-1, 10))
    end
    print("\n    ")
    for i in 1:60
        print(mod(i-1, 10))
    end
    println()

    for entry in sleep_table
        id, asleep = entry
        print("$id ")
        for a in asleep
            print(a)
        end
        println()
    end
end

function solve042(filename)
    records = readlines(filename)
    minutes_asleep = create_sleep_table(records)
    print_sleep_table(minutes_asleep)
    most_sleep = first(sort(
        collect(minutes_asleep), by=entry->maximum(entry[2]), rev=true))
    most_sleep_id = first(most_sleep)
    id_number = parse(Int, most_sleep_id[2:lastindex(most_sleep_id)])
    asleep = last(most_sleep)
    most_sleep_minute = argmax(asleep) - 1
    return id_number, most_sleep_minute, id_number * most_sleep_minute
end

function solve051(filename)
    polymer = readlines(filename)[1]
    remaining = react_polymer(polymer)
    return length(remaining)
end

function react_polymer(polymer)
    # println(polymer)
    remaining = Stack{Char}()
    current = 'a'
    for current in polymer
        # println(current)
        if length(remaining) > 0
            lastc = pop!(remaining)
            if lowercase(lastc) == lowercase(current) && lastc != current
                # lastc and current cancel, so do nothing and they are gone.
            else
                push!(remaining, lastc)
                push!(remaining, current)
            end
        else
            push!(remaining, current)
        end
        # println(remaining)
    end
    return remaining
end

function solve052(filename)
    polymer = readlines(filename)[1]
    polymer_lengths = Array{Int}(undef, 26)
    for entry in enumerate("abcdefghijklmnopqrstuvwxyz")
        i, c = entry
        stripped = strip_polymer(polymer, c)
        # println(reverse(collect(stripped)))
        remaining = react_polymer(reverse(collect(stripped)))
        n = length(remaining)
        println(entry..., ": ", n)
        polymer_lengths[i] = n
    end
    return minimum(polymer_lengths)
end

function strip_polymer(polymer, c)
    remaining = Stack{Char}()
    for current in polymer
        if lowercase(c) != lowercase(current)
            push!(remaining, current)
        end
    end
    # println(c)
    # println(remaining)
    return remaining
end

function solve061(filename)
    point_strs = readlines(filename)
    points = transpose(parse.(Int, hcat(split.(point_strs, ",")...)))
    # display(points)
    dist = Array{Int}(undef, size(points, 1))
    grid = zeros(Int, maximum(points[:,1])+2, maximum(points[:,2])+2)
    grid_x_axis, grid_y_axis = axes(grid)
    for ix in grid_x_axis, iy in grid_y_axis
        x, y = ix-1, iy-1
        for i in axes(points,1)
            dist[i] = manhattan_dist([x,y], points[i,:])
        end
        # Leave ties unclaimed.
        claimed = argmin(dist)
        if sum(dist .== dist[claimed]) > 1
            continue
        end
        grid[ix, iy] = claimed
    end
    # display(grid)

    # Look at boundary to find unbounded point regions.
    bounded = fill(true, size(points, 1))
    for ix in grid_x_axis
        iymin, iymax = firstindex(grid, 2), lastindex(grid, 2)
        if grid[ix, iymin] > 0
            bounded[grid[ix, iymin]] = false
        end
        if grid[ix, iymax] > 0
            bounded[grid[ix, iymax]] = false
        end
    end
    for iy in grid_y_axis
        ixmin, ixmax = firstindex(grid, 1), lastindex(grid, 1)
        if grid[ixmin, iy] > 0
            bounded[grid[ixmin, iy]] = false
        end
        if grid[ixmax, iy] > 0
            bounded[grid[ixmax, iy]] = false
        end
    end
    # display(bounded)

    areas = Array{Int}(undef, size(points, 1))
    for i in axes(points, 1)
        areas[i] = sum(grid .== i)
    end
    largest_bounded_area = maximum(areas[bounded])
    return largest_bounded_area
end

function manhattan_dist(x1, x2)
    abs(x1[1]-x2[1]) + abs(x1[2]-x2[2])
end

function solve062(filename)
    MAX_DIST = 10000
    point_strs = readlines(filename)
    points = transpose(parse.(Int, hcat(split.(point_strs, ",")...)))
    # display(points)
    dist = Array{Int}(undef, size(points, 1))
    grid = zeros(Int, maximum(points[:,1])+2, maximum(points[:,2])+2)
    grid_x_axis, grid_y_axis = axes(grid)
    for iy in grid_y_axis, ix in grid_x_axis
        x, y = ix-1, iy-1
        for i in axes(points,1)
            dist[i] = manhattan_dist([x,y], points[i,:])
        end
        if sum(dist) < MAX_DIST
            grid[ix, iy] = 1
        end
    end
    # display(grid)
    return sum(grid)
end

function solve071(filename)
    nodes, pred_dag, succ_dag = open(build_dag, filename)
    next_nodes = binary_minheap(Char)
    remaining_nodes = Set(nodes)
    completed_nodes = fill('0', length(remaining_nodes))
    schedule_nodes!(next_nodes, remaining_nodes, pred_dag)
    # display(pred_dag)
    # display(next_nodes)
    # display(remaining_nodes)
    # display(completed_nodes)
    for i in axes(completed_nodes, 1)
        n = pop!(next_nodes)
        completed_nodes[i] = n
        clear_pred!(pred_dag, n)
        schedule_nodes!(next_nodes, remaining_nodes, pred_dag)
        # display(pred_dag)
        # display(next_nodes)
        # display(remaining_nodes)
        # display(completed_nodes)
    end
    return string(completed_nodes...)
end

function schedule_nodes!(next_nodes, remaining_nodes, pred_dag)
    # Find workable nodes and queue them up.
    for n in copy(remaining_nodes)
        if length(pred_dag[n]) == 0
            push!(next_nodes, n)
            pop!(remaining_nodes, n)
        end
    end
end

function clear_pred!(pred_dag, n)
    for arcset in pred_dag
        orig, pred = arcset
        if n in pred
            pop!(pred, n)
        end
    end
end

function build_dag(input)
    nodes = Set{Char}()
    pred_dag = Dict{Char, Set{Char}}()
    succ_dag = Dict{Char, Set{Char}}()
    for line = eachline(input)
        step_re = r"Step ([A-Z]).*step ([A-Z]).*"
        pred, succ = match(step_re, line).captures
        pred, succ = pred[1], succ[1]  # String to Char.
        if haskey(pred_dag, succ)
            pred_dag[succ] = push!(pred_dag[succ], pred)
        else
            pred_dag[succ] = Set{Char}([pred])
        end
        if haskey(succ_dag, pred)
            succ_dag[pred] = push!(succ_dag[pred], succ)
        else
            succ_dag[pred] = Set{Char}([succ])
        end
        push!(nodes, pred)
        push!(nodes, succ)
    end
    for n in nodes
        if !haskey(pred_dag, n); pred_dag[n] = Set{Char}(); end
        if !haskey(succ_dag, n); succ_dag[n] = Set{Char}(); end
    end
    return nodes, pred_dag, succ_dag
end

function solve072(filename)
    N_WORKERS = 5
    TIME_OFFSET = 60
    CHAR_OFFSET = Int('A') - 1
    time_required = [Int(c) - CHAR_OFFSET + TIME_OFFSET for c in 'A':'Z']
    nodes, pred_dag, succ_dag = open(build_dag, filename)
    next_nodes = binary_minheap(Char)
    available_workers = Set(1:N_WORKERS)
    next_workers = PriorityQueue{Tuple{Int, Char, Int}, Int}()
    remaining_nodes = Set(nodes)
    completed_nodes = fill('0', length(remaining_nodes))
    schedule_nodes!(next_nodes, remaining_nodes, pred_dag)
    t = 0
    schedule_workers!(next_workers, next_nodes, available_workers, t, time_required)
    # display(pred_dag)
    # display(next_workers)
    # display(remaining_nodes)
    # display(completed_nodes)
    for i in axes(completed_nodes, 1)
        w, n, tf = dequeue!(next_workers)
        completed_nodes[i] = n
        push!(available_workers, w)
        t = tf
        clear_pred!(pred_dag, n)
        schedule_nodes!(next_nodes, remaining_nodes, pred_dag)
        schedule_workers!(next_workers, next_nodes, available_workers, t, time_required)
        # display(pred_dag)
        # display(next_workers)
        # display(remaining_nodes)
        # display(completed_nodes)
    end
    return string(completed_nodes...), t
end

function schedule_workers!(next_workers, next_nodes, available_workers, t, time_required)
    CHAR_OFFSET = Int('A') - 1
    while length(next_nodes) > 0 && length(available_workers) > 0
        n = pop!(next_nodes)
        w = pop!(available_workers)
        i = Int(n) - CHAR_OFFSET
        dt = time_required[i]
        enqueue!(next_workers, (w, n, t+dt)=>t+dt)
    end
end

function solve081(filename)
    CHILD = 1
    METADATA = 2
    HEADER1 = 3
    HEADER2 = 4
    inputs = readline(filename)
    tot::Int = 0
    next_mode = Stack{Int}()
    push!(next_mode, HEADER1)
    local mode::Int = 0
    n_child::Int = 0
    n_metadata::Int = 0
    for c in split(inputs)
        mode = pop!(next_mode)
        val = parse(Int, c)
        # println("Pre ", mode, " ", val, " ", next_mode)
        if mode == HEADER1
            # Reading number of children
            n_child = val
            push!(next_mode, HEADER2)
        elseif mode == HEADER2
            # Reading number of metadata
            n_metadata = val
            if n_metadata > 0
                push!(next_mode, repeat([METADATA], n_metadata)...)
            end
            if n_child > 0
                push!(next_mode, repeat([HEADER1], n_child)...)
            end
        elseif mode == METADATA
            tot += val
        end
        # println("Post ", tot, " ", next_mode)
    end
    return tot
end

mutable struct Node{T}
    val::T
    children::Array{T,1}
    metadata::Array{T,1}
end

function solve082(filename)
    CHILD = 1
    METADATA = 2
    HEADER1 = 3
    HEADER2 = 4
    inputs = readline(filename)
    tot::Int = 0
    tree = Node{Int}[]
    push!(tree, Node{Int}(-1, [], []))
    max_node_idx::Int = 1
    next_mode = Stack{Tuple{Int, Int, Int}}()
    push!(next_mode, (HEADER1, 0, max_node_idx))
    local mode::Int = 0
    n_child::Int = 0
    n_metadata::Int = 0
    for c in split(inputs)
        mode, parent, node_idx = pop!(next_mode)
        val = parse(Int, c)
        # println("Pre ", mode, " ", parent, " ", next_mode)
        # display(tree); println()
        if mode == HEADER1
            # Reading number of children
            n_child = val
            push!(next_mode, (HEADER2, parent, node_idx))
        elseif mode == HEADER2
            # Reading number of metadata
            n_metadata = val
            if n_metadata > 0
                push!(next_mode, repeat([(METADATA, node_idx, 0)], n_metadata)...)
            end
            if n_child > 0
                push!(next_mode, [(HEADER1, node_idx, max_node_idx + i) for i in n_child:-1:1]...)
                for i in 1:n_child
                    push!(tree, Node{Int}(-1, [], []))
                end
                max_node_idx += n_child
            end
            if parent > 0
                parent_node = tree[parent]
                push!(parent_node.children, node_idx)
            end
        elseif mode == METADATA
            parent_node = tree[parent]
            push!(parent_node.metadata, val)
        end
        # println("Post ", parent, " ", next_mode)
        # display(tree); println()
    end

    setval!(tree, tree[1])
    display(tree); println()
    return tree[1].val
end

function setval!(tree::Array{Node{T},1}, n::Node{T}) where T<:Integer
    if n.val >= 0
        return
    elseif length(n.children) == 0
        n.val = sum(n.metadata)
    else
        vals = similar(n.children)
        for tup in enumerate(n.children)
            idx, i = tup
            c = tree[i]
            if c.val < 0
                setval!(tree, c)
            end
            vals[idx] = c.val
        end
        n.val = 0
        for m in n.metadata
            if m <= length(n.children)
                n.val += vals[m]
            end
        end
    end
end

function solve091(args...)
    if length(args) == 1
        re = r"([0-9]*) players; last marble is worth ([0-9]*) points"
        input_str = string(open(readchomp, args[1]))
        println(input_str)
        input_m = match(re, input_str)
        show(input_m); println()
        if input_m !== nothing
            input = input_m.captures
        else
            error("Invalid input file.")
        end
    elseif length(args) == 2
        input = args
    else
        error("Must provide at least two extra arguments: N_PLAYERS and MAX_MARBLE")
    end

    n_players, max_marble = map(x -> parse(Int, x), input)
    # Increasing index is clockwise.
    marbles = OffsetArray{Int}(undef, 0:0)
    marbles[0] = 0
    current_marble_loc = 0
    current_player = 1
    scores = zeros(Int, n_players)
    for m = 1:max_marble
        if mod(m, 23) ≠ 0
            # Insert marble between marbles 1 and 2 clockwise from current.
            i = mod(current_marble_loc + 2, length(marbles))
            insert!(marbles, i, m)
            current_marble_loc = i
        else
            # Score marble and remove.
            scores[current_player] += m
            i_remove = mod(current_marble_loc - 7, length(marbles))
            scores[current_player] += marbles[i_remove]
            splice!(marbles, i_remove)
            current_marble_loc = i_remove
        end
        # println(current_player, ", ", m, ": ", circshift(marbles, 1))
        current_player = mod(current_player + 1, n_players+1)
        (current_player == 0) && (current_player = 1)
    end

    return maximum(scores)
end

function solve092(args...)
    println("solve092 ", args)
    if length(args) == 1
        re = r"([0-9]*) players; last marble is worth ([0-9]*) points"
        input_str = string(open(readchomp, args[1]))
        println(input_str)
        input_m = match(re, input_str)
        show(input_m); println()
        if input_m !== nothing
            input = input_m.captures
        else
            error("Invalid input file.")
        end
    elseif length(args) == 2
        input = args
    else
        error("Must provide at least two extra arguments: N_PLAYERS and MAX_MARBLE")
    end

    n_players, max_marble = map(x -> parse(Int, x), input)
    # Increasing index is clockwise.
    marbles = circular_list(0)
    current_player = 1
    scores = zeros(Int, n_players)
    println("Progress: (. = 10000 marbles)")
    for m = 1:max_marble
        mod(m, 10_000) == 0 && print(".")
        if mod(m, 23) ≠ 0
            # Insert marble between marbles 1 and 2 clockwise from current.
            marbles = insertafter!(next(marbles), m)
        else
            # Score marble and remove.
            scores[current_player] += m
            for i = 1:7; marbles = prev(marbles); end
            scores[current_player] += value(marbles)
            removeleftshift!(marbles)
        end
        # println(current_player, ", ", m, ": ", circshift(marbles, 1))
        current_player = mod(current_player + 1, n_players+1)
        (current_player == 0) && (current_player = 1)
    end
    println()
    return maximum(scores)
end


function add(obj::FieldVector, arg0::FieldVector)
    return jcall(obj, "add", FieldVector, (FieldVector,), arg0)
end

function append(obj::FieldVector, arg0::FieldElement)
    return jcall(obj, "append", FieldVector, (FieldElement,), arg0)
end

function append(obj::FieldVector, arg0::FieldVector)
    return jcall(obj, "append", FieldVector, (FieldVector,), arg0)
end

function copy(obj::FieldVector)
    return jcall(obj, "copy", FieldVector, ())
end

function dot_product(obj::FieldVector, arg0::FieldVector)
    return jcall(obj, "dotProduct", FieldElement, (FieldVector,), arg0)
end

function ebe_divide(obj::FieldVector, arg0::FieldVector)
    return jcall(obj, "ebeDivide", FieldVector, (FieldVector,), arg0)
end

function ebe_multiply(obj::FieldVector, arg0::FieldVector)
    return jcall(obj, "ebeMultiply", FieldVector, (FieldVector,), arg0)
end

function get_dimension(obj::FieldVector)
    return jcall(obj, "getDimension", jint, ())
end

function get_entry(obj::FieldVector, arg0::jint)
    return jcall(obj, "getEntry", FieldElement, (jint,), arg0)
end

function get_field(obj::FieldVector)
    return jcall(obj, "getField", Field, ())
end

function get_sub_vector(obj::FieldVector, arg0::jint, arg1::jint)
    return jcall(obj, "getSubVector", FieldVector, (jint, jint), arg0, arg1)
end

function map_add(obj::FieldVector, arg0::FieldElement)
    return jcall(obj, "mapAdd", FieldVector, (FieldElement,), arg0)
end

function map_add_to_self(obj::FieldVector, arg0::FieldElement)
    return jcall(obj, "mapAddToSelf", FieldVector, (FieldElement,), arg0)
end

function map_divide(obj::FieldVector, arg0::FieldElement)
    return jcall(obj, "mapDivide", FieldVector, (FieldElement,), arg0)
end

function map_divide_to_self(obj::FieldVector, arg0::FieldElement)
    return jcall(obj, "mapDivideToSelf", FieldVector, (FieldElement,), arg0)
end

function map_inv(obj::FieldVector)
    return jcall(obj, "mapInv", FieldVector, ())
end

function map_inv_to_self(obj::FieldVector)
    return jcall(obj, "mapInvToSelf", FieldVector, ())
end

function map_multiply(obj::FieldVector, arg0::FieldElement)
    return jcall(obj, "mapMultiply", FieldVector, (FieldElement,), arg0)
end

function map_multiply_to_self(obj::FieldVector, arg0::FieldElement)
    return jcall(obj, "mapMultiplyToSelf", FieldVector, (FieldElement,), arg0)
end

function map_subtract(obj::FieldVector, arg0::FieldElement)
    return jcall(obj, "mapSubtract", FieldVector, (FieldElement,), arg0)
end

function map_subtract_to_self(obj::FieldVector, arg0::FieldElement)
    return jcall(obj, "mapSubtractToSelf", FieldVector, (FieldElement,), arg0)
end

function outer_product(obj::FieldVector, arg0::FieldVector)
    return jcall(obj, "outerProduct", FieldMatrix, (FieldVector,), arg0)
end

function projection(obj::FieldVector, arg0::FieldVector)
    return jcall(obj, "projection", FieldVector, (FieldVector,), arg0)
end

function set(obj::FieldVector, arg0::FieldElement)
    return jcall(obj, "set", void, (FieldElement,), arg0)
end

function set_entry(obj::FieldVector, arg0::jint, arg1::FieldElement)
    return jcall(obj, "setEntry", void, (jint, FieldElement), arg0, arg1)
end

function set_sub_vector(obj::FieldVector, arg0::jint, arg1::FieldVector)
    return jcall(obj, "setSubVector", void, (jint, FieldVector), arg0, arg1)
end

function subtract(obj::FieldVector, arg0::FieldVector)
    return jcall(obj, "subtract", FieldVector, (FieldVector,), arg0)
end

function to_array(obj::FieldVector)
    return jcall(obj, "toArray", Vector{FieldElement}, ())
end


# This file contains specification and implementation
# Of the homoiconic DSL for defining rules and theories
using MatchCore

function rule_sym_map(ex)
    @smatch ex begin
        :($a => $b) => RewriteRule
        :($a |> $b) => DynamicRule
        :($a == $b) => EqualityRule
        :($a != $b) => UnequalRule
        :($a ≠ $b) => UnequalRule
        _ => error("Cannot parse rule from $ex")
    end
end

interp_dollar(x, mod::Module) = x
function interp_dollar(ex::Expr, mod::Module)
    if Meta.isexpr(ex, :$)
        mod.eval(ex.args[1])
    else
        Expr(ex.head, map(x -> interp_dollar(x, mod), ex.args)...)
    end
end

"""
Construct an `AbstractRule` from a quoted expression.
You can also use the [`@rule`] macro to
create a `Rule`.
"""
function Rule(e::Expr, mod::Module=@__MODULE__, resolve_fun=false)
    op = gethead(e)
    RuleType = rule_sym_map(e)
    l, r = e.args[Meta.isexpr(e, :call) ? (2:3) : (1:2)]

    l = interp_dollar(l, mod)

    if RuleType !== DynamicRule
        r = interp_dollar(r, mod)
    end

    lhs = Pattern(l, mod, resolve_fun)
    rhs = r

    if RuleType <: SymbolicRule
        rhs = Pattern(rhs, mod, resolve_fun)
    end

    return RuleType(lhs, rhs)
end

# fallback when defining theories and there's already a rule
function Rule(r::AbstractRule, mod::Module=@__MODULE__, resolve_fun=false)
    r
end

macro rule(e)
    e = macroexpand(__module__, e)
    e = rmlines(copy(e))
    Rule(e, __module__, false)
end

macro methodrule(e)
    e = macroexpand(__module__, e)
    e = rmlines(copy(e))
    Rule(e, __module__, true)
end

# Theories can just be vectors of rules!

macro theory(e)
    e = macroexpand(__module__, e)
    e = rmlines(e)
    # e = interp_dollar(e, __module__)
    if Meta.isexpr(e, :block)
        Vector{AbstractRule}(e.args .|> x -> Rule(x, __module__, false))
    else
        error("theory is not in form begin a => b; ... end")
    end
end

# TODO document this puts the function as pattern head instead of symbols
macro methodtheory(e)
    e = macroexpand(__module__, e)
    e = rmlines(e)
    # e = interp_dollar(e, __module__)
    if Meta.isexpr(e, :block)
        Vector{AbstractRule}(e.args .|> x -> Rule(x, __module__, true))
    else
        error("theory is not in form begin a => b; ... end")
    end
end

"""
A Theory is either a vector of [`Rule`](@ref) or
a compiled, callable function.
"""
const Theory = Union{Vector{<:AbstractRule}, Function}


# PART 5 – ASSESSMENT OF LOCAL METAL LOSS

# Determine Asessment Applicability
#Determine the assessment applicability
# @doc DesignCodeCriteria
# @doc MaterialToughness
# @doc CyclicService
# @doc Part5ComponentType
print("Begin -- Assessment Applicability and Component Type Checks\n")
creep_range = CreepRangeTemperature("Carbon Steel (UTS ≤ 414MPa (60 ksi))"; design_temperature=100.0, units="nmm-mm-mpa")
design = DesignCodeCriteria("ASME B&PV Code, Section VIII, Division 1")
toughness = MaterialToughness("Certain")
cyclic = CyclicService(100, "Meets Part 14")
x = Part5ComponentType("Cylindrical Vessel", vessel_orientation="horizontal", material="Carbon and Low Alloy Steels", D=24.75,Lss=120.0,H=0.0, NPS=3.0, design_temperature=500.0, units="lbs-in-psi")
part5_applicability = Part5AsessmentApplicability(x,design,toughness,cyclic,creep_range)

# For all assessments - determine the inspection data grid
M1 = [0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300]
M2 = [0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.100, 0.220, 0.280, 0.250, 0.240, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300]
M3 = [0.300, 0.300, 0.300, 0.300, 0.300, 0.215, 0.255, 0.215, 0.145, 0.275, 0.170, 0.240, 0.250, 0.250, 0.280, 0.290, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300]
M4 = [0.300, 0.300, 0.300, 0.300, 0.300, 0.170, 0.270, 0.190, 0.190, 0.285, 0.250, 0.225, 0.275, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300]
M5 = [0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300]
M6 = [0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300, 0.300]
CTPGrid = hcat(M6,M5,M4,M3,M2,M1) # build in descending order
CTPGrid = rotl90(CTPGrid) # rotate to correct orientation

# Level 1 fit for service
    annex2c_tmin_category = "Straight Pipes Subject To Internal Pressure" # ["Cylindrical Shell","Spherical Shell","Hemispherical Head","Elliptical Head","Torispherical Head","Conical Shell","Toriconical Head","Conical Transition","Nozzles Connections in Shells",
    # "Junction Reinforcement Requirements at Conical Transitions","Tubesheets","Flat head to cylinder connections","Bolted Flanges","Straight Pipes Subject To Internal Pressure","Boiler Tubes","Pipe Bends Subject To Internal Pressure",
    # "MAWP for External Pressure","Branch Connections","API 650 Storage Tanks"]
    equipment_group = "piping" # "vessel", "tank"
    flaw_location = "external" # "External","Internal"
    metal_loss_categorization = "LTA" # "LTA" or "Groove-Like Flaw"
    units = "lbs-in-psi" # "lbs-in-psi" or "nmm-mm-mpa"
    tnom = .3 # nominal or furnished thickness of the component adjusted for mill undertolerance as applicable.
    trd = .3 # uniform thickness away from the local metal loss location established by thickness measurements at the time of the assessment.
    FCAml = 0.0 # Future Corrosion Allowance applied to the region of metal loss.
    FCA = 0.0 # Future Corrosion Allowance applied to the region away from the metal loss (see Annex 2C, paragraph 2C.2.8).
    LOSS = 0.0 #the amount of uniform metal loss away from the local metal loss location at the time of the assessment.
    Do = 3.5 # Outside Diameter
    D = Do - 2*(tnom)
    P = 1480.0 # internal design pressure.
    S = 20000.0 # allowable stress.
    E = 1.0 # weld joint efficiency or quality factor from the original construction code, if unknown use 0.7.
    MA = 0.0 # mechanical allowances (thread or groove depth); for threaded components, the nominal thread depth (dimension h of ASME B.1.20.1) shall apply.
    Yb31 = 0.4 # coefficient from ASME B31 Piping codes used for determining the pipe wall thickness, the coefficient can be determined from the following table that is valid for tmin < Do / 6 Annex 2C .
    t = trd # thickness of the shell or pipe adjusted for mill tolerance, LOSS and FCA , or cylinder thickness at a conical transition for a junction reinforcement calculation adjusted for mill tolerance, LOSS and FCA , as applicable.
    tsl = 0.0 # supplemental thickness for mechanical loads other than pressure that result in longitudinal stress; this thickness is usually obtained from the results of a weight case in a stress analysis of the piping system (see paragraph 2C.2.7).
    spacings = 0.5 # spacings determine by visual inspection to adequately ccategorizse the corrosion -----------+ may add to CTP_Grid function for plotting purposes
    # Flaw dimensions
    s = 6.0 # longitudinal extent or length of the region of local metal loss based on future corroded thickness,
    c = 2.0 # circumferential extent or length of the region of local metal loss (see Figure 5.2 and Figure 5.10), based on future corroded thickness, tc .
    Ec = 1.0 # circumferential weld joint efficiency. note if damage on weld see # 2C.2.5 Treatment of Weld and Riveted Joint Efficiency, and Ligament Efficiency
    El = 1.0 # longitudinal weld joint efficiency. note if damage on weld see # 2C.2.5 Treatment of Weld and Riveted Joint Efficiency, and Ligament Efficiency
    RSFa = 0.9 # remaining strength factor - consult API 579 is go lower than 0.9

    # For all assessments determine far enough from structural discontinuity
    # Flaw-To-Major Structural Discontinuity Spacing
    L1msd = [12.0] # distance to the nearest major structural discontinuity.
    L2msd = [12.0] # distance to the nearest major structural discontinuity.
    L3msd = [12.0] # distance to the nearest major structural discontinuity.
    L4msd = [12.0] # distance to the nearest major structural discontinuity.
    L5msd = [12.0] # distance to the nearest major structural discontinuity.
    Lmsd = minimum([L1msd,L2msd,L3msd,L4msd,L5msd])
    if (Lmsd[1] >= (1.8*(sqrt(D*(trd - LOSS - FCA)))))
        print("Satisfied - Flaw is located far enough from structural discontinuity\n")
        lmsd_satisfied = 1
    else
        print("Not satisfied - Flaw is too close to a structural discontinuity - Conduct a level 3 assessment\n")
        lmsd_satisfied = 0
    end

    # Groove Like Flaw dimensions
    gl = .05 # length of the Groove-Like Flaw based on future corroded condition.
    gw = .4 # width of the Groove-Like Flaw based on future corroded condition.
    gr = 0.1 # radius at the base of a Groove-Like Flaw based on future corroded condition.
    β = 40.0 # see (Figure 5.4) :: orientation of the groove-like flaw with respect to the longitudinal axis or a parameter to compute an effective fracture toughness for a groove being evaluated as a crack-like flaw, as applicable.

# Perform level 1 assessment
if (part5_applicability[1] == 1 && lmsd_satisfied == 1) # begin level 1 assessment
    #let part_5_lta_output = Array{Any,2},
    part_5_lta_output = Part5LTALevel1(annex2c_tmin_category; equipment_group=equipment_group, flaw_location=flaw_location, metal_loss_categorization=metal_loss_categorization, units=units, tnom=tnom,
        trd=trd, FCA=FCA, FCAml=FCAml, LOSS=LOSS, Do=Do, D=D, P=P, S=S, E=E, MA=MA, Yb31=Yb31, tsl=tsl, spacings=spacings, s=s, c=c, El=El, Ec=Ec, RSFa=RSFa, gl=gl, gw=gw, gr=gr,β=β)
    #end # let end
elseif (part5_applicability[1] == 0 && lmsd_satisfied == 0)
    print("Level 1 Criteria Not Met - Perform Level 2 or 3 as applicable")
elseif (part5_applicability[1] == 1 && lmsd_satisfied == 0)
    print("Level 1 Criteria Not Met - Perform Level 2 or 3 as applicable")
elseif (part5_applicability[1] == 0 && lmsd_satisfied == 1)
    print("Level 1 Criteria Not Met - Perform Level 2 or 3 as applicable")
end

#=
# Level 2 Assessment
# STEP 1 – Determine the CTP (see paragraph 5.3.3.2).
# Conducted at the top of the script
# STEP 2 – Determine the wall thickness to be used in the assessment using Equation (5.3) or Equation (5.4), as applicable.
tc = trd - LOSS - FCA # wall thickness away from the damaged area adjusted for LOSS and FCA , as applicable. # eq (5.3)
tc = trd - FCA # wall thickness away from the damaged area adjusted for LOSS and FCA , as applicable. # eq (5.4)
# STEP 3 – Determine the minimum measured thickness, tmm , and the flaw dimensions s and c (see paragraph 5.3.3.2.b).
tmm = CTP_Grid(CTPGrid) # minimum measured thickness determined at the time of the inspection.
# STEP 4 – Determine the remaining thickness ratio, Rt , using Equation (5.5) and the longitudinal flaw length parameter, λ , using Equation (5.6).
Rt = (tmm-FCA) / tc # remaining thickness ratio. # (5.5)
lambda = (1.285*s)/(sqrt(D*tc)) # longitudinal flaw length parameter eq (5.6)
# STEP 5 – Check the limiting flaw size criteria in paragraph 5.4.2.2.e. If all of these requirements are satisfied, then proceed to STEP 6; otherwise, the flaw is not acceptable per the Level 2 Assessment procedure.
x = FlawSizeLimitCriteria("piping","lbs-in-psi")
flaw_size_accept = FlawSizeLevel1Acceptance(x,"piping")

RSFa = .9 # allowable remaining strength factor (see Part 2).
if (Rt < RSFa)
Q = round(1.123*((((1-Rt)/(1-Rt/RSFa))^2-1)^.5),digits=2) # factor used to determine the length for thickness averaging based on an allowable Remaining Strength Factor (see Part 2) and the remaining thickness ratio, Rt (see Table 4.8).
elseif (Rt >= RSFa)
    Q = 50.0
end
L = Q*(sqrt(Dml*tc)) # length for thickness averaging along the shell.

# If visual inspection or NDE methods are utilized to quantify the metal loss, an alternative spacing can be used as long as the metal loss on the component can be adequately characterized.
if (flaw_location == "Internal")
    Ls = minimum([L,(2*trd)]) # recommended spacing of thickness readings
    print("Recommended spacing of thickness readings = ",Ls)
elseif (flaw_location == "External")
    print("Can Determine alterante spacing providing component can be adequately characterized - Using Spacing = ", spacings)
end

# Piping
if (part5_applicability[1] == 1) # begin level 1 assessment
    print("Begin part 5 - Level 1 assessment - applicability criteria has been met")

end # end piping level 1
=#


using Test: @test, @testset, @inferred, @test_throws
import RestrictedBoltzmannMachines as RBMs

@testset "obs" begin
    X, Y = randn(10, 4), randn(5, 4)
    @test (@inferred RBMs._nobs(X, Y)) == 4
    @test_throws AssertionError RBMs._nobs(X, randn(5, 3))
    @test (@inferred RBMs._getobs([2, 3], X, Y)) == (X[:, [2, 3]], Y[:, [2, 3]])

    X, Y, Z = randn(10, 4), nothing, randn(5, 4)
    @test (@inferred RBMs._nobs(X, Y, Z)) == 4
    @test_throws AssertionError RBMs._nobs(X, randn(5, 3), nothing)
    @test (@inferred RBMs._getobs([2, 3], X, Y, Z)) == (X[:, [2, 3]], nothing, Z[:, [2, 3]])
end

@testset "minibatches" begin
    @test RBMs.minibatch_count(10; batchsize = 3) == 4
    @test RBMs.minibatches(7; batchsize=3, shuffle=false) == [[1,2,3], [4,5,6], [7,1,2]]
    @test RBMs.minibatches(7; batchsize=5, shuffle=false) == [[1,2,3,4,5], [6,7,1,2,3]]
    @test length(RBMs.minibatches(7; batchsize=3)) == RBMs.minibatch_count(7; batchsize=3)

    X, Y, Z = randn(10, 4), nothing, randn(5, 4)
    batches = RBMs.minibatches(X, Y, Z; batchsize=2, shuffle=false)
    @test length(batches) == RBMs.minibatch_count(4; batchsize=2) == 2
    @test length(batches) == RBMs.minibatch_count(X, Y, Z; batchsize=2)
    @test batches[1] == (X[:,1:2], nothing, Z[:,1:2])
    @test batches[2] == (X[:,3:4], nothing, Z[:,3:4])

    X = randn(7, 10)
    batches = RBMs.minibatches(X, X, copy(X); batchsize=3, shuffle=true)
    for (x1, x2, x3) in batches
        @test size(x1) == (7, 3)
        @test x1 == x2 == x3
    end

    batch = only(RBMs.minibatches([1,2,3], [1,2,3]'; batchsize=3))
    p = sortperm(batch[1])
    @test batch[1][p] == batch[2]'[p] == [1,2,3]
end

@testset "training_epochs" begin
    nsamples = 5421
    nupdates = 10403
    batchsize = 5
    nepochs = RBMs.training_epochs(; nsamples, nupdates, batchsize)
    @test RBMs.minibatch_count(nsamples; batchsize) * (nepochs - 1) ≤ nupdates
    @test RBMs.minibatch_count(nsamples; batchsize) * (nepochs + 1) ≥ nupdates
end


using JLD2
using Statistics

mutable struct IntrinsicRewardLogger <: LoggerKeyData
    intrinsic_reward::Array{Float64,1}
    log_interval::Int

    function IntrinsicRewardLogger(logger_init_info)
        num_logged_steps = logger_init_info[LoggerInitKey.TOTAL_STEPS]
        new(zeros(num_logged_steps), 1)
    end
end

function lg_step!(self::IntrinsicRewardLogger, env, agent, s, a, s_next, r, is_terminal, cur_step_in_episode, cur_step_total)
    if rem(cur_step_total, self.log_interval) == 0
        ind = fld(cur_step_total, self.log_interval)
        self.intrinsic_reward[ind] = get_reward(agent.intrinsic_reward, agent)
    end
end

function lg_episode_end!(self::IntrinsicRewardLogger, cur_step_in_episode, cur_step_total)
end

function save_log(self::IntrinsicRewardLogger, save_dict::Dict)
    save_dict[:intrinsic_reward] = self.intrinsic_reward
end

function extract_demon_weights(learner_weights,gvf_i,demon_learner)
    demon_weights = if demon_learner isa SRLearner
        immediate_rewards = learner_weights[2][gvf_i,:]
        num_SFs = demon_learner.feature_size
        ψ_weights = learner_weights[1][num_SFs*(gvf_i-1)+1:num_SFs*(gvf_i)]
        flattenall([immediate_rewards, ψ_weights])
    elseif demon_learner isa QLearner
        num_actions = demon_learner.num_actions
        gvf_w = learner_weights[(gvf_i-1)*num_actions + 1: gvf_i * num_actions,:]
        flattenall(gvf_w)
    else
        @warn "Not a valid type of demon learner for monitoring weight change contribution per demon"
    end
    return demon_weights
end

mutable struct WC_Demon_Logger <: LoggerKeyData
    wc_per_demon::Array{Float64,2}
    gvf_weights::Dict
    log_interval::Int
    function WC_Demon_Logger(logger_init_info)
        new(zeros(4,logger_init_info[LoggerInitKey.TOTAL_STEPS]), Dict(),1)
    end
end

function lg_start!(self::WC_Demon_Logger, env, agent)
    learner_weights = get_weights(agent.demon_learner)
    for gvf_i in 1:4
        gvf_weights = deepcopy(extract_demon_weights(learner_weights,gvf_i, agent.demon_learner))
        self.gvf_weights[gvf_i] = gvf_weights
    end
end

function lg_step!(self::WC_Demon_Logger, env, agent, s, a, s_next, r, is_terminal, cur_step_in_episode, cur_step_total)
    learner_weights = deepcopy(get_weights(agent.demon_learner))
    for gvf_i in 1:4
        gvf_weights = deepcopy(extract_demon_weights(learner_weights,gvf_i, agent.demon_learner))
        wc = sum(abs.(self.gvf_weights[gvf_i] - gvf_weights))
        self.wc_per_demon[gvf_i,cur_step_total] = wc
        self.gvf_weights[gvf_i] = gvf_weights
    end
end

function lg_episode_end!(self::WC_Demon_Logger, cur_step_in_episode, cur_step_total)
end

function save_log(self::WC_Demon_Logger, save_dict::Dict)
    save_dict[:wc_per_demon] = self.wc_per_demon
end


export Mixture, PackedMixture

_defaultNamesMixtures(N::Int) = ((Symbol[Symbol("c$i") for i in 1:N])...,)

"""
$(TYPEDEF)

A `Mixture` object for use with either a `<: AbstractPrior` or `<: AbstractRelative`.

Notes
- The internal data representation is a `::NamedTuple`, which allows total type-stability for all component types.
- Various construction helpers can accept a variety of inputs, including `<: AbstractArray` and `Tuple`.

DevNotes
- TODO on sampling see #1099 and #1094 and #1069

Example
```juila
# prior factor
msp = Mixture(PriorSphere1,
              [model=Normal(0,0.1), Uniform(-pi/1,pi/2)],
              [0.5;0.5])

addFactor!(fg, [:head], msp, tags=[:MAGNETOMETER;])

# Or relative
mlr = Mixture(LinearRelative,
              (correlator=AliasingScalarSampler(...), naive=Normal(0.5,5), lucky=Uniform(0,10)),
              [0.5;0.4;0.1])

addFactor!(fg, [:x0;:x1], mlr)
```
"""
struct Mixture{N, F<:FunctorInferenceType, S, T<:Tuple} <: FunctorInferenceType
  mechanics::F
  components::NamedTuple{S,T}
  diversity::Distributions.Categorical
  dims::Int
  labels::Vector{Int}
end

Mixture(f::Type{F},
        z::NamedTuple{S,T},
        c::Distributions.DiscreteNonParametric ) where {F<:FunctorInferenceType, S, T} = Mixture{length(z),F,S,T}(f(LinearAlgebra.I), z, c, size( rand(z[1],1), 1), zeros(Int, 0))
Mixture(f::F,
        z::NamedTuple{S,T},
        c::Distributions.DiscreteNonParametric ) where {F<:FunctorInferenceType, S, T} = Mixture{length(z),F,S,T}(f, z, c, size( rand(z[1],1), 1), zeros(Int, 0))
Mixture(f::Union{F,Type{F}},z::NamedTuple{S,T},
        c::AbstractVector{<:Real}) where {F<:FunctorInferenceType,S,T} = Mixture(f, z, Categorical([c...]) )
Mixture(f::Union{F,Type{F}},
        z::NamedTuple{S,T},
        c::NTuple{N,<:Real}) where {N,F<:FunctorInferenceType,S,T} = Mixture(f, z, [c...] )
Mixture(f::Union{F,Type{F}},
        z::Tuple,
        c::Union{<:Distributions.DiscreteNonParametric, <:AbstractVector{<:Real}, <:NTuple{N,<:Real}} ) where {F<:FunctorInferenceType, N} = Mixture(f,NamedTuple{_defaultNamesMixtures(length(z))}(z), c )
Mixture(f::Union{F,Type{F}},
        z::AbstractVector{<:SamplableBelief},
        c::Union{<:Distributions.DiscreteNonParametric, <:AbstractVector{<:Real}, <:NTuple{N,<:Real}} ) where {F <: FunctorInferenceType, N} = Mixture(f,(z...,), c )

function Base.resize!(mp::Mixture, s::Int)
  resize!(mp.labels, s)
end

_lengthOrNothing(val) = length(val)
_lengthOrNothing(val::Nothing) = 0

# TODO make in-place memory version
function sampleFactor( cf::CalcFactor{<:Mixture}, N::Int=1)
  #
  # TODO consolidate #927, case if mechanics has a special sampler
  # TODO slight bit of waste in computation, but easiest way to ensure special tricks in s.mechanics::F are included
  ## example case is old FluxModelsPose2Pose2 requiring velocity
  # FIXME better consolidation of when to pass down .mechanics, also see #1099 and #1094 and #1069

  cf_ = CalcFactor( cf.factor.mechanics, cf.metadata, 0, _lengthOrNothing(cf._legacyMeas), cf._legacyMeas, cf._legacyParams)
  smpls = [getSample(cf_) for _=1:N]
    # smpls = Array{Float64,2}(undef,s.dims,N)
  #out memory should be right size first
  length(cf.factor.labels) != N ? resize!(cf.factor.labels, N) : nothing
  cf.factor.labels .= rand(cf.factor.diversity, N)
  for i in 1:N
    mixComponent = cf.factor.components[cf.factor.labels[i]]
    # measurements relate to the factor's manifold (either tangent vector or manifold point)
    setPointsMani!(smpls[i], rand(mixComponent,1))
  end

  # TODO only does first element of meas::Tuple at this stage, see #1099
  smpls
end

function DistributedFactorGraphs.isPrior(::Mixture{N, F, S, T}) where {N,F,S,T}
  return F <: AbstractPrior
end

"""
$(TYPEDEF)

Serialization type for `Mixture`.
"""
mutable struct PackedMixture <: PackedInferenceType
  N::Int
  # store the packed type for later unpacking
  F_::String
  S::Vector{String}
  components::Vector{String}
  diversity::String
end

function convert(::Type{PackedMixture}, obj::Mixture{N,F,S,T}) where {N,F,S,T}
  allcomp = String[]
  for val in obj.components
    # TODO, likely to be difficult for non-standard "Samplable" types -- e.g. Flux models in RoME
    push!(allcomp, convert(PackedSamplableBelief, val))
  end
  # pm = DFG.convertPackedType(obj.mechanics)
  pm = convert(DFG.convertPackedType(obj.mechanics), obj.mechanics)
  sT = string(typeof(pm))
  PackedMixture( N, sT, string.(collect(S)), allcomp, convert(PackedSamplableBelief, obj.diversity) )
end
function convert(::Type{Mixture}, obj::PackedMixture)
  N = obj.N
  F1 = getfield(Main, Symbol(obj.F_))
  S = (Symbol.(obj.S)...,)
  F2 = DFG.convertStructType(F1)
  components = convert.(SamplableBelief, obj.components)
  diversity = convert(SamplableBelief, obj.diversity)
  tupcomp = (components...,)
  ntup = NamedTuple{S,typeof(tupcomp)}(tupcomp)
  Mixture(F2, ntup, diversity)
end

  #


using Pkg
Pkg.activate(".")

using Negotiations

params = parameter_set_from_config("test/test-config.yaml")
include("test/create-db.jl")
db = load_database("test.sqlite")
model = setup_model(params, db)
rule = BoundedConfidence(bc = 2.0, inertia = 10.0)
simulate(model, rule, 2, db, batchname = "test")


module BlackHoleStrengthTrigger

using ..Ahorn, Maple

const placements = Ahorn.PlacementDict(
    "Black Hole Strength" => Ahorn.EntityPlacement(
        Maple.BlackHoleStrengthTrigger,
        "rectangle"
    )
)

function Ahorn.editingOptions(trigger::Maple.BlackHoleStrengthTrigger)
    return Dict{String, Any}(
        "strength" => Maple.black_hole_trigger_strengths
    )
end

end


using AMDGPUnative
const AS = AMDGPUnative.AS
using GPUCompiler
using LLVM, LLVM.Interop
using HSARuntime
using InteractiveUtils
using SpecialFunctions
using Test

agent_name = HSARuntime.get_name(get_default_agent())
agent_isa = get_first_isa(get_default_agent())
@info "Testing using device $agent_name with ISA $agent_isa"

@testset "AMDGPUnative" begin

@testset "Core" begin
include("pointer.jl")
end

if AMDGPUnative.configured
    @test length(get_agents()) > 0
    if length(get_agents()) > 0
        include("synchronization.jl")
        include("trap.jl")
        @testset "Device" begin
            include("device/launch.jl")
            include("device/vadd.jl")
            include("device/memory.jl")
            include("device/indexing.jl")
            include("device/hostcall.jl")
            include("device/output.jl")
            include("device/globals.jl")
            if Base.libllvm_version >= v"7.0"
                include("device/math.jl")
            else
                @warn "Testing with LLVM 6; some tests will be disabled!"
                @test_skip "Math Intrinsics"
            end
        end
    end
else
    @warn("AMDGPUnative.jl has not been configured; skipping on-device tests.")
end

end


using Luxor

Drawing(400, 400, joinpath(dirname(@__DIR__), "images", "logo.png"))
origin()
setline(5)
sethue("white")
squircle(O, 185, 185, :fillpreserve)
sethue("black")
strokepath()

@layer begin
    translate(0, 30)
    fontface("AlegreyaSans-BoldItalic")
    fontsize(70)
    text("HUMANS", O - (0, 100), halign=:center)
    fontsize(40)
    text("OF", Point(-65, -50))
    julialogo(centered=true)
end

finish()


__precompile__(true)
module ShortStrings

using SortingAlgorithms
export ShortString, ShortString15, ShortString7, ShortString3, fsort, fsort!,
       @ss15_str, @ss7_str, @ss3_str

import Base:unsafe_getindex, ==

struct ShortString{T} <: AbstractString where T
    size_content::T
end

function ShortString{T}(s::String) where T
    sz = sizeof(s)
    if sz > sizeof(T) - 1 # the last byte is used to store the length
        throw(ErrorException("sizeof(::ShortString) must be shorter than or equal to $(sizeof(T) - 1) in length; you have supplied a string of size $sz"))
    end
    bits_to_wipe = 8(sizeof(T) - sz)
    content = (T(s |> pointer |> Ptr{T} |> Base.unsafe_load |> ntoh) >> bits_to_wipe) << bits_to_wipe
    ShortString{T}(content | T(sz))
end

String(s::ShortString) = String(reinterpret(UInt8, [s.size_content|>ntoh])[1:sizeof(s)])

Base.lastindex(s::ShortString) = Int(s.size_content & 0xf)
Base.iterate(s::ShortString, i::Integer) = iterate(String(s), i)
Base.iterate(s::ShortString) = iterate(String(s))
Base.sizeof(s::ShortString) = Int(s.size_content & 0xf)
Base.print(s::ShortString) = print(String(s))
Base.display(s::ShortString) = display(String(s))
Base.convert(::ShortString{T}, s::String) where T = ShortString{T}(s)
Base.convert(::String, ss::ShortString) = String(a) #reduce(*, ss)
Base.firstindex(::ShortString) = 1
Base.ncodeunits(s::ShortString) = ncodeunits(String(s))
Base.codeunit(s::ShortString, i) = codeunits(String(s), i)
Base.isvalid(s::ShortString, i::Integer) = isvalid(String(s), i)

Base.getindex(s::ShortString{T}, i::Integer) where T = begin
    print(i)
    Char((s.size_content << 8(i-1)) >> 8(sizeof(T)-1))
end
Base.collect(s::ShortString) = getindex.(s, 1:lastindex(s))

==(s::ShortString, b::String) = begin
    String(s)  == b
end

size_content(s::ShortString) = s.size_content

const ShortString15 = ShortString{UInt128}
const ShortString7 = ShortString{UInt64}
const ShortString3 = ShortString{UInt32}

# ss15"ShortString15"
macro ss15_str(s)
    :(ShortString15($s))
end

# ss7"Short7"
macro ss7_str(s)
    :(ShortString7($s))
end

# ss3"ss3"
macro ss3_str(s)
    :(ShortString3($s))
end

fsort(v::Vector{ShortString{T}}; rev = false) where T = sort(v, rev = rev, by = size_content, alg = RadixSort)
fsort!(v::Vector{ShortString{T}}; rev = false) where T = sort!(v, rev = rev, by = size_content, alg = RadixSort)

fsortperm(v::Vector{ShortString{T}}; rev = false) where T = sortperm(v, rev = rev)

end # module


export VaxInt16, VaxInt32

primitive type VaxInt16 <: VaxInt 16 end
primitive type VaxInt32 <: VaxInt 32 end

VaxInt16(x::UInt16) = reinterpret(VaxInt16, htol(x))
VaxInt16(x::Signed) = reinterpret(VaxInt16, htol(trunc(Int16,x)))

Base.convert(::Type{Int16},x::VaxInt16) = reinterpret(Int16,ltoh(x))
Base.convert(::Type{T},x::VaxInt16) where T <: Union{Int32,Int64,Int128,BigInt,AbstractFloat} = convert(T,convert(Int16,x))

Base.promote_rule(::Type{T},::Type{VaxInt16}) where T <: Union{Int8,VaxInt16} = Int16
Base.promote_rule(::Type{T},::Type{VaxInt16}) where T <: Union{Int16,Int32,Int64,Int128,BigInt,AbstractFloat} = T

Base.promote_type(::Type{VaxInt16}, ::Type{VaxInt16}) = Int16

VaxInt32(x::UInt32) = reinterpret(VaxInt32, htol(x))
VaxInt32(x::Signed) = reinterpret(VaxInt32, htol(trunc(Int32,x)))

Base.convert(::Type{Int32},x::VaxInt32) = reinterpret(Int32,ltoh(x))
Base.convert(::Type{T},x::VaxInt32) where T <: Union{Int16,Int64,Int128,AbstractFloat} = convert(T,convert(Int32,x))

Base.promote_rule(::Type{T},::Type{VaxInt32}) where T <: Union{Int8,Int16,VaxInt16,VaxInt32} = Int32
Base.promote_rule(::Type{T},::Type{VaxInt32}) where T <: Union{Int32,Int64,Int128,BigInt,AbstractFloat} = T

Base.promote_type(::Type{VaxInt32}, ::Type{VaxInt32}) = Int32


# # Modeling with JuMP

using PowerSystems
using JuMP


Pkg.add("Vega")
using Vega

#see also: https://vega.github.io/editor/#/examples/vega/word-cloud

corpus = [
"Julia is a high-level, high-performance dynamic programming language for technical computing, with syntax that is familiar to users of other technical computing environments. It provides a sophisticated compiler, distributed parallel execution, numerical accuracy, and an extensive mathematical function library. Julia’s Base library, largely written in Julia itself, also integrates mature, best-of-breed open source C and Fortran libraries for linear algebra, random number generation, signal processing, and string processing. In addition, the Julia developer community is contributing a number of external packages through Julia’s built-in package manager at a rapid pace. IJulia, a collaboration between the IPython and Julia communities, provides a powerful browser-based graphical notebook interface to Julia.",
"Julia programs are organized around multiple dispatch; by defining functions and overloading them for different combinations of argument types, which can also be user-defined. For a more in-depth discussion of the rationale and advantages of Julia over other systems, see the following highlights or read the introduction in the online manual."
]

corpus2 = ["2d", "mesh", "polygon", "2d", "scatter", "image", "subscene", "2d",
"contour", "Anthony", "2d", "contour", "2d", "contour", "2d", "heatmap", "updating",
"2d", "scatter", "animation", "2d", "text", "align", "rotation", "2d", "text",
"scatter", "3d", "similar", "animated", "meshscatter", "axi s", "mesh", "3d",
"views", "scatter", "axis", "lines", "3d", "meshscatter", "3d", "animated",
"video", "axis", "wireframe", "3d", "surface", "mesh", "cat", "3d", "linesegment",
"mesh", "3d", "glow", "scatter", "3d", "scatter", "axis", "marker", "3d", "animated",
"offset", "lines", "3d", "gif", " axis", "3d", "surface", "text", "volume", "3d",
"documentation", "2d", "heatmap", "documentation", "mesh", "cat", "3d",
"documentation", "texture", "mesh", "cat", "3d", "documentation", "axis",
"mesh", "3d", "documentation", "mesh", "cat", "wireframe", "3d", "documentation",
"wireframe", "3d", " documentation", "wireframe", "3d", "documentation", "surface",
"3d", "documentation", "surface", "image", "3d", "documentation", "surface", "2d",
"lines", "documentation", "meshscatter", "3d", "documentation", "2d", "scatter",
"documentation", "Makie.scatter", "Anthony", "2d", "scatter", "docume ntation",
"Makie.scatter", "Anthony", "2d", "scatter", "documentation", "2d", "scatter",
"VideoStream", "documentation", "linesegment", "scatter", " lines", "3d",
"documentation", "linestyle", "legend", "colorlegend", "2d", "documentation",
"legend", "meshscatter", "3d", "documentation", "VideoStream", "linesegment",
"slices", "volume", "3d layout", "heatmap", "documentation", "layout", "contour",
"slices", "volume", "gui", "animation", "documentation", "layout"]

wc = wordcloud(x = corpus2)
colorscheme!(wc, palette = ("Spectral", 11))


"""
    AbstractGraph

Abstract type

Subtypes
* AbstractNetwork
* AbstractLattice
"""
abstract type AbstractGraph end

"""
    AbstractNetwork

Abstract subtype of AbstractGraph

Subtypes
* Network
"""
abstract type AbstractNetwork <: AbstractGraph end

"""
    AbstractLattice

Abstract subtype of AbstractGraph

Subtypes
* Lattice2D
* Lattice3D
"""
abstract type AbstractLattice <: AbstractGraph end

"""
    Observables

Contains the observables associated to a percolation simulation.

Attributes
* `avg_cluster_size`     : Array where `avg_cluster_size[t]` is the average cluster size at step `t-1`
* `heterogeneity`        : Array where `heterogeneity[t]` is the number of unique cluster sizes at step `t-1`
* `largest_cluster_size` : Array where `C[t]` is the largest cluster size at step `t-1`
* `delta`                : See analysis_methods.compute_delta for more info
"""
mutable struct Observables
    avg_cluster_size::Array{Float64,1}
    heterogeneity::Array{Int,1}
    largest_cluster_size::Array{Int,1}
    delta::Tuple

    function Observables()
        avg_cluster_size = [1.0]
        heterogeneity = [1]
        largest_cluster_size = [1]
        delta = ()

        new(avg_cluster_size, heterogeneity, largest_cluster_size, delta)
    end
end

"""
    Network(N::Integer; seed::Integer=8)

This type represents a random network in which edges are allowed to be active between any two of the `N` nodes.
It houses information about the nodes, edges, clusters, and observables.

Arguments
* `N`               : Total number of nodes in the network
Keyword Arguments
* `seed`            : Seed value for the random number generator (default = 8)
Returns
* `g`               : A new instance of type Network
Attributes
* `N`               : Total number of nodes in the network
* `t`               : Current step in the evolution process, number of edges in the network
* `edges`           : Set of edges present in the network
* `cluster_ids`     : Array with nodes as indices and cluster IDs as values
* `clusters`        : Dictionary with cluster IDs as keys and clusters as values
* `cluster_sizes`   : Dictionary with cluster sizes as keys and cluster counts as values, i.e. cluster size distribution
* `rng`             : Random number generator
* `observables`     : Custom type containing observables associated with `g`
"""
mutable struct Network{T<:Integer} <: AbstractNetwork
    N::Int
    t::Int
    edges::Set{Tuple{T,T}}
    cluster_ids::Array{T,1}
    clusters::Dict{T,Set{T}}
    cluster_sizes::Dict{T,T}
    rng::MersenneTwister
    observables::Observables

    function Network{T}(N::Integer; seed::Integer = 8) where {T <: Integer}
        t = 0
        edges = Set()
        cluster_ids = collect(UnitRange{T}(1, N))
        clusters = Dict(UnitRange{T}(1, N) .=> Set.(UnitRange{T}(1, N)))
        cluster_sizes = Dict(T(1) => T(N))
        rng = MersenneTwister(seed)
        observables = Observables()

        new(N, t, edges, cluster_ids, clusters, cluster_sizes, rng, observables)
    end
end

"""
    Lattice2D(L::Integer; seed::Integer=8)

This type represents a 2D lattice in which edges are only allowed to be active between nearest neighbors.
It houses information about the nodes, edges, clusters, and observables.

Arguments
* `L`               : Side length of the square lattice
Keyword Arguments
* `seed`            : Seed value for the random number generator (default = 8)
Returns
* `g`               : A new instance of type Lattice2D
Attributes
* `L`               : Side length of the square lattice
* `N`               : Total number of nodes in the lattice, `N = L^2`
* `t`               : Current step in the evolution process, number of edges in the lattice
* `edges`           : Set of edges present in the lattice
* `cluster_ids`     : Array with nodes as indices and cluster IDs as values
* `clusters`        : Dictionary with cluster IDs as keys and clusters as values
* `cluster_sizes`   : Dictionary with cluster sizes as keys and cluster counts as values, i.e. cluster size distribution
* `rng`             : Random number generator
* `observables`     : Custom type containing observables associated with `g`
"""
mutable struct Lattice2D{T<:Integer} <: AbstractLattice
    L::Int
    N::Int
    t::Int
    edges::Set{Tuple{T,T}}
    cluster_ids::Array{T,2}
    clusters::Dict{T,Set{T}}
    cluster_sizes::Dict{T,T}
    rng::MersenneTwister
    observables::Observables

    function Lattice2D{T}(L::Integer; seed::Integer = 8) where {T <: Integer}
        N = L^2
        t = 0
        edges = Set()
        cluster_ids = reshape(collect(UnitRange{T}(1, N)), (L, L))
        clusters = Dict(UnitRange{T}(1, N) .=> Set.(UnitRange{T}(1, N)))
        cluster_sizes = Dict(T(1) => T(N))
        rng = MersenneTwister(seed)
        observables = Observables()

        new(L, N, t, edges, cluster_ids, clusters, cluster_sizes, rng, observables)
    end
end

"""
    Lattice3D(L::Integer; seed::Integer=8)

This type represents a 3D lattice in which edges are only allowed to be active between nearest neighbors.
It houses information about the nodes, edges, clusters, and observables.

Arguments
* `L`               : Side length of the cubic lattice
Keyword Arguments
* `seed`            : Seed value for the random number generator (default = 8)
Returns
* `g`               : A new instance of type Lattice3D
Attributes
* `L`               : Side length of the cubic lattice
* `N`               : Total number of nodes in the lattice, `N = L^3`
* `t`               : Current step in the evolution process, number of edges in the lattice
* `edges`           : Set of edges present in the lattice
* `cluster_ids`     : Array with nodes as indices and cluster IDs as values
* `clusters`        : Dictionary with cluster IDs as keys and clusters as values
* `cluster_sizes`   : Dictionary with cluster sizes as keys and cluster counts as values, i.e. cluster size distribution
* `rng`             : Random number generator
* `observables`     : Custom type containing observables associated with `g`
"""
mutable struct Lattice3D{T<:Integer} <: AbstractLattice
    L::Int
    N::Int
    t::Int
    edges::Set{Tuple{T,T}}
    cluster_ids::Array{T,3}
    clusters::Dict{T,Set{T}}
    cluster_sizes::Dict{T,T}
    rng::MersenneTwister
    observables::Observables

    function Lattice3D{T}(L::Integer; seed::Integer = 8) where {T <: Integer}
        N = L^3
        t = 0
        edges = Set()
        cluster_ids = reshape(collect(UnitRange{T}(1, N)), (L, L, L))
        clusters = Dict(UnitRange{T}(1, N) .=> Set.(UnitRange{T}(1, N)))
        cluster_sizes = Dict(T(1) => T(N))
        rng = MersenneTwister(seed)
        observables = Observables()

        new(L, N, t, edges, cluster_ids, clusters, cluster_sizes, rng, observables)
    end
end


filename() = "day19.txt"

# wraps patterns in parentheses
datum(x::String) = "($x)"
datum(x::Vector{String}) = datum(join(datum.(x)))
datum(x::Vector{Vector{String}}) = datum(join(datum.(x), "|"))

"""
The rules are parsed into a Dict that looks like this:

julia> read_data("day19_sample.txt")[1]
Dict{String,String} with 6 entries:
  "(2)" => "(((4)(4))|((5)(5)))"
  "(5)" => "(b)"
  "(1)" => "(((2)(3))|((3)(2)))"
  "(3)" => "(((4)(5))|((5)(4)))"
  "(4)" => "(a)"
  "(0)" => "((4)(1)(5))"
"""
function read_data(file)
    blocks = split(String(read(file)), "\n\n")

    rules = Dict(let v = string.(split(r, r":* "))
                    k = v[1]
                    if startswith(v[2], "\"")  # final term
                        datum(k) => datum(v[2][2:end-1])
                    elseif "|" ∈ v[2:end]      # multiple choices
                        d = findfirst(==("|"), v)
                        datum(k) => datum([v[2:d-1], v[d+1:end]])
                    else                       # single choice
                        datum(k) => datum(v[2:end])
                    end
                 end for r in split(blocks[1], "\n"))

    messages = split(blocks[2], "\n")

    return rules, messages
end

"""
Given the rules, resolve the variables into terminal strings "a" and "b"
and then return a `Regex` object.
"""
function make_regex(rules)
    str = rules["(0)"]
    while match(r"\([0-9]+\)", str) !== nothing
        for (k,v) in rules
            str = replace(str, k => v)
        end
    end
    for i in 1:10  # trim parentheses to reduce size of regex
        str = replace(str, r"\(([ab]*)\)" => s"\1")
    end
    str = "^" * str * "\$"
    # @show str
    return Regex(str)
end

function part1()
    rules, messages = read_data(filename())
    re = make_regex(rules)
    return count(m -> !isnothing(match(re, m)), messages)
end

function part2()
    rules, messages = read_data(filename())
    rules["(8)"] = "((42)+)"
    return sum(1:8) do i
        rules["(11)"] = "((42){$i}(31){$i})"
        re = make_regex(rules)
        count(m -> !isnothing(match(re, m)), messages)
    end
end

# For some reasons, regex recursion did not work: "((42)(?R)?(31))"
function part2b()
    rules, messages = read_data(filename())
    rules["(8)"] = "((42)+)"
    rules["(11)"] = "((42)(?1)?(31))"
    re = make_regex(rules)
    return count(m -> !isnothing(match(re, m)), messages)
end


using Compose
import Cairo, Fontconfig

imgs = [SVG("arrow.svg", 5cm, 5cm),
        PDF("arrow.pdf", 5cm, 5cm)]

X =  [0.047 0.87 0.95 0.93;
0.22 0.01 0.21 0.7;
0.86 0.85 0.95 0.21]

point_array = [[(x1,y1), (x2,y2)] for (x1,y1,x2,y2) in zip(X[:,1],X[:,2],X[:,3],X[:,4])]
img = compose(context(),
    (context(), line(point_array), stroke(["red","green","deepskyblue"]), arrow())
)

draw.(imgs, [img])


struct FashionMNIST <: GrayImage{MLImage} end
getModule(::FashionMNIST) = MLDatasets.FashionMNIST
size(::FashionMNIST) = (60000, 0, 10000)
problem(::FashionMNIST) = Classification


#unit-constants.jl
#testing each of the unum constant generators in four unum environments:
# Unum{0,0}, Unum{1,1}, Unum{4,6}, Unum{4,8}

#it's best to do this with a macro
macro uctest(T, f, ex, fr, fl, es, fs)
  creationstmt = :($f($T))

  arraytest = T.args[3] > 6 ? (:(x.fraction.a == $fr)) : (:(x.fraction == $fr))
  quote
    x = $creationstmt

    @test x.fsize == $fs
    @test x.esize == $es
    @test x.flags == $fl
    @test $arraytest
    @test x.exponent == $ex
  end
end

nan(T) = T(NaN)
inf(T) = Unums.inf(T)

@uctest Unum{0,0} zero z64 z64 z16 z16 z16
@uctest Unum{1,1} zero z64 z64 z16 z16 z16
@uctest Unum{4,6} zero z64 z64 z16 z16 z16
@uctest Unum{4,8} zero z64 [z64, z64, z64, z64] z16 z16 z16

@uctest Unum{0,0} one z64 t64 z16 z16 z16
@uctest Unum{1,1} one o64 z64 z16 o16 z16
@uctest Unum{4,6} one o64 z64 z16 o16 z16
@uctest Unum{4,8} one o64 [z64, z64, z64, z64] z16 o16 z16
@uctest Unum{0,8} one z64 [t64, z64, z64, z64] z16 z16 z16

@uctest Unum{4,6} neg_one o64 z64 0x0002 o16 z16

@uctest Unum{0,0} nan o64 t64                                     0x0001 z16 z16
@uctest Unum{1,1} nan 0x0000_0000_0000_0003 0xC000_0000_0000_0000 0x0001 0x0001 0x0001
@uctest Unum{4,6} nan 0x0000_0000_0000_FFFF 0xFFFF_FFFF_FFFF_FFFF 0x0001 0x000F 0x003F
@uctest Unum{4,8} nan 0x0000_0000_0000_FFFF [f64, f64, f64, f64]  0x0001 0x000F 0x00FF

@uctest Unum{0,0} inf o64 t64                                     0x0000 z16 z16
@uctest Unum{1,1} inf 0x0000_0000_0000_0003 0xC000_0000_0000_0000 0x0000 0x0001 0x0001
@uctest Unum{4,6} inf 0x0000_0000_0000_FFFF 0xFFFF_FFFF_FFFF_FFFF 0x0000 0x000F 0x003F
@uctest Unum{4,8} inf 0x0000_0000_0000_FFFF [f64, f64, f64, f64]  0x0000 0x000F 0x00FF

@uctest Unum{0,0} neg_inf o64 t64                                     0x0002 z16 z16
@uctest Unum{1,1} neg_inf 0x0000_0000_0000_0003 0xC000_0000_0000_0000 0x0002 0x0001 0x0001
@uctest Unum{4,6} neg_inf 0x0000_0000_0000_FFFF 0xFFFF_FFFF_FFFF_FFFF 0x0002 0x000F 0x003F
@uctest Unum{4,8} neg_inf 0x0000_0000_0000_FFFF [f64, f64, f64, f64]  0x0002 0x000F 0x00FF

@uctest Unum{0,0} mmr 0x0000_0000_0000_0001 0x0000_0000_0000_0000                  0x0001    z16    z16
@uctest Unum{1,1} mmr 0x0000_0000_0000_0003 0x8000_0000_0000_0000                  0x0001 0x0001 0x0001
@uctest Unum{4,6} mmr 0x0000_0000_0000_FFFF 0xFFFF_FFFF_FFFF_FFFE                  0x0001 0x000F 0x003F
@uctest Unum{4,8} mmr 0x0000_0000_0000_FFFF [f64, f64, f64, 0xFFFF_FFFF_FFFF_FFFE] 0x0001 0x000F 0x00FF

@uctest Unum{0,0} neg_mmr 0x0000_0000_0000_0001 0x0000_0000_0000_0000                  0x0003    z16    z16
@uctest Unum{1,1} neg_mmr 0x0000_0000_0000_0003 0x8000_0000_0000_0000                  0x0003 0x0001 0x0001
@uctest Unum{4,6} neg_mmr 0x0000_0000_0000_FFFF 0xFFFF_FFFF_FFFF_FFFE                  0x0003 0x000F 0x003F
@uctest Unum{4,8} neg_mmr 0x0000_0000_0000_FFFF [f64, f64, f64, 0xFFFF_FFFF_FFFF_FFFE] 0x0003 0x000F 0x00FF

@uctest Unum{0,0} big_exact 0x0000_0000_0000_0001 0x0000_0000_0000_0000                  0x0000    z16    z16
@uctest Unum{1,1} big_exact 0x0000_0000_0000_0003 0x8000_0000_0000_0000                  0x0000 0x0001 0x0001
@uctest Unum{4,6} big_exact 0x0000_0000_0000_FFFF 0xFFFF_FFFF_FFFF_FFFE                  0x0000 0x000F 0x003F
@uctest Unum{4,8} big_exact 0x0000_0000_0000_FFFF [f64, f64, f64, 0xFFFF_FFFF_FFFF_FFFE] 0x0000 0x000F 0x00FF

@uctest Unum{0,0} neg_big_exact 0x0000_0000_0000_0001 0x0000_0000_0000_0000                  0x0002    z16    z16
@uctest Unum{1,1} neg_big_exact 0x0000_0000_0000_0003 0x8000_0000_0000_0000                  0x0002 0x0001 0x0001
@uctest Unum{4,6} neg_big_exact 0x0000_0000_0000_FFFF 0xFFFF_FFFF_FFFF_FFFE                  0x0002 0x000F 0x003F
@uctest Unum{4,8} neg_big_exact 0x0000_0000_0000_FFFF [f64, f64, f64, 0xFFFF_FFFF_FFFF_FFFE] 0x0002 0x000F 0x00FF

@uctest Unum{0,0} sss z64 z64                   0x0001    z16    z16
@uctest Unum{1,1} sss z64 z64                   0x0001 0x0001 0x0001
@uctest Unum{4,6} sss z64 z64                   0x0001 0x000F 0x003F
@uctest Unum{4,8} sss z64 [z64, z64, z64, z64]  0x0001 0x000F 0x00FF

@uctest Unum{0,0} neg_sss z64 z64                    0x0003    z16    z16
@uctest Unum{1,1} neg_sss z64 z64                    0x0003 0x0001 0x0001
@uctest Unum{4,6} neg_sss z64 z64                    0x0003 0x000F 0x003F
@uctest Unum{4,8} neg_sss z64 [z64, z64, z64, z64]   0x0003 0x000F 0x00FF

@uctest Unum{0,0} small_exact z64 t64                   0x0000    z16    z16
@uctest Unum{1,1} small_exact z64 0x4000_0000_0000_0000 0x0000 0x0001 0x0001
@uctest Unum{4,6} small_exact z64 o64                   0x0000 0x000F 0x003F
@uctest Unum{4,8} small_exact z64 [z64, z64, z64, o64]  0x0000 0x000F 0x00FF

@uctest Unum{0,0} neg_small_exact z64 t64                   0x0002    z16    z16
@uctest Unum{1,1} neg_small_exact z64 0x4000_0000_0000_0000 0x0002 0x0001 0x0001
@uctest Unum{4,6} neg_small_exact z64 o64                   0x0002 0x000F 0x003F
@uctest Unum{4,8} neg_small_exact z64 [z64, z64, z64, o64]  0x0002 0x000F 0x00FF

#this doesn't seem to always work.
@test is_mmr(mmr(Unum{0,0}))
@test is_mmr(mmr(Unum{1,1}))
@test is_mmr(mmr(Unum{2,2}))
@test is_mmr(mmr(Unum{3,3}))
@test is_mmr(mmr(Unum{3,5}))
@test is_mmr(mmr(Unum{4,6}))
@test is_mmr(mmr(Unum{4,7}))
#turns out that the generation of the top_mask implicitly converted an UInt16 to
#Int64 by adding one, which has a different top_mask definition.


using MechanicalSketch
import MechanicalSketch: get, getinverse, textextents,  Point, rect, label,
    EM, TXTOFFV, WI, background, sethue, text, background, @colorant_str,
    lumin, finish, empty_figure, PALETTE, color_from_palette

let

this_fig = empty_figure(filename = joinpath(@__DIR__, "test_2.png"))
midcol = MechanicalSketch.HSL(get(PALETTE, 0.5))
bckcol = MechanicalSketch.RGB(MechanicalSketch.HSL(midcol.h, midcol.s, midcol.l * 0.5))
background(bckcol)
width_px = this_fig.width
height_px = this_fig.height
for (i, c)  in  enumerate(0:0.08:1)
    colo = get(PALETTE, c)
    r, g, b = sethue(colo)
    lightness = lumin(colo) / 100
    gamma = 2.2
    lumin2 = 0.2126 * r^gamma + 0.7152 * g^gamma + 0.0722 * b^gamma
    posx = width_px * (-0.5 + 0.1 * c)
    posy = height_px * - 0.5 + EM *  i
    stri = "Da jeg var på vei til kirken "
    downleftpttext = Point(posx, posy)
    x_bearing, y_bearing, twidth, theight, x_advance, y_advance = textextents(stri)
    downleftptrect = Point(posx, posy + TXTOFFV)
    rect( downleftptrect , twidth * 3, -EM, :fill)
    sethue(lightness > 0.6 ? "black" : "white")
    text(stri,  downleftpttext)
    downleftpt = Point(posx + twidth, posy)
    sethue(lumin2 > 0.5 ? "black" : "white")
    text(stri * string(c),  downleftpt)
end
stri = " α"
sethue( get(PALETTE, 0.15))
label("2" * "E", :E, Point(0 , 2EM))
sethue(get(PALETTE, getinverse(PALETTE, colorant"red" )))
label("N" * stri, :N, Point(0, 1EM), leader = true, offset = EM, leaderoffsets = [2., 3.])
label("S" * stri, :S, Point(0 , 1EM), leader = true, offset = EM, leaderoffsets = [-2., -3.])
sethue(color_from_palette("red"))
label("E" * stri, :E, Point(0, 1EM), leader = true, offset = EM, leaderoffsets = [2., 3.])
label("W" * stri, :W, Point(0 , 1EM), leader = true, offset = EM, leaderoffsets = [-2., -3.])
sethue(get(PALETTE, getinverse(PALETTE, colorant"green" )))
label("5N" * stri, :N, Point(-WI/4, 5EM), leader = true, offset = EM, leaderoffsets = [2., 3.])
label("5S" * stri, :S, Point(-WI/4, 5EM), leader = true, offset = EM, leaderoffsets = [-2., -3.])
sethue(color_from_palette("darkblue"))
label("5E" * stri, :E, Point(-WI/4, 5EM), leader = true, offset = EM, leaderoffsets = [2., 3.])
label("5W" * stri, :W, Point(-WI/4 , 5EM), leader = true, offset = EM, leaderoffsets = [-2., -3.])
sethue(get(PALETTE, getinverse(PALETTE, colorant"blue" )))
label("7" * stri, :W, Point(WI/4 , 7EM), leader = true, offset = EM, leaderoffsets = [-2., -3.])

finish()
end


module TopicTiling
include("Utls.jl")
using Statistics
using .Utls

"""
    TopicTiling.SegmentObject(window_size, smooth_window_size, lda_model, dictionary)
TopicTiling is an extension of TextTiling that uses the topic IDs of words in a sentence to calculate the similarity between blocks.
# Arguments
- `window_size`: Sliding window size.
- `smooth_window_size`: Window size for smoothing depth scores.
- `lda_model`: Trained LDA topic model.
- `dictionary`: A dictionary showing word-id mappings.
"""
mutable struct SegmentObject
    window_size::Int
    smooth_window_size::Int
    lda_model::Any
    dictionary::Any
end

function preprocessing(seg::SegmentObject, tokenized_document)
    n = length(tokenized_document)
    seg.window_size = maximum([minimum([seg.window_size, n / 3]), 1])
    return [
        Utls.count_elements(
            aggregate_topic_id(
                seg.lda_model.get_document_topics(
                    seg.dictionary.doc2bow(tokenized_document[i]),
                    per_word_topics = true,
                )[2],
            ),
        ) for i = 1:n
    ]
end

function aggregate_topic_id(ids)
    sentence_ids = []
    for i in ids
        append!(sentence_ids, string.(i[2]))
    end
    return sentence_ids
end

function calculate_gap_score(seg::SegmentObject, preprocessed_document)
    n = length(preprocessed_document)
    gap_score = [0.0 for _ = 1:n]

    for i = 1:n
        sz = minimum([minimum([i, n - i]), seg.window_size])
        left_side, right_side = Utls.SentenceElements(Dict{String,Int}()),
        Utls.SentenceElements(Dict{String,Int}())

        for j = Int(i - sz + 1):Int(i)
            Utls.merge_elements(left_side, preprocessed_document[j])
        end

        for j = Int(i + 1):Int(i + sz)
            Utls.merge_elements(right_side, preprocessed_document[j])
        end

        gap_score[i] =
            Utls.calculate_cosin_similarity(left_side.elements_dct, right_side.elements_dct)
    end
    return gap_score
end

function calculate_depth_score(seg::SegmentObject, gap_score)
    n = length(gap_score)
    depth_score = [0.0 for _ = 1:n]

    for i = 1:n
        if i <= seg.window_size || i + seg.window_size > n
            continue
        end

        ptr = i - 1
        while ptr >= 1 && gap_score[ptr] >= gap_score[ptr+1]
            ptr -= 1
        end
        lval = gap_score[ptr+1]

        ptr = i + 1
        while ptr < n && gap_score[ptr] >= gap_score[ptr-1]
            ptr += 1
        end
        rval = gap_score[ptr-1]

        depth_score[i] = 1 / 2 * (lval - gap_score[i] + rval - gap_score[i])
    end
    return depth_score
end

function smoothing(seg::SegmentObject, depth_score)
    n = length(depth_score)
    smooth_depth_score = [0.0 for _ = 1:n]

    for i = 1:n
        if i - seg.smooth_window_size < 1 || i + seg.smooth_window_size >= n
            smooth_depth_score[i] = depth_score[i]
        else
            smooth_depth_score[i] =
                mean(depth_score[(i-seg.smooth_window_size):(i+seg.smooth_window_size)])
        end
    end
    return smooth_depth_score
end

function determine_boundaries(seg::SegmentObject, smooth_depth_score, num_topics)
    n = length(smooth_depth_score)
    boundaries = ["0" for _ = 1:n]
    cutoff_threshold = mean(smooth_depth_score) - std(smooth_depth_score) / 2.0

    depth_tuples = [(x, y) for (x, y) in zip(smooth_depth_score, [i for i = 1:n])]
    depth_tuples = reverse(sort(depth_tuples))

    if num_topics == Nothing
        for x in depth_tuples
            if x[1] > cutoff_threshold
                boundaries[x[2]] = "1"
            end
        end
    else
        for (i, x) in enumerate(depth_tuples)
            if x[1] > cutoff_threshold && i <= num_topics - 1
                boundaries[x[2]] = "1"
            end
        end
    end
    return join(boundaries[1:end-1])
end

"""
    TopicTiling.segment(seg, document, [num_topics]) -> String
Performs the splitting of the document entered in the `document` argument.
# Arguments
- `seg`: Segment object.
- `document`: The document to be text segmented.
- `num_topics`: num_topics is the number of topics in the document. If this value is specified, segment boundaries are determined by the number of num_topics, starting with the highest depth score.

# Examples
```jldoctest
using TextSegmentation

# LDA Topic Model
pygensim = pyimport("gensim")
# train_document
# Data to be used when training the LDA topic model.
# Data from the same domain as the text to be segmented is preferred.
tokenized_train_document = [Utls.tokenize(i) for i in train_document]
dictionary = pygensim.corpora.Dictionary(tokenized_train_document)
corpus = [dictionary.doc2bow(text) for text in tokenized_train_document]
lda_model = pygensim.models.ldamodel.LdaModel(
    corpus = corpus,
    id2word = dictionary,
    minimum_probability = 0.0001,
    num_topics = 3,
    random_state=1234,
)

# TopicTiling
window_size = 2
smooth_window_size = 1
num_topics = 3
to = TopicTiling.SegmentObject(window_size, smooth_window_size, lda_model, dictionary)
result = TopicTiling.segment(to, document, num_topics)
println(result)
000110000000
```
"""
function segment(seg, test_document, num_topics=Nothing)
    tokenized_test_document = [Utls.tokenize(i) for i in test_document]
    preprocessed_document = preprocessing(seg, tokenized_test_document)
    gap_score = calculate_gap_score(seg, preprocessed_document)
    depth_score = calculate_depth_score(seg, gap_score)
    smooth_depth_score = smoothing(seg, depth_score)
    return determine_boundaries(seg, smooth_depth_score, num_topics)
end
end