hsr(s :: AbstractString, sp :: AbstractString) = ∘((f -> make(f.([s, sp])...)).([identity, (s) -> join(collect(s), '.')])...)

make(s :: AbstractString, sp :: AbstractString) = (x) -> hsr0(s, sp, x)

function hsr0(s :: AbstractString, sp :: AbstractString, orig :: AbstractString)
    i = 1
    r = ""
    lo = lowercase(orig)
    ls = lowercase(s)
    while true
        p = findnext(ls, lo, i)
        if p === nothing
            break
        end
        if p.start > i
            r *= orig[i:p.start-1]
        end
        i = p.stop + 1
        r *= caseawarenessconvert(orig[p], sp)
    end
    if i <= length(orig)
        r *= orig[i:end]
    end
    return r
end

function caseawarenessconvert(s :: AbstractString, sp :: AbstractString)
    fs = ((c) -> if islowercase(c)
                    lowercase
                elseif isuppercase(c)
                    uppercase
                else
                    identity
                end).(collect(s))
    lsp = collect(sp)
    l = length(sp) > 1 ? (n -> 1 + round(Int, (length(fs) - 1) * (n - 1) / (length(lsp) - 1))).(1:length(lsp)) : 1:1
    join(broadcast((f, c) -> f(c), fs[l], lsp))
end

function test()
    f = hsr("windows", "harmony")
    println(f("Windows 10 is the newest W.I.N.D.O.W.S version"))
    println(f("WindoWS 2000 is built on NT"))
    println(f("WINdows is funny"))
end

test()


using FastGroupBy, Compat
import Base.Forward
import SortingAlgorithms: RADIX_MASK, RADIX_SIZE, uint_mapping

"""
sumby that uses much less memory but is slower

It saves memory by not creating a separate copy of by and val vectors but instead
follows a chain of assignment for the swap operations.

"""
function sumby_lessmem_chain{T, S<:Number}(by::AbstractVector{T},  val::AbstractVector{S})
  o = Forward
  lo = 1
  hi = length(by)

  if lo >= hi;  return by;  end

  if !isbits(T)
      error("Radix sort only sorts bits types (got $T)")
  end

  # Init
  iters = ceil(Integer, sizeof(T)*8/RADIX_SIZE)
  bin = zeros(UInt32, 2^RADIX_SIZE, iters)
  if lo > 1;  bin[1,:] = lo-1;  end

  # Histogram for each element, radix
  for i = lo:hi
    v = uint_mapping(o, by[i])
    for j = 1:iters
        idx = @compat(Int((v >> (j-1)*RADIX_SIZE) & RADIX_MASK)) + 1
        @inbounds bin[idx,j] += 1
    end
  end

  # Sort!
  len = hi-lo+1
  for j = 1:iters
      # Unroll first data iteration, check for degenerate case
      v = uint_mapping(o, by[hi])
      idx = @compat(Int((v >> (j-1)*RADIX_SIZE) & RADIX_MASK)) + 1

      # are all values the same at this radix?
      if bin[idx,j] == len;  continue;  end

      cbin = cumsum(bin[:,j])

      # chain assignment to save memory
      touched = BitArray(hi) # to keep track of which cell has been touched
      i = hi
      while i > 0
          begin_index = i # this is where the chain starts
          tmp = (by[i], val[i])
          v = uint_mapping(o, tmp[1])
          idx = @compat(Int((v >> (j-1)*RADIX_SIZE) & RADIX_MASK)) + 1

          while true
              ci = cbin[idx]
              new_tmp = (by[ci], val[ci])
              by[ci], val[ci] = tmp[1], tmp[2]
              tmp = new_tmp
              touched[ci] = true # keep track of which number is done
              cbin[idx] -= 1

              if ci == begin_index # the chain has ended
                  break;
              end
              v = uint_mapping(o, tmp[1])
              idx = @compat(Int((v >> (j-1)*RADIX_SIZE) & RADIX_MASK)) + 1
          end

          while i > 0 && touched[i]
              i -= 1
          end
      end
  end

  sumby_contiguous(by, val)
end


"""
    process_streaks(df::AbstractDataFrame; photometry = false)

Take as input a processed pokes datafame `df` of 1 single session and returns a dataframe were each row are information about a poking streak.
A poking streak ends when the animal changes side. For photometry data a set of other columns regarding indexes for traces allignment are computed.
"""

function process_streaks(df::AbstractDataFrame; photometry = false)
    vals_per_streak = [:Stim, :StimFreq, :Wall, :Protocol, :Block, :StreakInBlock, :Side, :ReverseStreak]
    vals_per_day = [:Box,:StimDay];
    booleans=[:Reward,:Stim,:Wall,:Correct,:StimDay]#columns to convert to Bool
    for x in booleans
        if x in propertynames(df)
            df[:,x] = eltype(df[:,x]) == Bool ? df[:,x] : occursin.("true",df[:,x],)
        end
    end
    streak_table = combine(groupby(df, :Streak)) do dd
        lr_idx = findlast(dd[!,:Reward] .== 1).== nothing ? 0 : findlast(dd[!,:Reward] .== 1)
        dt = DataFrame(
        Num_pokes = size(dd,1),
        Num_Rewards = length(findall(dd[!,:Reward].==1)),
        Start_Reward = dd[1,:Reward],
        Last_Reward = lr_idx,
        Last_Reward_Time = lr_idx == 0 ? dd[1,:PokeOut] -  dd[1,:PokeIn] : dd[lr_idx,:PokeOut] -  dd[1,:PokeIn],
        Prev_Reward = findlast(dd[!,:Reward] .== 1).== nothing ? 0 : findprev(dd[!,:Reward] .==1, findlast(dd[!,:Reward] .==1)-1),
        Trial_Duration = (dd[end,:PokeOut]-dd[1,:PokeIn]),
        Poking_time = sum(dd.PokeDur),
        Start = (dd[1,:PokeIn]),
        Stop = (dd[end,:PokeOut]),
        PreInterpoke = size(dd,1) > 1 ? maximum(skipmissing(dd[!,:PreInterpoke])) : missing,
        PostInterpoke = size(dd,1) > 1 ? maximum(skipmissing(dd[!,:PostInterpoke])) : missing,
        CorrectStart = dd[1,:Correct],
        CorrectLeave = !dd[end,:Correct]
        )

        for v in vals_per_streak
            if v in propertynames(dd)
                dt[!,v] .= dd[1, v]
            end
        end

        return dt
    end
    streak_table[!,:Prev_Reward] = [x .== nothing ? 0 : x for x in streak_table[:,:Prev_Reward]]
    streak_table[!,:AfterLast] = streak_table[!,:Num_pokes] .- streak_table[!,:Last_Reward];
    streak_table[!,:AfterLast_Duration] = streak_table[!,:Trial_Duration] .- streak_table[!,:Last_Reward_Time];
    streak_table[!,:BeforeLast] = streak_table[!,:Last_Reward] .- streak_table[!,:Prev_Reward].-1;
    prov = lead(streak_table[!,:Start],default = 0.0) .- streak_table[!,:Stop];
    streak_table[!,:Travel_to] = [x.< 0 ? 0 : x for x in prov]

    for v in vals_per_day
        if v in propertynames(df)
            streak_table[!,v] .= df[1, v]
        end
    end

    # for photometry datas a set of indexes are extracted to allign the data in the pokes dataframe
    # This indexes are to be reported in the streaks dataframe as well
    if photometry
        frames = combine(groupby(df, :Streak)) do dd
            dd = DataFrame(
            In = dd[1,:In],
            Out = dd[end,:Out],
            LR_In = findlast(dd[:Reward])==0 ? NaN : dd[findlast(dd[:Reward]),:In],
            LR_Out = findlast(dd[:Reward])==0 ? NaN : dd[findlast(dd[:Reward]),:Out]
            )
        end
        streak_table[:In] = frames[:In]
        streak_table[:Out] = frames[:Out]
        streak_table[:LR_In] = frames[:LR_In]
        streak_table[:LR_Out] = frames[:LR_Out]
    end

    return streak_table
end


module custom_module
  using LinearAlgebra

  function squareMeBaby(A)
    #  A = unsafe_wrap(Array, data, len)
    ## Square array and return the result
    println(typeof(A))

    #  A[:]=[i*i for i in A]
    #  return A
    B = A.*A
    return B
    #  return map(x -> x*x, A)
  end

  function mutateMeByTen!(A)
    ## Multiple array in place by ten
    lmul!(10, A)
  end

  export dummy
  export squareMeBaby
  export mutateMeByTen!

  function dummy()
    println("Julia says... Hello, world ! Function dummy() from module custom_module has been executed !")
  end

end
## TODO Fix this
const julia_dummy = @cfunction(custom_module.dummy, (Cvoid), ())

function addMeBaby(x, y)
  return x+y
end
const julia_addMeBabyInt = @cfunction(addMeBaby, Cint, (Cint, Cint,))
#  const julia_addMeBabyFloat = @cfunction(addMeBaby, Cfloat, (Cfloat, Cfloat,))


if VERSION >= v"0.7.0"
	import Random: randperm, shuffle!, shuffle, seed!
	import Printf.@printf
	import LinearAlgebra
	import LinearAlgebra: norm, Diagonal, dot
	import Statistics: var, mean, std
	using UnicodePlots
	import Base.minimum
end


## Lifted operators

import Base: promote_op, scalarmin, scalarmax
using Compat

if isdefined(Base, :fieldname) && Base.fieldname(Nullable, 1) == :hasvalue # Julia 0.6
    _Nullable(R, x, hasvalue::Bool) = Nullable{R}(x, hasvalue)
else
    _Nullable(R, x, hasvalue::Bool) = Nullable{R}(x, !hasvalue)
end

"""
    null_safe_op(f::Any, ::Type...)::Bool

Returns whether an operation `f` can safely be applied to any value of the passed type(s).
Returns `false` by default.

Custom types should implement methods for some or all operations `f` when applicable:
returning `true` means that the operation may be called on any bit pattern without
throwing an error (though returning invalid or nonsensical results is not a problem).
In particular, this means that the operation can be applied on the whole domain of the
type *and on uninitialized objects*. As a general rule, these proporties are only true for
safe operations on `isbits` types.

Types declared as safe can benefit from higher performance for operations on nullable: by
always computing the result even for null values, a branch is avoided, which helps
vectorization.
"""
null_safe_op(f::Any, ::Type...) = false

@compat const SafeSignedInts = Union{Int128,Int16,Int32,Int64,Int8}
@compat const SafeUnsignedInts = Union{Bool,UInt128,UInt16,UInt32,UInt64,UInt8}
@compat const SafeInts = Union{SafeSignedInts,SafeUnsignedInts}
@compat const SafeFloats = Union{Float16,Float32,Float64}

# Float types appear in both since they promote to themselves,
# and therefore can't fail due to conversion of negative numbers
@compat const SafeSigned = Union{SafeSignedInts,SafeFloats}
@compat const SafeUnsigned = Union{SafeUnsignedInts,SafeFloats}
@compat const SafeTypes = Union{SafeInts,SafeFloats}

# Unary operators

# Note this list does not include sqrt since it can raise an error,
# nor cbrt (for which there is no functor on Julia 0.4)
for op in (:+, :-, :abs, :abs2)
    @eval begin
        null_safe_op{T<:SafeTypes}(::typeof($op), ::Type{T}) = true
    end
end

null_safe_op{T<:SafeInts}(::typeof(~), ::Type{T}) = true
null_safe_op{T<:SafeTypes}(::typeof(cbrt), ::Type{T}) = true
null_safe_op(::typeof(!), ::Type{Bool}) = true

# Temporary workaround until JuliaLang/julia#18803
null_safe_op(::typeof(cbrt), ::Type{Float16}) = false

for op in (:+, :-, :!, :~, :abs, :abs2, :sqrt, :cbrt)
    @eval begin
        @inline function Base.$op{S}(x::Nullable{S})
            R = promote_op($op, S)
            if null_safe_op($op, S)
                _Nullable(R, $op(x.value), !isnull(x))
            else
                isnull(x) ? Nullable{R}() :
                            Nullable{R}($op(x.value))
            end
        end
        Base.$op(x::Nullable{Union{}}) = Nullable()
    end
end

# Binary operators

# Note this list does not include ^, ÷ and %
# Operations between signed and unsigned types are not safe: promotion to unsigned
# gives an InexactError for negative numbers
for op in (:+, :-, :*, :/, :&, :|, :<<, :>>, :(>>>),
           :(==), :<, :>, :<=, :>=,
           :scalarmin, :scalarmax,
           :isless)
    @eval begin
        # to fix ambiguities
        null_safe_op{S<:SafeFloats,
                     T<:SafeFloats}(::typeof($op), ::Type{S}, ::Type{T}) = true
        null_safe_op{S<:SafeSigned,
                     T<:SafeSigned}(::typeof($op), ::Type{S}, ::Type{T}) = true
        null_safe_op{S<:SafeUnsigned,
                     T<:SafeUnsigned}(::typeof($op), ::Type{S}, ::Type{T}) = true
    end
end

for op in (:+, :-, :*, :/, :%, :÷, :&, :|, :^, :<<, :>>, :(>>>),
           :(==), :<, :>, :<=, :>=,
           :scalarmin, :scalarmax)
    @eval begin
        @inline function Base.$op{S,T}(x::Nullable{S}, y::Nullable{T})
            R = promote_op(Base.$op, S, T)
            if null_safe_op(Base.$op, S, T)
                _Nullable(R, Base.$op(x.value, y.value), !(isnull(x) | isnull(y)))
            else
                (isnull(x) | isnull(y)) ? Nullable{R}() :
                                          Nullable{R}(Base.$op(x.value, y.value))
            end
        end
        Base.$op(x::Nullable{Union{}}, y::Nullable{Union{}}) = Nullable()
        Base.$op{S}(x::Nullable{Union{}}, y::Nullable{S}) = Nullable{S}()
        Base.$op{S}(x::Nullable{S}, y::Nullable{Union{}}) = Nullable{S}()
    end
end

if !method_exists(isless, Tuple{Nullable{Int}, Nullable{Int}})
    function Base.isless{S,T}(x::Nullable{S}, y::Nullable{T})
        # NULL values are sorted last
        if null_safe_op(isless, S, T)
            (!isnull(x) & isnull(y)) |
            (!isnull(x) & !isnull(y) & isless(x.value, y.value))
        else
            if isnull(x)
                return false
            elseif isnull(y)
                return true
            else
                return isless(x.value, y.value)
            end
        end
    end
    Base.isless(x::Nullable{Union{}}, y::Nullable{Union{}}) = false
    Base.isless(x::Nullable{Union{}}, y::Nullable) = false
    Base.isless(x::Nullable, y::Nullable{Union{}}) = !isnull(x)
end


##################################################################################################
## Read length
##################################################################################################
"""

    `fastq_len_store(FASTQ)`

    Store length of reads in FASTQ file.

    # Examples
    ```julia-repl
    julia> CpelNano.fastq_len_store(fastq)
    ```
"""
function fastq_len_store(fastq::String, outdir::String)::Nothing

    # Get prefix
    pref = split(basename(fastq), ".fastq")[1]

    # Loop over records in FASTQ file
    lens = Vector{Int64}()
    open(FASTQ.Reader, fastq) do reader
        for record in reader
            # Get lengths
            push!(lens, length(FASTQ.sequence(record)))
        end
    end

    # Write lengths
    print_log("Storing lengths...")
    open("$(outdir)/$(pref)_read_lens.txt", "w") do io
        writedlm(io, lens)
    end

    # Return nothing
    return nothing

end
"""
    `gen_read_len(NREADS)`

    Returns a vector of read lengths following an exponential distribution obtained performing
    ML estimation to read length of dataset from:

        https://github.com/nanopore-wgs-consortium/NA12878/blob/master/Genome.md

    # Examples
    ```julia-repl
    julia> CpelNano.gen_read_len(10)
    ```
"""
function gen_read_len(n_reads::Int64)::Vector{Int64}

    # Return read lengths
    return Int.(round.(rand(Exponential(8484.84), n_reads)))

end
##################################################################################################
## Generate Nucleotide Sequences
##################################################################################################
"""

    `gen_seq(seq_len,n)`

    Generates a random sequence of length `seq_len` with `n` CpG sites.

    # Examples
    ```julia-repl
    julia> CpelNano.gen_seq(10,2)
    "CCGGGGCGGA"
    ```
"""
function gen_seq(seq_len::Int64, n::Int64)::String

    # Create random sequence
    seq = randstring("ACGT", seq_len)

    # Get rid of coincidental CpGs
    seq = replace(seq, r"CG" => (c) -> "C" * randstring("AT", 1))

    # Determine spacing required
    spacing = floor(Int64, seq_len / n)
    spacing >= 2 || return ""

    # Add equally spaced CG's
    seq = split(seq, "")
    ind = floor(Int64, spacing / 2):spacing:seq_len
    for i in ind
        seq[i:(i + 1)] = ["C","G"]
    end

    # Return sequence with n CpG sites
    return join(seq)

end
"""

    `gen_read(seq,read_len)`

    Returns a random read of length `read_len` from sequence `seq`.

    # Examples
    ```julia-repl
    julia> CpelNano.gen_read("CCGGGGCGGA",5)
    (5,"GGCGG")
    ```
"""
function gen_read(chr::String, fasta::String, read_len::Int64)::Tuple{Int64,String}

    # Get position
    fa_record = CpelNano.get_chr_fa_rec(chr, fasta)
    chrstart = fa_record.sequence[1]
    chrend = fa_record.sequence[length(fa_record.sequence)];
    pos = rand(chrstart:(chrend - read_len + 1))

    # Extract read
    seq = convert(String, FASTA.sequence(fa_record, pos:(pos + read_len - 1)));

    # Return tuple
    return pos, seq

end
"""

    `count_cpg(seq)`

    Count the number of CGs in sequence `seq`.

    # Examples
    ```julia-repl
    julia> CpelNano.count_cpg("CCGGGGCGGA")
    2
    ```
"""
function count_cpg(seq::String)::Int64
    ind = minimum.(findall(r"CG", seq))
    return length(ind)
end
"""

    `count_mpg(seq)`

    Count the number of MGs in sequence `seq`.

    # Examples
    ```julia-repl
    julia> CpelNano.count_mpg("CMGGGGCGGA")
    1
    ```
"""
function count_mpg(seq::String)::Int64
    ind = minimum.(findall(r"MG", seq))
    return length(ind)
end
"""

    `meth_seq(seq,methyl_vec)`

    Converts a CG in `seq` to MG based on its methylation status from the corresponding index in `methyl_vec`.

    # Examples
    ```julia-repl
    julia> CpelNano.meth_seq("CCGGGGCGGA",CpelNano.metro_hast_inst(2,10000,0.5,0.5))
    "CMGGGGMGGA"
    ```
"""
function meth_seq(seq::String, methyl_vec::Vector{Int8})::String

    # Get number of CpG sites and spacing
    n = length(methyl_vec)
    spacing = floor(Int64, length(seq) / n)

    # Find indices of C's in CG context
    ind = minimum.(findall(r"CG", seq))
    length(ind) == n || return ""

    # Modify sequence
    seq = split(seq, "")
    seq[ind[methyl_vec .== Int8(1)]] .= "M"

    # Return modified sequence
    return join(seq)

end
"""
    `aug_mod_file(reg_file,chr_name,fasta)`

    Augments model file. If analysis region is in `reg_file`,uses parameter vector from `reg_file`. Otherwise,
    randomly samples parameter vector from analysis regions in `reg_file`. This function produces a text file
    with suffix `.aug`.

    # Examples
    ```julia-repl
    julia> CpelNano.aug_mod_file(reg_file,chr_name,fasta)
    ```
"""
function aug_mod_file(reg_file::String, chr_name::String, fasta::String)::Nothing

    # Read in model file
    reg_lines = readdlm(reg_file)

    # Get FASTA record for chr
    fa_record = get_chr_fa_rec(chr_name, fasta)

    # Initialize vectors to store parameter vector and partitions from model file
    model_part = Vector{UnitRange{Int64}}()
    ϕhat_1 = Vector{Float64}()
    ϕhat_2 = Vector{Float64}()
    ϕhat_3 = Vector{Float64}()

    # Read in parameter vectors and partitions from model files
    for i in 1:size(reg_lines)[1]
        chrst = reg_lines[i,2]
        chrend = reg_lines[i,3]
        ϕhat = parse.(Float64, split(reg_lines[i,4], ','));
        push!(model_part, chrst:chrend)
        push!(ϕhat_1, ϕhat[1])
        push!(ϕhat_2, ϕhat[2])
        push!(ϕhat_3, ϕhat[3])
    end

    # Obtain full set of partitions from whole chromosome
    config = CpelNano.CpelNanoConfig()
    chr_part = CpelNano.get_chr_part(chr_name, config, fasta)

    # Initialize vector to store analysis regions
    model_regs = Vector{CpelNano.RegStruct}()

    # Iterate through every partition in the chromosome
    for i in 1:length(chr_part)

        # Check if partition is in model file
        modelpart_index = findfirst(x -> x == chr_part[i], model_part)

        # If partition is not in model file,then randomly sample parameter vector from model file
        if isnothing(modelpart_index)
            # Randomly sample parameters
            new_ϕhat_1 = ϕhat_1[rand(1:length(ϕhat_1))]
            new_ϕhat_2 = ϕhat_2[rand(1:length(ϕhat_2))]
            new_ϕhat_3 = ϕhat_2[rand(1:length(ϕhat_3))]
        else
            # Keep the estimated vectors
            new_ϕhat_1 = ϕhat_1[modelpart_index]
            new_ϕhat_2 = ϕhat_2[modelpart_index]
            new_ϕhat_3 = ϕhat_3[modelpart_index]
        end

        # Construct RegStruct object for this analysis region
        reg_data = CpelNano.RegStruct()
        reg_data.chr = chr_name
        reg_data.chrst = minimum(chr_part[i])
        reg_data.chrend = maximum(chr_part[i])
        reg_data.ϕhat = [new_ϕhat_1,new_ϕhat_2,new_ϕhat_3]

        # Get group info assuming singletons
        CpelNano.get_grp_info!(reg_data, fa_record, 1)

        # Set processed flag as true to store
        reg_data.proc = true

        # Add RegStruct to list of RegStructs for the chromosome
        push!(model_regs, reg_data)

    end

    # Store
    write_output_ϕ("$(reg_file).aug", model_regs)

    # Return nothing
    return nothing

end
"""
    `reads_reg_chr(reg_file,chr_name,fasta,fa_record)`

    Returns a list of RegStructs for every partition in chromosome given `reg_file` containing all
    estimation regions in the chromosome. WARNING: this function assumes there is an existing model
    for each single region.

    # Examples
    ```julia-repl
    julia> model_regs = CpelNano.read_reg_chr(reg_file,chr_name,fasta,fa_record)
    ```
"""
function read_reg_chr(reg_file::String, chr_name::String, fasta::String, fa_record::FASTA.Record)::Vector{CpelNano.RegStruct}

    # Read in model file
    reg_lines = readdlm(reg_file)

    # Initialize vectors to store parameter vector and partitions from model file
    model_part = Vector{UnitRange{Int64}}()
    ϕhat_1 = Vector{Float64}()
    ϕhat_2 = Vector{Float64}()
    ϕhat_3 = Vector{Float64}()

    # Read in parameter vectors and partitions from model files
    for i in 1:size(reg_lines)[1]
        chrst = reg_lines[i,2]
        chrend = reg_lines[i,3]
        ϕhat = parse.(Float64, split(reg_lines[i,4], ','));
        push!(model_part, chrst:chrend)
        push!(ϕhat_1, ϕhat[1])
        push!(ϕhat_2, ϕhat[2])
        push!(ϕhat_3, ϕhat[3])
    end

    # Initialize vector to store analysis regions
    model_regs = Vector{CpelNano.RegStruct}()

    # Obtain full set of partitions from whole chromosome
    config = CpelNano.CpelNanoConfig();
    chr_part = CpelNano.get_chr_part(chr_name, config, fasta);

    # Iterate through every partition in the chromosome
    for i in 1:length(chr_part)

        # Get model index
        modelpart_index = findfirst(x -> x == chr_part[i], model_part)

        # Get corresponding ϕ
        new_ϕhat_1 = ϕhat_1[modelpart_index]
        new_ϕhat_2 = ϕhat_2[modelpart_index]
        new_ϕhat_3 = ϕhat_3[modelpart_index]

        # Construct RegStruct object for this analysis region
        reg_data = CpelNano.RegStruct()
        reg_data.chr = chr_name
        reg_data.chrst = minimum(chr_part[i])
        reg_data.chrend = maximum(chr_part[i])
        reg_data.ϕhat = [new_ϕhat_1,new_ϕhat_2,new_ϕhat_3]

        # Get group info assuming singletons
        CpelNano.get_grp_info!(reg_data, fa_record, 1)

        # Add RegStruct to list of RegStructs for the chromosome
        push!(model_regs, reg_data)

    end

     # Return list of RegStructs in chromosome in same order as chromosome partitions
    return model_regs

end
"""
    `linearsearch_readstart(startpos,parts)`

    Find index of analysis region corresponding to start of read `startpos` given list of chromosome partitions `parts`

    # Examples
    ```julia-repl
    julia> CpelNano.linearsearch_readstart(14996959,CpelNano.get_chr_part("chr22",CpelNano.CpelNanoConfig(),"/Users/sandeepk/Downloads/chr22.fa"))
    5000
    ```
"""
function linearsearch_readstart(startpos::Int64, parts::Vector{UnitRange{Int64}})::Int64
    for i in 1:length(parts)
        chrst = parts[i][1]
        chrend = parts[i][length(parts[i])]
        if startpos >= chrst && startpos <= chrend
            return i
        end
    end
    return -1
end
"""
    `linearsearch_readend(startpos,idx,parts)`

    Find index of analysis region corresponding to end of read `startpos` given list of chromosome partitions `parts`
    and index of analysis region corresponding to start of read `idx`

    # Examples
    ```julia-repl
    julia> CpelNano.linearsearch_readend(20999958,5000,CpelNano.get_chr_part("chr22",CpelNano.CpelNanoConfig(),"/Users/sandeepk/Downloads/chr22.fa"))
    7001
    ```
"""
function linearsearch_readend(endpos::Int64, idx::Int64, parts::Vector{UnitRange{Int64}})::Int64
    for i in idx:length(parts)
        chrst = parts[i][1]
        chrend = parts[i][length(parts[i])]
        if endpos >= chrst && endpos <= chrend
            return i
        end
    end
    return -1
end
"""
    `get_methyl_vector(rd_st,rd_end,startidx,endidx,model_regs)`

    Obtain methylation vector for a given read given start of read `rd_st`,end of read `rd_end`,
    index of analysis region corresponding to start of read `startidx`,index of analysis region
    corresponding to end of read `endidx`,and list of RegStructs in chromosome `model_regs`

    # Examples
    ```julia-repl
    julia> CpelNano.get_methyl_vector(rd_st,rd_end,startidx,endidx,model_regs)
    ```
"""
function get_methyl_vector(rd_st::Int64, rd_end::Int64, startidx::Int64, endidx::Int64, model_regs::Vector{RegStruct})::Tuple{Vector{Int8},Vector{Int64}}

    # Initialize
    x_out = Vector{Int8}()
    cpg_pos = Vector{Int64}()

    # Concatenate all αs and βs
    @inbounds for i in startidx:endidx

        # Get region struct
        rs = model_regs[i]

        # If no CpG sites, skip
        length(rs.cpg_pos) > 0 || continue

        # Get parameter vectors
        α, β = CpelNano.get_αβ_from_ϕ(rs.ϕhat, rs)

        # Generate realization x
        x = length(rs.cpg_pos) == 1 ? [Int8(1)] : CpelNano.gen_x_mc(α, β)
        x_out = vcat(x_out, x)

        # Add CpG sites
        cpg_pos = vcat(cpg_pos, rs.cpg_pos)

    end

    # Return nothing
    length(cpg_pos) > 0 || return [], []

    # Keep only overlapping part
    fst_ind = findfirst(x -> x >= rd_st, cpg_pos)
    isnothing(fst_ind) && return [], []
    lst_ind = findlast(x -> x <= rd_end, cpg_pos)
    isnothing(lst_ind) && return [], []
    cpg_pos = cpg_pos[fst_ind:lst_ind]
    x_out = x_out[fst_ind:lst_ind]

    # Return x & cpg positions
    return x_out, cpg_pos

end
##################################################################################################
## Markov chain sampler of fine grain model
##################################################################################################
"""
    `get_log_g_x(n,α,β)`

    Compute function log(g_n(x_n)).

    # Examples
    ```julia-repl using CpelNano
    julia> CpelNano.get_log_g_x(1,zeros(10),zeros(9))
    (256.0,256.0)
    ```
"""
function get_log_g_x(n::Int64, α::Vector{Float64}, β::Vector{Float64})::NTuple{2,Float64}

    # Get size
    N = length(α)

    # If n=N return 0 (log(1))
    n == N && return 0.0, 0.0

    # Define u_n for xn = ±1
    unm1 = [-(α[n + 1] - β[n]) / 2.0; +(α[n + 1] - β[n]) / 2.0]
    unp1 = [-(α[n + 1] + β[n]) / 2.0; +(α[n + 1] + β[n]) / 2.0]

    # Define u_N
    uN = [-α[end] / 2.0; α[end] / 2.0]

    # If longer gap...
    n == (N - 1) && return log_vec_vec_mult(unm1, uN), log_vec_vec_mult(unp1, uN)

    # Define Wn
    Wnm1 = [-(α[n + 1] - β[n]) / 2.0 + β[n + 1] - α[n + 2] / 2.0 -(α[n + 1] - β[n]) / 2.0 - β[n + 1] + α[n + 2] / 2.0;
            +(α[n + 1] - β[n]) / 2.0 - β[n + 1] - α[n + 2] / 2.0 +(α[n + 1] - β[n]) / 2.0 + β[n + 1] + α[n + 2] / 2.0]
    Wnp1 = [-(α[n + 1] + β[n]) / 2.0 + β[n + 1] - α[n + 2] / 2.0 -(α[n + 1] + β[n]) / 2.0 - β[n + 1] + α[n + 2] / 2.0;
            +(α[n + 1] + β[n]) / 2.0 - β[n + 1] - α[n + 2] / 2.0 +(α[n + 1] + β[n]) / 2.0 + β[n + 1] + α[n + 2] / 2.0]

    # If longer gap...
    n == (N - 2) && return log_vec_vec_mult(log_vec_mat_mult(unm1, Wnm1), uN), log_vec_vec_mult(log_vec_mat_mult(unp1, Wnp1), uN)

    # Define Wn for n'>n+1
    Wns = Vector{Array{Float64,2}}()
    @inbounds for l = (n + 2):(N - 1)
        W = [-α[l] / 2.0 + β[l] - α[l + 1] / 2.0 -α[l] / 2.0 - β[l] + α[l + 1] / 2.0;
             +α[l] / 2.0 - β[l] - α[l + 1] / 2.0 +α[l] / 2.0 + β[l] + α[l + 1] / 2.0]
        push!(Wns, W)
    end

    # Return log g_l(x_l) for x_l∈{-1,+1}
    Wns_prod = mult_log_mats(Wns)
    gl_m1 = log_vec_vec_mult(log_vec_mat_mult(log_vec_mat_mult(unm1, Wnm1), Wns_prod), uN)
    gl_p1 = log_vec_vec_mult(log_vec_mat_mult(log_vec_mat_mult(unp1, Wnp1), Wns_prod), uN)

    # return log unm1'*Wnm1*prod(Wns)*uN,log unp1'*Wnp1*prod(Wns)*uN
    return gl_m1, gl_p1

end
"""
    `gen_x_mc(α,β)`

    Generate an realization of X=[X1,…,XN] using α_n and β_n,or X̄=[X̄1,…,X̄L] if using α_l and β_l.

    # Examples
    ```julia-repl
    julia> CpelNano.gen_x_mc(ones(2),ones(1))
    2-element Array{Int8,1}:
     1
     1
    ```
"""
function gen_x_mc(α::Vector{Float64}, β::Vector{Float64})::Vector{Int8}

    # Size of x
    N = length(α)

    # Initialize x
    x = zeros(Int8, N)

    # Sample first CpG site (y1/(y1+y2))
    g = get_log_g_x(1, α, β)
    log_y1 = g[1] - α[1]
    log_y2 = g[2] + α[1]
    c = max(log_y1, log_y2)
    log_den = c + log(exp(log_y1 - c) + exp(log_y2 - c))
    p = exp(log_y2 - log_den)
    x[1] = rand() < p ? Int8(1) : Int8(-1)
        # print_log("g: $(g)")
        # print_log("p: $(p)")
        # readline()

    # Loop over all CpG sites
    @inbounds for n in 2:(N - 1)

        # Get g
        g = get_log_g_x(n, α, β)
            # g = get_g_x(n,α,β)

        # Get transition probability
        log_y1 = g[1] - α[n] - β[n - 1] * x[n - 1]
        log_y2 = g[2] + α[n] + β[n - 1] * x[n - 1]
        c = max(log_y1, log_y2)
        log_den = c + log(exp(log_y1 - c) + exp(log_y2 - c))
        p = exp(log_y2 - log_den)
            # p = g[2]*exp(α[n]+β[n-1]*x[n-1])
            # p /= (p + g[1]*exp(-α[n]-β[n-1]*x[n-1]))
            # print_log("p: $(p)")

        # Sample n-th CpG site
        x[n] = rand() < p ? Int8(1) : Int8(-1)

        # Print for debug
        # print_log("g: $(g)")
        # print_log("p: $(p)")
        # readline()

    end

    # Sample last CpG site
    log_y1 = -α[N] - β[N - 1] * x[N - 1]
    log_y2 = α[N] + β[N - 1] * x[N - 1]
    c = max(log_y1, log_y2)
    log_den = c + log(exp(log_y1 - c) + exp(log_y2 - c))
    p = exp(log_y2 - log_den)
    x[N] = rand() < p ? Int8(1) : Int8(-1)
        # p = exp(α[N]+β[N-1]*x[N-1])
        # p /= (p + exp(-α[N]-β[N-1]*x[N-1]))
        # x[N] = rand()<p ? Int8(1) : Int8(-1)

    # Return x in final state
    return x

end
##################################################################################################
## Permutation testing simulations
##################################################################################################
"""

    `gen_grp_comp_data(S1,S2,ϕ1,ϕ2,SEED)`

    Simulates data for group comparison.

    # Examples
    ```julia-repl
    julia> CpelNano.gen_grp_comp_data(s1,s2,ϕ1,ϕ2,seed)
    ```
"""
function gen_grp_comp_data(s1::Int64, s2::Int64, ϕ1::Vector{Float64}, ϕ2::Vector{Float64}, nse_sc::Float64, seed::Int64)::Tuple{Vector{RegStruct},Vector{RegStruct}}

    # Configuration
    config = CpelNano.CpelNanoConfig()

    # Random genomic properties
    Random.seed!(seed)
    chrst = 1
    chrend = 3000
    cpg_pos = collect(25:25:2975)
    dn = fill(25.0, length(cpg_pos) - 1)
    ρn = rand(0.001:0.001:0.1, length(cpg_pos))

    ## Region genomic properties

    # Group 2
    mods_g1 = Vector{RegStruct}()
    @inbounds for s in 1:s1

        # Parameter vectors
        ϕ1_s = ϕ1 .+ nse_sc .* [rand(-0.1:0.01:0.1),rand(-0.5:0.01:0.5),rand(0.0:0.01:0.2)]

        # Init structure
        rs1 = RegStruct();
        rs1.cpg_pos = cpg_pos; rs1.N = length(cpg_pos); rs1.chrst = chrst;
        rs1.chrend = chrend; rs1.ϕhat = ϕ1_s; rs1.ρn = ρn; rs1.dn = dn;

        # Re-scale models
        rscle_grp_mod!(rs1)

        # Get statistical summaries
        get_stat_sums!(rs1, config)

        # Push
        push!(mods_g1, rs1)

    end

    # Group 2
    mods_g2 = Vector{RegStruct}()
    @inbounds for s in 1:s2

        # Parameter vectors
        ϕ2_s = ϕ2 .+ nse_sc .* [rand(-0.1:0.01:0.1),rand(-0.5:0.01:0.5),rand(0.0:0.01:0.2)]

        # Init structure
        rs2 = RegStruct();
        rs2.cpg_pos = cpg_pos; rs2.N = length(cpg_pos); rs2.chrst = chrst;
        rs2.chrend = chrend; rs2.ϕhat = ϕ2_s; rs2.ρn = ρn; rs2.dn = dn;

        # Re-scale model
        rscle_grp_mod!(rs2)

        # Get statistical summaries
        get_stat_sums!(rs2, config)

        # Push
        push!(mods_g2, rs2)

    end

    # Return tuple of arrays
    return mods_g1, mods_g2

end
###################################################################################################################
# Simulate 1. X̄~q(x̄;ϕ) 2. Y|X̄ ~ 𝒩(μ(X̄),σ(X̄))
###################################################################################################################
"""
    `cpel_samp_ont(m,Nl,αs,βs,pobs,μ_m,σ_m,μ_u,σ_u,𝒜s,ℬs)`

    Simulate entire model where observable is distributed according to a Gaussian distribution. The same
    Gaussian distributions are assumed regardless of the k-mer. The mean is given in terms of mA. Here we
    assume that we are passed

        α_l = (a + b⋅ρ̄_l)⋅N_l ,β_l = c/d_l

    where ρ̄_l and N_l is the average density and the number of CpG sites in the l-th group,respectively.

    # Examples
    ```julia-repl
    julia> L = 10; M = 25; αs = fill(0.0,L); βs = fill(0.0,L-1);
    julia> pobs = 1.0; μ_m = 60.0; σ_m = 1.0; μ_u = 70.0; σ_u = 1.0;
    julia> data = CpelNano.cpel_samp_ont(M,αs,βs,pobs,μ_m,σ_m,μ_u,σ_u);
    ```
"""
function cpel_samp_ont(m::Int64,α::Vector{Float64},β::Vector{Float64},pobs::Float64,μ_m::Float64,σ_m::Float64,
    μ_u::Float64,σ_u::Float64)::RegStruct

    # Distributions p(y|x̄)
    p_y_x_m = Normal(μ_m, σ_m)
    p_y_x_u = Normal(μ_u, σ_u)

    # Generate calls
    L = length(α)
    calls = Vector{Vector{MethCallCpgGrp}}()
    while length(calls) < m
        # Init call vector
        call = [MethCallCpgGrp() for l = 1:L]
        # Get x̄ realization
        xcpel = gen_x_mc(α, β)
        # Fill call vector
        for l in 1:L
            # Check if group is observed
            rand() < pobs || continue
            # Methylation call (convert to bool)
            xbar = xcpel[l] == 1
            # Sample normal observable
            y = xbar ? rand(p_y_x_m) : rand(p_y_x_u)
            # Compute log p(y_l|x̄_l)
            log_pyx_u = log(pdf(p_y_x_u, y))
            log_pyx_m = log(pdf(p_y_x_m, y))
            # Set call for x̄_l
            call[l].obs = true
            call[l].log_pyx_u = log_pyx_u
            call[l].log_pyx_m = log_pyx_m
        end
        push!(calls, call)
    end

    # Create simulation structure
    sim_struct = CpelNano.RegStruct(calls)

    # Return data object
    return sim_struct

end
"""
    `cpel_samp_wgbs(m,Nl,αs,βs,pobs,μ_m,σ_m,μ_u,σ_u,𝒜s,ℬs)`

    Simulate entire model where observable is distributed according to a Gaussian distribution. The same
    Gaussian distributions are assumed regardless of the k-mer. The mean is given in terms of mA. Here we
    assume that we are passed

        α_l = (a + b⋅ρ̄_l)⋅N_l ,β_l = c/d_l

    where ρ̄_l and N_l is the average density and the number of CpG sites in the l-th group,respectively.

    # Examples
    ```julia-repl
    julia> L = 20; M = 20; αs = fill(0.0,L); βs = fill(0.0,L-1); pobs = 0.5;
    julia> data = CpelNano.cpel_samp_wgbs(M,αs,βs,pobs);
    ```
"""
function cpel_samp_wgbs(m::Int64, α::Vector{Float64}, β::Vector{Float64}, pobs::Float64)::RegStruct

    # Generate calls
    L = length(α)
    calls = Vector{Vector{MethCallCpgGrp}}()
    while length(calls) < m
        # Init call vector
        call = [MethCallCpgGrp() for l = 1:L]
        # Init obs counter
        obs_count = 15
        # Get x̄ realization
        xcpel = gen_x_mc(α, β)
        # Fill call vector
        for l in 1:L
            # Check if group is observed
            if rand() < pobs && obs_count >= 15
                obs_count = 0
            end
            # Increase obs counter
            obs_count += 1
            obs_count < 10 || continue
            # Methylation call (convert to bool)
            xbar = xcpel[l] == 1
            # Compute log p(y_l|x̄_l)
            log_pyx_u = xbar ? log_pyx_wrong_x : log_pyx_right_x
            log_pyx_m = xbar ? log_pyx_right_x : log_pyx_wrong_x
            # Set call for x̄_l
            call[l].obs = true
            call[l].log_pyx_u = log_pyx_u
            call[l].log_pyx_m = log_pyx_m
        end
        push!(calls, call)
    end

    # Create simulation structure
    sim_struct = CpelNano.RegStruct(calls)

    # Return data object
    return sim_struct

end
"""
    `bin_calls!(RS)`

    Binarizes calls.

    # Examples
    ```julia-repl
    julia> CpelNano.bin_calls!(rs)
    ```

"""
function bin_calls!(rs::CpelNano.RegStruct)::Nothing

    # Loop over calls
    for call in rs.calls
        # Loop over sites
        for meth_call in call
            # If observed
            if meth_call.obs
                if meth_call.log_pyx_u >= meth_call.log_pyx_m
                    meth_call.log_pyx_u = log_pyx_right_x
                    meth_call.log_pyx_m = log_pyx_wrong_x
                else
                    meth_call.log_pyx_u = log_pyx_wrong_x
                    meth_call.log_pyx_m = log_pyx_right_x
                end
            end
        end
    end

    # Return nothing
    return nothing

end
##################################################################################################
## Metropolis Hastings (NOT USED)
##################################################################################################
"""
    `init_lattice(lat_length)`

     Randomly initializes vector of length `lat_length` with 1 and -1 representing methylated
     and unmethylated CGs respectively.

    # Examples
    ```julia-repl
    julia> CpelNano.init_lattice(2)
    2-element Array{Int8,1}:
     -1
     -1
    ```
"""
function init_lattice(lat_length::Int64)::Vector{Int8}

    # Return initialized lattice
    return rand([Int8(-1),Int8(1)], lat_length)

end
"""
    `get_ΔU(lattice,i,α,β)`

     Computes the change in potential energy U(X) by flipping the methylation state of the `i`th CG
     in a lattice with vectors `[α1,α2,…,αN]` and `[β1,β2,…,β_{N-1}]`.

    # Examples
    ```julia-repl
    julia> CpelNano.get_ΔU([Int8(1),Int8(1)],2,[0.0,0.0],[0.0])
    0.0
    ```
"""
function get_ΔU(lattice::Vector{Int8}, i::Int64, α::Vector{Float64}, β::Vector{Float64})::Float64

    # Modified lattice
    mod_lattice = copy(lattice)
    mod_lattice[i] = -mod_lattice[i]

    # Original contribution of i-th CpG site
    U = -α[i] * lattice[i]
    if i == 1
        U -= β[1] * lattice[i] * lattice[i + 1]
    elseif i == length(lattice)
        U -= β[end] * lattice[i] * lattice[i - 1]
    else
        U -= β[i - 1] * lattice[i - 1] * lattice[i] + β[i] * lattice[i] * lattice[i + 1]
    end

    # Original contribution of i-th CpG site
    Uflip = -α[i] * mod_lattice[i]
    if i == 1
        Uflip -= β[1] * mod_lattice[i] * mod_lattice[i + 1]
    elseif i == length(mod_lattice)
        Uflip -= β[end] * mod_lattice[i] * mod_lattice[i - 1]
    else
        Uflip -= β[i - 1] * mod_lattice[i - 1] * mod_lattice[i] + β[i] * mod_lattice[i] * mod_lattice[i + 1]
    end

    # Note: if ΔU>0 → new config is less likely under θ
# Return ΔU
    return Uflip - U

end
"""
    `metro_hast_inst(lat_length,n_iters,α,β)`

    Run MCMC for `n_iters` iterations on a lattice of length `lat_length` with vectors `[α1,α2,…,αN]`
    and `[β1,β2,…,β_{N-1}]`.

    # Examples
    ```julia-repl
    julia> CpelNano.metro_hast_inst(2,1000,[1.0,1.0],[1.0])
    2-element Array{Int8,1}:
     1
     1
    ```
"""
function metro_hast_inst(lat_length::Int64, n_iters::Int64, α::Vector{Float64}, β::Vector{Float64})::Vector{Int8}

    # Initialize lattice
    lattice = init_lattice(lat_length)

    # Iterate til convergence of MC chain
    @inbounds for i in 1:n_iters
        @inbounds for j in 1:lat_length
            # Get energy Hamiltonian differential
            ΔU = get_ΔU(lattice, j, α, β)
            # If flipping leads to lower energy,flip
            if ΔU < 0
                lattice[j] = -lattice[j]
            else
                # If flipping leads to ΔU>0,flip based on weighted likelihood from Boltzmann distribution
                if rand() < exp(-ΔU)
                    lattice[j] = -lattice[j]
                end
            end
        end
    end

    # Return lattice in final state
    return lattice

end


# This file was generated, do not modify it. # hide
cap_mean = combine(cap_gr, :capacity_mw => mean)
cap_sum = combine(cap_gr, :capacity_mw => sum)
first(cap_sum, 3)


struct TextTree
    children::Vector{Vector{Int}}
    text::Vector{String}
end

function TextTree(D::DESPOT)
    lenb = length(D.children)
    lenba = length(D.ba_children)
    len = lenb + lenba
    children = Vector{Vector{Int}}(len)
    text = Vector{String}(len)
    for b in 1:lenb
        children[b] = D.children[b] .+ lenb
        text[b] = @sprintf("o:%-5s U:%6.2f, l:%6.2f, μ:%6.2f, l₀:%6.2f, |Φ|:%3d",
                           b==1 ? "<root>" : string(D.obs[b]),
                           D.U[b],
                           D.l[b],
                           D.mu[b],
                           D.l_0[b],
                           length(D.scenarios[b]))
    end
    for ba in 1:lenba
        children[ba+lenb] = D.ba_children[ba]
        text[ba+lenb] = @sprintf("a:%-5s l:%6.2f μ:%6.2f, ρ:%6.2f", D.ba_action[ba], ba_l(D, ba), D.ba_mu[ba], D.ba_rho[ba])
    end
    return TextTree(children, text)
end

struct TreeView
    t::TextTree
    root::Int
    depth::Int
end

TreeView(D::DESPOT, b::Int, depth::Int) = TreeView(TextTree(D), b, depth)

Base.show(io::IO, tv::TreeView) = shownode(io, tv.t, tv.root, tv.depth, "", "")

function shownode(io::IO, t::TextTree, n::Int, depth::Int, item_prefix::String, prefix::String)
    print(io, item_prefix)
    print(io, @sprintf("[%-4d]", n))
    print(io, " $(t.text[n])")
    if depth <= 0
        println(io, " ($(length(t.children[n])) children)")
    else
        println(io)
        if !isempty(t.children[n])
            for c in t.children[n][1:end-1]
                shownode(io, t, c, depth-1, prefix*"├──", prefix*"│  ")
            end
            shownode(io, t, t.children[n][end], depth-1, prefix*"└──", prefix*"   ")
        end
    end
end


import Base: dotview, view, unsafe_view, copyto!, eachindex, _maybe_reshape_parent, reshape, to_shape, SubArray, compute_stride1, axes, check_parent_index_match, IndexStyle, LinearIndices
using Base: unalias, index_ndims, @_inline_meta, @boundscheck, ViewIndex, OneTo, rdims, viewindexing, ensure_indexable, index_dimsum, fill_to_length
using Base.Broadcast: Broadcasted

### k[1:2,3:4] .= 0 => materialize!(dotview(k,1:2,3:4),broadcasted(identity,0))

# The following adapted from base: views.jl broadcast.jl subarray.jl
# Much of this will be unnecessary if we can inherit from AbstractArray

dotview(A::KnetArray,I...) = view(A,I...)

function view(A::KnetArray, I::Vararg{Any,N}) where {N}
    @_inline_meta
    J = map(i->unalias(A,i), to_indices(A, I))
    #TODO: @boundscheck checkbounds(A, J...)
    unsafe_view(_maybe_reshape_parent(A, index_ndims(J...)), J...)
end

function unsafe_view(A::KnetArray, I::Vararg{ViewIndex,N}) where {N}
    @_inline_meta
    SubArray(A, I)
end

eachindex(::IndexLinear, A::KnetArray) = (@_inline_meta; OneTo(length(A)))
LinearIndices(A::KnetArray) = LinearIndices(axes(A))

_maybe_reshape_parent(A::KnetArray, ::NTuple{1, Bool}) = reshape(A, Val(1))
_maybe_reshape_parent(A::KnetArray{<:Any,1}, ::NTuple{1, Bool}) = reshape(A, Val(1))
_maybe_reshape_parent(A::KnetArray{<:Any,N}, ::NTuple{N, Bool}) where {N} = A
_maybe_reshape_parent(A::KnetArray, ::NTuple{N, Bool}) where {N} = reshape(A, Val(N))

reshape(parent::KnetArray{T,N}, ndims::Val{N}) where {T,N} = parent
reshape(parent::KnetArray, ndims::Val{N}) where N = reshape(parent, rdims(Val(N), axes(parent)))
reshape(parent::KnetArray, shp::Tuple{Union{Integer,OneTo}, Vararg{Union{Integer,OneTo}}}) = reshape(parent, to_shape(shp))

function SubArray(parent::KnetArray, indices::Tuple)
    @_inline_meta
    SubArray(IndexStyle(viewindexing(indices), IndexStyle(parent)), parent, ensure_indexable(indices), index_dimsum(indices...))
end

IndexStyle(::KnetArray)=IndexLinear()

compute_stride1(parent::KnetArray, I::NTuple{N,Any}) where {N} =
    (@_inline_meta; compute_stride1(1, fill_to_length(axes(parent), OneTo(1), Val(N)), I))

function axes(A::KnetArray{T,N}, d) where {T,N}
    @_inline_meta
    d <= N ? axes(A)[d] : OneTo(1)
end

check_parent_index_match(parent::KnetArray{T,N}, ::NTuple{N, Bool}) where {T,N} = nothing

# dotview(P::KnetArray{T,N},I...) where {T,N} =SubArray{T,N,typeof(P),typeof(I),false}(P,I,0,0)
# check_parent_index_match(parent::KnetArray{T,N}, ::NTuple{N, Bool}) where {T,N} = nothing


#https://www.sciencedirect.com/topics/mathematics/euclidean-distance#:~:text=The%20weighted%20Euclidean%20distance%20measure,element%20of%20the%20average%20profile.

dispersion = abs(a + b)
@propagate_inbounds function EuclidDist(a, b) #sumInterval subtracts 1 (complies with julia logic)
    cond = dispersion - 1
    @inbounds if cond >= 1
        return cond
    end
end

@benchmark sumInterval(1, 10)


module PredictiveControl

################################################################################
# Main exports from this package
################################################################################
export ConstrainedTimeInvariantLQR,
       PrimalFullCondensing, PrimalNoCondensing, DualFullCondensing

# Analysis functions
export conditionBound

# Preconditioners
export circulantblockpreconditioner, sdpblockpreconditoner

################################################################################
# Packages we need
################################################################################
using BlockArrays
using ControlSystems
using ControlSystems: nstates, ninputs
using COSMO
using FFTW
using JuMP
using LinearAlgebra
using Logging
using MathOptInterface
using MatrixEquations
using OSQP

const MOI = MathOptInterface

# Overload these functions for our own types
import LinearAlgebra: cond

include( "utilities.jl" )

include( "types/problemTypes.jl" )
include( "types/clqr.jl" )

include( "matrices/fullCondensing.jl" )
include( "matrices/preconditioning.jl" )

include( "analysis/conditioning.jl")

# Solvers
include( "solvers/fastGradientMethod/fgm.jl" )
using .FastGradientMethod
const FGM = FastGradientMethod

export FGM, fastgradientmethod

# Solver interfaces
include( "solvers/interfaces/fgm_interface.jl" )
include( "solvers/interfaces/jump_interface.jl" )
include( "solvers/interfaces/osqp_interface.jl" )

end # module PredictiveControl


using POMDPs, StatsBase, POMDPToolbox, SARSOP, QMDP, AutomotiveDrivingModels, JLD, Parameters, Distributions

include("../../admenv/crosswalk/occluded_crosswalk_env.jl")
include("../../admenv/crosswalk/pomdp_types.jl")
include("../../admenv/crosswalk/spaces.jl")
include("../../admenv/crosswalk/transition.jl")
include("../../admenv/crosswalk/observation.jl")
include("../../admenv/crosswalk/belief.jl")
include("../../admenv/crosswalk/adm_helpers.jl")
include("../../admenv/crosswalk/decomposition.jl")

include("config.jl")
include("ego_control.jl")
include("sensor.jl")
include("policy.jl")
include("state_estimation.jl")
include("constant_pedestrian.jl")
include("simulation.jl")
include("baseline_policy.jl")

config = EvalConfig(time_out = 200,
                    n_ped=2,
                    n_episodes = 3)

params = EnvParams(ped_rate = 0.9)
env = CrosswalkEnv(params)

pomdp = OCPOMDP()
pomdp.collision_cost = float(ARGS[1])

solver = SARSOPSolver(randomization=true, precision = 0.001, timeout = 3600.)

policy = solve(solver, pomdp,
               SARSOP.create_policy(solver, pomdp, "crosswalk$(abs(pomdp.collision_cost)).policy"),
               "model$(round(abs(pomdp.collision_cost))).pomdpx" )


struct Features
    """
    A type for holding all the fixed quantities during a simulation. This involves
    simulation constants for detection probabiliites, constraints for consistency
    with data and some data based measures like the local cases population size,
    duration of sims and the length of the observation perod.
    """
    # constraints for the simulation
    max_forecast_cases::Int
    N::Int
    T_observed::Int
    T_end::Int
    # number of days for each consistency check
    τ::Int
    min_cases::Vector{Int}
    max_cases::Vector{Int}
    idxs_limits::Vector{UnitRange{Int64}}
    state::String
    # truncations for the various bits and pieces
    omicron_start_day::Int
    omicron_only_day::Int
end

mutable struct Realisation
    """
    A type for holding the realisations of the simulations. This is a cleaner way of
    holding the information for the three different matrices used. Z is for infections
    D is for observed cases and U is for undetected cases.
    """
    Z::Array{Int}
    Z_historical::Array{Int}
    D::Array{Int}
    U::Array{Int}

    function Realisation(
        sim_duration,
    )
        """
        Initialising a state array object with arrays of zeros.
        We pad the infection array Z with 35 days to account for
        infections occuring prior to the simulation period. We do this
        separately (and not in a struct) as the arrays are large.
        """

        Z = zeros(Int, sim_duration + 30, 3)
        Z_historical = zeros(Int, sim_duration + 30)
        D = zeros(Int, sim_duration, 3)
        U = zeros(Int, sim_duration, 3)

        return new(Z, Z_historical, D, U)
    end
end

mutable struct Results
    """
    A type for holding the realisations of the simulations. This is a cleaner way of
    holding the information for the three different matrices used. Z is for infections
    D is for observed cases and U is for undetected cases.
    """
    Z::Array{Int}
    Z_historical::Array{Int}
    D::Array{Int}
    U::Array{Int}

    function Results(
        sim_duration,
        nsims,
    )
        """
        Initialising a state array object with arrays of zeros.
        We pad the infection array Z with 35 days to account for
        infections occuring prior to the simulation period. We do this
        separately (and not in a struct) as the arrays are large.
        """

        Z = zeros(Int, sim_duration + 30, 3, nsims)
        Z_historical = zeros(Int, sim_duration + 30, nsims)
        D = zeros(Int, sim_duration, 3, nsims)
        U = zeros(Int, sim_duration, 3, nsims)

        return new(Z, Z_historical, D, U)
    end
end

struct Constants{S, T}
    """
    A type for the dynamical / simulation constants
    """
    p_detect::S
    p_symp::S
    p_detect_given_symp::S
    p_detect_given_asymp::S
    p_symp_given_detect::S
    α_s::S
    α_a::S
    p_detect_import::S
    prior_alpha::T
    prior_beta::T
    ϕ::T
    consistency_multiplier::T

    function Constants(start_date, end_date)
        """
        Constructor for the older version of the model. This assumes a simpler form of the CAR and
        doesn't have jurisdiction dependent assumptions in relation to the CAR.
        """

        # date vectors to indicate which period we're in. Noting that this is +5 days (mean inc
        # period) compared to the ranges used in the fitting. This is to deal with detection being
        # related to detection of cases and not infection dates.
        CAR_normal_before = Dates.Date(start_date):Dates.Day(1):Dates.Date("2021-12-09")
        CAR_low = Dates.Date("2021-12-10"):Dates.Day(1):Dates.Date(end_date)
        # all dates
        CAR_dates = [
            CAR_normal_before;
            CAR_low
        ]

        # initialise the
        CAR = 0.75 * ones(Float64, length(CAR_dates))
        # idx = CAR_dates .>= Dates.Date("2021-12-10")
        idx = CAR_dates .>= CAR_low[begin]
        CAR[idx] .= 0.5

        p_symp = zeros(Float64, length(CAR_dates))
        p_detect_given_asymp = zeros(Float64, length(CAR_dates))
        p_detect_given_symp = zeros(Float64, length(CAR_dates))
        p_symp_given_detect = zeros(Float64, length(CAR_dates))
        p_detect_import = zeros(Float64, length(CAR_dates))

        for (i, d) in enumerate(CAR_dates)
            p_detect = CAR[i]

            if d <= Dates.Date("2021-11-01")
                p_symp[i] = 0.7
            else
                p_symp[i] = 0.4
            end

            if d <= Dates.Date("2021-11-01")
                # p_symp = 0.7 and p_d = 0.75
                p_detect_given_symp[i] = 0.9375
                p_detect_given_asymp[i] = 0.3125
                p_detect_import[i] = 0.98
            elseif d <= Dates.Date("2021-12-10")
                # p_symp = 0.4 and p_d = 0.75
                p_detect_given_symp[i] = 0.9
                p_detect_given_asymp[i] = 0.65
                p_detect_import[i] = p_detect
            else
                # p_symp = 0.4 and p_d = 0.5 noting that the line defined by
                # p(d|s) = -p(a)/p(s) p(d|a) + p(d)/p(s)
                # translates vertically when the CA changes and so we just find the point on
                # the reduced CA line which is closest to the previous values
                # (p(d|a), p(d|s)) = (0.65, 0.9)
                p_detect_given_asymp[i] = 0.3615
                p_detect_given_symp[i] = 0.7076
                p_detect_import[i] = p_detect
            end

            p_symp_given_detect[i] = p_detect_given_symp[i] * p_symp[i] / p_detect

        end

        p_detect = CAR

        γ = 0.5     # relative infectiousness of asymptomatic
        α_s = 1 ./ (p_symp .+ γ * (1 .- p_symp))    # contribution of TP attributable to symptomatic
        α_a = γ * α_s   # contribution of TP attributable to asymptomatic

        # for checking consistency when we miss an outbreak
        consistency_multiplier = 5.0
        # prior parametes for the import model
        prior_alpha = 0.5
        prior_beta = 0.1
        # ema smoothing factor for the import model
        ϕ = 0.5

        # use this to ensure the Constants type is correct
        S = typeof(p_symp)
        T = typeof(prior_alpha)

        return new{S, T}(
            p_detect,
            p_symp,
            p_detect_given_symp,
            p_detect_given_asymp,
            p_symp_given_detect,
            α_s,
            α_a,
            p_detect_import,
            prior_alpha,
            prior_beta,
            ϕ,
            consistency_multiplier,
        )
    end

    function Constants(start_date, end_date, state)
        """
        Newer version constructor. This is distinguished from the other by the extra parameter
        `state` which enables us to produce jurisdiction specific scenarios.
        """
        # date vectors to indicate which period we're in
        CAR_normal_before = Dates.Date(start_date):Dates.Day(1):Dates.Date("2021-12-07")
        CAR_low = Dates.Date("2021-12-08"):Dates.Day(1):Dates.Date("2022-01-17")
        CAR_normal_after = Dates.Date("2022-01-18"):Dates.Day(1):Dates.Date(end_date)
        # all dates
        CAR_dates = [
            CAR_normal_before;
            CAR_low;
            CAR_normal_after
        ]

        # initialise the
        CAR = 0.75 * ones(Float64, length(CAR_dates))

        # if we are one of the jurisdictions that got overwhelmed, apply time varying CAR
        if state in ("NSW", "VIC", "ACT", "QLD")
            idx = (CAR_dates .>= CAR_low[begin]) .& (CAR_dates .<= CAR_low[end])
            CAR[idx] .= 0.333
            idx = CAR_dates .> CAR_low[end]
            CAR[idx] .= 0.75
        end

        p_symp = zeros(Float64, length(CAR_dates))
        p_detect_given_asymp = zeros(Float64, length(CAR_dates))
        p_detect_given_symp = zeros(Float64, length(CAR_dates))
        p_symp_given_detect = zeros(Float64, length(CAR_dates))
        p_detect_import = zeros(Float64, length(CAR_dates))

        for (i, d) in enumerate(CAR_dates)
            p_detect = CAR[i]

            if d <= Dates.Date("2021-11-01")
                p_symp[i] = 0.7
            else
                p_symp[i] = 0.4
            end

            if d <= Dates.Date("2021-11-01")
                # p_symp = 0.7 and p_d = 0.75
                p_detect_given_symp[i] = 0.9375
                p_detect_given_asymp[i] = 0.3125
                p_detect_import[i] = 0.98
            elseif d <= Dates.Date("2021-12-07") || d >= Dates.Date("2022-01-18")
                # p_symp = 0.4 and p_d = 0.75
                p_detect_given_symp[i] = 0.9
                p_detect_given_asymp[i] = 0.65
                p_detect_import[i] = p_detect
            else
                # p_symp = 0.4 and p_d = 0.333
                p_detect_given_asymp[i] = 0.16885
                p_detect_given_symp[i] = 0.57923
                p_detect_import[i] = p_detect
            end

            p_symp_given_detect[i] = p_detect_given_symp[i] * p_symp[i] / p_detect

        end

        p_detect = CAR

        γ = 0.5     # relative infectiousness of asymptomatic
        α_s = 1 ./ (p_symp .+ γ * (1 .- p_symp))
        α_a = γ * α_s

        # other parameters
        consistency_multiplier = 5.0
        # prior parametes for the import model
        prior_alpha = 0.5
        prior_beta = 0.1
        # ema smoothing factor
        ϕ = 0.5

        S = typeof(p_symp)
        T = typeof(prior_alpha)

        return new{S, T}(
            p_detect,
            p_symp,
            p_detect_given_symp,
            p_detect_given_asymp,
            p_symp_given_detect,
            α_s,
            α_a,
            p_detect_import,
            γ,
            prior_alpha,
            prior_beta,
            ϕ,
            consistency_multiplier,
        )
    end
end

struct IndividualTypeMap
    S::Int
    A::Int
    I::Int

    function IndividualTypeMap()
        """
        Constructor for the mapping between type symbols S, A, I and integers.
        """
        S = 1
        A = 2
        I = 3

        return new(S, A, I)

    end
end

struct Forecast
    """
    This type wraps all the information relating to the forecast in a single accessible
    object. This drastically simplifies the layout of the code at no performance hit.
    """
    sim_features::Features
    sim_realisation::Realisation
    sim_results::Results
    sim_constants::Constants
    individual_type_map::IndividualTypeMap
end

struct JurisdictionAssumptions
    """
    This type holds all the assumptions for each jurisdiction. This
    involves the assumed starting dates, population sizes and initial conditions. It also
    includes key dates we don't have stored elsewhere.
    """
    simulation_start_dates::Dict{String, String}
    pop_sizes::Dict{String, Int}
    initial_conditions::Dict{
        String,
        NamedTuple{(:S, :A, :I), Tuple{Int64, Int64, Int64}},
    }
    omicron_start_date::Date
    omicron_dominant_date::Date

    function JurisdictionAssumptions()
        simulation_start_dates = Dict{String, String}(
            "NSW" => "2021-06-01",
            "QLD" => "2021-11-20",
            "SA" => "2021-11-01",
            "TAS" => "2021-11-01",
            "WA" => "2021-12-15",
            "ACT" => "2021-12-01",
            "NT" => "2021-12-01",
            "VIC" => "2021-08-01",
        )

        # date we want to apply increase in cases due to Omicron
        # put a small delay on this as we use the infection dates to index things but in
        # the fitting we used the confirmation dates
        omicron_start_date = Dates.Date("2021-11-15")
        omicron_dominant_date = Dates.Date("2021-12-15") - Dates.Day(5)

        pop_sizes = Dict{String, Int}(
            "NSW" => 8189266,
            "QLD" => 5221170,
            "SA" => 1773243,
            "TAS" => 541479,
            "VIC" => 6649159,
            "WA" => 2681633,
            "ACT" => 432266,
            "NT" => 246338,
        )

        initial_conditions = Dict{
            String, NamedTuple{(:S, :A, :I), Tuple{Int64, Int64, Int64}}
        }(
            "NSW" => (S = 0, A = 0, I = 0),
            "QLD" => (S = 1, A = 1, I = 0),
            "SA" => (S = 0, A = 0, I = 0),
            "TAS" => (S = 0, A = 0, I = 0),
            "VIC" => (S = 5, A = 5, I = 0),
            "WA" => (S = 0, A = 0, I = 0),
            "ACT" => (S = 12, A = 10, I = 0),
            "NT" => (S = 0, A = 0, I = 0),
        )

        return new(
            simulation_start_dates,
            pop_sizes,
            initial_conditions,
            omicron_dominant_date,
        )

    end
end


module FerriteViz

using Makie
using Tensors
import Ferrite
import LinearAlgebra

abstract type AbstractPlotter end

include("utils.jl")
include("makieplotting.jl")

export MakiePlotter
export ferriteviewer

end


using Pythia
pythia = newPythia()

icxx"""$pythia->readString("Beams:idA = 11");"""
icxx"""$pythia->readString("Beams:idB = -11");"""
icxx"""$pythia->readString("Beams:eCM = 10.");"""
icxx"""$pythia->readString("WeakSingleBoson:ffbar2ffbar(s:gm) = on");"""
icxx"""$pythia->init();"""

# Check if this is a e⁺e⁻ -> qqbar process
function isqqbar(e)
    # Look through the particles to find those in the hard process
    for p in particles(e)
        if abs(icxx"$p.status();") == 23
            id = icxx"$p.id();"
            # quarks
            return 1 <= abs(id) <= 5
        end
    end
end

count_hadrons(e) = sum([icxx"$p.isHadron();" for p in particles(e)])

energies = linspace(10,120,6)
nhadrons = map(energies) do eCM
   pythia = newPythia()

   icxx"""$pythia->readString("Beams:idA = 11");"""
   icxx"""$pythia->readString("Beams:idB = -11");"""
   readString(pythia,"Beams:eCM = $eCM")
   icxx"""$pythia->readString("WeakSingleBoson:ffbar2ffbar(s:gm) = on");"""
   icxx"""$pythia->init();"""

   mean(filter(x->x != -1, map(e->isqqbar(e) ? count_hadrons(e) : -1, take(events(pythia),1000))))
end
plot(x = energies, y = nhadron, Geom.point, Geom.smooth(method=:loess,smoothing=0.9), Guide.xlabel("√s [GeV]"), Guide.ylabel("⟨Nhadron⟩"))


abstract type MTDSModel end

# Common interface
is_amortized(m::MTDSModel) = error(format("It's not known whether {:s} uses amortized inference. "*
 "Please implement `is_amortized(m)` for this type.", get_strtype_wo_params(m)))
has_x0_encoding(m::MTDSModel) = error(format("It's not known whether {:s} uses an x0 (recurrent "*
 "state) encoder. Please implement `has_x0_encoding(m)` for this type.", get_strtype_wo_params(m)))

forward(m::MTDSModel, x0, z, U; T_steps=size(U, 2)) =
    error(format("Not Implemented Yet: {:s}. Please implement a forward proc (given z).", string(typeof(m))))
encode(m::MTDSModel, Y, U; kl_coeff=1f0, stochastic=true, logstd_prior=nothing, T_enc=size(U, 2)) =
    error(format("Not Implemented Yet: {:s}. Please implement an encode proc.", string(typeof(m))))

function reconstruct(m::MTDSModel, Y, U; T_steps=size(Y, 2), T_enc=T_steps, stochastic=true)
    @argcheck size(Y,2) == size(U,2)
    (smpz, μ_z, σ_z), (smpx0, μ_x0, σ_x0) = encode(m, Y, U; T_enc=T_enc, stochastic=stochastic)
    forward(m, smpx0, smpz, U; T_steps=T_steps)
end

function forward_multiple_z(m::MTDSModel, z, u::AbstractMatrix; maxbatch=200)
    M = size(z, 2)
    ŷs = map(Iterators.partition(1:M, maxbatch)) do cbatch
        u_rep = repeat(u, outer=(1, 1, length(cbatch)));
        m(z[:, cbatch], u_rep)
    end
    cat(ŷs..., dims=3)   # problematic if length(ŷs) is too large, but there's no nice `reduce` version in Julia yet.
end
forward_multiple_z(m::MTDSModel, z, u::AbstractVector; maxbatch=200) = forward_multiple_z(m, z, unsqueeze(u,1); maxbatch=maxbatch)

do_importance_smp(m::MTDSModel, y, u, tT=80, M=200) = error(format("Not implemented yet for model type: {:s}", get_strtype_wo_params(m)))

nllh(m::MTDSModel, args...; kwargs...) = error(format("Not implemented yet for model type: {:s}", get_strtype_wo_params(m)))
nllh_of_recon_batchavg(m::MTDSModel, args...; kwargs...) = error(format("Not implemented yet for model type: {:s}", get_strtype_wo_params(m)))
elbo(m::MTDSModel, args...; kwargs...) = error(format("Not implemented yet for model type: {:s}", get_strtype_wo_params(m)))
elbo_w_kl(m::MTDSModel, args...; kwargs...) = error(format("Not implemented yet for model type: {:s}", get_strtype_wo_params(m)))

check_inputs_outputs(m::MTDSModel, Ys, Us) = error(format("Not implemented yet for model type: {:s}", get_strtype_wo_params(m)))


using Test
using Printf
using FileIO

# ScatterPatch depends on this
include("../src/image.jl")
include("../src/synthResult.jl")

# the tested code
include("../src/scatterPatch.jl")

include("tinyImage.jl")

@testset "ScatterPatch" begin

    # This mimics the startup of Resynthesizer
    testImage = createTestMaskedImage()
    testSynthResult = initialSynthResult(testImage)
    testSortedOffsets = createSortedOffsets(testImage.image)

    @debug "Sorted offset" testSortedOffsets

    # scatter patch distinguished point (origin) is upper left of synth
    testDistinguishedPoint = PointInMaskedImage( CartesianIndex(3,2), testImage)

    testScatterPatch = prepareScatterPatch(testDistinguishedPoint, testSynthResult, testSortedOffsets)

    # Original C algorithm.
    # The patchPoint origin is in the ScatterPatch (at offset 0,0)
    # We also know it should be the first element, since ordered by offset
    # @test any(neighbor.offset == CartesianIndex(0,0) for neighbor in testScatterPatch))
    #@test testScatterPatch[1].offset == CartesianIndex(0,0)

    # The patchPoint origin *not* in the ScatterPatch (at offset 0,0)
    @test ! any( neighbor.offset == CartesianIndex(0,0) for neighbor in testScatterPatch )

    # The point above  the origin is in the ScatterPatch ( at offset -1,0)
    # We know its the third element.
    # More generally, use any()
    @test testScatterPatch[2].offset == CartesianIndex(-1,0)

    # The point below the origin is not in the scatterPatch
    # because it is in the synth and has no value (not synthesized)
    @test ! any( neighbor.offset == CartesianIndex(1,0) for neighbor in testScatterPatch )

    # The point left of the origin is in the scatterPatch
    # because it is in the context and has a value
    @test any( neighbor.offset == CartesianIndex(0, -1) for neighbor in testScatterPatch )

    # The points are distinct
    # Specifically, the origin is not repeated
    # TODO

end


module RoundingEmulator

export add_up, add_down, sub_up, sub_down, mul_up, mul_down, div_up, div_down, sqrt_up, sqrt_down

include("rounding.jl")

end # module


module CorneaDetection

using ImageIO,VideoIO
using Images,ImageFiltering,ImageTransformations,Interpolations
using Optim,Statistics,ProgressMeter,StatsBase
using JLD2,CSV,DataFrames

import Base: dirname, length, isempty, getindex, iterate, get, fullname, summary, show
export findpurkinje,detect,detectfolder
export resize,summarize,intensity,darkframes,goodframes,makemovie
export Subject,Visit,Trial,subject,visit,trial,ImageFolder,images
export dataroot,dirname,fullname,filenames,shortname
export numvisits,numtrials,numframes,length,visits,trials,summary,results
export saveopt

include("utils.jl")
include("parameters.jl")
include("summarize.jl")
include("optimize.jl")
include("detect.jl")
include("show.jl")
include("interface.jl")

end # module


using Test
using Dates
using ExtensibleScheduler
using TimeFrames

@testset "TimeFrameTrigger" begin
    tf = tf"5Min"
    trigger = Trigger(tf)

    n_executed = 0

    function sample_action()
        n_executed += 1
    end

    dt_start = DateTime(2010, 1, 3, 23, 58)
    dt = dt_start
    clock = SimClock(dt)
    sched = BlockingScheduler(clock=clock, delayfunc=NoSleep)
    action = Action(sample_action)

    add(sched, action, trigger)
    run_pending(sched)
    @test n_executed == 0

    for i in 1:13
        dt += Dates.Minute(1)
        #println("Clock set to $dt")
        set(clock, dt)
        run_pending(sched)
    end

    @test n_executed == 3

end


# Support for the DVS128 camera.

const DVS128_SIZE = (128,128)

""" Event class for the DVS128 """
struct DVS128
    pol::Bool
    x::Int8
    y::Int8
    ext::Bool
end

function Base.convert(::Type{DVS128}, a::Union{UInt16,UInt32})
    # Unpack the DVS128 fields from a 16-bit or 32-bit address:
    pol = a & 0x1
    x = (a >> 1) & 0x7F
    y = (a >> 8) & 0x7F
    ext = (a >> 15) & 0x1
    DVS128(pol, x, y, ext)
end

event_coord(e::DVS128) = (e.x, e.y)
event_pol(e::DVS128) = e.pol
event_location(e::DVS128, imsize) = (DVS128_SIZE[1]-e.y, e.x+1)

image_size(e::Type{DVS128}) = DVS128_SIZE

# There's no positive way to identify a DVS128 event.
# These are normally not mixed with other event types
# anyways. They can be found in 1.0 and 2.0 files.
isevent(::Type{DVS128}, a::Event{UInt16}) = true
isevent(::Type{DVS128}, a::Event{UInt32}) = true


__precompile__(true)

module NGSIM

using AutomotiveDrivingModels
using DataFrames
using Distributions

export
    NGSIMRoadway,
    RoadwayInputParams,

    ROADWAY_80,
    ROADWAY_101,

    NGSIMTrajdata,
    VehicleSystem,
    FilterTrajectoryResult,

    NGSIM_TIMESTEP,
    NGSIM_TRAJDATA_PATHS,
    TRAJDATA_PATHS,

    carsinframe,
    load_ngsim_trajdata,
    get_corresponding_roadway,
    filter_trajectory!,
    symmetric_exponential_moving_average!,
    load_trajdata,
    convert_raw_ngsim_to_trajdatas

include("roadway.jl")
include("ngsim_trajdata.jl")
include("trajdata.jl")

end # module


struct Layer{T}
    geom::T
    options::Dict{Symbol,Any}
end

Layer(geom; kwargs...) = Layer(geom, Dict{Symbol,Any}(kwargs))

function chloropleth{T}(
        geom::T,
        cvalues::Vector{<:Real},
        cgradient;
        kwargs...
    )
    options = Dict{Symbol,Any}(kwargs)
    minv = minimum(cvalues); rangev = maximum(cvalues) - minv
    options[:color] = if rangev < 1e-4
        cgradient[cvalues[1]]
    else
        [cgradient[minv + (v-minv)/rangev] for v in cvalues]
    end
    Layer{T}(geom, options)
end

function chloropleth(
        df::DataFrames.DataFrame,
        column::Symbol,
        cgradient;
        geometry::Symbol = :geometry0,
        kwargs...
    )
    chloropleth(df[geometry], df[column], cgradient; kwargs...)
end

RecipesBase.@recipe function f(layer::Layer)
    _get(opt::Any, i::Int) = opt
    _get(opt::Vector, i::Int) = opt[i]

    aspect_ratio --> get(layer.options, :aspect_ratio, 1)
    legend --> get(layer.options, :legend, :false)
    grid --> get(layer.options, :grid, false)
    haskey(layer.options, :framestyle) && (framestyle --> layer.options[:framestyle])
    haskey(layer.options, :ticks) && (ticks --> layer.options[:ticks])

    for (i,g) in enumerate(layer.geom)
        RecipesBase.@series begin
            gtype = GeoInterface.geotype(g)
            haskey(layer.options, :label) && (label := _get(layer.options[:label], i))
            if gtype == :Point || gtype == :MultiPoint
                seriestype := :scatter
                haskey(layer.options, :alpha) && (markeralpha := _get(layer.options[:alpha], i))
                haskey(layer.options, :color) && (markercolor := _get(layer.options[:color], i))
                haskey(layer.options, :shape) && (markershape := _get(layer.options[:shape], i))
                haskey(layer.options, :size)  && (markersize  := _get(layer.options[:size], i))
                haskey(layer.options, :strokealpha) && (markerstrokealpha  := _get(layer.options[:strokealpha],i))
                haskey(layer.options, :strokecolor) && (markerstrokecolor  := _get(layer.options[:strokecolor],i))
                haskey(layer.options, :strokestyle) && (markerstrokestyle  := _get(layer.options[:strokestyle],i))
                haskey(layer.options, :strokewidth)  && (markerstrokewidth := _get(layer.options[:strokewidth],i))
            elseif gtype == :LineString || gtype == :MultiLineString
                seriestype := :path
                haskey(layer.options, :arrow) && (arrow     := _get(layer.options[:arrow], i))
                haskey(layer.options, :alpha) && (linealpha := _get(layer.options[:alpha], i))
                haskey(layer.options, :color) && (linecolor := _get(layer.options[:color], i))
                haskey(layer.options, :style) && (linestyle := _get(layer.options[:style], i))
                haskey(layer.options, :width) && (linewidth := _get(layer.options[:width], i))
            elseif gtype == :Polygon || gtype == :MultiPolygon
                seriestype := :shape
                haskey(layer.options, :alpha) && (fillalpha := _get(layer.options[:alpha], i))
                haskey(layer.options, :color) && (fillcolor := _get(layer.options[:color], i))
            else
                warn("unknown geometry type: $gtype")
            end
            GeoInterface.shapecoords(g)
        end
    end
end


# function to allocate incidents from the current queue of each department
# to a second queue that checks whether other departments could help out
function fill_exchange_queue!(sim_data::DataFrame,
                                incident_queue::Array{Union{Missing,Int64},3},
                                exchange_queue::Array{Union{Missing,Int64},1},
                                max_queue::Int64,
                                card_districts::Int64,
                                priority::Int64)
    place_in_queue = 1
    exchange_queue .= missing
    for i = 1:card_districts
        for j = 1:max_queue
            # allocate the incident from the main queues to the exchange queue
            if ismissing(incident_queue[j,i,priority]) == false
                current_case = incident_queue[j,i,priority]
                if place_in_queue > length(exchange_queue)
                    error("w_aus is too small!")
                    break
                else
                    if sim_data[current_case,:cars_missing] > 0
                        exchange_queue[place_in_queue] = current_case
                        place_in_queue += 1
                    end
                end
            else
                break
            end
        end
    end
end


module GVFHordes

import Lazy: @forward

export GVFParamFuncs, AbstractGVF, GVF, Horde

"""
    GVFParamFuncs

Module containing and the GVF parameter function types. Cleaner to keep these in a seperate namespace where the user can decide to `using` if desired.
"""
module GVFParamFuncs

import Base.get

using Random
using ..GVFHordes

export
    AbstractCumulant,
    FeatureCumulant,
    PredictionCumulant,
    ScaledCumulant,
    FunctionalCumulant
include("gvf/cumulant.jl")

export
    AbstractDiscount,
    ConstantDiscount,
    StateTerminationDiscount
include("gvf/discount.jl")

export
    AbstractPolicy,
    NullPolicy,
    PersistentPolicy,
    RandomPolicy,
    FucntionalPolicy
include("gvf/policy.jl")

end

"""
    AbstractGVF

This is the base type for a General Value Function. See the "Horde: A scalable real-time architecture for learning knowledge from unsupervised sensorimotor interaction" paper for more details.
"""
abstract type AbstractGVF end

"""
    get(gvf::AbstractGVF, state_t, action_t, state_tp1, action_tp1, preds_tp1)

Get the parameters for the cumulant, discount, and probability of taking an action given the parameters.
"""
function Base.get(gvf::AbstractGVF, state_t, action_t, state_tp1, action_tp1, preds_tp1) end

function cumulant(gvf::AbstractGVF) end
function discount(gvf::AbstractGVF) end
function policy(gvf::AbstractGVF) end

"""
    GVF{C<:AbstractCumulant, D<:AbstractDiscount, P<:AbstractPolicy} <: AbstractGVF

A realized version of a GVF where the cumulant, discount, and policies can be any structure following the AbstractCumulant, AbstractDiscount, or AbstractPolicy api respectively.
"""
struct GVF{C<:GVFParamFuncs.AbstractCumulant,
           D<:GVFParamFuncs.AbstractDiscount,
           P<:GVFParamFuncs.AbstractPolicy} <: AbstractGVF
    cumulant::C
    discount::D
    policy::P
end

cumulant(gvf::GVF) = gvf.cumulant
discount(gvf::GVF) = gvf.discount
policy(gvf::GVF) = gvf.policy

function Base.get(gvf::GVF, state_t, action_t, state_tp1, action_tp1, preds_tp1)

    c = get(gvf.cumulant, state_tp1, action_tp1, preds_tp1)
    γ = get(gvf.discount, state_t, action_t, state_tp1, action_tp1, preds_tp1)
    π_prob = get(gvf.policy, state_t, action_t)
    return c, γ, π_prob
end

"""
    AbstractHorde

An abstract collection of GVFs.
"""
abstract type AbstractHorde end

"""
    Horde{T<:AbstractGVF} <: AbstractHorde

The simplest implementation of a horde as a collection of AbstractGVFs.
"""
struct Horde{T<:AbstractGVF} <: AbstractHorde
    gvfs::Vector{T}
end

merge(h1::Horde, h2::Horde) =
    Horde([deepcopy(h1.gvfs); deepcopy(h2.gvfs)])

function Base.get(gvfh::Horde, state_t, action_t, state_tp1, action_tp1, preds_tp1)
    C = map(gvf -> get(cumulant(gvf), state_tp1, action_tp1, preds_tp1), gvfh.gvfs)
    Γ = map(gvf -> get(discount(gvf), state_t, action_t, state_tp1, action_tp1, preds_tp1), gvfh.gvfs)
    Π_probs = map(gvf -> get(policy(gvf), state_t, action_t), gvfh.gvfs)
    return C, Γ, Π_probs
end

function Base.get!(C::Array{T, 1}, Γ::Array{F, 1}, Π_probs::Array{H, 1}, gvfh::Horde, state_t, action_t, state_tp1, action_tp1, preds_tp1) where {T, F, H}
    C .= map(gvf -> get(cumulant(gvf), state_tp1, action_tp1, preds_tp1), gvfh.gvfs)
    Γ .= map(gvf -> get(discount(gvf), state_t, action_t, state_tp1, action_tp1, preds_tp1), gvfh.gvfs)
    Π_probs .= map(gvf -> get(policy(gvf), state_t, action_t), gvfh.gvfs)
    return C, Γ, Π_probs
end

Base.get(gvfh::Horde, state_tp1, preds_tp1) =
    get(gvfh::Horde, nothing, nothing, state_tp1, nothing, preds_tp1)

Base.get(gvfh::Horde, state_t, action_t, state_tp1) =
    get(gvfh::Horde, state_t, action_t, state_tp1, nothing, nothing)

Base.get(gvfh::Horde, state_t, action_t, state_tp1, preds_tp1) =
    get(gvfh::Horde, state_t, action_t, state_tp1, nothing, preds_tp1)

@forward Horde.gvfs Base.length

end # module


"""
# SMM.jl
This is the documentation for SMM.jl

"""

module SMM

  # Dependencies
  #-------------
  using BlackBoxOptim
  using Optim
  using JLD2
  using Plots
  using CSV
  using ProgressMeter
  using Calculus
  using CompEcon
  using Distributions
  using DataFrames

  @static if VERSION < v"0.7.0"
      using DataStructures
  else
      using OrderedCollections
  end

  # Exports from BlackBoxOptim
  #---------------------------
  export best_candidate

  # Includes
  #---------
  # Types
  #------
  include("types.jl");

  # API
  #----
  include("api.jl")

  # General functions, useful at several places
  #---------------------------------------------
  include("generic.jl")

  # Functions to load and save
  #--------------------------
  include("save_load.jl")

  # Functions to minimize the objective function
  #---------------------------------------------
  include("optimize.jl")

  # Functions to do inference
  #---------------------------------------------
  include("econometrics.jl")

  # Functions to do plots
  #---------------------------------------------
  include("plots.jl")

  # Exports
  #--------
  # Functions and types in types.jl
  #----------------------------------
  export SMMOptions, SMMProblem
  export default_function, rosenbrock2d
  export is_global_optimizer, is_local_optimizer
  export convert_to_optim_algo, convert_to_fminbox
  export is_bb_optimizer, is_optim_optimizer

  # Functions and types in api.jl
  #-------------------------------

  # Functions and types in generic.jl
  #----------------------------------
  export set_simulate_empirical_moments!, construct_objective_function!
  export set_priors!, set_empirical_moments!, set_Sigma0!
  export set_bbSetup!, generate_bbSearchRange
  export create_lower_bound, create_upper_bound
  export set_global_optimizer!
  export create_grid, create_grid_stochastic, generate_std, latin_hypercube_sampling
  export get_now

  # Functions and types in save_load.jl
  #------------------------------------
  export read_priors, read_empirical_moments
  export saveSMMOptim, loadSMMOptim

  # Functions and types in optimize.jl
  #-----------------------------------
  export smmoptimize!, smm_minimizer
  export smm_refine_globalmin!, smm_local_minimizer
  export smm_local_minimum
  export smm_localmin, local_to_global!

  # Functions in econometrics.jl
  #-----------------------------
  export calculate_D, calculate_Avar!, calculate_se, calculate_t, calculate_pvalue, calculate_CI
  export summary_table

  # Functions in plots.jl
  #----------------------
  export smm_slices

end # module


"""
    _outlength(A, I, J)
    _outlength(u, I)
Determine the size of the output for an operation like extract or range-based indexing.
"""
function _outlength(A, I, J)
    if I == ALL
        Ilen = size(A, 1)
    else
        Ilen = length(I)
    end
    if J == ALL
        Jlen = size(A, 2)
    else
        Jlen = length(J)
    end
    return Ilen, Jlen
end

function _outlength(u, I)
    if I == ALL
        wlen = size(u)
    else
        wlen = length(I)
    end
    return wlen
end

"""
    extract!(C::GBMatrix, A::GBMatOrTranspose, I, J; kwargs...)::GBMatrix
    extract!(C::GBVector, A::GBVector, I; kwargs...)::GBVector

Extract a submatrix or subvector from `A` into `C`.

# Arguments
- `C::Union{GBVector, GBMatrix}`: the submatrix or subvector extracted from `A`.
- `A::GBArray`
- `I` and `J`: A colon, scalar, vector, or range indexing A.

# Keywords
- `mask::Union{Nothing, GBArray} = nothing`: mask where
    `size(M) == (max(I), max(J))`.
- `accum::Union{Nothing, AbstractBinaryOp} = nothing`: binary accumulator operation
    where `C[i,j] = accum(C[i,j], T[i,j])` where T is the result of this function before accum is applied.
- `desc::Union{Nothing, Descriptor} = nothing`

# Returns
- `GBMatrix` or `GBVector`: the modified array `C`, now containing the matrix `A[I, J]` or
    `A[I]` for a vector.

# Throws
- `GrB_DIMENSION_MISMATCH`: If `size(C) != (max(I), max(J))` or `size(C) != size(mask)`.
"""
extract!

function extract!(
    C::GBMatrix, A::GBMatOrTranspose, I, J;
    mask = nothing, accum = nothing, desc = nothing
)
    I, ni = idx(I)
    J, nj = idx(J)
    I isa Number && (I = UInt64[I])
    J isa Number && (J = UInt64[J])
    mask === nothing && (mask = C_NULL)
    desc = _handledescriptor(desc; in1 = A)
    libgb.GrB_Matrix_extract(C, mask, getaccum(accum, eltype(C)), parent(A), I, ni, J, nj, desc)
    return C
end

function extract!(
    C::GBMatrix, A::GBMatOrTranspose, ::Colon, J;
    mask = nothing, accum = nothing, desc = nothing
)
    return extract!(C, A, ALL, J; mask, accum, desc)
end

function extract!(
    C::GBMatrix, A::GBMatOrTranspose, I, ::Colon;
    mask = nothing, accum = nothing, desc = nothing
)
    return extract!(C, A, I, ALL; mask, accum, desc)
end

function extract!(
    C::GBMatrix, A::GBMatOrTranspose, ::Colon, ::Colon;
    mask = nothing, accum = nothing, desc = nothing
)
    return extract!(C, A, ALL, ALL; mask, accum, desc)
end

"""
    extract(A::GBMatOrTranspose, I, J; kwargs...)::GBMatrix
    extract(A::GBVector, I; kwargs...)::GBVector

Extract a submatrix or subvector from `A`

# Arguments
- `A::GBArray`: the array being indexed.
- `I` and `J`: A colon, scalar, vector, or range indexing A.

# Keywords
- `mask::Union{Nothing, GBArray} = nothing`: mask where
    `size(M) == (max(I), max(J))`.
- `accum::Union{Nothing, AbstractBinaryOp} = nothing`: binary accumulator operation
    where `C[i,j] = accum(C[i,j], T[i,j])` where T is the result of this function before accum is applied.
- `desc::Descriptor = nothing`

# Returns
- `GBMatrix`: the submatrix `A[I, J]`.

# Throws
- `GrB_DIMENSION_MISMATCH`: If `(max(I), max(J)) != size(mask)`.
"""
extract

function extract(
    A::GBMatOrTranspose, I, J;
    mask = nothing, accum = nothing, desc = nothing
)
    Ilen, Jlen = _outlength(A, I, J)
    C = similar(A, Ilen, Jlen)
    return extract!(C, A, I, J; mask, accum, desc)
end

function extract(
    A::GBMatOrTranspose, ::Colon, J;
    mask = nothing, accum = nothing, desc = nothing
)
    return extract(A, ALL, J; mask, accum, desc)
end

function extract(
    A::GBMatOrTranspose, I, ::Colon;
    mask = nothing, accum = nothing, desc = nothing
)
    return extract(A, I, ALL; mask, accum, desc)
end

function extract(
    A::GBMatOrTranspose, ::Colon, ::Colon;
    mask = nothing, accum = nothing, desc = nothing
)
    return extract(A, ALL, ALL; mask, accum, desc)
end

function Base.getindex(
    A::GBMatOrTranspose, ::Colon, j;
    mask = nothing, accum = nothing, desc = nothing
)
    return extract(A, ALL, j; mask, accum, desc)
end

function extract!(
    w::GBVector, u::GBVector, I;
    mask = nothing, accum = nothing, desc = nothing
)
    I, ni = idx(I)
    desc = _handledescriptor(desc)
    mask === nothing && (mask = C_NULL)
    libgb.GrB_Matrix_extract(w, mask, getaccum(accum, eltype(w)), u, I, ni, UInt64[1], 1, desc)
    return w
end

function extract!(
    w::GBVector, u::GBVector, ::Colon;
    mask = nothing, accum = nothing, desc = nothing
)
    return extract!(w, u, ALL; mask, accum, desc)
end

function extract(
    u::GBVector, I;
    mask = nothing, accum = nothing, desc = nothing
)
    wlen = _outlength(u, I)
    w = similar(u, wlen)
    return extract!(w, u, I; mask, accum, desc)
end

function extract(u::GBVector, ::Colon; mask = nothing, accum = nothing, desc = nothing)
    extract(u, ALL; mask, accum, desc)
end