@testset "simulate to maxtime" begin

    rng = MersenneTwister(12)
    input = MultilevelInput(
        modulesize=4,
        fixedmu=true,
        b=0.1,
        d=0.1,
        bdrate=0.1,
        clonalmutations=0,
        maxtime=400,
        maxmodules=1000,
        branchrate=3/365,
        branchfraction=0.2,
        μ=1
    )
    population = multilevel_simulation(input, rng, :fixedtime)
    @test age(population) < 400

    b, d = 0.1, 0.0
    modulesize = 6
    branchinitsize = 2
    branchrate = 1/5
    maxtime = 100

    moduletracker = SomaticEvolution.initialize_population(
        modulesize,
        clonalmutations=0
    )[1]

    rng = MersenneTwister(12)

    moduletracker, newmoduletracker =
        SomaticEvolution.module_simulate_to_branching!(
            moduletracker,
            maxtime,
            b,
            d,
            b,
            branchrate,
            modulesize,
            branchinitsize,
            2,
            rng
    )
    @test moduletracker.tvec[end] <= maxtime
    @test moduletracker.Nvec[end] == modulesize - branchinitsize
    @test newmoduletracker.Nvec[end] == branchinitsize
    @test newmoduletracker.tvec[end] == moduletracker.tvec[end]

    newmaxtime = moduletracker.tvec[end] + 0.1
    moduletracker, newmoduletracker =
        SomaticEvolution.module_simulate_to_branching!(
            moduletracker,
            newmaxtime,
            b,
            d,
            b,
            branchrate,
            modulesize,
            branchinitsize,
            2,
            rng
    )
    @test newmoduletracker === nothing
end

input = MultilevelInput(
    modulesize=4,
    fixedmu=true,
    b=0.1,
    d=0,
    bdrate=0.01,
    clonalmutations=0,
    maxtime=1*365,
    branchrate=3/365,
    branchfraction=0.2,
    μ=1
)

mt1 = SomaticEvolution.ModuleTracker(
    [1, 2, 3, 4, 4, 4, 4, 4, 4, 4, 3, 4, 4, 4, 4],
    [0.0, 14.437821887595856, 38.636412371018324, 38.80067279227201, 39.07905423063021, 56.24183190197407, 61.57555456465887, 93.99343227770015, 174.7383395589789, 177.87796865140686, 187.1205185574646, 190.3156218593258, 222.64188808574013, 235.8640994797343, 242.08245554076214],
    Cell[Cell([3, 4, 20], 1), Cell([3, 5, 6, 8, 10, 17, 18, 19], 1), Cell([3, 4, 21, 22, 23], 1), Cell([1, 24], 1)],
    SomaticEvolution.CloneTracker[], 1, 0)
mt2 = SomaticEvolution.ModuleTracker(
    [1, 2, 3, 4, 4, 4, 3],
    [187.1205185574646, 207.5419622150601, 209.92705453342273, 216.2145518957684, 243.24240821372297, 252.31115142542512, 257.22794191422554],
    Cell[Cell([1, 13, 25, 26, 27], 1), Cell([1, 13], 1), Cell([1, 13, 25, 28, 29], 1)],
    SomaticEvolution.CloneTracker[], 2, 1)
mt3 = SomaticEvolution.ModuleTracker(
    [1],
    [257.22794191422554],
    Cell[Cell([1], 1)], SomaticEvolution.CloneTracker[], 3, 2)

@testset "module sampling" begin
    rng = MersenneTwister(12)
    moduletracker = deepcopy(mt1)
    moduletracker, newmoduletracker =
        SomaticEvolution.sample_new_module!(moduletracker, 2, 1, 1.0, rng)
    @test length(moduletracker) == 3
    @test length(newmoduletracker) == 1
    @test newmoduletracker.id == 2
    @test newmoduletracker.Nvec == [1]
    @test newmoduletracker.tvec == [1.0]
end

population = SomaticEvolution.Population(input, [mt1, mt2, mt3])

#check that statistics are calculated correctly
@testset "mutation statistics" begin
    @test mutations_per_cell(mt1) == [3, 8, 5, 2]
    @test average_mutations_per_module(population) ≈ [4.5, 4, 1]
    @test all(average_mutations(population, true) .≈ (3.875, 5.267857142857143))
    @test clonal_mutations(population) == [0, 2, 1]
    @test clonal_mutation_ids(population) == [[], [1, 13], [1]]
    @test pairwise_fixed_differences_matrix(population, diagonals=true) == [0 0 0; 2 2 0; 1 1 1]
    @test pairwise_fixed_differences_matrix(population, [2,3]) == [0 0; 1 0]
    @test pairwise_fixed_differences(population) == Dict{Int64, Int64}(1=>2, 2=>1)
    @test pairwise_fixed_differences(population, [2,3]) == Dict{Int64, Int64}(1=>1)
    @test shared_fixed_mutations(population) == Dict{Int64, Int64}(1=>1, 2=>1)
    @test shared_fixed_mutations(population, [2,3]) == Dict{Int64, Int64}(1=>1, 2=>1)
    @test all(pairwise_fixed_differences_statistics(population, clonal=true) .≈ (1.3333333333333333,0.33333333333333333,1,1))
    @test newmoduletimes(population) ≈ [0.0, 187.1205185574646, 257.22794191422554]
    @test all(cellpopulationsize(population, 50) .≈ ([0,50,100,150,200,250],[1, 4, 4, 4, 5, 8]))
end

@testset "updates" begin
    rng = MersenneTwister(12)
    populationtracker = [deepcopy(mt1), deepcopy(mt1), deepcopy(mt3)]

    #kills only cell in module 3 => population size goes to 2
    SomaticEvolution.deathupdate!(populationtracker, 4, 3, rng)
    @test length(populationtracker) == 2

    #one module splits into two modules of length two
    SomaticEvolution.modulebranchingupdate!(populationtracker, 4, 2, 1.0, rng)
    @test length(populationtracker) == 3
    @test sum(map(x -> length(x.cells), populationtracker)) == 8
    @test sum(map(x -> x.Nvec[end], populationtracker)) == 8

    #birth update one module gets an extra cell
    SomaticEvolution.birthupdate!(populationtracker, 4, 2, 1.0, 1, rng)
    @test sum(map(x -> length(x.cells), populationtracker)) == 9
    @test sum(map(x -> x.Nvec[end], populationtracker)) == 9

    #moran update population size stays the same
    SomaticEvolution.moranupdate!(populationtracker, 4, 3, 1.0, 1, rng)
    @test sum(map(x -> length(x.cells), populationtracker)) == 9
    @test sum(map(x -> x.Nvec[end], populationtracker)) == 9

    N0 = populationtracker[1].Nvec[end]
    @test N0 == length(populationtracker[1].cells)
    L = length(populationtracker[1].Nvec)
    @test L == length(populationtracker[1].tvec)
    SomaticEvolution.updatemodulehistory!(populationtracker[1], 1, 20.0)
    @test populationtracker[1].tvec[end] == 20.0
    @test populationtracker[1].Nvec[end] == 1 + N0
    @test length(populationtracker[1].Nvec) == length(populationtracker[1].tvec) == L + 1
    @test_throws ErrorException SomaticEvolution.check_module_number(populationtracker, 1)

end

@testset "simulate to fixed size" begin
    b = 0.1
    d = 0.0
    bdrate = 0.01
    modulesize = 4
    branchinitsize = 1
    branchrate = 3/365
    maxtime = 1000
    maxmodules = 5
    populationtracker = SomaticEvolution.initialize_population(modulesize, clonalmutations=0)
    mutID = SomaticEvolution.getmutID(populationtracker[1].cells) #get id of next mutation
    t = 0
    rng = MersenneTwister(1)

    #check first update is correct
    transitionrates = SomaticEvolution.get_transitionrates(
        populationtracker,
        b,
        d,
        bdrate,
        branchrate,
        modulesize
    )
    #only tranisition with non-zero rate is birth which has rate b=0.1
    @test transitionrates == [0.0, 0.1, 0.0, 0.0]
    populationtracker, mutID, t =
        SomaticEvolution.update_population!(
            populationtracker,
            b,
            d,
            bdrate,
            branchrate,
            modulesize,
            branchinitsize,
            mutID,
            0,
            maxtime,
            rng
    )
    #popsize is now 2
    @test populationtracker[1].Nvec[end] == length(populationtracker[1].cells)
    @test length(populationtracker[1].Nvec) === length(populationtracker[1].tvec) == 2
    #each cell has one mutation
    @test length(populationtracker[1].cells[1].mutations) == length(populationtracker[1].cells[2].mutations) == 1

    #check transition rates for population of modules m1, m2, m3
    transitionrates = SomaticEvolution.get_transitionrates(
        [mt1, mt2, mt3],
        b,
        d,
        bdrate,
        branchrate,
        modulesize
    )
    @test all(transitionrates .≈ [0.04, 0.4, 0.0, 3/365])

    #check transitionrates for non-zero death rate
    transitionrates = SomaticEvolution.get_transitionrates(
        [mt1, mt2, mt3],
        b,
        0.01,
        bdrate,
        branchrate,
        modulesize
    )
    @test all(transitionrates .≈ [0.04, 0.4, 0.04, 3/365])

    #check only homeostatic modules are chosen for Moran and module branching and that prob of choosing cells is equal
    @test all(SomaticEvolution.choose_homeostaticmodule([mt1,mt2,mt3], 4, rng)==mt1 for i in 1:10)
    @test isapprox(mean(reduce(hcat, [[v for v in SomaticEvolution.choose_homeostaticmodule_cells([mt1,mt2,mt3], 4, rng)[2:3]] for i in 1:10000]), dims=2), [2.5, 2.5], atol=0.1)
    #check prob of choosing non-homeostatic modules is proportional to number of cells
    @test isapprox(mean(length(SomaticEvolution.choose_growingmodule_cell([mt1,mt2,mt3], 4, rng)[1].cells) for i in 1:100000), 2.5, atol=0.1)

    #check multilevel_simulation function
    rng = MersenneTwister(12)
    input = MultilevelInput(
            modulesize=4,
            fixedmu=true,
            b=0.1,
            d=0.01,
            bdrate=0.01,
            clonalmutations=0,
            maxtime=20*365,
            maxmodules=10,
            branchrate=3/365,
            branchfraction=0.2,
            μ=1
    )
    population = multilevel_simulation(input, rng, :normal)
    @test length(population) == 10
    input = MultilevelInput(
            modulesize=4,
            fixedmu=true,
            b=0.1,
            d=0.1,
            bdrate=0.1,
            clonalmutations=0,
            maxtime=365,
            maxmodules=1000,
            branchrate=3/365,
            branchfraction=0.2,
            μ=1
    )
    population = multilevel_simulation(input, rng, :normal)
    @test age(population) <= 365

end


module SolarLEDs

export main, spaceship_dj, sun_wind_switch

using LibSerialPort, Blink, Interact, COBS, ProgressMeter, Flatten, StaticArrays

#remove this
#=function get_port_list(;nports_guess::Integer=64)
    ports = sp_list_ports()
    port_list = String[]
    for port in unsafe_wrap(Array, ports, nports_guess, own=false)
        port == C_NULL && return port_list
        push!(port_list, sp_get_port_name(port))
    end
    sp_free_port_list(ports)
    return port_list
end=#

const N_UPLOAD_ATTEMPTS = 10
const BUTTON_LABELS = vcat(string.(0:9), "#", "*", "→", "↑", "←", "↓", "OK")
const N_BUTTONS = length(BUTTON_LABELS)
const BAUD = 115200

function tobytes(x::Integer)
    low = UInt8(x & 0xFF)
    high = UInt8((x >> 8) & 0xFF)
    return low, high
end
tobytes(x::Number) = tobytes(Int(x))
struct Sun
    int::UInt8
    low::UInt8
    high::UInt8
    siz::UInt8
end
function Sun(card, pos, rad, int, n_leds_per_strip)
    pos -= 1 + rad
    if card == :NS
        pos += n_leds_per_strip
    end
    low, high = tobytes(pos)
    return Sun(int, low, high, 2rad + 1)
end
function guiaxes(setup)
    card = radiobuttons(setup.cardinals)
    pos = slider(setup.elevations, value = first(setup.elevations))
    rad = slider(setup.radii, value = first(setup.radii))
    int = slider(setup.intensities, value = first(setup.intensities))
    on(pos) do p
        a = p - 1
        if rad[] > a
            rad[] = a
        end
        a = setup.n_leds_per_strip - p[]
        if rad[] > a
            rad[] = a
        end
    end
    on(rad) do r
        a = r + 1
        if pos[]  < a
            pos[] = a
        end
        a = setup.n_leds_per_strip - r
        if pos[] > a
            pos[] = a
        end
    end
    output = map(Sun, card, pos, rad, int, setup.n_leds_per_strip)
    wdg = Widget{:sun}(["card" => card, "pos" => pos, "rad" => rad, "int" => int], output = output)
    @layout! wdg vbox( pad(1em, hbox("Cardinal axes", :card)), pad(1em, hbox("Position", :pos)), pad(1em, hbox("Radius", :rad)), pad(1em, hbox("Intensity", :int)))
end

function uploaded(sp, msg)
    encode(sp, msg)
    @showprogress 0 "Uploading..." for i in 1:100
        if bytesavailable(sp) > 0
            return true
        else
            sleep(0.1)
        end
    end
    return false
end
function attemptupload(sp, msg)
    for attempt = 1:N_UPLOAD_ATTEMPTS
        if uploaded(sp, msg)
            msg2 = decode(sp)
            if msg2 == msg
                return nothing
            end
        end
        println("attempt #$attempt to upload data failed...")
    end
    error("failed to upload data after $N_UPLOAD_ATTEMPTS attempts")
end

struct Setup
    nsuns::Int
    n_leds_per_strip::Int
    cardinals::Vector{Symbol}
    elevations::Vector{Int}
    radii::Vector{Int}
    intensities::Vector{Int}
end

function main(; n_leds_per_strip::Int = 150, cardinals = [:EW, :NS], elevations = 1:150, radii = 0:25, intensities = 0:255)
    if 0 ∉ intensities
        push!(intensities, 0)
    end
    if !issorted(elevations)
        sort!(elevations)
    end
    if !issorted(radii)
        sort!(radii)
    end
    if !issorted(intensities)
        sort!(intensities)
    end
    nsuns = 4
    @assert 0 < nsuns "number of suns must be larger than zero"
    @assert 0 < n_leds_per_strip "number of LEDs per strip must be larger than zero"
    main(Setup(nsuns, n_leds_per_strip, cardinals, elevations, radii, intensities))
end

sun_wind_switch() = main(n_leds_per_strip = 150, elevations = [5, 17, 36, 48, 59, 63, 65, 66, 68, 69, 71, 73, 74, 76, 77, 79, 83, 94, 106, 125, 137], radii = [0], intensities = [255])
spaceship_dj() = main(n_leds_per_strip = 73, elevations = [12, 29, 46, 64], radii = [0,1,2], intensities = 0:255, cardinals = [:NA])

function returnport()
    for port in get_port_list()
        sp = LibSerialPort.sp_get_port_by_name(port)
        if occursin(r"arduino", LibSerialPort.sp_get_port_usb_manufacturer(sp))
            LibSerialPort.sp_free_port(sp)
            return port
        end
        LibSerialPort.sp_free_port(sp)
    end
    return nothing
end

function main(setup)

    port = returnport()
    isnothing(port) && error("no ports were detected...")
    serialport = open(port, BAUD)

    bottoms = Dict(l => guisuns(serialport, setup) for l in BUTTON_LABELS)
    top = tabs(BUTTON_LABELS)
    bottom = map(top) do l
        bottoms[l]
    end
    #=download = button("Download")
    on(download) do _
    l = top[]
    i = UInt8(findfirst(isequal(l), BUTTON_LABELS) - 1)
    uploaded([0x02, i])
    msg = decode(serialport)
    setgui(bottoms[l], msg)
    # bottom[] = bottom[]
    end=#
    upload = button("Upload")
    on(upload) do _
        l = top[]
        i = UInt8(findfirst(isequal(l), BUTTON_LABELS) - 1)
        msg = vcat(0x01, i, bottom[][])
        attemptupload(serialport, msg)
    end
    uploadall = button("Upload all")
    on(uploadall) do _
        for (i,l) in enumerate(BUTTON_LABELS)
            b = bottoms[l]
            msg = vcat(0x01, i - 1, b[])
            attemptupload(serialport, msg)
        end
    end
    reset = button("Reset")
    on(reset) do _
        flush(serialport)
        uploaded(serialport, [UInt8(3)])
        @assert Bool(decode(serialport)[])
    end
    w = Window()
    body!(w, dom"div"(hbox(pad(1em, uploadall), pad(1em, upload), pad(1em, reset)), top, bottom))

    return nothing
end

function guisuns(sp, setup)
    suns = [guiaxes(setup) for i in 1:setup.nsuns]
    output = map(suns...) do ss...
        SVector(flatten(ss))
    end
    on(output) do pl
        encode(sp, [0x00; pl])
    end
    d = Dict("sun$i" => s for (i,s) in enumerate(suns))
    wgt = Widget{:suns}(d, output = output)
    @layout! wgt hbox(:sun1, :sun2, :sun3, :sun4)
end

### TODO
# implement downloading
# maybe prepare hosting this to a server for mobile interface

end # module


"""
    MRCData{T<:Number,N,EH<:MRCExtendedHeader,D<:AbstractArray{T,N}} <: AbstractArray{T,N}

Container for electron density data loaded from an MRC file.

Each instance of `MRCData` carries with it a header and extended header, respectively
accessed with [`header`](@ref) and [`extendedheader`](@ref).

If changes are made to the header, extended header, or the data, call [`updateheader!`](@ref)
before writing to file to ensure the header is stil consistent with the data.

    MRCData()

Create an empty array.

    MRCData(header[, extendedheader])

Create an array whose size is indicated by the entries in `header`.

    MRCData(size)
    MRCData(size...)

Create an array of the specified size.
"""
struct MRCData{T<:Number,N,EH,D} <: AbstractArray{T,N}
    header::MRCHeader
    extendedheader::EH
    data::D
end
function MRCData(header, extendedheader, data::AbstractArray{T,N}) where {T,N}
    return MRCData{T,N,typeof(extendedheader),typeof(data)}(header, extendedheader, data)
end
function MRCData(header=MRCHeader(), extendedheader=MRCExtendedHeader())
    data_size = size(header)
    data_length = prod(data_size)
    dtype = datatype(header)
    dims = ndims(header)
    s = ntuple(i -> data_size[i], dims)
    data = Array{dtype,dims}(undef, s)
    return MRCData(header, extendedheader, data)
end
function MRCData(size::NTuple{3,<:Integer})
    header = MRCHeader()
    for i in 1:3
        setproperty!(header, (:nx, :ny, :nz)[i], size[i])
    end
    return MRCData(header)
end
MRCData(size::Integer...) = MRCData(size)

"""
    header(data::MRCData) -> MRCHeader

Get header.
"""
header(d::MRCData) = d.header

"""
    extendedheader(data::MRCData) -> MRCExtendedHeader

Get extended header.
"""
extendedheader(d::MRCData) = d.extendedheader

"""
    updateheader!(data::MRCData; statistics = true)

Update the header stored in `data` from the data and its extended header.
Set `statistics=false` to avoid computing summary statistics from the data.
"""
function updateheader!(d::MRCData; statistics=true)
    h = header(d)

    # update size
    s = size(d)
    for i in eachindex(s)
        setproperty!(h, (:nx, :ny, :nz)[i], s[i])
    end

    # update space group
    if length(s) == 2
        h.nz = 1
        h.mz = 1
        if h.ispg < 0 || h.ispg > 230
            h.ispg = 0
        end
    elseif length(s) == 3 && h.ispg == 0
        h.ispg == 1
    end

    # misc
    h.mode = TYPE_TO_MODE[eltype(d)]
    h.nsymbt = bytesize(extendedheader(d))
    h.nlabl = sum(s -> length(rstrip(s)) > 0, h.label)

    # update statistics
    if statistics
        h.dmin, h.dmax = extrema(d)
        dmean = mean(d)
        h.dmean, h.rms = dmean, stdm(d, dmean; corrected=false)
    end

    return h
end

# Array overloads
@inline Base.parent(d::MRCData) = d.data

@inline Base.getindex(d::MRCData, idx::Int...) = getindex(parent(d), idx...)

@inline Base.setindex!(d::MRCData, val, idx::Int...) = setindex!(parent(d), val, idx...)

Base.size(d::MRCData) = size(parent(d))
Base.size(d::MRCData, i) = size(parent(d), i)

Base.ndims(d::MRCData) = ndims(parent(d))

Base.length(d::MRCData) = length(parent(d))

Base.iterate(d::MRCData, idx...) = iterate(parent(d), idx...)

Base.lastindex(d::MRCData) = lastindex(parent(d))

function Base.similar(d::MRCData)
    return typeof(d)(header(d), extendedheader(d), similar(parent(d)))
end

function Base.IndexStyle(::Type{A}) where {T,N,EH,D,A<:MRCData{T,N,EH,D}}
    return Base.IndexStyle(D)
end

"""
    eachmaprow(d::MRCData)

Return an iterator over map rows.
"""
eachmaprow(d::MRCData) = eachslice(d; dims=header(d).mapr)

"""
    eachmapcol(d::MRCData)

Return an iterator over columns.
"""
eachmapcol(d::MRCData) = eachslice(d; dims=header(d).mapc)

"""
    eachmapsection(d::MRCData)

Return an iterator over sections.
"""
eachmapsection(d::MRCData) = eachslice(d; dims=header(d).maps)

"""
    eachstackunit(d::MRCData)

Return an iterator over elements of stacks.
"""
function eachstackunit(d::MRCData)
    h = header(d)
    mrcdtype = mrcdatatype(h)
    return if mrcdtype ∈ (Volume, Image)
        (d for _ in 1:1)
    elseif mrcdtype == ImageStack
        eachmapsection(d)
    else
        error("Volume stacks not currently supported.")
    end
end


function global_update_backup_swap!(mc::AbstractDQMC)
  # swap current stack and greens to backup stack and greens (more efficient than copying)
  mc.s.gb_u_stack, mc.s.u_stack = mc.s.u_stack, mc.s.gb_u_stack
  mc.s.gb_d_stack, mc.s.d_stack = mc.s.d_stack, mc.s.gb_d_stack
  mc.s.gb_t_stack, mc.s.t_stack = mc.s.t_stack, mc.s.gb_t_stack
  mc.s.gb_greens, mc.s.greens = mc.s.greens, mc.s.gb_greens # this is greens at time slice == p.slices
  mc.s.gb_log_det, mc.s.log_det = mc.s.log_det, mc.s.gb_log_det # this is logdet of greens at time slice == p.slices + 1
end

@inline function global_update_perform_shift!(mc::AbstractDQMC)
    @inbounds @views begin
      @simd for k in 1:mc.p.opdim
        mc.p.hsfield[k,:,:] .+= randuniform(mc.p.box_global)
      end
    end
end

function global_update(mc::AbstractDQMC)
  p = mc.p
  s = mc.s

  @assert (s.current_slice == p.slices) && (s.direction == -1)

  S_old = p.boson_action
  # if p.all_checks && !isapprox(S_old,calc_boson_action(p, l)) warn("Incorrect boson action found during attempt to do global update.") end

  global_update_backup_swap!(mc) # save current stack etc.
  s.gb_hsfield = copy(p.hsfield)

  global_update_perform_shift!(mc)
  build_stack(mc)
  propagate(mc)
  # now we have s.greens = G_{p.slices} (up to one down-wrap) and s.log_det = logdet of G_{p.slices + 1} for new globally shifted conf

  p.boson_action = calc_boson_action(mc)

  p_boson = exp(-(p.boson_action - S_old)) # exp_delta_S_boson

  # calculate detratio = fermion accept. prob.
  p_fermion = exp(s.gb_log_det - s.log_det)

  p_acc = p_boson * real(p_fermion)

  # @printf("p_boson %.2e\n",abs.(p_boson))
  # @printf("p_fermion %.2e\n",abs.(p_fermion))
  # @printf("p_acc %.2e\n",abs.(p_acc))
  # println("")

  if p_acc > 1.0 || rand() < p_acc
    return 1
  else
    # undo global move
    p.boson_action = S_old
    p.hsfield = s.gb_hsfield
    global_update_backup_swap!(mc)
    return 0
  end

end


###
### Random Sequence Generator
###
###
### Random LongSequence generator.
###
### This file is a part of BioJulia.
### License is MIT: https://github.com/BioJulia/BioSequences.jl/blob/master/LICENSE.md

import Random: Sampler, rand!
const DEFAULT_RNG = StableRNG(rand(Int) & typemax(Int))

"""
    SamplerUniform{T}

Uniform sampler of type T. Instantiate with a collection of eltype T containing
the elements to sample.

# Examples
```
julia> sp = SamplerUniform(rna"ACGU");
```
"""
struct SamplerUniform{T} <: Sampler{T}
    elems::Vector{T}

    function SamplerUniform{T}(elems) where {T}
        elemsvector = convert(Vector{T}, collect(elems))
        if length(elemsvector) < 1
            throw(ArgumentError("elements collection must be non-empty"))
        end
        return new(elemsvector)
    end
end

SamplerUniform(elems) = SamplerUniform{eltype(elems)}(elems)
Base.eltype(::Type{SamplerUniform{T}}) where {T} = T
Base.rand(rng::AbstractRNG, sp::SamplerUniform) = rand(rng, sp.elems)
Base.rand(sp::SamplerUniform) = rand(DEFAULT_RNG, sp.elems)
const DefaultAASampler = SamplerUniform(aa"ACDEFGHIKLMNPQRSTVWY")

"""
    SamplerWeighted{T}

Weighted sampler of type T. Instantiate with a collection of eltype T containing
the elements to sample, and an orderen collection of probabilities to sample
each element except the last. The last probability is the remaining probability
up to 1.

# Examples
```
julia> sp = SamplerWeighted(rna"ACGUN", fill(0.2475, 4));
```
"""
struct SamplerWeighted{T} <: Sampler{T}
    elems::Vector{T}
    probs::Vector{Float64}

    function SamplerWeighted{T}(elems, probs) where {T}
        elemsvector = convert(Vector{T}, collect(elems))
        probsvector = convert(Vector{Float64}, collect(probs))
        if !isempty(probsvector)
            probsum = sum(probsvector)
            if probsum > 1.0
                throw(ArgumentError("sum of probabilties cannot exceed 1.0"))
            elseif minimum(probsvector) < 0.0
                throw(ArgumentError("probabilities must be non-negative"))
            end
        else
            probsum = 0.0
        end
        if length(elemsvector) != length(probsvector) + 1
            throw(ArgumentError("length of elems must be length of probs + 1"))
        end
        # Even with float weirdness,  we can guarantee x + (1.0 - x) == 1.0,
        # when 0 ≤ x ≤ 1, as there's no exponent, and it works like int addition
        return new(elemsvector, push!(probsvector, 1.0 - probsum))
    end
end

SamplerWeighted(elems, probs) = SamplerWeighted{eltype(elems)}(elems, probs)
Base.eltype(::Type{SamplerWeighted{T}}) where {T} = T

function Base.rand(rng::AbstractRNG, sp::SamplerWeighted)
    r = rand(rng)
    j = 1
    @inbounds cumulative_prob = sp.probs[j]
    while cumulative_prob < r
        j += 1
        @inbounds cumulative_prob += sp.probs[j]
    end
    return @inbounds sp.elems[j]
end
Base.rand(sp::SamplerWeighted) = rand(DEFAULT_RNG, sp)

"""
    randseq([rng::AbstractRNG], A::Alphabet, sp::Sampler, len::Integer)

Generate a LongSequence{A} of length `len` with elements drawn from
the given sampler.

# Example:
```
# Generate 1000-length RNA with 4% chance of N, 24% for A, C, G, or U
julia> sp = SamplerWeighted(rna"ACGUN", fill(0.24, 4))
julia> seq = randseq(RNAAlphabet{4}(), sp, 50)
50nt RNA Sequence:
CUNGGGCCCGGGNAAACGUGGUACACCCUGUUAAUAUCAACNNGCGCUNU
```
"""
function randseq(rng::AbstractRNG, A::Alphabet, sp::Sampler, len::Integer)
    seq = LongSequence{typeof(A)}(len)
    @inbounds for i in 1:len
        letter = rand(rng, sp)
        unsafe_setindex!(seq, letter, i)
    end
    return seq
end
randseq(A::Alphabet, sp::Sampler, len::Integer) = randseq(DEFAULT_RNG, A, sp, len)

"""
    randseq([rng::AbstractRNG], A::Alphabet, len::Integer)

Generate a LongSequence{A} of length `len` from the specified alphabet, drawn
from the default distribution. User-defined alphabets should implement this
method to implement random LongSequence generation.

For RNA and DNA alphabets, the default distribution is uniform across A, C, G,
and T/U.
For AminoAcidAlphabet, it is uniform across the 20 standard amino acids.
For a user-defined alphabet A, default is uniform across all elements of
`symbols(A)`.

# Example:
```
julia> seq = randseq(AminoAcidAlphabet(), 50)
50aa Amino Acid Sequence:
VFMHSIRMIRLMVHRSWKMHSARHVNFIRCQDKKWKSADGIYTDICKYSM
```
"""
function randseq(rng::AbstractRNG, A::NucleicAcidAlphabet{4}, len::Integer)
    seq = LongSequence{typeof(A)}(len)
    data = seq.data
    rand!(rng, data)
    @inbounds for i in eachindex(data)
        nuc = 0x1111111111111111
        mask = data[i]
        nuc = ((nuc & mask) << 1) | (nuc & ~mask)
        mask >>>= 1
        nuc = ((nuc & mask) << 2) | (nuc & ~mask)
        data[i] = nuc
    end
    return seq
end
randseq(A::NucleicAcidAlphabet{4}, len::Integer) = randseq(DEFAULT_RNG, A, len)

function randseq(rng::AbstractRNG, ::AminoAcidAlphabet, len::Integer)
    return randseq(rng, AminoAcidAlphabet(), DefaultAASampler, len)
end
randseq(A::AminoAcidAlphabet, len::Integer) = randseq(DEFAULT_RNG, A, len)

# Generic fallback method.
function randseq(rng::AbstractRNG, A::Alphabet, len::Integer)
    if length(A) == 1 << bits_per_symbol(A)
        return randseq_allbits(rng, A, len)
    else
        letters = symbols(A)
        sampler = SamplerUniform{eltype(A)}(letters)
        return randseq(rng, A, sampler, len)
    end
end
randseq(A::Alphabet, len::Integer) = randseq(DEFAULT_RNG, A, len)

function randseq_allbits(rng::AbstractRNG, A::Alphabet, len::Integer)
    vector = rand(rng, UInt64, seq_data_len(typeof(A), len))
    return LongSequence{typeof(A)}(vector, 1:len, false)
end

"""
    randdnaseq([rng::AbstractRNG], len::Integer)

Generate a random LongSequence{DNAAlphabet{4}} sequence of length `len`,
with bases sampled uniformly from [A, C, G, T]
"""
randdnaseq(rng::AbstractRNG, len::Integer) = randseq(rng, DNAAlphabet{4}(), len)
randdnaseq(len::Integer) = randseq(DEFAULT_RNG, DNAAlphabet{4}(), len)

"""
    randrnaseq([rng::AbstractRNG], len::Integer)

Generate a random LongSequence{RNAAlphabet{4}} sequence of length `len`,
with bases sampled uniformly from [A, C, G, U]
"""
randrnaseq(rng::AbstractRNG, len::Integer) = randseq(rng, RNAAlphabet{4}(), len)
randrnaseq(len::Integer) = randseq(DEFAULT_RNG, RNAAlphabet{4}(), len)

"""
    randaaseq([rng::AbstractRNG], len::Integer)

Generate a random LongSequence{AminoAcidAlphabet} sequence of length `len`,
with amino acids sampled uniformly from the 20 standard amino acids.
"""
randaaseq(rng::AbstractRNG, len::Integer) = randseq(rng, AminoAcidAlphabet(), len)
randaaseq(len::Integer) = randseq(DEFAULT_RNG, AminoAcidAlphabet(), len)


## GCMGridSpec function with default gridName argument:

GCMGridSpec() = GCMGridSpec("LLC90")

## GCMGridSpec function with gridName argument:

"""
    GCMGridSpec(gridName)

Set global variables in the module scope for grDir, nFaces, grTopo, ioSize,
facesSize, ioPrec using hard-coded values for LLC90, CS32, LL360 (for now).
"""
function GCMGridSpec(gridName)

global grDir, nFaces, grTopo, ioSize, facesSize, ioPrec;

if gridName=="LLC90";
    grDir="GRID_LLC90/";
    nFaces=5;
    grTopo="llc";
    ioSize=[90 1170];
    facesSize=[(90, 270), (90, 270), (90, 90), (270, 90), (270, 90)]
    ioPrec=Float64;
elseif gridName=="CS32";
    grDir="GRID_CS32/";
    nFaces=6;
    grTopo="cs";
    ioSize=[32 192];
    facesSize=[(32, 32), (32, 32), (32, 32), (32, 32), (32, 32), (32, 32)]
    ioPrec=Float32;
elseif gridName=="LL360";
    grDir="GRID_LL360/";
    nFaces=1;
    grTopo="ll";
    ioSize=[360 160];
    facesSize=[(360, 160)]
    ioPrec=Float32;
else;
    error("unknown gridName case");
end;

return "GCMGridSpec: passed"

end

## GCMGridLoad function

"""
    GCMGridLoad()

Loads grid variables from files located in grDir set by GCMGridSpec.

Grid variables are XC, XG, YC, YG, RAC, RAZ, DXC, DXG, DYC, DYG, hFacC,
hFacS, hFacW, Depth based on the MITgcm naming convention.
"""
function GCMGridLoad()

    global XC, XG, YC, YG, RAC, RAZ, DXC, DXG, DYC, DYG, hFacC, hFacS, hFacW, Depth;

    list0=("XC","XG","YC","YG","RAC","RAZ","DXC","DXG","DYC","DYG","hFacC","hFacS","hFacW","Depth");
    for ii=1:length(list0);
        tmp1=read_bin(grDir*list0[ii]*".data",ioPrec);
        tmp2=Symbol(list0[ii]);
        @eval (($tmp2) = ($tmp1))
    end

    return "GCMGridLoad: passed"

end

"""
    GCMGridOnes(grTp,nF,nP)

Define all-1 grid variables instead of using GCMGridSpec & GCMGridLoad.
"""
function GCMGridOnes(grTp,nF,nP)

    global grDir, nFaces, grTopo, ioSize, facesSize, ioPrec;
    global XC, XG, YC, YG, RAC, RAZ, DXC, DXG, DYC, DYG, hFacC, hFacS, hFacW, Depth;

    grDir=""
    grTopo=grTp
    nFaces=nF
    ioSize=[nP nP*nF]
    facesSize=Array{NTuple{2, Int},1}(undef,nFaces)
    facesSize[:].=[(nP,nP)]
    ioPrec=Float32

    list0=("XC","XG","YC","YG","RAC","RAZ","DXC","DXG","DYC","DYG","hFacC","hFacS","hFacW","Depth");
    for ii=1:length(list0);
        tmp1=fill(1.,nP,nP*nF);
        tmp1=convert2gcmfaces(tmp1);
        tmp2=Symbol(list0[ii]);
        @eval (($tmp2) = ($tmp1))
    end

    return "GCMGridOnes: passed"

end


#=

JuliaBolt is ported from Neobolt: Neo4j Bolt connector for Python
https://github.com/neo4j-drivers/neobolt

Original notices:

# Copyright (c) 2002-2018 "Neo4j,"
# Neo4j Sweden AB [http://neo4j.com]
#
# This file is part of Neo4j.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

=#

#####################################
#             Structure             #
#####################################

mutable struct Structure
    tag::UInt8
    fields::Array{Any}

    Structure(tag, fields) = new(tag, fields)
    Structure(tag) = new(tag, [])
end

function Base.getindex(structure::Structure, i::Int)
    return Base.getindex(structure.fields, i)
end

function Base.setindex(structure::Structure, v, i::Int)
    return Base.setindex(structure.fields, v, i)
end

function Base.firstindex(structure::Structure)
    return Base.firstindex(structure.fields)
end

function Base.lastindex(structure::Structure)
    return Base.lastindex(structure.fields)
end

# Structure ----------------------- #

Base.:(==)(x::Structure, y::Structure) = x.tag == y.tag && x.fields == y.fields

# Structure ----------------------- #
# # # # # # # # # # # # # # # # # # #

#####################################
#              strpack              #
#####################################

function struct_pack(value::UInt8)
    return [hton(value)]
end

# strpack ------------------------- #

function struct_pack(value::Union{UInt16, UInt32, Int16, Int32, Int64, Float64})
    return copy(reinterpret(UInt8, [hton(value)]))
end

# strpack ------------------------- #

function struct_unpack(t::DataType, v::AbstractArray{UInt8})
    _value = reinterpret(t, v)
    return ntoh(_value[1])
end

# strpack ------------------------- #
# # # # # # # # # # # # # # # # # # #

include("packer.jl")
include("unpacker.jl")


export phasequery
ETYPE_MAPPING = Dict(
    "eq"=>"earthquake",
    "qb"=> "quarry blast",
    "sn"=> "sonic boom",
    "nt"=> "nuclear blast",
    "uk"=> "not reported",
)

MAGTYPE_MAPPING = Dict(
    "b"=> "Mb",
    "e"=> "Me",
    "l"=> "ML",
    "s"=> "MS",
    "c"=> "Mc",
    "n"=> "",
    "w"=> "Mw",
    "h"=> "Mh",
    "d"=> "Md",
    "lr"=> "Mlr",
)

POLARITY_MAPPING = Dict(
    "c."=> 'U',
    ".c"=> 'U',
    "u."=> 'U',
    "d."=> 'D',
    ".d"=> 'D',
    "r."=> ' ',
    ".."=> ' ',
    ".."=> ' ',
    ""  => ' ',
)

function read_phase(phasestream::AbstractArray)
    pstr = String(phasestream)
    parr = split(pstr,"\n")
    parr = strip.(parr[1:end-1])
    event_header = String(parr[1])
    parr = parr[2:end]
    N = length(parr)

    # get event info from first line of phase file
    evid, etype, origin_time, eqmag, eqloc = parse_event_header(event_header,N)

    # need to wrap this
    df = phasestring2df(parr)
    EQ = parse_phases(df)

    # make SeisHdr
    hdr = SeisHdr(id=evid,loc=eqloc,mag=eqmag,ot=origin_time,typ=etype)
    return SeisEvent(hdr=hdr,data=EQ)
end

function parse_event_header(header::String,N::Int)
    dateformater = Dates.DateFormat("y/m/d,H:M:S.s")
    evid, etype, geo, origin_time, lat, lon, depth, mag, magtype = split(header)
    evid = String(evid)
    lat = parse(Float64,lat)
    lon = parse(Float64,lon)
    depth = parse(Float64,depth)
    mag = parse(Float32,mag)
    origin_time = DateTime(origin_time,dateformater)
    etype = ETYPE_MAPPING[String(etype)]

    # make magnitude
    eqmag = EQMag(val=mag,scale=MAGTYPE_MAPPING[magtype],src="SCSN",nst=N)

    # make location
    eqloc = EQLoc(lat=lat,lon=lon,dep=depth,nst=N)
    return evid, etype, origin_time, eqmag, eqloc
end

"""
    phasequery(eventid,eventtime)

Use S3 to query SCEDC-pds phase arrivals for eventid == `eventid`.

# Arguments
- `eventid::Int`: SCSN event ID.
- `eventtime::TimeType`: Time of event.

Returns a `SeisIO.Quake.SeisEvent`
"""
function phasequery(aws::AWSConfig,eventid::Int,eventtime::TimeType)
    daynum = lpad(dayofyear(eventtime),3,'0')
    yr = year(eventtime)
    query = "event_phases/$yr/$(yr)_$daynum/$eventid.phase"
    phasestream = Array{UInt8}(undef,0)
    try
        phasestream = s3_get(aws,"scedc-pds",query)
    catch e
        throw(e,"Event $eventid on $eventtime could not be found.")
    end

    return read_phase(phasestream)
end
phasequery(a...) = phasequery(global_aws_config(region="us-west-2"),a...)
phasequery(d::Dict,a...) = phasequery(global_aws_config(region=d[:region]),a...)

function phasestring2df(phasestring::AbstractArray)
    ns = [:NET,:STA,:CHAN,:LOC,:LAT,:LON,:ELEV,:PHASE,:MOTION,:ONSET,:QUAL,:DIST,:OFFSET]
    df = DataFrame(permutedims(String.(hcat(split.(phasestring)...))),ns)
    df[!,[:LAT,:LON,:ELEV,:QUAL,:DIST,:OFFSET]] .= parse.(Float64,df[!,[:LAT,:LON,:ELEV,:QUAL,:DIST,:OFFSET]])
    df[!,:LOC] .= replace.(df[!,:LOC],"--"=>"00")
    df[!,:POL] = map(x -> POLARITY_MAPPING[x], df[!,:MOTION])
    return df
end

function parse_phases(df::DataFrame)
    # construct EventChannel
    N = size(unique(df,[:NET,:STA,:CHAN]),1)
    EQ = EventTraceData(N)
    for (ii,group) in enumerate(groupby(df,[:NET,:STA,:CHAN]))
        EQ[ii] = df2phacat(group)
    end
    return EQ
end

function df2phacat(df::AbstractDataFrame)
    N = size(df,1)
    if N == 0
        throw(ErrorException("Empty DataFrame."))
    end
    id = join(df[1,[:NET,:STA,:LOC,:CHAN]],'.')
    loc = GeoLoc(lat=df[1,:LAT],lon=df[1,:LON],el=df[1,:ELEV])
    dist = df[1,:DIST]
    phacat = Dict(df[ii,:PHASE]=>SeisPha(d=df[ii,:DIST],tt=df[ii,:OFFSET],pol=df[ii,:POL]) for ii = 1:N)
    return EventChannel(id=id,loc=loc,dist=dist,pha=phacat)
end


#============================ Global Parameters ===============================#
const MAX_MOVES = 256
const MAX_PLY = 128
const MATE = 32000
const MAX_QUIET_TRACK = 92
ABORT_SIGNAL = Base.Threads.Atomic{Bool}(false)

#============================== Search Parameters =============================#

const Q_FUTILITY_MARGIN = 193
const RAZOR_DEPTH = 1
const RAZOR_MARGIN = 325
const BETA_PRUNE_DEPTH = 8
const BETA_PRUNE_MARGIN = 172
const SEE_PRUNE_DEPTH = 8
const SEE_QUIET_MARGIN = -190
const SEE_NOISY_MARGIN = -25
const FUTILITY_PRUNE_DEPTH = 8
const FUTILITY_MARGIN = 392
const FUTILITY_MARGIN_NOHIST = 300
const FUTILITY_LIMIT = @SVector [12000, 6000]
const COUNTER_PRUNE_DEPTH = @SVector [4, 3]
const COUNTER_PRUNE_LIMIT = @SVector [0, -500]
const FOLLOW_PRUNE_DEPTH = @SVector [4, 3]
const FOLLOW_PRUNE_LIMIT = @SVector [-1000, -3000]
const WINDOW_DEPTH = 5
const LATE_MOVE_COUNT = @SVector [SVector{10}([0, 2, 4,  7, 11, 16, 22, 29, 37, 46]), SVector{10}([0, 4, 7, 12, 20, 30, 42, 56, 73, 92])]
const LATE_MOVE_PRUNE_DEPTH = 9
const LMRTABLE = init_reduction_table()
const PROBCUT_DEPTH = 5
const PROBCUT_MARGIN = 190

#============================ Piece square tables =============================#

const PAWN_PSQT = SVector{8}([
    SVector{4}([makescore(  0,  0), makescore(  0,  0), makescore(  0,  0), makescore( 0,   0)]),
    SVector{4}([makescore(  -5, -6), makescore( 48, 15), makescore( 18, 27), makescore( 5,  3)]),
    SVector{4}([makescore(-24,-15), makescore( -3, 3), makescore( -19,-11), makescore( -27, -9)]),
    SVector{4}([makescore(-31, 6), makescore(-14, 8), makescore(  -19, 9), makescore( 10, -33)]),
    SVector{4}([makescore( -16, 23), makescore(  14, -2), makescore(0,-11), makescore( -10,-24)]),
    SVector{4}([makescore( 4, 14), makescore(15, 23), makescore(13, -15), makescore( -30, -37)]),
    SVector{4}([makescore( 33,  28), makescore( -6, 25), makescore( 1, 12), makescore(24, -47)]),
    SVector{4}([makescore(  0,  0), makescore(  0,  0), makescore(  0,  0), makescore( 0,   0)])])

const KNIGHT_PSQT = SVector{8}([
    SVector{4}([makescore(-8, -37), makescore(-42, -59), makescore(-96, -14), makescore(-80, 32)]),
    SVector{4}([makescore( -28, -42), makescore(-108, -4), makescore(-1, -56), makescore(-26, 6)]),
    SVector{4}([makescore( -33, 7), makescore(49, -40), makescore( 16,  -15), makescore( -14, 23)]),
    SVector{4}([makescore( -9, 16), makescore( 29, -10), makescore( 66, 14), makescore( 32,  38)]),
    SVector{4}([makescore( 27, 5), makescore( 54, 19), makescore( 13, 49), makescore( 72,  43)]),
    SVector{4}([makescore( 32, -39), makescore( 74, 1), makescore( 94, 2), makescore( 130,  -18)]),
    SVector{4}([makescore( -84, -33), makescore(-32, 10), makescore( 86, -24), makescore( -3, 2)]),
    SVector{4}([makescore(-175,-117), makescore(29, -25), makescore(-46, 9), makescore( 40, 1)])])

const BISHOP_PSQT = SVector{8}([
    SVector{4}([makescore(-103, -41), makescore( -82, 59), makescore(-49,-15), makescore(-53,22)]),
    SVector{4}([makescore(-33, -62), makescore( 3, -58), makescore( 12,-23), makescore(  -9,  6)]),
    SVector{4}([makescore(-8, -16), makescore( -22, 7), makescore( -1,  -48), makescore( 11, 5)]),
    SVector{4}([makescore( 32, 8), makescore( 88,  -83), makescore( 10,  10), makescore( 46, -17)]),
    SVector{4}([makescore( 29, 21), makescore( 21,  1), makescore( 23, 4), makescore( 66, -32)]),
    SVector{4}([makescore( 21, 44), makescore(  60,  -21), makescore( 23,  -20), makescore( 17, -15)]),
    SVector{4}([makescore(-47, -50), makescore(-42, -37), makescore(  -14,  20), makescore( -13,  36)]),
    SVector{4}([makescore(-32, -80), makescore( -10, 31), makescore(-47,-16), makescore(-3,-28)])])

const ROOK_PSQT = SVector{8}([
    SVector{4}([makescore(-25,-17), makescore(-47,12), makescore(35,-20), makescore(31,-37)]),
    SVector{4}([makescore(-93,1), makescore(-58,-4), makescore(-9, -27), makescore( -42, -21)]),
    SVector{4}([makescore(-56, -15), makescore(45,-15), makescore( -44, -22), makescore( -16, -39)]),
    SVector{4}([makescore(-26, -5), makescore( -9, 19), makescore( -14, 2), makescore(16,  -29)]),
    SVector{4}([makescore(-41, 20), makescore(-12, -7), makescore( 43, 21), makescore( -15, 13)]),
    SVector{4}([makescore(-19, 8), makescore( 85, -4), makescore( -4, -1), makescore(-23, 20)]),
    SVector{4}([makescore(  57,  -14), makescore( 20, 17), makescore( 38, 1), makescore(-31, 25)]),
    SVector{4}([makescore( 38, 23), makescore(111, 18), makescore( -13, 48), makescore(-93, 66)])])

const QUEEN_PSQT = SVector{8}([
    SVector{4}([makescore( 0,-70), makescore(-5,-60), makescore(-5,-45), makescore( 5,-25)]),
    SVector{4}([makescore( 0,-55), makescore( 5,-30), makescore(10,-20), makescore(10, -5)]),
    SVector{4}([makescore( 0,-40), makescore( 5,-20), makescore(10,-10), makescore(10,  0)]),
    SVector{4}([makescore( 5,-25), makescore( 5,  0), makescore(10, 10), makescore(10, 25)]),
    SVector{4}([makescore( 0,-30), makescore(15, -5), makescore(10, 10), makescore( 5, 20)]),
    SVector{4}([makescore(-5,-40), makescore(10,-20), makescore( 5,-10), makescore(10,  0)]),
    SVector{4}([makescore(-5,-50), makescore( 5,-25), makescore(10,-25), makescore(10,-10)]),
    SVector{4}([makescore( 0,-75), makescore( 0,-50), makescore( 0,-40), makescore( 0,-35)])])

const KING_PSQT = SVector{8}([
    SVector{4}([makescore(216,  0), makescore(264, 36), makescore(216, 68), makescore(160, 60)]),
    SVector{4}([makescore(224, 44), makescore(240, 80), makescore(188,108), makescore(144,108)]),
    SVector{4}([makescore(156, 72), makescore(208,104), makescore(136,136), makescore( 96,140)]),
    SVector{4}([makescore(132, 80), makescore(152,124), makescore(112,136), makescore( 80,136)]),
    SVector{4}([makescore(124, 76), makescore(144,132), makescore( 84,160), makescore( 56,160)]),
    SVector{4}([makescore(100, 72), makescore(116,136), makescore( 64,148), makescore( 24,152)]),
    SVector{4}([makescore(72,  40), makescore(96,  96), makescore( 52, 92), makescore( 28,104)]),
    SVector{4}([makescore(48,   8), makescore(72,  48), makescore( 36, 60), makescore(  0, 64)])])

const PSQT = @SVector [PAWN_PSQT, KNIGHT_PSQT, BISHOP_PSQT, ROOK_PSQT, QUEEN_PSQT, KING_PSQT]

#=============================== PIECE VALUES =================================#

const PVALS = @SVector [makescore(211, 245), makescore(949, 863), makescore(953, 962), makescore(1220, 1490), makescore(2711, 2840), makescore(15000, 15000)]
const PVALS_MG = SVector{6}(scoreMG.(PVALS))
const PVALS_EG = SVector{6}(scoreEG.(PVALS))

#=============================== Tempo Bonus ==================================#

const TEMPO_BONUS = 22

#============================= Rook Evaluation ================================#

const ROOK_OPEN_FILE_BONUS = makescore(114, 10)
const ROOK_SEMIOPEN_FILE_BONUS = makescore(42, 5)
const ROOK_KING_FILE_BONUS = 10
const ROOK_ON_QUEEN_FILE = makescore(20, 26)

#============================= Pawn Evaluation ================================#

const PAWN_SHIELD_BONUS = makescore(13, 10)
const DOUBLE_PAWN_PENALTY = makescore(34, 84)
const ISOLATED_PAWN_PENALTY = makescore(17, 34)
const PAWN_DEFEND_PAWN_BONUS = 10
const WEAK_PAWN_PENALTY = 25
const PASS_PAWN_THREAT = SVector{7}([makescore(0, 0), makescore(6, 23), makescore(7, 37), makescore(3, 1), makescore(16, 29), makescore(142, 149), makescore(237, 216)])
const CONNECTED_PAWN_PSQT = SVector{8}([
    SVector{4}([makescore(   0,  0), makescore(   0,  0), makescore(   0,  0), makescore(   0,  0)]),
    SVector{4}([makescore(   0,-10), makescore(  10,  0), makescore(   5,  0), makescore(   5, 15)]),
    SVector{4}([makescore(  15,  0), makescore(   30, 0), makescore(  20, 10), makescore(  25, 15)]),
    SVector{4}([makescore(  10,  0), makescore(  25,  5), makescore(  10, 10), makescore(  15, 20)]),
    SVector{4}([makescore(  15,  8), makescore(  20, 15), makescore(  25, 20), makescore(  30, 20)]),
    SVector{4}([makescore(  60, 25), makescore(  51, 50), makescore(  70, 55), makescore(  85, 60)]),
    SVector{4}([makescore( 110,  0), makescore( 205, 10), makescore( 230, 30), makescore( 240, 50)]),
    SVector{4}([makescore(   0,  0), makescore(   0,  0), makescore(   0,  0), makescore(   0,  0)])])
const WEAK_UNOPPOSED = makescore(22, 52)
const WEAK_LEVER = makescore(31, 34)
const BACKWARD_PAWN = makescore(37, 11)

#=========================== Knight Evaluation ================================#

const KNIGHT_TRAP_PENALTY = makescore(50, 50)
const KNIGHT_RAMMED_BONUS = makescore(6, 14)
const KNIGHT_OUTPOST_BONUS = makescore(20, 61)
const KNIGHT_POTENTIAL_OUTPOST_BONUS = makescore(8, 29)

#=========================== Bishop Evaluation ================================#

const BISHOP_TRAP_PENALTY = makescore(80, 80)
const BISHOP_COLOR_PENALTY = makescore(3, 14)
const BISHOP_RAMMED_COLOR_PENALTY = makescore(3, 31)
const BISHOP_PAIR_BONUS = makescore(55, 77)
const BISHOP_OUTPOST_BONUS = makescore(50, 29)
const BISHOP_CENTRAL_CONTROL = makescore(60, 33)

#============================= King Evaluation ================================#

const CASTLE_OPTION_BONUS = makescore(10, 0)
const KING_PAWN_SHIELD_BONUS = 12
const KING_FLANK_ATTACK = makescore(11, 3)
const PAWNLESS_FLANK = makescore(19, 111)

const KING_SHELTER_OFFFILE = SVector{8}([
SVector{8}([makescore(-11, 1), makescore(12, -25), makescore(18, -9), makescore(9, 2), makescore(4, 2), makescore(0, 0), makescore(-4, -31), makescore(-45, 15)]),
SVector{8}([makescore(14, -9), makescore(18, -18), makescore(0, -6), makescore(-14, 2), makescore(-27, 12), makescore(-63, 60), makescore(81, 72), makescore(-24, 0)]),
SVector{8}([makescore(30, -5), makescore(11, -9), makescore(-27, 5), makescore(-11, -9), makescore(-18, -5), makescore(-11, 0), makescore(0, 58), makescore(-12, -3)]),
SVector{8}([makescore(2, 9), makescore(18, -11), makescore(2, -11), makescore(12, -21), makescore(21, -34), makescore(-54, 3), makescore(-124, 45), makescore(3, -8)]),
SVector{8}([makescore(-15, 5), makescore(2, -5), makescore(-25, 0), makescore(-18, 3), makescore(-18, -7), makescore(-38, -2), makescore(28, -17), makescore(-7, -3)]),
SVector{8}([makescore(40, -18), makescore(18, -17), makescore(-20, 0), makescore(-12, -18), makescore(3, -23), makescore(14, -20), makescore(36, -27), makescore(-23, 0)]),
SVector{8}([makescore(21, -13), makescore(-2, -16), makescore(-22, -3), makescore(-18, -9), makescore(-27, -8), makescore(-34, 27), makescore(0, 38), makescore(-9, 0)]),
SVector{8}([makescore(-9, -12), makescore(4, -18), makescore(5, 0), makescore(-3, 9), makescore(-11, 12), makescore(-9, 32), makescore(-171, 77), makescore(-17, 11)])
])

const KING_SHELTER_ONFILE = SVector{8}([
SVector{8}([makescore(0, 0), makescore(-13, -25), makescore(2, -20), makescore(-40, 14), makescore(-24, 0), makescore(1, 40), makescore(-169, -10), makescore(-48, 5)]),
SVector{8}([makescore(0, 0), makescore(22, -23), makescore(5, -9), makescore(-20, -3), makescore(-3, -14), makescore(24, 64), makescore(-186, -5), makescore(-40, 1)]),
SVector{8}([makescore(0, 0), makescore(30, -13), makescore(-4, -9), makescore(5, -19), makescore(13, -9), makescore(-89, 45), makescore(-86, -75), makescore(-20, -7)]),
SVector{8}([makescore(0, 0), makescore(-5, 7), makescore(-4, 0), makescore(-19, 0), makescore(-29, 0), makescore(-100, 30), makescore(5, -43), makescore(-24, -9)]),
SVector{8}([makescore(0, 0), makescore(10, 0), makescore(10, -7), makescore(12, -13), makescore(12, -28), makescore(-59, 13), makescore(-106, -63), makescore(-3, -9)]),
SVector{8}([makescore(0, 0), makescore(4, -10), makescore(-20, 0), makescore(-29, -8), makescore(15, -26), makescore(-40, 1), makescore(53, 36), makescore(-20, -8)]),
SVector{8}([makescore(0, 0), makescore(20, -17), makescore(10, -15), makescore(-10, -9), makescore(-29, 7), makescore(-10, 13), makescore(-58, -50), makescore(-33, 10)]),
SVector{8}([makescore(0, 0), makescore(10, -40), makescore(17, -29), makescore(-20, -6), makescore(-19, 16), makescore(-7, 20), makescore(-230, -57), makescore(-24, 0)])
])

const KING_STORM_UNBLOCKED = SVector{4}([
SVector{8}([makescore(-7, 26), makescore(116, -10), makescore(-28, 23), makescore(-22, 5), makescore(-17, 0), makescore(-10, -7), makescore(-19, 2), makescore(-25, -5)]),
SVector{8}([makescore(-6, 46), makescore(54, 9), makescore(-22, 21), makescore(-8, 8), makescore(-7, 2), makescore(2, -7), makescore(-4, 0), makescore(-14, 0)]),
SVector{8}([makescore(5, 35), makescore(14, 20), makescore(-26, 18), makescore(-14, 8), makescore(0, 0), makescore(4, 0), makescore(8, -10), makescore(0, -4)]),
SVector{8}([makescore(-5, 22), makescore(13, 18), makescore(-19, 8), makescore(-17, 0), makescore(-16, 0), makescore(4, -14), makescore(0, -10), makescore(-16, 0)])
])

const KING_STORM_BLOCKED = SVector{4}([
SVector{8}([makescore(0, 0), makescore(-18, -19), makescore(-20, -4), makescore(15, -20), makescore(7, -10), makescore(0, -20), makescore(-6, -4), makescore(14, 26)]),
SVector{8}([makescore(0, 0), makescore(-19, -37), makescore(-6, -11), makescore(32, -14), makescore(-4, -5), makescore(10, -26), makescore(-10, -13), makescore(-20, 0)]),
SVector{8}([makescore(0, 0), makescore(-31, -52), makescore(-30, -9), makescore(8, -14), makescore(0, -5), makescore(-11, -17), makescore(-16, -18), makescore(-14, 3)]),
SVector{8}([makescore(0, 0), makescore(-5, -20), makescore(-19, -20), makescore(-14, -7), makescore(-7, -10), makescore(1, -28), makescore(70, -14), makescore(10, 18)])
])

#============================ Queen Evaluation ================================#

#============================ Other Evaluation ================================#

const SPACE_BONUS = 4
const PIN_BONUS = 21
const HANGING_BONUS = makescore(6, 19)
const THREAT_BY_MINOR = @SVector [makescore(16, 24), makescore(60, 86), makescore(42, 74), makescore(131, 7), makescore(89, 123)]
const THREAT_BY_ROOK = @SVector [makescore(10, 31), makescore(58, 36), makescore(44, 88), makescore(1, 6), makescore(120, 45)]
const LAZY_THRESH = 2000
const MINOR_KING_PROTECTION = makescore(8, 6)
const RESTRICTION_BONUS = makescore(19, 2)
const KNIGHT_SAFE_CHECK = 182
const QUEEN_SAFE_CHECK  = 148
const BISHOP_SAFE_CHECK = 87
const ROOK_SAFE_CHECK   = 137
const KD_QUEEN  = 44
const KD_ROOK   = 9
const KD_BISHOP = 9
const KD_KNIGHT = 1
const THREAT_BY_PAWN = makescore(129, 28)
const THREAT_BY_PUSH = makescore(62, 43)
const THREAT_BY_KING = makescore(8, 26)

const KING_FLANK_DEFEND = makescore(10, 1)
const KING_BOX_WEAK = makescore(7, 16)
const UNSAFE_CHECK = makescore(18, 0)
const KNIGHT_ON_QUEEN = makescore(15, 10)

#========================= Mobility Evaluation ================================#

const KNIGHT_MOBILITY = @SVector [makescore(-60,-80), makescore(-50,-55), makescore(-10,-30), makescore( -5,-15),
    makescore(  5,  10), makescore( 15, 15), makescore( 20, 25), makescore( 30, 30), makescore( 35, 35)]

const BISHOP_MOBILITY = @SVector [makescore(-50,-60), makescore(-20,-25), makescore( 15, -5), makescore( 25, 15), makescore( 40, 25),
    makescore( 50, 40), makescore( 55, 55), makescore( 65, 60), makescore( 65, 65), makescore( 70, 75), makescore( 80, 80),
    makescore( 80, 85), makescore( 90, 90), makescore( 100, 100)]

const ROOK_MOBILITY = @SVector [makescore(-60,-75), makescore(-30,-20), makescore(-15, 30), makescore(-10, 55), makescore( -5, 70), makescore( 0, 80),
      makescore(  10,110), makescore( 15,120), makescore( 30,130), makescore( 30,140), makescore( 30,155), makescore( 40,165),
      makescore( 45,165), makescore( 50,170), makescore( 60,170)]

const QUEEN_MOBILITY = @SVector [makescore(-40,-35), makescore(-20,-15), makescore(  5,  10), makescore(  5, 20), makescore( 15, 35), makescore( 20, 55),
      makescore( 30, 60), makescore( 40, 75), makescore( 45, 80), makescore( 50, 90), makescore( 55, 95), makescore( 60,105),
      makescore( 60,115), makescore( 65,120), makescore( 70,125), makescore( 70,125), makescore( 70,130), makescore( 75,135),
      makescore( 80,140), makescore( 90,145), makescore( 90,150), makescore( 100,165), makescore(100,170), makescore(100,175),
      makescore(105,185), makescore(110,190), makescore(115,205), makescore(115,210)]

#============================ SCALING FACTORS =================================#

const SCALE_OCB_BISHOPS = 64
const SCALE_OCB_ONE_KNIGHT = 106
const SCALE_OCB_ONE_ROOK = 96
const SCALE_DRAW = 0
const SCALE_NORMAL = 128


# All functions related to reproduction.

"""
    reproduce!(patch)

Reproduction of individuals in a patch (default function).
"""
function reproduce!(patch::Patch) #TODO: refactor!
    #TODO This is one of the most compute-intensive functions in the model - optimise?
    for ind in patch.community
        ind.marked && continue # individual might not have established yet
        ind.size < ind.traits["repsize"] && continue
        metaboffs = setting("fertility") * ind.size^(-1/4) * exp(-act/(boltz*patch.temp))
        noffs = rand(Poisson(metaboffs))
        noffs < 1 && continue
        partners = findmate(patch.community, ind)
        if length(partners) < 1 && rand() < ind.traits["selfing"]
            partners = [ind]
        elseif length(partners) < 1
            continue
        end
        numpartners = Integer(round(ind.traits["numpollen"]))
        if numpartners == 0
            #XXX happens really often?
            # `numpollen` not handled specifically in `createtraits()`?
            @simlog("Individual cannot reproduce, `numpollen` too low.", 'd')
            continue
        end
        for ptn in 1:numpartners
            partner = rand(partners)
            # subtract offspring mass from parent #XXX is that sensible, does it make a difference?
            parentmass = ind.size - noffs * ind.traits["seedsize"]
            if parentmass <= 0
                break
            else
                ind.size = parentmass
            end
            append!(patch.seedbank, createoffspring(noffs, ind, partner))
        end
    end
    @simlog("Patch $(patch.id): $(length(patch.seedbank)) offspring", 'd')
end

"""
    greproduce!(patch)

Reproduction of individuals in a patch with global mating.
"""
function greproduce!(patch::Patch)
    for ind in patch.community
        ind.marked && continue # individual might not have established yet
        ind.size < ind.traits["repsize"] && continue
        metaboffs = setting("fertility") * ind.size^(-1/4) * exp(-act/(boltz*patch.temp))
        noffs = rand(Poisson(metaboffs))
        noffs < 1 && continue
        partners = findmate([(map(x -> x.community, world)...)...], ind)
        if length(partners) < 1 && rand() < ind.traits["selfing"]
            partners = [ind]
        elseif length(partners) < 1
            continue
        end
        numpartners = Integer(round(ind.traits["numpollen"]))
        for ptn in 1:numpartners
            partner = rand(partners, 1)[1]
            parentmass = ind.size - noffs * ind.traits["seedsize"] # subtract offspring mass from parent
            if parentmass <= 0
                continue
            else
                ind.size = parentmass
            end
            append!(patch.seedbank, createoffspring(noffs, ind, partner))
        end
    end
    @simlog("Patch $(patch.id): $(length(patch.seedbank)) offspring", 'd')
end

"""
    reproduce!(world)

Carry out reproduction on all patches.
"""
function reproduce!(world::Array{Patch,1})
    for patch in world
        if setting("globalmating")
            greproduce!(patch)
        else
            (patch.isisland || !setting("static")) && reproduce!(patch) # pmap(!,patch) ???
        end
    end
end

"""
    findmate(population, individual)

Find a reproduction partner for the given individual in the given population.
"""
function findmate(population::AbstractArray{Individual, 1}, ind::Individual)
    #XXX This function is pretty expensive
    indstate = ind.marked
    ind.marked = true
    mates = Individual[]
    startidx = rand(eachindex(population))
    mateidx = startidx
    while true
        mate = population[mateidx]
        if !mate.marked && iscompatible(mate, ind)
            push!(mates, mate)
            break
        end
        mateidx += 1
        mateidx > length(eachindex(population)) && (mateidx = 1)
        mateidx == startidx && break
    end
    ind.marked = indstate
    mates
end

"""
    createoffspring(noffs, individual, partner, dimorphism)

The main reproduction function. Take two organisms and create the given number
of offspring individuals. Returns an array of individuals.
"""
function createoffspring(noffs::Integer, ind::Individual, partner::Individual, dimorphism::Bool=false)
    #TODO This is a very compute-intensive function - optimise?
    offspring = Individual[]
    heterozygosity = setting("heterozygosity")
    for i in 1:noffs # pmap? this loop could be factorized!
        # offspring have different genomes due to recombination
        partnergenome = meiosis(partner.genome, false, heterozygosity)
        mothergenome = meiosis(ind.genome, true, heterozygosity)
        (isempty(partnergenome) || isempty(mothergenome)) && continue
        genome = vcat(partnergenome,mothergenome)
        if setting("degpleiotropy") == 0 && setting("linkage") == "none"
            traits = gettraitdictfast(genome, setting("traitnames"))
        else
            traits = gettraitdict(genome, setting("traitnames"))
        end
        marked = true
        fitness = 0.0
        newpartner = 0
        newsize = ind.traits["seedsize"]
        newid = rand(UInt64) #XXX Is this collision-safe?
        sex = hermaphrodite
        if dimorphism
            rand(Bool) ? sex = male : sex = female
        end
        lineage = ind.lineage
        if lineage != partner.lineage
            # hybrids are assigned the lineage of the parent they are phenotypically most similar to
            idiff = sum(t -> abs(traits[t]-ind.traits[t]), keys(traits))
            pdiff = sum(t -> abs(traits[t]-partner.traits[t]), keys(traits))
            (pdiff < idiff) && (lineage = partner.lineage)
        end
        push!(offspring, Individual(lineage, genome, traits, marked, fitness,
                                    fitness, newsize, sex, newpartner, newid))
    end
    offspring
end


# this writes to disk the items that Octave needs to do II or BII estimation
include("dataprep.jl")
mZs, sZs, beta, sErrors = CreatePreprocessItems()
writedlm("beta", beta)
writedlm("mZs", mZs)
writedlm("sZs", sZs)
writedlm("sErrors", sErrors)
using MXNet
items = mx.load_checkpoint("dsge30040", 20)
alpha1 = copy(items[2][:fullyconnected0_bias])
alpha2 = copy(items[2][:fullyconnected1_bias])
alpha3 = copy(items[2][:fullyconnected2_bias])
beta1 = copy(items[2][:fullyconnected0_weight])
beta2 = copy(items[2][:fullyconnected1_weight])
beta3 = copy(items[2][:fullyconnected2_weight])
writedlm("alpha1", alpha1)
writedlm("alpha2", alpha2)
writedlm("alpha3", alpha3)
writedlm("beta1", beta1)
writedlm("beta2", beta2)
writedlm("beta3", beta3)


function drawFiller(ctx::Cairo.CairoContext, camera::Camera, filler::Filler; alpha=nothing)
    if ctx.ptr != C_NULL
        x, y = Int(filler.x) * 8, Int(filler.y) * 8
        w, h = Int(filler.w) * 8, Int(filler.h) * 8

        Cairo.save(ctx)

        translate(ctx, -camera.x, -camera.y)
        scale(ctx, camera.scale, camera.scale)
        translate(ctx, x, y)

        Ahorn.drawRectangle(ctx, 0, 0, w, h, colors.filler_room_fill)

        Cairo.restore(ctx)
    end
end

function fillerVisible(camera::Camera, width::Int, height::Int, filler::Filler)
    actuallX = camera.x / camera.scale
    actuallY = camera.y / camera.scale

    actuallWidth = width / camera.scale
    actuallHeight = height / camera.scale

    x, y = Int(filler.x) * 8, Int(filler.y) * 8
    w, h = Int(filler.w) * 8, Int(filler.h) * 8

    cameraRect = Rectangle(actuallX, actuallY, actuallWidth, actuallHeight)
    fillerRect = Rectangle(x, y, w, h)

    return checkCollision(cameraRect, fillerRect)
end


export decode

mutable struct Decoder
    worddict::Dict
    chardict::Dict
    tagdict::Dict
    nn
end

struct Sample
    w
    batchdims_w
    c
    batchdims_c
    t
end

macro logtime()
    return :( Dates.format(now(), "yyyy-mm-dd HH:MM:SS") )
end

function create_batch(samples::Vector{Sample}, batchsize::Int)
    batches = Sample[]
    for i = 1:batchsize:length(samples)
        range = i:min(i+batchsize-1,length(samples))
        s = samples[range]
        w = cat(2, map(x -> x.w, s)...)
        c = cat(2, map(x -> x.c, s)...)
        batchdims_w = cat(1, map(x -> x.batchdims_w, s)...)
        batchdims_c = cat(1, map(x -> x.batchdims_c, s)...)
        t = s[1].t == nothing ? nothing : cat(1, map(x -> x.t, s)...)

        w = BACKEND(w)
        batchdims_w = sort(batchdims_w, rev=true)
        c = BACKEND(c)
        batchdims_c = BACKEND(batchdims_c)
        t = BACKEND(t)

        push!(batches, Sample(w, batchdims_w, c, batchdims_c, t))
    end
    batches
end

function Decoder(config::Dict, iolog)
    words = h5read(config["wordvec_file"], "words")
    wordembeds = h5read(config["wordvec_file"], "vectors")
    worddict = Dict(words[i] => i for i=1:length(words))
    chardict, tagdict = initvocab(config["train_file"])
    charembeds = Normal(0,0.01)(Float32, 20, length(chardict))
    traindata = readdata(config["train_file"], worddict, chardict, tagdict)
    testdata = readdata(config["test_file"], worddict, chardict, tagdict)
    nepochs = haskey(config, "nepochs") ? config["nepochs"] : 1
    nlayers = haskey(config, "nlayers") ? config["nlayers"] : 1
    winsize_c = haskey(config, "winsize_c") ? config["winsize_c"] : 2
    winsize_w = haskey(config, "winsize_w") ? config["winsize_w"] : 5
    droprate = haskey(config, "droprate") ? config["droprate"] : 0.2

    procname = @sprintf("NER.Convolution[%s]", haskey(config, "jobid") ? config["jobid"] : "-")
    @printf(iolog, "%s %s training - train:%d test:%d words:%d, chars:%d tags:%d\n", @logtime, procname,
            length(traindata), length(testdata), length(worddict), length(chardict), length(tagdict))
    @printf(iolog, "%s %s arguments - nepochs:%d, nlayers:%d droprate:%f winsize_c:%d winsize_w:%d\n", @logtime, procname,
            nepochs, nlayers, droprate, winsize_c, winsize_w)
    flush(iolog)

    nn = NN(wordembeds, charembeds, length(tagdict);
            nlayers=nlayers, winsize_c=winsize_c, winsize_w=winsize_w, droprate=droprate)

    info("#Training examples:\t$(length(traindata))")
    info("#Testing examples:\t$(length(testdata))")
    info("#Words1:\t$(length(worddict))")
    info("#Chars:\t$(length(chardict))")
    info("#Tags:\t$(length(tagdict))")

    batchsize = config["batchsize"]
    testdata = create_batch(testdata, batchsize)

    opt = SGD()
    for epoch = 1:nepochs
        println("Epoch:\t$epoch")

        #opt.rate = LEARN_RATE / BATCHSIZE
        opt.rate = config["learning_rate"] * batchsize / sqrt(batchsize) / (1 + 0.05*(epoch-1))
        println("Learning rate: $(opt.rate)")

        @printf(iolog, "%s %s begin epoch %d - learnrate:%.5f\n", @logtime, procname,
                epoch, opt.rate)
        flush(iolog)

        shuffle!(traindata)
        batches = create_batch(traindata, batchsize)
        prog = Progress(length(batches))
        loss = 0.0
        for i in 1:length(batches)
            s = batches[i]
            z = nn(s, true)
            loss += sum(z.data)
            params = gradient!(z)
            foreach(opt, params)
            ProgressMeter.next!(prog)
        end
        loss /= length(batches)
        println("Loss:\t$loss")

        # test
        println("Testing...")
        preds = Int[]
        golds = Int[]
        for s in testdata
            z = nn(s, false)
            append!(preds, z)
            append!(golds, s.t)
        end
        length(preds) == length(golds) || throw("Length mismatch: $(length(preds)), $(length(golds))")

        preds = BIOES.decode(preds, tagdict)
        golds = BIOES.decode(golds, tagdict)
        prec, recall, fval = fscore(golds, preds)

        @printf(iolog, "%s %s end epoch %d - loss:%.4e fval:%.5f prec:%.5f recall:%.5f\n", @logtime, procname,
                epoch, loss, fval, prec, recall)
        flush(iolog)
        println()
    end

    @printf(iolog, "%s %s training complete\n", @logtime, procname)

    Decoder(worddict, chardict, tagdict, nn)
end

function initvocab(path::String)
    chardict = Dict{String,Int}()
    tagdict = Dict{String,Int}()
    lines = open(readlines, path)
    for line in lines
        isempty(line) && continue
        items = Vector{String}(split(line,"\t"))
        word = strip(items[1])
        chars = Vector{Char}(word)
        for c in chars
            c = string(c)
            if haskey(chardict, c)
                chardict[c] += 1
            else
                chardict[c] = 1
            end
        end

        tag = strip(items[2])
        haskey(tagdict,tag) || (tagdict[tag] = length(tagdict)+1)
    end

    chars = String[]
    for (k,v) in chardict
        v >= 3 && push!(chars,k)
    end
    chardict = Dict(chars[i] => i for i=1:length(chars))
    chardict["UNKNOWN"] = length(chardict) + 1

    chardict, tagdict
end

function decode(dec::Decoder, config::Dict)
    testdata = readdata(config["test_file"], dec.worddict, dec.chardict, dec.tagdict)
    testdata = create_batch(testdata, 10)
    id2tag = Array{String}(length(dec.tagdict))
    for (k, v) in dec.tagdict
        id2tag[v] = k
    end

    preds = Int[]
    for x in testdata
        y = dec.nn(x, false)
        append!(preds, y)
    end

    lines = open(readlines, config["test_file"])
    i = 1
    for line in lines
        if !isempty(strip(line))
            tag = id2tag[preds[i]]
            println("$line\t$tag")
            i += 1
        else
            println("")
        end
    end
end

function readdata(path::String, worddict::Dict, chardict::Dict, tagdict::Dict)
    samples = Sample[]
    words, tags = String[], String[]
    unkword = worddict["UNKNOWN"]
    unkchar = chardict["UNKNOWN"]

    lines = open(readlines, path)
    push!(lines, "")
    for i = 1:length(lines)
        line = lines[i]
        if isempty(line)
            isempty(words) && continue
            wordids = Int[]
            charids = Int[]
            batchdims_c = Int[]
            for w in words
                #w0 = replace(lowercase(w), r"[0-9]", '0')
                id = get(worddict, lowercase(w), unkword)
                push!(wordids, id)

                chars = Vector{Char}(w)
                ids = map(chars) do c
                    get(chardict, string(c), unkchar)
                end
                append!(charids, ids)
                push!(batchdims_c, length(ids))
            end
            batchdims_w = [length(words)]
            w = reshape(wordids, 1, length(wordids))
            c = reshape(charids, 1, length(charids))
            t = isempty(tags) ? nothing : map(t -> tagdict[t], tags)
            push!(samples, Sample(w,batchdims_w,c,batchdims_c,t))
            empty!(words)
            empty!(tags)
        else
            items = Vector{String}(split(line,"\t"))
            word = strip(items[1])
            @assert !isempty(word)
            push!(words, word)
            if length(items) >= 2
                tag = strip(items[2])
                push!(tags, tag)
            end
        end
    end
    samples
end

function fscore(golds::Vector{T}, preds::Vector{T}) where T
    set = intersect(Set(golds), Set(preds))
    count = length(set)
    prec = round(count/length(preds), 5)
    recall = round(count/length(golds), 5)
    fval = round(2*recall*prec/(recall+prec), 5)
    println("Prec:\t$prec")
    println("Recall:\t$recall")
    println("Fscore:\t$fval")
    prec, recall, fval
end


"""
    struct NystromMesh{N,T} <: AbstractMesh{N,T}

A mesh data structure for solving boundary integral equation using Nyström
methods.

A `NystromMesh` can be constructed from a `mesh::AbstractMesh` and a dictionary
`etype2qrule` mapping element types in `mesh` (the keys) to an appropriate
quadrature rule for integration over elements of that type (the value).

The degrees of freedom of a `NystromMesh` are associated to its quadrature nodes
`qnodes::Vector{SVector{N,T}}`.
"""
Base.@kwdef struct NystromMesh{N,T} <: AbstractMesh{N,T}
    elements::OrderedDict{DataType,Any} = OrderedDict{DataType,Any}()
    etype2qrule::OrderedDict{DataType,AbstractQuadratureRule}= OrderedDict{DataType,AbstractQuadratureRule}()
    # quadrature info
    qnodes::Vector{SVector{N,T}} = Vector{SVector{N,T}}()
    qweights::Vector{T} = Vector{T}()
    qnormals::Vector{SVector{N,T}} = Vector{SVector{N,T}}()
    # mappings from elements --> dofs and entities --> elements
    elt2dof::OrderedDict{DataType,Matrix{Int}} = OrderedDict{DataType,Matrix{Int}}()
    ent2elt::OrderedDict{AbstractEntity,OrderedDict{DataType,Vector{Int}}} = OrderedDict{AbstractEntity,OrderedDict{DataType,Vector{Int}}}()
end

function Base.append!(msh1::NystromMesh,msh2::NystromMesh)
    Ω = domain(msh2)
    @assert ambient_dimension(msh1) == ambient_dimension(msh2)
    @assert eltype(msh1) == eltype(msh2)
    # extract relevant quadrature rules
    etype2dof = OrderedDict{DataType,Vector{Int}}()
    for (E,q) in etypes(msh2,Ω)
        if haskey(msh1.elements,E)
            etype2dof[E]        = msh1.elt2dof[E] |> vec
            @assert msh1.etype2qrule[E] == q
        else
            etype2dof[E]        = Int[]
            msh1.elements[E]    = E[]
            msh1.etype2qrule[E] = q
        end
    end
    # loop over entities and extract the information
    for ent in entities(msh2)
        haskey(msh1.ent2elt,ent) && continue # entity already meshed
        msh1.ent2elt[ent] = OrderedDict{DataType,Vector{Int}}()
        for (E,v) in ent2elt(msh2,ent)
            # add dofs information
            msh2_dofs  = msh2.elt2dof[E][:,v]
            idx_start = length(msh1.qnodes) + 1
            append!(msh1.qnodes,msh2.qnodes[msh2_dofs])
            append!(msh1.qweights,msh2.qweights[msh2_dofs])
            if !isempty(qnormals(msh2)) # otherwise assume it did not have normal stored
                append!(msh1.qnormals,msh2.qnormals[msh2_dofs])
            end
            idx_end = length(msh1.qnodes) # end index of dofs added
            append!(etype2dof[E],collect(idx_start:idx_end))
            # append elements
            idx_start = length(msh1.elements[E]) + 1
            append!(msh1.elements[E],msh2.elements[E][v])
            idx_end   = length(msh1.elements[E])
            # and mapping ent -> elt
            msh1.ent2elt[ent][E] = collect(idx_start:idx_end)
        end
    end
    # finally do some rehshaping
    for (E,v) in etype2dof
        dof_per_el = size(msh2.elt2dof[E],1)
        msh1.elt2dof[E] = reshape(v,dof_per_el,:)
    end
    return msh1
end

# getters
qnodes(m::NystromMesh)   = m.qnodes
qweights(m::NystromMesh) = m.qweights
qnormals(m::NystromMesh) = m.qnormals
elements(m::NystromMesh) = m.elements

# various mappings
elt2dof(m::NystromMesh)  = m.elt2dof
elt2dof(m::NystromMesh,E::DataType) = m.elt2dof[E]
ent2elt(m::NystromMesh) = m.ent2elt
ent2elt(m::NystromMesh,ent::AbstractEntity) = m.ent2elt[ent]
etype2qrule(m)   = m.etype2qrule
etype2qrule(m,E) = etype2qrule(m)[E]

Geometry.domain(mesh::NystromMesh)   = Domain(entities(mesh))
Geometry.entities(mesh::NystromMesh) = collect(keys(mesh.ent2elt))

function dom2elt(mesh::NystromMesh,domain::Domain)
    dict = OrderedDict{DataType,Vector{Int}}()
    for ent in entities(domain)
        other = ent2elt(mesh,ent)
        mergewith!(append!,dict,other)
    end
    return dict
end

function dom2dof(mesh::NystromMesh,domain::Domain)
    idxs = Int[]
    for ent in entities(domain)
        dict = ent2elt(mesh,ent)
        for (E,tags) in dict
            append!(idxs,view(elt2dof(mesh,E),:,tags))
        end
    end
    return idxs
end
dom2dof(mesh,ent::AbstractEntity) = dom2dof(mesh,Domain(ent))
ent2dof(mesh,ent::AbstractEntity) = dom2dof(mesh,ent)

Base.length(m::NystromMesh) = length(qnodes(m))

function etypes(m::NystromMesh,Ω::Domain=domain(m))
    ee = Tuple{DataType,AbstractQuadratureRule}[]
    for ent in entities(Ω)
        dict = ent2elt(m,ent)
        EE    = keys(dict)
        for E in EE
            q = etype2qrule(m,E)
            push!(ee,(E,q))
        end
    end
    unique!(ee)
    return ee
end

function NystromMesh(mesh::GenericMesh,Ω::Domain,e2qrule,compute_normal::Bool=true)
    N,T        = ambient_dimension(mesh), eltype(mesh)
    # initialize empty fields
    nys_mesh = NystromMesh{N,T}(;etype2qrule=e2qrule)
    # loop over entities
    for ent in entities(Ω)
        dict    = OrderedDict{DataType,Vector{Int}}()
        submesh = view(mesh,ent)
        for E in etypes(submesh)
            haskey(e2qrule,E) || error("no quadrature rule found for element of type $E")
            qrule = e2qrule[E]
            if haskey(nys_mesh.elements,E)
                istart = length(nys_mesh.elements[E])+1
            else
                istart = 1
            end
            _build_nystrom_mesh!(nys_mesh,submesh,E,qrule,compute_normal)
            iend   = length(nys_mesh.elements[E])
            dict[E] = collect(istart:iend)
        end
        nys_mesh.ent2elt[ent] = dict # new entry
    end
    return nys_mesh
end

@noinline function _build_nystrom_mesh!(nys_mesh,submesh,E,qrule,compute_normal)
    els::Vector{E}         = get!(nys_mesh.elements,E,Vector{E}())
    el2qnodes   = get!(nys_mesh.elt2dof,E,Matrix{Int}(undef,0,0))
    el2qnodes   = vec(el2qnodes)
    x̂,ŵ         = qrule() #nodes and weights on reference element
    for el in ElementIterator(submesh,E)
        # compute quadrature on element
        x = map(el,x̂)
        w = map(zip(x̂,ŵ)) do (x̂,ŵ)
            ŵ*measure(el,x̂)
        end
        # add element
        push!(els,el)
        # append quadrature information
        istart = length(qnodes(nys_mesh)) + 1
        append!(qweights(nys_mesh),w)
        append!(qnodes(nys_mesh),x)
        if compute_normal
            ν = map(x->normal(el,x),x̂)
            append!(qnormals(nys_mesh),ν)
        end
        iend   = length(qnodes(nys_mesh))
        # add information to recover qnodes from index of el
        append!(el2qnodes,collect(istart:iend))
    end
    el2qnodes = reshape(el2qnodes,length(x̂),:)
    nys_mesh.elt2dof[E] = el2qnodes
    return nys_mesh
end

NystromMesh(submesh::SubMesh,args...) = NystromMesh(submesh.mesh,submesh.domain,args...)

function NystromMesh(mesh::SubMesh;order,compute_normal::Bool=true)
    etype2qrule = Mesh._qrule_for_mesh(mesh,order)
    NystromMesh(mesh,etype2qrule,compute_normal)
end

function NystromMesh(mesh::GenericMesh,Ω::Domain;quad_rule,compute_normal::Bool=true)
    etype2qrule = OrderedDict(E=>quad_rule for E in etypes(view(mesh,Ω)))
    NystromMesh(mesh,Ω,etype2qrule,compute_normal)
end

# construct a mesh as a restriction to a domain
function Base.getindex(msh2::NystromMesh,Ω::Domain)
    N,T      = ambient_dimension(msh2), eltype(msh2)
    # initialize empty fields
    msh1 = NystromMesh{N,T}()
    # extract relevant quadrature rules
    etype2dof = OrderedDict{DataType,Vector{Int}}()
    for (E,q) in etypes(msh2,Ω)
        msh1.etype2qrule[E] = q
        etype2dof[E] = Int[]
        msh1.elements[E] = E[]
    end
    # loop over entities and extract the information
    for ent in Ω
        @assert ent in domain(msh2)
        msh1.ent2elt[ent] = OrderedDict{DataType,Vector{Int}}()
        for (E,v) in msh2.ent2elt[ent]
            # add dofs information
            msh2_dofs  = msh2.elt2dof[E][:,v]
            idx_start = length(msh1.qnodes) + 1
            append!(msh1.qnodes,msh2.qnodes[msh2_dofs])
            append!(msh1.qweights,msh2.qweights[msh2_dofs])
            if !isempty(qnormals(msh2))
                append!(msh1.qnormals,msh2.qnormals[msh2_dofs])
            end
            idx_end = length(msh1.qnodes)
            append!(etype2dof[E],collect(idx_start:idx_end))
            # append elements
            idx_start = length(msh1.elements[E]) + 1
            append!(msh1.elements[E],msh2.elements[E][v])
            idx_end   = length(msh1.elements[E])
            # and mapping ent -> elt
            msh1.ent2elt[ent][E] = collect(idx_start:idx_end)
        end
    end
    # finally do some rehshaping
    for (E,v) in etype2dof
        dof_per_el = size(msh2.elt2dof[E],1)
        msh1.elt2dof[E] = reshape(v,dof_per_el,:)
    end
    return msh1
end
Base.getindex(m::NystromMesh,ent::AbstractEntity) = m[Domain(ent)]

function _elt2dof(m::NystromMesh,Ω::Domain)
    out = OrderedDict{DataType,Matrix{Int}}()
    for ent in Ω
        dict = ent2elt(m,ent)
        for (E,v) in dict
            dofs = get!(out,E,Matrix{Int}(undef,0,0))
            # make into vec to append
            dofs = vec(dofs)
            dof_per_el = size(elt2dof(m,E),1)
            append!(dofs,elt2dof(m,E)[:,v])
            # convert back to matrix shape
            dofs = reshape(dofs,dof_per_el,:)
            out[E] = dofs
        end
    end
    return out
end

"""
    near_interaction_list(pts,Y;atol)

Given a target points `pts` and a `Y::NystromMesh`, return a dictionary with keys
given by element types of the source mesh `Y`. To each key, which represents an
element type, we associate a vector whose `i`-th entry encodes
information about the points in `X` which are close to a given element of
the key type.
"""
function near_interaction_list(pts,Y::NystromMesh;atol)
    dict = OrderedDict{DataType,Vector{Vector{Tuple{Int,Int}}}}()
    for (E,Q) in etypes(Y)
        ilist = _etype_near_interaction_list(pts,Y,E,atol)
        push!(dict,E=>ilist)
    end
    return dict
end

function _etype_near_interaction_list(pts,Y,E,atol)
    ilist = Vector{Vector{Tuple{Int,Int}}}()
    e2n   = elt2dof(Y,E)
    npts,nel = size(e2n)
    for n in 1:nel
        ynodes = qnodes(Y)[e2n[:,n]]
        inear  = _near_interaction_list(pts,ynodes,atol)
        push!(ilist,inear)
    end
    ilist
end

function _near_interaction_list(pts,ynodes,atol)
    ilist    = Tuple{Int,Int}[]
    for (i,x) in enumerate(pts)
        d,j = findmin([norm(x-y) for y in ynodes])
        if d ≤ atol
            push!(ilist,(i,j))
        end
    end
    return ilist
end

elements(m::NystromMesh,E::DataType) = m.elements[E]


# move
"""
    move(data::Data, move_vec::Array)

Do this will move the model uniformly respect to `move_vec` in unit of Angstrom

# Example
```julia-repl
data_cell = genr_cell([1 1 1])
str = Si3N4()
data_atom = genr_atom(data_cell, str)
data_move = move(data_atom, [10 10 10])
```
"""
function move(data::Data_Unit, move_vec)
    # Motion of box
    data_new = Data_Unit(data)
    if typeof(move_vec) == Array{Int64,2}
        move_vec = conv(move_vec, 1)
    end
    data_new.data_basic.box_size = broadcast(+, data_new.data_basic.box_size, move_vec)
    # Motion of atoms
    for atom = 1 : data_new.data_basic.num_atoms
        data_new.vec_atom[atom].coord += move_vec
    end
    data_new
end

function move(data::Data_Sum, move_vec)
    # Motion of box
    data_new = Data_Sum(data)
    if typeof(move_vec) == Array{Int64,2}
        move_vec = conv(move_vec, 1)
    end
    data_new.data_basic.box_size = broadcast(+, data_new.data_basic.box_size, move_vec)
    # Motion of atoms
    for atom = 1 : data_new.data_basic.num_atoms
        data_new.vec_atom[atom].coord += move_vec
    end
    data_new
end

# select and remove

function find(vec_unit::Union{Vector{T}, Int64}, list_atom) where T <: Unit
    len = length(vec_unit)
    if typeof(vec_unit) == Int64
        return 0
    else
        atom = [vec_unit[n].atom for n = 1:len]
        list = 0
        id = 0
        for i in atom
            id += 1
            for j in i
                if sum(list_atom .== j) == 1
                    list = vcat(list, id)
                    break
                end
            end
        end
        list[2:end]
    end
end

"""
    select(data_cell::Data_Cell; mode::String, para)

Do this will return a list of id of cells in specific region controled by variables `mode` and `para`

# Arguments
- `data_cell`:
- `mode`: "cylinder" and "sphere" are supported currently.
- `para`: parameters needed to describe a region correspond to each modes

# Parameters List

- `"cylinder"`: `[radius dim]`, `dim` represents which axies cylinder will be prallel to
- `"sphere"`: `[radius]`
- `"plane"`: `[vec_plane, radius, origin]`, `vec_plane` is a vector prependicular to plane that you want to cut, `radius` is the cutoff radius cells whose projection to `vec_plane` over it will be selected, `origin` is the origin point of `vec_plane`, `[0, 0, 0]` is set defaultly

# Example
```julia-repl
data_cell = genr_cell([10 10 2])
data_select = select(data_cell, mode="cylinder", para=[3, 3])
```
"""
function select(data_cell::Data_Cell; mode::String, para)
    list_mode = split("cylinder sphere plane")
    if !in(mode, list_mode)
        error(join(["Error, mode: ",mode," is not supported."]))
    end

    center = data_cell.cell_vec ./ 2

    cell_list = 0

    if mode == "cylinder"
        for cell in 1:data_cell.num_cells
            if dist(data_cell.cell_mat[cell, :], center, para[2]) <= para[1]
                cell_list = vcat(cell_list, cell)
            end
        end
    elseif mode == "sphere"
        for cell in 1:data_cell.num_cells
            if dist(data_cell.cell_mat[cell, :], center) <= para[1]
                cell_list = vcat(cell_list, cell)
            end
        end
    elseif mode == "plane"
        vec = norm_vec(para[1:3])
        dist_proj = para[4]
        if length(para) == 4
            org = [0, 0, 0]
        else
            org = para[5:7]
        end
        for cell = 1:data_cell.num_cells
            dist_now = (data_cell.cell_mat[cell, :] .- org)' * vec
            if dist_now >= dist_proj
                cell_list = vcat(cell_list, cell)
            end
        end
    end
    if length(cell_list) == 1
        error("No cell has been selected")
    end
    return cell_list[2:end]
end

"""
    select(data_atom::Data; mode::String, para)

Do this will return a list of id of atoms in specific region controled by variables `mode` and `para`

# Arguments
- `data_atom`:
- `mode`: "cylinder" and "sphere" are supported currently.
- `para`: parameters needed to describe a region correspond to each modes

# Parameters List

- `"cylinder"`: `[radius, dim]`, `dim` represents which axies cylinder will be prallel to
- `"sphere"`: `[radius]`
- `"plane"`: `[vec_plane, radius, origin]`, `vec_plane` is a vector prependicular to plane that you want to cut, `radius` is the cutoff radius atoms whose projection to `vec_plane` over it will be selected, `origin` is the origin point of `vec_plane`, `[0, 0, 0]` is set defaultly

# Example
```julia-repl
data_cell = genr_cell([10 10 2])
str = Si3N4()
data_atom = genr_atom(data_cell, str)
data_select = select(data_atom, mode="cylinder", para=[10, 3])
```
"""
function select(data_unit::Data_Unit; mode::String, para)
    list_mode = split("cylinder sphere plane")
    if !in(mode, list_mode)
        error(join(["Error, mode: ",mode," is not supported."]))
    end

    box_size = data_unit.data_basic.box_size
    box_tilt = data_unit.data_basic.box_tilt
    center = (max(data_unit.vec_atom, "coord")-min(data_unit.vec_atom, "coord")) ./ 2
    #center[1] += box_tilt[1] + box_tilt[2]
    #center[2] += box_tilt[3]

    atom_list = 0

    if mode == "cylinder"
        for atom = 1:data_unit.data_basic.num_atoms
            if dist(data_unit.vec_atom[atom], center, para[2]) <= para[1]
                atom_list = vcat(atom_list, atom)
            end
        end
    elseif mode == "sphere"
        for atom = 1:data_unit.data_basic.num_atoms
            if dist(data_unit.vec_atom[atom], center) <= para[1]
                atom_list = vcat(atom_list, atom)
            end
        end
    elseif mode == "plane"
        vec = norm_vec(para[1:3])
        dist_proj = para[4]
        if length(para) == 4
            org = [0, 0, 0]
        else
            org = para[5:7]
        end
        for atom = 1:data_unit.data_basic.num_atoms
            dist_now = (data_unit.vec_atom[atom].coord .- org)' * vec
            #dist_now = sqrt(dist_now' * dist_now)
            if dist_now >= dist_proj
                atom_list = vcat(atom_list, atom)
            end
        end
    end

    if length(atom_list) == 1
        error("No atom has been selected")
    end
    return atom_list[2:end]
end

"""
    remove!(data_cell::Data_Cell, list_cell::Array)

Do this will remove all cells in `list_cell` from `data_cell`

# Example
```julia-repl
data_cell = genr_cell([10 10 2])
data_select = select(data_cell, mode="cylinder", para=[3, 3])
data_new = remove!(data_cell, data_select)
```
"""
function remove!(data_cell::Data_Cell, list_cell)
    # Reading Input
    cell_mat = data_cell.cell_mat
    len = size(cell_mat)[1]
    # Deleting Cells
    judge = trues(len)
    for id in list_cell
        judge .&= 1:len .!= id
    end
    data_cell.cell_mat = cell_mat[judge, :]
    data_cell.num_cells = size(data_cell.cell_mat)[1]

    # Output
    data_cell
end

"""
    remove!(data::Data, list_atom::Array)

Do this will remove all atoms in `list_atom` themselves and other infomation related to them  from `data`

# Example
```julia-repl
data_cell = genr_cell([10 10 2])
str = Si3N4()
data_atom = genr_atom(data_cell, str)
data_select = select(data_atom, mode="cylinder", para=[3, 3])
data_new = remove!(data_atom, data_select)
```
"""
function remove!(data::Data, list_atom::Array)
    fields = fieldnames(typeof(data))
    name_fields = [string(fields[n]) for n = 1:length(fields)]
    num_fields = length(fields)
    list_fields = findall(x->occursin("vec", x), name_fields)

    for field in  list_fields
        # Find all elements that need to be removed
        list_id = find(getfield(data, fields[field]), list_atom)

        if list_id == 0
            continue
        else
            # Changing # of specifc field
            num_ids = length(list_id)
            para_now = name_fields[field][findall(x->in('_', x), name_fields[field])[1]+1 : end]
            para_now = Meta.parse(join(["num_", para_now, "s"]))
            para_result = getfield(data.data_basic, para_now) - num_ids
            setfield!(data.data_basic, para_now, para_result)

            # Changing Vector of each field
            setfield!(data, fields[field], remove!(getfield(data, fields[field]), list_id)) # remove!(vec::Vector{T} , id::Array) where T<:Data_Unit
        end
    end
    sort_data!(data, list_atom)
end

function remove!(vec::Union{Vector{T}, Int64}, id::Array) where T <: Unit
    len = length(vec)
    judge = trues(len)
    for i in id
        judge .&= 1 : len .!= i
    end
    vec[judge]
end

function remove!(mat; id=1, dim=1)
    if dim == 1
        result = mat[1:end .!= id, :]
    elseif dim == 2
        result = mat[:, 1:end .!= id]
    else
        error("Error, dim should be 1 or 2, representing row or column respectively!")
    end
    result
end

"""
    remove_else!(data::Data, list_atom::Array)

Do this will remove all atoms but atoms in `list_atom` themselves and other infomation related to them  from `data`

# Example
```julia-repl
data_cell = genr_cell([10 10 2])
str = Si3N4()
data_atom = genr_atom(data_cell, str)
data_select = select(data_atom, mode="cylinder", para=[3, 3])
data_new = remove_else!(data_atom, data_select)
```
"""
function remove_else!(data::Data, list_atom::Array)
    atom_all = get_data(data.vec_atom, "atom")
    atom_else = setdiff(atom_all, list_atom) # Notice: order of `atom_all` and `atom_else` is vital
    remove!(data, atom_else)
end


module Kash

using FASTX
using BioSequences
using BSON
using MinHash

# Need to use CodecZlib for Gzip integration!

include("kmersketch.jl")
include("serialize.jl")

export FastaIterator,
    CanonicalKmerIterator,
    KmerSketcher,
    KmerHashes,
    update!,
    kmer_minhash,
    kmer_minhash_each,
    intersectionlength

end # module


using ClusterManagers, DSGE

fn = dirname(@__FILE__)

# What do you want to do?
run_full_forecast = true
do_histforecast   = true
do_shockdecs      = true
do_irfs           = true
make_plots        = true
make_packet       = true

# Initialize model objects and desired settings
m = Model1002("ss10")
m <= Setting(:sampling_method, :SMC)
usual_settings!(m, "190808"; cdvt = "190808", fcast_date = quartertodate("2019-Q3"))
m <= Setting(:saveroot, "$(fn)/../../save/") # set save root
m <= Setting(:dataroot, "$(fn)/../../save/input_data/") # set data root
m <= Setting(:date_forecast_end, quartertodate("2030-Q1"))
forecast_string = "" # to identify different forecasts from each other
model_string = "m1002/ss10" # used further down to construct file paths

# Override input file locations for saved estimation output
overrides = forecast_input_file_overrides(m)
overrides[:full] = "$(fn)/../../save/output_data/m1002/ss10/estimate/raw/smcsave_vint=190712.h5"

# Load data from SMC cloud estimation and guarantee saved parameters are
# a draw from the weighted particle distribution
weighted_data = load("$(fn)/../../save/output_data/m1002/ss10/estimate/raw/smc_cloud_vint=190712.jld2")
Random.seed!(1793)
params = Matrix{Float64}(DSGE.get_vals(weighted_data["cloud"])[:,resample(weighted_data["W"][:,end])]')
h5open("$(fn)/../../save/output_data/m1002/ss10/estimate/raw/smcsave_vint=190712.h5", "w") do file
    write(file, "smcparams", params)
end

# Full-distribution forecast
if run_full_forecast
    if get_setting(m, :sampling_method) == :SMC
        m <= Setting(:forecast_block_size, 500)
    end

    output_vars = Vector{Symbol}(undef,0)
    if do_histforecast
        # Write data to create historical and forecast output
        output_vars = vcat(output_vars, [:histpseudo, :histobs, :histstdshocks,
                                         :hist4qpseudo, :hist4qobs, :histutpseudo,
                                         :forecastpseudo, :forecastobs, :forecastutpseudo,
                                         :forecast4qpseudo, :forecast4qobs, :forecaststdshocks])
    end
    if do_shockdecs
        # Shock decompositions of forecasts
        output_vars = vcat(output_vars, [:dettrendobs, :dettrendpseudo, :trendobs,
                                         :trendpseudo, :shockdecpseudo, :shockdecobs])
    end
    if do_irfs
        # Impulse response to all shocks, including for endogenous states
        output_vars = vcat(output_vars, [:irfstates, :irfobs, :irfpseudo])
    end

    my_procs = addprocs(20)
    @everywhere using DSGE, OrderedCollections
    usual_forecast(m, :full, :none, output_vars,
                       est_override = overrides[:full],
                       forecast_string = forecast_string,
                       density_bands = [.5, .6, .68, .7, .8, .9])
end

# Packet
if make_packet
    gr()

    output_vars = [:forecastobs, :forecastpseudo]
    if do_shockdecs
        output_vars = vcat(output_vars, [:shockdecobs, :shockdecpseudo])
    end
    if do_irfs
        # we do not plot irfs for endogenous states b/c too many!
        output_vars = vcat(output_vars, [:irfobs, :irfpseudo])
        sections = [:estimation, :forecast, :irf]
    else
        sections = [:estimation, :forecast]
    end
    if make_plots
        plot_standard_packet(m, :full, :none, output_vars,
                             forecast_string = forecast_string,
                             sections = sections)
    end

    # Choose forecast centered or the standard packet.
    # Note that these functions write to the same tex file,
    # so you can use only one for a given data vintage!
    # write_forecast_centric_packet(m, :full, :none, output_vars,
    #                               sections = sections, forecast_string = forecast_string)
    write_standard_packet(m, :full, :none, output_vars,
                                  sections = sections, forecast_string = forecast_string)
    moment_tables(m)
end

rmprocs(my_procs)
nothing


# MIT License
#
# Copyright (c) 2018 Martin Biel
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

# Quadratic decision function #
# ========================== #
mutable struct QuadraticDecisionConstraintBridge{T, S} <: MOIB.Constraint.AbstractBridge
    constraint::CI{MOI.ScalarQuadraticFunction{T}, S}
    decision_function::QuadraticDecisionFunction{T}
    set::S
end

function MOIB.Constraint.bridge_constraint(::Type{QuadraticDecisionConstraintBridge{T,S}},
                                           model,
                                           f::QuadraticDecisionFunction{T},
                                           set::S) where {T, S}
    # All decisions have been mapped to the
    # variable part terms at this point.
    # Add the bridged constraint
    constraint = MOI.add_constraint(model,
                                    MOI.ScalarQuadraticFunction(
                                        f.variable_part.affine_terms,
                                        f.variable_part.quadratic_terms,
                                        zero(T)),
                                    MOIU.shift_constant(set, -f.variable_part.constant))
    # Save the constraint index, the decision function, and the set, to allow modifications
    return QuadraticDecisionConstraintBridge{T, S}(constraint, f, set)
end

function MOI.supports_constraint(::Type{<:QuadraticDecisionConstraintBridge{T}},
                                 ::Type{<:QuadraticDecisionFunction{T}},
                                 ::Type{<:MOI.AbstractScalarSet}) where T
    return true
end
function MOIB.added_constrained_variable_types(::Type{<:QuadraticDecisionConstraintBridge})
    return Tuple{DataType}[]
end
function MOIB.added_constraint_types(::Type{QuadraticDecisionConstraintBridge{T, S}}) where {T, S}
    return [(MOI.ScalarQuadraticFunction{T}, S)]
end
function MOIB.Constraint.concrete_bridge_type(::Type{<:QuadraticDecisionConstraintBridge{T}},
                                              ::Type{QuadraticDecisionFunction{T}},
                                              S::Type{<:MOI.AbstractScalarSet}) where T
    return QuadraticDecisionConstraintBridge{T, S}
end

MOI.get(b::QuadraticDecisionConstraintBridge{T, S}, ::MOI.NumberOfConstraints{MOI.ScalarQuadraticFunction{T}, S}) where {T, S} = 1
MOI.get(b::QuadraticDecisionConstraintBridge{T, S}, ::MOI.ListOfConstraintIndices{MOI.ScalarQuadraticFunction{T}, S}) where {T, S} = [b.constraint]

function MOI.get(model::MOI.ModelLike, attr::MOI.ConstraintFunction,
                 bridge::QuadraticDecisionConstraintBridge{T,S}) where {T,S}
    return bridge.decision_function
end

function MOI.get(model::MOI.ModelLike, attr::MOI.ConstraintSet,
                 bridge::QuadraticDecisionConstraintBridge{T,S}) where {T,S}
    return bridge.set
end

function MOI.get(model::MOI.ModelLike, attr::Union{MOI.ConstraintPrimal, MOI.ConstraintDual},
                 bridge::QuadraticDecisionConstraintBridge{T,S}) where {T,S}
    return MOI.get(model, attr, bridge.constraint)
end

function MOI.get(model::MOI.ModelLike, ::DecisionIndex,
                 bridge::QuadraticDecisionConstraintBridge{T}) where T
    return bridge.constraint
end

function MOI.delete(model::MOI.ModelLike, bridge::QuadraticDecisionConstraintBridge)
    MOI.delete(model, bridge.constraint)
    return nothing
end

function MOI.set(model::MOI.ModelLike, ::MOI.ConstraintFunction,
                 bridge::QuadraticDecisionConstraintBridge{T,S},
                 f::QuadraticDecisionFunction{T}) where {T,S,F}
    # Update bridge functions and function constant
    bridge.decision_function = f
    # Change the function of the bridged constraints
    MOI.set(model, MOI.ConstraintFunction(), bridge.constraint,
            MOI.ScalarQuadraticFunction(
                f.variable_part.affine_terms,
                f.variable_part.quadratic_terms,
                zero(T)))
    # Shift constraint set
    MOI.set(model, MOI.ConstraintSet(), bridge.constraint,
            MOIU.shift_constant(bridge.set, -f.variable_part.constant))
    return nothing
end

function MOI.set(model::MOI.ModelLike, ::MOI.ConstraintSet,
                 bridge::QuadraticDecisionConstraintBridge{T,S}, change::S) where {T,S}
    f = bridge.decision_function
    bridge.set = change
    # Shift constraint set
    MOI.set(model, MOI.ConstraintSet(), bridge.constraint,
            MOIU.shift_constant(bridge.set, -f.variable_part.constant))
    return nothing
end

function MOI.modify(model::MOI.ModelLike, bridge::QuadraticDecisionConstraintBridge{T,S}, change::MOI.ScalarConstantChange) where {T,S}
    f = bridge.decision_function
    # Modify constant of variable part
    lq.variable_part.constant = change.new_constant
    # Shift constraint set
    MOI.set(model, MOI.ConstraintSet(), bridge.constraint,
            MOIU.shift_constant(bridge.set, -f.variable_part.constant))
    return nothing
end

function MOI.modify(model::MOI.ModelLike, bridge::QuadraticDecisionConstraintBridge{T,S}, change::MOI.ScalarCoefficientChange) where {T,S}
    f = bridge.decision_function
    # Update variable part
    modify_coefficient!(f.variable_part.affine_terms, change.variable, change.new_coefficient)
    # Modify variable part of objective as usual
    F = MOI.ScalarQuadraticFunction{T}
    MOI.modify(model, MOI.ObjectiveFunction{F}(), change)
    return nothing
end