using HDF5

export smapseries

const SMAPMISSING = -9999.0f0
const SMAPGEODATA = "Geophysical_Data"
const SMAPCRS = ProjString("+proj=cea +lon_0=0 +lat_ts=30 +x_0=0 +y_0=0 +datum=WGS84 +units=m +no_defs +ellps=WGS84 +towgs84=0,0,0")
const SMAPSIZE = (3856, 1624)
const SMAPDIMTYPES = (X, Y)

# Dataset wrapper ###############################################################
# Becauase SMAP is just one of many HDF5 formats,
# we wrap it in SMAPhdf5 and SMAPvar wrappers

struct SMAPhdf5{T}
    ds::T
end

missingval(ds::SMAPhdf5) = SMAPMISSING
layerkeys(ds::SMAPhdf5) = keys(ds)
filekey(ds::SMAPhdf5, key::Nothing) = first(keys(ds))

function DD.dims(wrapper::SMAPhdf5)
    dataset = parent(wrapper)
    proj = read(HDF5.attributes(HDF5.root(dataset)["EASE2_global_projection"]), "grid_mapping_name")
    if proj == "lambert_cylindrical_equal_area"
        # There are matrices for lookup but all rows/colums are identical.
        # For performance and simplicity we just take a vector slice for each dim.
        extent = HDF5.attributes(HDF5.root(dataset)["Metadata/Extent"])
        lonbounds = read(extent["westBoundLongitude"]), read(extent["eastBoundLongitude"])
        latbounds = read(extent["southBoundLatitude"]), read(extent["northBoundLatitude"])
        lonvec = HDF5.root(dataset)["cell_lon"][:, 1]
        latvec = HDF5.root(dataset)["cell_lat"][1, :]
        lonlookup = Mapped(lonvec;
           order=ForwardOrdered(),
           span=Irregular(lonbounds),
           sampling=Intervals(Center()),
           crs=SMAPCRS,
           mappedcrs=EPSG(4326),
           dim=X(),
        )
        latlookup = Mapped(latvec;
            order=ReverseOrdered(),
            span=Irregular(latbounds),
            sampling=Intervals(Center()),
            crs=SMAPCRS,
            mappedcrs=EPSG(4326),
            dim=Y(),
        )
        return X(lonlookup), Y(latlookup)
    else
        error("projection $proj not supported")
    end
end

# TODO actually add metadata to the dict
DD.metadata(wrapper::SMAPhdf5) = Metadata{SMAPfile}(Dict())

function DD.layerdims(ds::SMAPhdf5)
    keys = cleankeys(layerkeys(ds))
    # All dims are the same
    NamedTuple{keys}(map(_ -> SMAPDIMTYPES, keys))
end

function DD.layermetadata(ds::SMAPhdf5)
    keys = cleankeys(layerkeys(ds))
    NamedTuple{keys}(map(_ -> DD.metadata(ds), keys))
end

Base.keys(ds::SMAPhdf5) = cleankeys(keys(parent(ds)[SMAPGEODATA]))
Base.parent(wrapper::SMAPhdf5) = wrapper.ds
Base.getindex(wrapper::SMAPhdf5, key) = SMAPvar(wrapper.ds[_smappath(key)])

_smappath(key::Key) = SMAPGEODATA * "/" * string(key)

struct SMAPvar{DS} <: AbstractArray{Float32,2}
    ds::DS
end

Base.parent(wrapper::SMAPvar) = wrapper.ds
Base.eltype(wrapper::SMAPvar) = eltype(parent(wrapper))
Base.size(wrapper::SMAPvar) = SMAPSIZE
Base.ndims(wrapper::SMAPvar) = length(SMAPSIZE)
Base.getindex(wrapper::SMAPvar, args...) = getindex(parent(wrapper), args...)
Base.setindex!(wrapper::SMAPvar, args...) = setindex!(parent(wrapper), args...)
Base.Array(wrapper::SMAPvar) = Array(parent(wrapper))
Base.collect(wrapper::SMAPvar) = collect(parent(wrapper))

DA.eachchunk(var::SMAPvar) = DA.GridChunks(var, size(var))
DA.haschunks(var::SMAPvar) = DA.Unchunked()

# Raster ######################################################################

function FileArray(ds::SMAPhdf5, filename::AbstractString; key, kw...)
    FileArray(ds[key], filename; key, kw...)
end
function FileArray(var::SMAPvar, filename::AbstractString; key, kw...)
    T = eltype(var)
    N = ndims(var)
    eachchunk = DA.eachchunk(var)
    haschunks = DA.haschunks(var)
    FileArray{SMAPfile,T,N}(filename, SMAPSIZE; key, eachchunk, haschunks, kw...)
end

function Base.open(f::Function, A::FileArray{SMAPfile}; kw...)
    _open(SMAPfile, filename(A); key=key(A), kw...) do var
        f(RasterDiskArray{SMAPfile}(var))
    end
end

DA.writeblock!(A::RasterDiskArray{SMAPfile}, v, r::AbstractUnitRange...) = A[r...] = v

haslayers(::Type{SMAPfile}) = true

# Stack ########################################################################

@deprecate SMAPstack(args...; kw...) RasterStack(args...; source=SMAPfile, kw...)

function FileStack{SMAPfile}(ds::SMAPhdf5, filename::AbstractString; write=false, keys)
    keys = map(Symbol, keys isa Nothing ? layerkeys(ds) : keys) |> Tuple
    type_size_ec_hc = map(keys) do key
        var = RasterDiskArray{SMAPfile}(ds[key])
        eltype(var), size(var), DA.eachchunk(var), DA.haschunks(var)
    end
    layertypes = map(x->x[1], type_size_ec_hc)
    layersizes = map(x->x[2], type_size_ec_hc)
    eachchunk = map(x->x[3], type_size_ec_hc)
    haschunks = map(x->x[4], type_size_ec_hc)
    FileStack{SMAPfile,keys}(filename, layertypes, layersizes, eachchunk, haschunks, write)
end
function OpenStack(fs::FileStack{SMAPfile,K}; kw...) where K
    ds = h5open(filename(fs); kw...)
    OpenStack{SMAPfile,K}(SMAPhdf5(ds))
end
Base.close(os::OpenStack{SMAPfile}) = close(dataset(os))

# Series #######################################################################

"""
    smapseries(filenames::AbstractString; kw...)
    smapseries(filenames::Vector{<:AbstractString}, dims=nothing; kw...)

[`RasterSeries`](@ref) loader for SMAP files and whole folders of files,
organised along the time dimension. Returns a [`RasterSeries`](@ref).

# Arguments

- `filenames`: A `String` path to a directory of SMAP files,
    or a vector of `String` paths to specific files.
- `dims`: `Tuple` containing `Ti` dimension for the series.
    Automatically generated form `filenames` unless passed in.

# Keywords

- `kw`: Passed to `RasterSeries`.
"""
function smapseries(dir::AbstractString; kw...)
    smapseries(joinpath.(dir, filter_ext(dir, ".h5")); kw...)
end
function smapseries(filenames::Vector{<:AbstractString}, dims=nothing; kw...)
    if dims isa Nothing
        usedpaths = String[]
        timeseries = []
        errors = []
        for filename in filenames
            try
                t = _smap_timefromfilename(filename)
                push!(timeseries, t)
                push!(usedpaths, filename)
            catch e
                push!(errors, e)
            end
        end
        # Use the first files time dim as a template, but join vals into an array of times.
        dims = (_smap_timedim(timeseries),)
    else
        usedpaths = filenames
    end
    # Show errors after all files load, or you can't see them:
    if length(errors) > 0
        println("Some errors thrown during file load: ")
        println.(errors)
    end
    RasterSeries(usedpaths, dims; child=RasterStack, duplicate_first=true, kw...)
end

# Utils ########################################################################

function _open(f, ::Type{SMAPfile}, filepath::AbstractString; key=nothing, kw...)
    h5open(filepath; kw...) do ds
        _open(f, SMAPfile, SMAPhdf5(ds); key, kw...)
    end
end
function _open(f, ::Type{SMAPfile}, ds::SMAPhdf5; key=nothing, kw...)
    cleanreturn(f(key isa Nothing ? ds : ds[key]))
end

function _smap_timefromfilename(filename::String)
    dateformat = DateFormat("yyyymmddTHHMMSS")
    dateregex = r"SMAP_L4_SM_gph_(\d+T\d+)_"
    datematch = match(dateregex, filename)
    if datematch !== nothing
        return DateTime(datematch.captures[1], dateformat)
    else
        error("Date/time not correctly formatted in path: $filename")
    end
end

_smap_timedim(t::DateTime) = _smap_timedim(t:Hour(3):t)
function _smap_timedim(times::AbstractVector)
    Ti(Sampled(times, ForwardOrdered(), Regular(Hour(3)), Intervals(Start()), NoMetadata()))
end


# Install Julia 0.6.4 and then run
# include("install.jl") (Only once)
# to install all required packages.

# It is not advised to run this file directly. Rather, start a fresh
# Julia session and run the code between the "================================"
# delimeters. Afterwards, start a fresh Julia session and run the next block.
# This is to ensure that the number of worker processes is correct. Also,
# each block will require a couple of hours to finish.

# For customization, the parameter arguments can be changed/supplied in the
# of the L-shaped solvers in dayahead_benchmark.jl. Use `?LShapedSolver`
# in Julia prompt after `using LShapedSolvers` for a docstring on its usage.
# The parameters and default value of each algorithm is accessed by for example
# `?LShaped`.

# Serial benchmarks. Used to generate Fig. 2
# ================================ #
include("dayahead_benchmark.jl")
serial_benchmark = prepare_sbenchmark()
warmup_benchmarks!(serial_benchmark)
run_benchmarks!(serial_benchmark) # Takes a couple of hours
save_results!(serial_benchmark,"serial_benchmarks.json")
# Plotting
using Plots
pyplot()
p = plot(TimePlot(serial_benchmark)) # Creates Fig. 2
savefig(p,"serial_benchmarks.pdf")
# ================================ #

# Distributed benchmarks. Used to generate Fig. 3 and Fig. 4
# ================================ #
# No worker cores (serial)
include("dayahead_benchmark.jl")
distributed_benchmark = prepare_dbenchmark()
warmup_benchmarks!(distributed_benchmark)
run_benchmarks!(distributed_benchmark) # Takes a couple of hours
save_results!(distributed_benchmark,"scaling_1.json")
# ================================ #
# 2 worker cores
addprocs(2)
include("dayahead_benchmark.jl")
distributed_benchmark = prepare_dbenchmark()
warmup_benchmarks!(distributed_benchmark)
run_benchmarks!(distributed_benchmark) # Takes a couple of hours
save_results!(distributed_benchmark,"scaling_2.json")
# ================================ #
# 4 worker cores
addprocs(4)
include("dayahead_benchmark.jl")
distributed_benchmark = prepare_dbenchmark()
warmup_benchmarks!(distributed_benchmark)
run_benchmarks!(distributed_benchmark) # Takes a couple of hours
save_results!(distributed_benchmark,"scaling_4.json")
# ================================ #
# 8 worker cores
addprocs(8)
include("dayahead_benchmark.jl")
distributed_benchmark = prepare_dbenchmark()
warmup_benchmarks!(distributed_benchmark)
run_benchmarks!(distributed_benchmark) # Takes a couple of hours
save_results!(distributed_benchmark,"scaling_8.json")
# ================================ #
# 16 worker cores
addprocs(16)
include("dayahead_benchmark.jl")
distributed_benchmark = prepare_dbenchmark()
warmup_benchmarks!(distributed_benchmark)
run_benchmarks!(distributed_benchmark) # Takes a couple of hours
save_results!(distributed_benchmark,"scaling_16.json")
# ================================ #
# Collect all distributed benchmarks and plot
distributed_benchmark = load_results!("scaling_1.json")
distributed_benchmark.results["1"] = copy(distributed_benchmark.results["1000"])
delete!(distributed_benchmark.results, "1000")
scaling_2 = load_results!("scaling_2.json")
distributed_benchmark.results["2"] = copy(scaling_2.results["1000"])
scaling_4 = load_results!("scaling_4.json")
distributed_benchmark.results["4"] = copy(scaling_4.results["1000"])
scaling_8 = load_results!("scaling_8.json")
distributed_benchmark.results["8"] = copy(scaling_8.results["1000"])
scaling_16 = load_results!("scaling_16.json")
distributed_benchmark.results["16"] = copy(scaling_16.results["1000"])
save_results!(distributed_benchmark,"distributed_benchmarks.json")
using Plots
pyplot()
p = plot(TimePlot(distributed_benchmark)) # Creates Fig. 3
savefig(p,"strong_scaling_comptime.pdf")
p = plot(ScalingPlot(distributed_benchmark)) # Creates Fig. 4
savefig(p,"strong_scaling_efficiency.pdf")
# ================================ #


mutable struct EDF_FileHeader
    # Header
    Version::Clong                      # 8 ascii : version of this data format (0)
    Patient::String                # 80 ascii : local patient identification (mind item 3 of the additional EDF+ specs)
    Comment::String                 # 80 ascii : local recording identification (mind item 4 of the additional EDF+ specs)
#     start_date::String                  # 8 ascii : startdate of recording (dd.mm.yy) (mind item 2 of the additional EDF+ specs)
#     start_time::String                  # 8 ascii : starttime of recording (hh.mm.ss)

    Year::Cint
    Month::Cint
    Day::Cint
    Hour::Cint
    Minute::Cint
    Second::Cint

    Padding::Vector{Cuchar}             # 8 ascii : number of bytes in header record
                                        # 44 ascii : reserved
    NumRecord::Cint                     # 8 ascii : number of data records (-1 if unknown, obey item 10 of the additional EDF+ specs)
    SampleDuration::Cfloat              # 8 ascii : duration of a data record, in seconds
    NumChannel::Cint                    # 4 ascii : number of signals (ns) in data record

    ChannelLabels::Vector{String}     # ns * 16 ascii : ns * label (e.g. EEG Fpz-Cz or Body temp) (mind item 9 of the additional EDF+ specs)
    ChannelType::Vector{String}       # ns * 80 ascii : ns * transducer type (e.g. AgAgCl electrode)
    PhysicalDim::Vector{String}       # ns * 8 ascii : ns * physical dimension (e.g. uV or degreeC)
    PhysicalMin::Vector{Cfloat}       # ns * 8 ascii : ns * physical minimum (e.g. -500 or 34)
    PhysicalMax::Vector{Cfloat}       # ns * 8 ascii : ns * physical maximum (e.g. 500 or 40)
    DigitalMin::Vector{Cshort}        # ns * 8 ascii : ns * digital minimum (e.g. -2048)
    DigitalMax::Vector{Cshort}        # ns * 8 ascii : ns * digital maximum (e.g. 2047)

    Prefiltering::Vector{String}      # ns * 80 ascii : ns * prefiltering (e.g. HP:0.1Hz LP:75Hz)
    NumSamplesInRecord::Vector{Cshort}           # ns * 8 ascii : ns * nr of samples in each data record
    NumReservedSamplesInRecord::Vector{Cshort}  # ns * 32 ascii : ns * reserved

    EDF_FileHeader(file::IOStream) = begin
        _x = new()

        _x.Version = read(file, Int64)
        _x.Patient = read(file, 80) |> String |> strip
        _x.Comment = read(file, 80) |> String |> strip

        _x.Day, _x.Month, _x.Year = read(file, 8) |> String |> strip |>
            x->match(r"(\d{2})\.(\d{2})\.(\d{2})", x) |> x-> (x[1], x[2], x[3]) .|> Meta.parse
        _x.Hour, _x.Minute, _x.Second = read(file, 8) |> String |> strip |>
            x->match(r"(\d{2})\.(\d{2})\.(\d{2})", x) |> x-> (x[1], x[2], x[3]) .|> Meta.parse
        _x.Padding = read(file, 52)

        _x.NumRecord = read(file, 8) |> String |> Meta.parse
        _x.SampleDuration = read(file, 8) |> String |> Meta.parse
        _x.NumChannel = read(file, 4) |> String |> Meta.parse

        _x.ChannelLabels = read(file, 16*_x.NumChannel) |> x->reshape(x, (16, _x.NumChannel)) |> b-> mapreduce(x->String([x]), *, b, dims=1)[:] .|> strip
        _x.ChannelType = read(file, 80*_x.NumChannel) |> x->reshape(x, (80, _x.NumChannel)) |> b-> mapreduce(x->String([x]), *, b, dims=1)[:] .|> strip
        _x.PhysicalDim = read(file, 8*_x.NumChannel) |> x->reshape(x, (8, _x.NumChannel)) |> b-> mapreduce(x->String([x]), *, b, dims=1)[:] .|> strip
        _x.PhysicalMin = read(file, 8*_x.NumChannel) |> x->reshape(x, (8, _x.NumChannel)) |> b-> mapreduce(x->String([x]), *, b, dims=1)[:] .|> strip .|> Meta.parse
        _x.PhysicalMax = read(file, 8*_x.NumChannel) |> x->reshape(x, (8, _x.NumChannel)) |> b-> mapreduce(x->String([x]), *, b, dims=1)[:] .|> strip .|> Meta.parse
        _x.DigitalMin  = read(file, 8*_x.NumChannel) |> x->reshape(x, (8, _x.NumChannel)) |> b-> mapreduce(x->String([x]), *, b, dims=1)[:] .|> strip .|> Meta.parse
        _x.DigitalMax  = read(file, 8*_x.NumChannel) |> x->reshape(x, (8, _x.NumChannel)) |> b-> mapreduce(x->String([x]), *, b, dims=1)[:] .|> strip .|> Meta.parse

        _x.Prefiltering = read(file, 80*_x.NumChannel) |> x->reshape(x, (80, _x.NumChannel)) |> b-> mapreduce(x->String([x]), *, b, dims=1)[:] .|> strip
        _x.NumSamplesInRecord = read(file, 8*_x.NumChannel) |> x->reshape(x, (8, _x.NumChannel)) |> b-> mapreduce(x->String([x]), *, b, dims=1)[:] .|> strip .|> Meta.parse
        _x.NumReservedSamplesInRecord = read(file, 32*_x.NumChannel) |> x->reshape(x, (32, _x.NumChannel)) |> b-> mapreduce(x->String([x]), *, b, dims=1)[:] .|> strip .|> Meta.parse .|> x -> (x==nothing) ? 0 : x

        _x
    end
end

mutable struct EDFData
    FILE_HEADER::EDF_FileHeader
    CONTINUOUS_DATA::Array{Int16, 2}

    ##### misc #####
    Freq::Float32
#     Tspec::Vector{Float32}
    PhysicalUnit::Vector{Float32}

    RecordTime::String
    RecordDate::String

    EDFData(filename::String) = begin
        _f = open(filename, "r")
        _x = new()

        _x.FILE_HEADER = EDF_FileHeader(_f)

        _raw_data = zeros(Int16, _x.FILE_HEADER.NumRecord * _x.FILE_HEADER.NumSamplesInRecord[1], _x.FILE_HEADER.NumChannel)
#         _raw_reserved = zeros(Int16, _x.FILE_HEADER.NumChannel, _x.FILE_HEADER.NumRecord * _x.FILE_HEADER.NumReservedSamplesInRecord[1])

        _step = sum(_x.FILE_HEADER.NumSamplesInRecord)
        _chstep = _x.FILE_HEADER.NumSamplesInRecord[1]
        for _record_idx = 1:_x.FILE_HEADER.NumRecord
            record_data = read!(_f, Array{Int16, 2}(undef, _chstep, _x.FILE_HEADER.NumChannel))
            _raw_data[_chstep*(_record_idx-1)+1:_chstep*_record_idx, :] = record_data
        end
        _x.CONTINUOUS_DATA = _raw_data

        _x.Freq = _chstep / _x.FILE_HEADER.SampleDuration
        _x.PhysicalUnit = ( _x.FILE_HEADER.PhysicalMax .-  _x.FILE_HEADER.PhysicalMin) ./ ( _x.FILE_HEADER.DigitalMax -  _x.FILE_HEADER.DigitalMin)

        _header = _x.FILE_HEADER
        _x.RecordTime = "$(_header.Year)-$(_header.Month)-$(_header.Day) $(_header.Hour):$(_header.Minute):$(_header.Second)"
        _x.RecordDate = "$(_header.Year)-$(_header.Month)-$(_header.Day)"

        _x
    end
end

function export_isplit(_edf::EDFData; Comment = nothing, _dir="./iSplit")

    _index_json = joinpath(_dir, "channel_index.json")
    if !isfile(_index_json)
        open(_index_json, "w") do file
            write(file, "{}")
        end
    end

    channel_index = open(_index_json, "r") do file
        JSON.parse(file)
    end

    _record_time = _edf.RecordTime
    _record_date = _edf.RecordDate

    _comment = (Comment == nothing) ? Dict() : Comment

    for (chidx, chlabel) in enumerate(_edf.FILE_HEADER.ChannelLabels)

        _comment["label"] = chlabel
        _value = _edf.CONTINUOUS_DATA[:, chidx]
        _idx = hash(_value) |> repr

        if haskey(channel_index, @sprintf("channel%03d", chidx))
            if _idx in channel_index[@sprintf("channel%03d", chidx)]
                continue
            end
        else
            channel_index[@sprintf("channel%03d", chidx)] = Vector{String}(undef, 0)
        end

        _isplit_name = joinpath(_dir, @sprintf("channel%03d.jld", chidx))

        if !isfile(_isplit_name)
            jldopen(_isplit_name, "w") do file
                write(file, "Name", @sprintf("Channel%03d", chidx))
                write(file, "Comment", "")

            end
        end

        jldopen(_isplit_name, "r+") do file
            _entry = iSplitUnit(
                chidx, _record_date, _record_time, _comment,
                _edf.PhysicalUnit[chidx], _edf.Freq, _value
            )
            write(file, _idx, _entry)
        end

        push!(channel_index[@sprintf("channel%03d", chidx)], _idx)
    end

    open(_index_json, "w") do file
        write(file, JSON.json(channel_index))
    end
end


#=
 * sais
 * Copyright (c) 2008-2010 Yuta Mori All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person
 * obtaining a copy of this software and associated documentation
 * files (the "Software"), to deal in the Software without
 * restriction, including without limitation the rights to use,
 * copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following
 * conditions:
 *
 * The above copyright notice and this permission notice shall be
 * included in all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
 * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
 * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
 * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 =#

struct IntVector <: AbstractVector{Int}
    vec::Array{Int,1}
    off::Int
end
Base.size(v::IntVector) = (length(v.vec)-v.off,)
Base.getindex(v::IntVector, key) = v.vec[v.off+Int(key)]
Base.setindex!(v::IntVector, value, key) = v.vec[v.off+Int(key)] = value

# TODO:
# - refactor code to simplify
# - build user interface for string operations

function getcounts(T::AbstractVector{<:Integer}, C::IntVector, n::Int, k::Int)
    for i = 1:k
        C[i] = 0
    end
    for i = 1:n
        C[T[i]+1] += 1
    end
end

function getbuckets(C::IntVector, B::IntVector, k::Int, isend::Bool)
    s = 0
    if isend != false
        for i = 1:k
            s += C[i]
            B[i] = s
        end
    else
        for i = 1:k
            s += C[i]
            B[i] = s - C[i]
        end
    end
end

function sais(
    T::AbstractVector{<:Integer},
    SA::IndexVector,
    fs::Int,
    n::Int,
    k::Int,
    isbwt::Bool,
)
    pidx = 0
    flags = 0
    if k <= 256
        C = IntVector(zeros(Int, k), 0)
        if k <= fs
            B = IntVector(SA, n + fs - k)
            flags = 1
        else
            B = IntVector(zeros(Int, k), 0)
            flags = 3
        end
    elseif k <= fs
        C = IntVector(SA, n + fs - k)
        if k <= fs - k
            B = IntVector(SA, n + fs - 2k)
            flags = 0
        elseif k <= 1024
            B = IntVector(zeros(Int, k), 0)
            flags = 2
        else
            B = C
            flags = 8
        end
    else
        C = B = IntVector(zeros(Int, k), 0)
        flags = 4 | 8
    end
    # stage 1
    getcounts(T, C, n, k)
    getbuckets(C, B, k, true)
    for i = 1:n
        SA[i] = 0
    end
    b = -1
    i = j = n
    m = 0
    c0 = c1 = T[n]
    i -= 1
    while 1 <= i && ((c0 = T[i]) >= c1)
        c1 = c0
        i -= 1
    end
    while 1 <= i
        c1 = c0
        i -= 1
        while 1 <= i && ((c0 = T[i]) <= c1)
            c1 = c0
            i -= 1
        end
        if 1 <= i
            0 <= b && (SA[b+1] = j)
            b = (B[c1+1] -= 1)
            j = i - 1
            m += 1
            c1 = c0
            i -= 1
            while 1 <= i && ((c0 = T[i]) >= c1)
                c1 = c0
                i -= 1
            end
        end
    end
    if 1 < m
        LMSsort(T, SA, C, B, n, k)
        name = LMSpostproc(T, SA, n, m)
    elseif m == 1
        SA[b+1] = j + 1
        name = 1
    else
        name = 0
    end
    # stage 2
    if name < m
        newfs = n + fs - 2m
        if flags & (1 | 4 | 8) == 0
            if (k + name) <= newfs
                newfs -= k
            else
                flags |= 8
            end
        end
        j = 2m + newfs
        for i = (m+(n>>1)):-1:(m+1)
            if SA[i] != 0
                SA[j] = SA[i] - 1
                j -= 1
            end
        end
        RA = IntVector(SA, m + newfs)
        sais(RA, SA, newfs, m, name, false)

        i = n
        j = 2m
        c0 = c1 = T[n]
        while 1 <= (i -= 1) && ((c0 = T[i]) >= c1)
            c1 = c0
        end
        while 1 <= i
            c1 = c0
            while 1 <= (i -= 1) && ((c0 = T[i]) <= c1)
                c1 = c0
            end
            if 1 <= i
                SA[j] = i
                j -= 1
                c1 = c0
                while 1 <= (i -= 1) && ((c0 = T[i]) >= c1)
                    c1 = c0
                end
            end
        end
        for i = 1:m
            SA[i] = SA[m + SA[i] + 1]
        end
        if flags & 4 != 0
            C = B = IntVector(zeros(Int, k), 0)
        end
        if flags & 2 != 0
            B = IntVector(zeros(Int, k), 0)
        end
    end
    # stage 3
    flags & 8 != 0 && getcounts(T, C, n, k)
    if 1 < m
        getbuckets(C, B, k, true)
        i = m - 1
        j = n
        p = SA[m]
        c1 = T[p+1]
        while true
            c0 = c1
            q = B[c0+1]
            while q < j
                j -= 1
                SA[j+1] = 0
            end
            while true
                j -= 1
                SA[j+1] = p
                i -= 1
                i < 0 && break
                p = SA[i+1]
                c1 = T[p+1]
                c1 != c0 && break
            end
            i < 0 && break
        end
        while 0 < j
            j -= 1
            SA[j+1] = 0
        end
    end
    if isbwt == false
        induceSA(T, SA, C, B, n, k)
    else
        computeBWT(T, SA, C, B, n, k)
    end
    return SA
end

function LMSsort(
    T::AbstractVector{<:Integer},
    SA::IndexVector,
    C::IntVector,
    B::IntVector,
    n::Int,
    k::Int,
)
    C == B && getcounts(T, C, n, k)
    getbuckets(C, B, k, false)
    j = n - 1
    c1 = T[j+1]
    b = B[c1+1]
    j -= 1
    SA[b+1] = T[j+1] < c1 ? ~j : j
    b += 1
    for i = 1:n
        if 0 < (j = SA[i])
            if (c0 = T[j+1]) != c1
                B[c1+1] = b
                c1 = c0
                b = B[c1+1]
            end
            j -= 1
            SA[b+1] = T[j+1] < c1 ? ~j : j
            b += 1
            SA[i] = 0
        elseif j < 0
            SA[i] = ~j
        end
    end
    C == B && getcounts(T, C, n, k)
    getbuckets(C, B, k, true)
    c1 = 0
    b = B[c1+1]
    for i = n:-1:1
        if 0 < (j = SA[i])
            c0 = T[j+1]
            if c0 != c1
                B[c1+1] = b
                c1 = c0
                b = B[c1+1]
            end
            j -= 1
            b -= 1
            SA[b+1] = T[j+1] > c1 ? ~(j + 1) : j
            SA[i] = 0
        end
    end
end

function LMSpostproc(T::AbstractVector{<:Integer}, SA::IndexVector, n::Int, m::Int)
    i = 1
    while (p = SA[i]) < 0
        SA[i] = ~p
        i += 1
    end
    if i - 1 < m
        j = i
        i += 1
        while true
            if (p = SA[i]) < 0
                SA[j] = ~p
                j += 1
                SA[i] = 0
                j - 1 == m && break
            end
            i += 1
        end
    end

    i = j = n
    c0 = c1 = T[n]
    while 1 <= (i -= 1) && ((c0 = T[i]) >= c1)
        c1 = c0
    end
    while 1 <= i
        c1 = c0
        while 1 <= (i -= 1) && ((c0 = T[i]) <= c1)
            c1 = c0
        end
        if 1 <= i
            SA[m + (i >> 1) + 1] = j - i
            j = i + 1
            c1 = c0
            while 1 <= (i -= 1) && ((c0 = T[i]) >= c1)
                c1 = c0
            end
        end
    end
    name = 0
    q = n
    qlen = 0
    for i = 1:m
        p = SA[i]
        plen = SA[m + (p >> 1) + 1]
        diff = true
        if plen == qlen && (q + plen < n)
            j = 0
            while j < plen && T[p+j+1] == T[q+j+1]
                j += 1
            end
            j == plen && (diff = false)
        end
        if diff != false
            name += 1
            q = p
            qlen = plen
        end
        SA[m + (p >> 1) + 1] = name
    end
    return name
end

function induceSA(
    T::AbstractVector{<:Integer},
    SA::IndexVector,
    C::IntVector,
    B::IntVector,
    n::Int,
    k::Int,
)
    C == B && getcounts(T, C, n, k)
    getbuckets(C, B, k, false)
    j = n - 1
    c1 = T[j+1]
    b = B[c1+1]
    SA[b+1] = 0 < j && T[j] < c1 ? ~j : j
    b += 1
    for i = 1:n
        j = SA[i]
        SA[i] = ~j
        if 0 < j
            j -= 1
            if (c0 = T[j+1]) != c1
                B[c1+1] = b
                c1 = c0
                b = B[c1+1]
            end
            SA[b+1] = 0 < j && T[j] < c1 ? ~j : j
            b += 1
        end
    end
    C == B && getcounts(T, C, n, k)
    getbuckets(C, B, k, true)
    c1 = 0
    b = B[c1+1]
    for i = n:-1:1
        if 0 < (j = SA[i])
            j -= 1
            c0 = T[j+1]
            if c0 != c1
                B[c1+1] = b
                c1 = c0
                b = B[c1+1]
            end
            b -= 1
            SA[b+1] = j == 0 || T[j] > c1 ? ~j : j
        else
            SA[i] = ~j
        end
    end
end

function computeBWT(
    T::AbstractVector{<:Integer},
    SA::IndexVector,
    C::IntVector,
    B::IntVector,
    n::Int,
    k::Int,
)
    pidx = -1
    C == B && getcounts(T, C, n, k)
    getbuckets(C, B, k, false)
    j = n - 1
    c1 = T[j+1]
    b = B[c1+1]
    SA[b+1] = 0 < j && T[j] < c1 ? ~j : j
    b += 1
    for i = 1:n
        if 0 < (j = SA[i])
            j -= 1
            c0 = T[j+1]
            SA[i] = ~c0
            if c0 != c1
                B[c1+1] = b
                c1 = c0
                b = B[c1+1]
            end
            SA[b+1] = 0 < j && T[j] < c1 ? ~j : j
            b += 1
        elseif j != 0
            SA[i] = ~j
        end
    end
    C == B && getcounts(T, C, n, k)
    getbuckets(C, B, k, true)
    c1 = 0
    b = B[c1+1]
    for i = n:-1:1
        if 0 < (j = SA[i])
            j -= 1
            c0 = T[j+1]
            SA[i] = c0
            if c0 != c1
                B[c1+1] = b
                c1 = c0
                b = B[c1+1]
            end
            b -= 1
            SA[b+1] = 0 < j && T[j] > c1 ? ~(T[j]) : j
        elseif j != 0
            SA[i] = ~j
        else
            pidx = i - 1
        end
    end
    return pidx
end


#!/usr/bin/env julia

# A sript to draw for secret santa and send pairings via e-mail using msmtp.

# Usage: julia secretsanta.jl <table> [<mail account>]

# <table> is a csv table with columns "Name", "Pair", and "Address".
# Use the "Pair" columns to prevent drawing, e.g, partners names by assigning
# partners an identical id.

using Random
using CSV
using Dates

"""
    readtable(f)

Reads a csv file f containing information of secret santa participants.
Order of columns is: names, pairs, e-mail addresses.
"""
function readtable(tablefile)
    if isfile(tablefile)
        table = CSV.read(tablefile, comment="#")
    else
        error("$tablefile is not a valid file.")
    end
    table
end

"""
    getsecretsanta(n, p)

Takes two vectors with names, n, and corresponding pairings, p,
and returns a permutated names vector for secret santa.
"""
function getsecretsanta(names, pairs)
    santaindex = shuffle(eachindex(names))
    while any(pairs.== pairs[santaindex])
        santaindex = shuffle(eachindex(names))
    end
    secretsanta = names[santaindex]
end

"""
    mailsecretsanta(t, a)

Notifies participants via e-mail using information in table `t`.
This requires a set-up msmtp. Choose your prefered mail account `a`.
"""
function mailsecretsanta(table, mailaccount = "default")
    if length(size(table)) > 1
        for i in 1:size(table, 1)
            name = table.Name[i]
            mail = table.Address[i]
            santa = table.Santa[i]
            message = "To: $mail\r\nSubject: Weihnachtswichteln $(Dates.year(Dates.today()))\r\n\r\nHallo $name,\r\n\r\nDu darfst heuer $santa bewichteln.\r\n\r\nLG\r\nDein Weihnachtswichtelskript\r\n\r\nBesuch mich auf https://github.com/lleiding/secretsanta.jl\r\n"
            run(pipeline(`echo $message`, `msmtp -a $mailaccount $mail`))
        end
    else
        name = table.Name
        mail = table.Address
        santa = table.Santa
        message = "To: $mail\r\nSubject: Weihnachtswichteln $(Dates.year(Dates.today()))\r\n\r\nHallo $name,\r\n\r\nDu darfst heuer $santa bewichteln.\r\n\r\nLG\r\nDein Weihnachtswichtelskript\r\n"
        run(pipeline(`echo $message`, `msmtp -a $mailaccount $mail`))
    end
end

"""
    runsecretsanta(f, a)

Runs the secret santa script using information in file `f`.
Mailing requires a set-up msmtp. Choose your prefered mail account `a`.
"""
function runsecretsanta(filename, mailaccount)
    table = readtable(filename)
    table.Santa = getsecretsanta(table[:,1], table[:,2])
    mailsecretsanta(table, mailaccount)
end

runsecretsanta(ARGS[1], ARGS[2])


#= MIT License

Copyright (c) 2020, 2021 Uwe Fechner

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE. =#

module Utils

# data structures for the flight state and the flight log
# functions for creating a demo flight state, demo flight log, loading and saving flight logs
# function se() for reading the settings
# in addition helper functions for working with rotations

using Rotations, StaticArrays, StructArrays, RecursiveArrayTools, Arrow, YAML
export SysState, ExtSysState, SysLog, MyFloat

export demo_state, demo_syslog, demo_log, load_log, syslog2extlog, save_log, rot3d, se

const MyFloat = Float32               # type to use for postions
const DATA_PATH = "./data"            # path for log files and other data

mutable struct Settings
    log_file::String
    model::String
    segments::Int64       # number of tether segments
    sample_freq::Int64
    time_lapse::Float64
    zoom::Float64
    fixed_font::String
end
const SETTINGS = [Settings("","",0,0,0,0,"")]

# getter function for the Settings struct
function se()
    if SETTINGS[1].segments == 0
        # load settings from YAML
        dict = YAML.load_file(joinpath(DATA_PATH, "settings.yaml"))
        SETTINGS[1].log_file=dict["system"]["log_file"]
        SETTINGS[1].model=dict["kite"]["model"]
        SETTINGS[1].segments=dict["system"]["segments"]
        SETTINGS[1].sample_freq=dict["system"]["sample_freq"]
        SETTINGS[1].time_lapse=dict["system"]["time_lapse"]
        SETTINGS[1].zoom=dict["system"]["zoom"]
        SETTINGS[1].fixed_font=dict["system"]["fixed_font"]
    end
    return SETTINGS[1]
end

# basic system state; one of these will be saved per time step
struct SysState
    time::Float64                          # time since start of simulation in seconds
    orient::MVector{4, Float32}            # orientation of the kite (quaternion)
    elevation::MyFloat                     # elevation angle in radians
    azimuth::MyFloat                       # azimuth angle in radians
    l_tether::MyFloat                      # tether length [m]
    v_reelout::MyFloat                     # reel out velocity [m/s]
    force::MyFloat                         # tether force [N]
    depower::MyFloat                       # depower settings
    v_app::MyFloat                         # apparent wind speed [m/s]
    X::MVector{se().segments+1, MyFloat}   # vector of particle positions in x
    Y::MVector{se().segments+1, MyFloat}   # vector of particle positions in y
    Z::MVector{se().segments+1, MyFloat}   # vector of particle positions in z
end

# extended SysState containing derived values for plotting
struct ExtSysState
    time::Float64                          # time since launch in seconds
    orient::UnitQuaternion{Float32}        # orientation of the kite
    X::MVector{se().segments+1, MyFloat}   # vector of particle positions in x
    Y::MVector{se().segments+1, MyFloat}   # vector of particle positions in y
    Z::MVector{se().segments+1, MyFloat}   # vector of particle positions in z
    x::MyFloat                             # kite position in x
    y::MyFloat                             # kite position in y
    z::MyFloat                             # kite position in z
end

# flight log, containing the basic data as struct of arrays
# and in addition an extended view on the data that includes derived/ calculated values for plotting
# finally meta data like the file name of the log file is included
struct SysLog
    name::String
    syslog::StructArray{SysState}    # struct of vectors
    extlog::StructArray{ExtSysState} # struct of vectors, containing derived values
end

# functions
"""
Calculate the rotation of reference frame (ax, ay, az) so that it matches the reference frame (bx, by, bz).
All parameters must be 3-element vectors. Both refrence frames must be orthogonal,
all vectors must already be normalized.
Source: http://en.wikipedia.org/wiki/User:Snietfeld/TRIAD_Algorithm
"""
function rot3d(ax, ay, az, bx, by, bz)
    R_ai = hcat(ax, az, ay)
    R_bi = hcat(bx, bz, by)
    return R_bi * R_ai'
end

# create a demo state with a given height and time
function demo_state(height=6.0, time=0.0)
    a = 10
    X = range(0, stop=10, length=(se().segments)+1)
    Y = zeros(length(X))
    Z = (a .* cosh.(X./a) .- a) * height/ 5.430806
    r_xyz = RotXYZ(pi/2, -pi/2, 0)
    q = UnitQuaternion(r_xyz)
    orient = MVector{4, Float32}(q.w, q.x, q.y, q.z)
    elevation = asin(Z[end]/X[end])
    return SysState(time, orient, elevation,0.,0.,0.,0.,0.,0.,X, Y, Z)
end

# create a demo flight log with given name [String] and duration [s]
function demo_syslog(name="Test flight"; duration=10)
    max_height = 6.03
    steps   = Int(duration * se().sample_freq) + 1
    time_vec = Vector{Float64}(undef, steps)
    myzeros = zeros(MyFloat, steps)
    elevation = Vector{Float64}(undef, steps)
    orient_vec = Vector{MVector{4, Float32}}(undef, steps)
    X_vec = Vector{MVector{se().segments+1, MyFloat}}(undef, steps)
    Y_vec = Vector{MVector{se().segments+1, MyFloat}}(undef, steps)
    Z_vec = Vector{MVector{se().segments+1, MyFloat}}(undef, steps)
    for i in range(0, length=steps)
        state = demo_state(max_height * i/steps, i/se().sample_freq)
        time_vec[i+1] = state.time
        orient_vec[i+1] = state.orient
        elevation[i+1] = asin(state.Z[end]/state.X[end])
        X_vec[i+1] = state.X
        Y_vec[i+1] = state.Y
        Z_vec[i+1] = state.Z
    end
    return StructArray{SysState}((time_vec, orient_vec, elevation, myzeros,myzeros,myzeros,myzeros,myzeros,myzeros, X_vec, Y_vec, Z_vec))
end

# extend a flight systom log with the fieds x, y, and z (kite positions) and convert the orientation to the type UnitQuaternion
function syslog2extlog(syslog)
    x_vec = @view VectorOfArray(syslog.X)[end,:]
    y_vec = @view VectorOfArray(syslog.Y)[end,:]
    z_vec = @view VectorOfArray(syslog.Z)[end,:]
    orient_vec = Vector{UnitQuaternion{Float32}}(undef, length(syslog.time))
    for i in range(1, length=length(syslog.time))
        orient_vec[i] = UnitQuaternion(syslog.orient[i])
    end
    return StructArray{ExtSysState}((syslog.time, orient_vec, syslog.X, syslog.Y, syslog.Z, x_vec, y_vec, z_vec))
end

# create an artifical log file for demonstration purposes
function demo_log(name="Test_flight"; duration=10)
    syslog = demo_syslog(name, duration=duration)
    return SysLog(name, syslog, syslog2extlog(syslog))
end

function save_log(flight_log::SysLog)
    Arrow.ArrowTypes.registertype!(SysState, SysState)
    filename=joinpath(DATA_PATH, flight_log.name) * ".arrow"
    Arrow.write(filename, flight_log.syslog, compress=:lz4)
end

function load_log(filename::String)

    Arrow.ArrowTypes.registertype!(SysState, SysState)
    Arrow.ArrowTypes.registertype!(MVector{4, Float32}, MVector{4, Float32})
    if isnothing(findlast(isequal('.'), filename))
        fullname = joinpath(DATA_PATH, filename) * ".arrow"
    else
        fullname = joinpath(DATA_PATH, filename)
    end
    table = Arrow.Table(fullname)
    myzeros = zeros(MyFloat, length(table.time))
    syslog = StructArray{SysState}((table.time, table.orient, table.elevation, table.azimuth, table.l_tether, table.v_reelout, table.force, table.depower, table.v_app, table.X, table.Y, table.Z))
    return SysLog(basename(fullname[1:end-6]), syslog, syslog2extlog(syslog))
end

function test(save=false)
    if save
        log_to_save=demo_log()
        save_log(log_to_save)
    end
    return(load_log("Test_flight.arrow"))
end

end


const MOVE_STRAIGHT_INDEX = 1
const TURN_RIGHT_INDEX = 2
const TURN_LEFT_INDEX = 3

next_lane(lane::Lane, roadway::Roadway, direction::Int) = roadway[lane.exits[clamp(direction,1,length(lane.exits))].target.tag]

prev_lane(lane::Lane, roadway::Roadway, direction::Int) = roadway[lane.entrances[clamp(direction,1,length(lane.entrances))].target.tag]

next_lane_point(lane::Lane, roadway::Roadway, direction::Int) = roadway[lane.exits[clamp(direction,1,length(lane.exits))].target]

prev_lane_point(lane::Lane, roadway::Roadway, direction::Int) = roadway[lane.entrances[clamp(direction,1,length(lane.entrances))].target]

function move_along_with_direction(roadind::RoadIndex, roadway::Roadway, Δs::Float64; direction::Int = 1)

    lane = roadway[roadind.tag]
    curvept = lane[roadind.ind, roadway]

    if curvept.s + Δs < 0.0
        if has_prev(lane)
            pt_lo = prev_lane_point(lane, roadway, direction)
            pt_hi = lane.curve[1]
            s_gap = norm(VecE2(pt_hi.pos - pt_lo.pos))

            if curvept.s + Δs < -s_gap
                lane_prev = prev_lane(lane, roadway, direction)
                curveind = curveindex_end(lane_prev.curve)
                roadind = RoadIndex(curveind, lane_prev.tag)
                return move_along_with_direction(roadind, roadway, Δs + curvept.s + s_gap, direction=direction)
            else # in the gap between lanes
                t = (s_gap + curvept.s + Δs) / s_gap
                curveind = CurveIndex(0, t)
                RoadIndex(curveind, lane.tag)
            end

        else # no prev lane, return the beginning of this one
            curveind = CurveIndex(1, 0.0)
            return RoadIndex(curveind, roadind.tag)
        end
    elseif curvept.s + Δs > lane.curve[end].s
        if has_next(lane)
            pt_lo = lane.curve[end]
            pt_hi = next_lane_point(lane, roadway, direction)
            s_gap = norm(VecE2(pt_hi.pos - pt_lo.pos))

            if curvept.s + Δs ≥ pt_lo.s + s_gap # extends beyond the gap
                curveind = lane.exits[min(length(lane.exits),direction)].target.ind
                roadind = RoadIndex(curveind, lane.exits[min(length(lane.exits),direction)].target.tag)
                return move_along_with_direction(roadind, roadway, Δs - (lane.curve[end].s + s_gap - curvept.s),direction=direction)
            else # in the gap between lanes
                t = (Δs - (lane.curve[end].s - curvept.s)) / s_gap
                curveind = CurveIndex(0, t)
                RoadIndex(curveind, lane.exits[clamp(direction,1,length(lane.exits))].target.tag)
            end
        else # no next lane, return the end of this lane
            curveind = curveindex_end(lane.curve)
            return RoadIndex(curveind, roadind.tag)
        end
    else
        if Δs >= 0.0
            if roadind.ind.i == 0
                ind = get_curve_index(CurveIndex(1,0.0), lane.curve, curvept.s+Δs)
            elseif roadind.ind.i == length(lane.curve)
                ind = get_curve_index(curveindex_end(lane.curve), lane.curve, curvept.s+Δs)
            else
                ind = get_curve_index(roadind.ind, lane.curve, Δs)
            end
            RoadIndex(ind, roadind.tag)
        else
            if roadind.ind.i == 0
                ind = get_curve_index(CurveIndex(1,0.0), lane.curve, curvept.s+Δs)
            elseif roadind.ind.i == length(lane.curve)
                ind = get_curve_index(curveindex_end(lane.curve), lane.curve, Δs)
            else
                ind = get_curve_index(roadind.ind, lane.curve, Δs)
            end
            RoadIndex(ind, roadind.tag)
        end
    end
end

function in_lanes(posG::VecSE2, roadway::Roadway)
    projections = Array{RoadProjection}(undef, 0)
    for seg in roadway.segments
        for lane in seg.lanes
            roadproj = proj(posG, lane, roadway, move_along_curves=false)
            targetlane = roadway[roadproj.tag]
            footpoint = targetlane[roadproj.curveproj.ind, roadway]
            dist = normsquared(VecE2(posG - footpoint.pos))
            if dist < targetlane.width/2.0
                push!(projections,roadproj)
            end
        end
    end

    return projections
end


"""
        BoundaryFunction{B, X1, X2}(func, parameters=nothing)

A wrapper for the user-defined boundary condition function `func`, on the
boundary specified by symbol `B` and at location `(X1, X2)`, and with `parameters`.

Example
=======

julia> using Oceananigans, Oceananigans.BoundaryConditions, Oceananigans.Fields

julia> top_tracer_flux = BoundaryFunction{:z, Cell, Cell}((x, y, t) -> cos(2π*x) * cos(t))
(::BoundaryFunction{:z,Cell,Cell,var"#7#8",Nothing}) (generic function with 1 method)

julia> top_tracer_bc = BoundaryCondition(Flux, top_tracer_flux);

julia> flux_func(x, y, t, p) = cos(p.k * x) * cos(p.ω * t); # function with parameters

julia> parameterized_u_velocity_flux = BoundaryFunction{:z, Face, Cell}(flux_func, (k=4π, ω=3.0))
(::BoundaryFunction{:z,Face,Cell,typeof(flux_func),NamedTuple{(:k, :ω),Tuple{Float64,Float64}}}) (generic function with 1 method)

julia> top_u_bc = BoundaryCondition(Flux, parameterized_u_velocity_flux);
"""
struct BoundaryFunction{B, X1, X2, F, P} <: Function
    func :: F
    parameters :: P

    function BoundaryFunction{B, X1, X2}(func, parameters=nothing) where {B, X1, X2}
        B ∈ (:x, :y, :z) || throw(ArgumentError("The boundary B at which the BoundaryFunction is
                                                to be applied must be either :x, :y, or :z."))
        return new{B, X1, X2, typeof(func), typeof(parameters)}(func, parameters)
    end
end

@inline call_boundary_function(func, ξ, η, t, ::Nothing) = func(ξ, η, t)
@inline call_boundary_function(func, ξ, η, t, parameters) = func(ξ, η, t, parameters)

@inline (bc::BoundaryFunction{:x, Y, Z})(j, k, grid, clock, state) where {Y, Z} =
    call_boundary_function(bc.func, ynode(Y, j, grid), znode(Z, k, grid), clock.time, bc.parameters)

@inline (bc::BoundaryFunction{:y, X, Z})(i, k, grid, clock, state) where {X, Z} =
    call_boundary_function(bc.func, xnode(X, i, grid), znode(Z, k, grid), clock.time, bc.parameters)

@inline (bc::BoundaryFunction{:z, X, Y})(i, j, grid, clock, state) where {X, Y} =
    call_boundary_function(bc.func, xnode(X, i, grid), ynode(Y, j, grid), clock.time, bc.parameters)

#####
##### Convenience constructors
#####

""" Returns the location of `loc` on the boundary `:x`, `:y`, or `:z`. """
boundary_location(::Val{:x}, loc) = (loc[2], loc[3])
boundary_location(::Val{:y}, loc) = (loc[1], loc[3])
boundary_location(::Val{:z}, loc) = (loc[1], loc[2])

""" Returns a boundary function at on the boundary `B` at the appropriate
    location for a tracer field. """
TracerBoundaryFunction(B, args...) = BoundaryFunction{B, Cell, Cell}(args...)

""" Returns a boundary function at on the boundary `B` at the appropriate
    location for u, the x-velocity field. """
function UVelocityBoundaryFunction(B, args...)
    loc = boundary_location(Val(B), (Face, Cell, Cell))
    return BoundaryFunction{B, loc[1], loc[2]}(args...)
end

""" Returns a boundary function at on the boundary `B` at the appropriate
    location for v, the y-velocity field. """
function VVelocityBoundaryFunction(B, args...)
    loc = boundary_location(Val(B), (Cell, Face, Cell))
    return BoundaryFunction{B, loc[1], loc[2]}(args...)
end

""" Returns a boundary function at on the boundary `B` at the appropriate
    location for w, the z-velocity field. """
function WVelocityBoundaryFunction(B, args...)
    loc = boundary_location(Val(B), (Cell, Cell, Face))
    return BoundaryFunction{B, loc[1], loc[2]}(args...)
end

"""
    TracerBoundaryCondition(bctype, B, args...)

Returns a `BoundaryCondition` of type `bctype`, that applies the function
`func` to a tracer on the boundary `B`, which is one of `:x, :y, :z`.
The boundary function has the signature

    `func(ξ, η, t)`

where `t` is time, and `ξ` and `η` are coordinates along the
boundary, eg: `(y, z)` for `B = :x`, `(x, z)` for `B = :y`, or
`(x, y)` for `B = :z`.
"""
TracerBoundaryCondition(bctype, B, args...) =
    BoundaryCondition(bctype, TracerBoundaryFunction(B, args...))

"""
    UVelocityBoundaryCondition(bctype, B, args...)

Returns a `BoundaryCondition` of type `bctype`, that applies the function
`func` to `u`, the `x`-velocity field, on the boundary `B`, which is one
of `:x, :y, :z`. The boundary function has the signature

    `func(ξ, η, t)`

where `t` is time, and `ξ` and `η` are coordinates along the
boundary, eg: `(y, z)` for `B = :x`, `(x, z)` for `B = :y`, or
`(x, y)` for `B = :z`.
"""
UVelocityBoundaryCondition(bctype, B, args...) =
    BoundaryCondition(bctype, UVelocityBoundaryFunction(B, args...))

"""
    VVelocityBoundaryCondition(bctype, B, args...)

Returns a `BoundaryCondition` of type `bctype`, that applies the function
`func` to `v`, the `y`-velocity field, on the boundary `B`, which is one
of `:x, :y, :z`. The boundary function has the signature

    `func(ξ, η, t)`

where `t` is time, and `ξ` and `η` are coordinates along the
boundary, eg: `(y, z)` for `B = :x`, `(x, z)` for `B = :y`, or
`(x, y)` for `B = :z`.
"""
VVelocityBoundaryCondition(bctype, B, args...) =
    BoundaryCondition(bctype, VVelocityBoundaryFunction(B, args...))

"""
    VVelocityBoundaryCondition(bctype, B, args...)

Returns a `BoundaryCondition` of type `bctype`, that applies the function
`func` to `w`, the `z`-velocity field, on the boundary `B`, which is one
of `:x, :y, :z`. The boundary function has the signature

    `func(ξ, η, t)`

where `t` is time, and `ξ` and `η` are coordinates along the
boundary, eg: `(y, z)` for `B = :x`, `(x, z)` for `B = :y`, or
`(x, y)` for `B = :z`.
"""
WVelocityBoundaryCondition(bctype, B, args...) =
    BoundaryCondition(bctype, WVelocityBoundaryFunction(B, args...))


module StarTIN
using Libdl
using startin_jll
# libstartin = joinpath(pwd(), "../startin/target/release/libstartin.dylib")

struct Star
    pt::Vector{Float64} # length 3
    link::Vector{UInt}
end

struct Triangulation
    stars::Vector{Star}
    snaptol::Float64
    cur::UInt
    is_init::Bool
    jump_and_walk::Bool
    robust_predicates::Bool
    removed_indices::Vector{UInt}
end

mutable struct DT
    ptr::Ptr{Triangulation}
    function DT()
        ptr = ccall((:new, libstartin), Ptr{Triangulation}, ())
        dt = new(ptr)
        finalizer(destroy!, dt)
        dt
    end
end

function Base.insert!(t::DT, points::Matrix{Float64})
    w, _ = size(points)
    w == 3 || error("Point array should be 3 dimensional, got $w dimensions.")
    res = ccall((:insert, libstartin), Cint, (Ptr{Triangulation}, Cint, Ref{Cdouble}), t.ptr, length(points), points)
    res == 0 || @warn "$res duplicate points encountered."
    res
end

function Base.insert!(t::DT, point::Vector{Float64})
    w = length(point)
    w == 3 || error("Point array should be 3 dimensional, got $w dimensions.")
    res = ccall((:insert_one_pt, libstartin), Cint, (Ptr{Triangulation}, Cdouble, Cdouble, Cdouble), t.ptr, point[1], point[2], point[3])
    res == 0 && @warn "Error inserting point."
    res
end

function Base.delete!(t::DT, pointid)
    ccall((:remove, libstartin), Cint, (Ptr{Triangulation}, Cint), t.ptr, pointid)
end

function info(t::DT)
    ccall((:debug, libstartin), Cint, (Ptr{Triangulation},), t.ptr)
end

function get_snap_tolerance(t::DT)
    ccall((:get_snap_tolerance, libstartin), Cdouble, (Ptr{Triangulation},), t.ptr)
end
function set_snap_tolerance!(t::DT, tol::Float64)
    ccall((:set_snap_tolerance, libstartin), Cdouble, (Ptr{Triangulation}, Cdouble), t.ptr, tol)
end

function interpolate_nn(t::DT, x::Float64, y::Float64)
    ccall((:interpolate_nn, libstartin), Cdouble, (Ptr{Triangulation}, Cdouble, Cdouble), t.ptr, x, y)
end
function interpolate_linear(t::DT, x::Float64, y::Float64)
    ccall((:interpolate_linear, libstartin), Cdouble, (Ptr{Triangulation}, Cdouble, Cdouble), t.ptr, x, y)
end
function interpolate_nni(t::DT, x::Float64, y::Float64)
    ccall((:interpolate_nni, libstartin), Cdouble, (Ptr{Triangulation}, Cdouble, Cdouble), t.ptr, x, y)
end
function interpolate_laplace(t::DT, x::Float64, y::Float64)
    ccall((:interpolate_laplace, libstartin), Cdouble, (Ptr{Triangulation}, Cdouble, Cdouble), t.ptr, x, y)
end

function write!(fn::AbstractString, t::DT)
    ccall((:write_obj, libstartin), Cint, (Ptr{Triangulation}, Cstring), t.ptr, fn)
end

function destroy!(t::DT)
    @debug "Cleaning up memory."
    ccall((:destroy, libstartin), Cint, (Ptr{Triangulation},), t.ptr)
end

export DT
export interpolate_nn
export interpolate_nni
export interpolate_linear
export interpolate_laplace
export get_snap_tolerance
export set_snap_tolerance!
export write!
export destroy!
export info

end


function FieldApsideDetector(arg0::FieldOrbit)
    return FieldApsideDetector((FieldOrbit,), arg0)
end

function FieldApsideDetector(arg0::RealFieldElement, arg1::FieldOrbit)
    return FieldApsideDetector((RealFieldElement, FieldOrbit), arg0, arg1)
end

function g(obj::FieldApsideDetector, arg0::FieldSpacecraftState)
    return jcall(obj, "g", RealFieldElement, (FieldSpacecraftState,), arg0)
end


push!(LOAD_PATH, joinpath(@__DIR__, ".."))

using PkgBenchmark
using Oceananigans
using Benchmarks

baseline = BenchmarkConfig(id="master")
script = joinpath(@__DIR__, "benchmarkable_incompressible_model.jl")
resultfile = joinpath(@__DIR__, "regression_benchmarks.json")

judge(Oceananigans, baseline, script=script, resultfile=resultfile, verbose=true)


using Knet, Test, Base.Iterators, Printf, LinearAlgebra, Random,  IterTools, CUDA
Knet.atype() = KnetArray{Float32}
include("/scratch/users/bberkay14/workfolder/language_to_logic/seq2tree/seq2tree/atis/tree.jl")
include("/scratch/users/bberkay14/workfolder/language_to_logic/seq2tree/seq2tree/atis/util.jl")
Knet.atype() = KnetArray{Float32}
using JLD2
using ArgParse
using Distributions

struct Memory; w; end

struct Attention; wquery; wattn; scale; end

struct Embed; w; end

function Embed(opt, vocabsize::Int, embedsize::Int)

    return Embed(param(rand(Uniform(-opt["init_weight"], opt["init_weight"]),embedsize,vocabsize)))
end

function (l::Embed)(x)

    return l.w[:,x]
end

struct Linear; w; b; end

function Linear(opt, inputsize::Int, outputsize::Int)

    return Linear(param(rand(Uniform(-opt["init_weight"], opt["init_weight"]),outputsize,inputsize)),param0(outputsize))
end

function (l::Linear)(x)

    return (mmul(l.w , x) .+ l.b)
end

struct L2L
    enc_lstm
    dec_lstm
    memory
    attention
    srcembed
    tgtembed
    dropout
    projection
end

function L2L(src_input_size, tgt_input_size, opt)
    srcembed = Embed(opt, src_input_size, opt["rnn_size"])
    tgtembed = Embed(opt, tgt_input_size, opt["rnn_size"])
    enc_lstm = RNN(opt["rnn_size"], opt["rnn_size"]; dropout=0, rnnType=:lstm)
    #input feeding deneme için aşağıdaki kodu aç
    #dec_lstm = RNN(opt["rnn_size"], 3*opt["rnn_size"]; dropout=0, rnnType=:lstm)
    dec_lstm = RNN(opt["rnn_size"], 2*opt["rnn_size"]; dropout=0, rnnType=:lstm)
    memory =1
    attention = Attention(1, param(rand(Uniform(-opt["init_weight"], opt["init_weight"]),opt["rnn_size"],2*opt["rnn_size"])), param(1) )

    dropout = opt["dropoutrec"]
    projection = Linear(opt, opt["rnn_size"],  tgt_input_size)
    return L2L(enc_lstm, dec_lstm, memory, attention, srcembed, tgtembed, dropout, projection)
end

function (m::Memory)(x)
    values = permutedims(x, [1, 3, 2])

    keys = mmul(m.w, values)
    return keys, values
end
# You can use the following helper function for scaling and linear transformations of 3-D tensors:
mmul(w,x) = (w == 1 ? x : w == 0 ? 0 : reshape(w * reshape(x,size(x,1),:), (:, size(x)[2:end]...)))

function encode(l::L2L, src, prev_c, prev_h)
    l.enc_lstm.h, l.enc_lstm.c = prev_c, prev_h
    src_embedding = l.srcembed(src)

    src_embedding = dropout(src_embedding,l.dropout; drop=true)
    src_encoded = l.enc_lstm(src_embedding)

    #s.decoder.h = permutedims(reshape(permutedims(src_encoded[:,:,end], (2,1)), (size(src_encoded[:,:,end], 2),Int64(size(src_encoded[:,:,end], 1) /2) ,2)), (2,1,3))
    l.dec_lstm.h = l.enc_lstm.h
    l.dec_lstm.c = l.enc_lstm.c
    return l.enc_lstm.c, l.enc_lstm.h
end

function (a::Attention)(cell, mem)
    query = mmul(a.wquery, cell)
    keys, vals = mem
    #attention_scores = bmm( permutedims(query, (3,1,2)) , keys)
    attention_scores = bmm( reshape(query, (size(query,3),size(query,1),size(query,2))) , keys)
    scaled_attention = a.scale .* attention_scores
    scaled_attention = softmax(scaled_attention; dims=2)
    context_vector = bmm(vals, permutedims(scaled_attention, (2,1,3)))

    concatenated_vectors = vcat(cell , permutedims(context_vector, (1,3,2)) )

    final_vector = mmul(a.wattn , concatenated_vectors)

    return final_vector
end

#function decode(l::L2L, tgt, mem, prev, parent_h)
function decode(l::L2L, tgt,  parent_h)
    tgt_embedding = l.tgtembed(tgt)
    #input feeding istiyorsan aşağıdaki kodu aç
    #input = vcat(tgt_embedding, parent_h,prev)
    input = vcat(tgt_embedding, parent_h)
    tgt_embedding = dropout(tgt_embedding, l.dropout; drop=true)
    decoder_output = l.dec_lstm(input)
    #attention_vector = l.attention(decoder_output, mem)
    #return attention_vector
    return l.dec_lstm.c, l.dec_lstm.h
end

function attention_decode(l::L2L, src_encoded, decoder_output)
    mem = l.memory(src_encoded)
    attention_vector = l.attention(decoder_output, mem)
    projected = l.projection(attention_vector)
    return projected
end

function eval_training(opt, train_loader,l::L2L, using_gpu, word_manager, form_manager)
    #encoder_optimizer.zero_grad()
    #decoder_optimizer.zero_grad()
    #attention_decoder_optimizer.zero_grad()
    enc_batch, enc_len_batch, dec_tree_batch = random_batch(train_loader)
    enc_max_len =  size(enc_batch, 2)

    #enc_outputs = zeros((enc_batch.size(0), enc_max_len, encoder.hidden_size), requires_grad=True)
    enc_outputs = param(size(enc_batch,1), enc_max_len, opt["rnn_size"]; init=zeros)
    #param((size(enc_batch,1), enc_max_len, opt["rnn_size"]); init=zeros)
    #if using_gpu:
    #    enc_outputs = enc_outputs.cuda()
    #enc_s = {}
    enc_s = Dict()
    for j in range(0,stop=opt["enc_seq_length"] )
        enc_s[j] = Dict()
    end

    dec_s = Dict()
    for i in range(0,stop=opt["dec_seq_length"] )
        dec_s[i] = Dict()
        for j in range(0,stop=opt["dec_seq_length"])
            dec_s[i][j] = Dict()
	end
    end

    for i in range(1, stop=2)
        #param a dimension belirtirken, dimensionlar knette farklı olabilir BAK
        enc_s[0][i] = param( opt["rnn_size"], opt["batch_size"]; init=zeros)

    end

    for i in range(1, stop=enc_max_len )
	println("Here")
	println(enc_s[i])
	println(length(enc_s))
	println(length(enc_s[i]))
	println(enc_batch)
	println(i)
        enc_s[i][1], enc_s[i][2] = encode(l, enc_batch[:,i], enc_s[i-1][1], enc_s[i-1][2])
        enc_outputs[:, i, :] = enc_s[i][2]
    end

    # tree decode
    queue_tree = Dict()
    for i in range(1, stop=opt["batch_size"])
        queue_tree[i] = []

        push!(queue_tree[i],Dict("tree"=>dec_tree_batch[i], "parent"=>0, "child_index"=>1))
    end
    loss = 0
    cur_index, max_index = 1,1
    dec_batch = Dict()
    #print(queue_tree[1][0]["tree"].to_string());exit()
    while (cur_index <= max_index)
        #print(cur_index)
        # build dec_batch for cur_index
        max_w_len = -1
        batch_w_list = []
        for i in range(1, stop=opt["batch_size"])
            w_list = []
            if (cur_index <= size(queue_tree[i],1))
                t = queue_tree[i][cur_index ]["tree"]
                for ic in range(1, stop=t.num_children)
                    #print("children ")
                    #print(ic +1)
                    if t.children[ic] isa Tree
                        push!(w_list, 4)
                        push!(queue_tree[i], Dict("tree"=>t.children[ic], "parent"=>cur_index, "child_index"=>ic))
                    else
                        push!(w_list, t.children[ic])
                    end
		end
                if size(queue_tree[i],1) > max_index
                    max_index = size(queue_tree[i],1)
                end
            end
            if length(w_list) > max_w_len
                max_w_len = length(w_list)
            end
            push!(batch_w_list, w_list)
        end
        #dec_batch[cur_index] = torch.zeros((opt.batch_size, max_w_len + 2), dtype=torch.long)
        #dec_batch[cur_index] = param(opt["batch_size"], max_w_len + 2; init=zeros)
	dec_batch[cur_index] = Int64.(zeros(opt["batch_size"], max_w_len + 2))
        for i in range(1, stop=opt["batch_size"])
            w_list = batch_w_list[i]
            if length(w_list) > 0
                for j in range(1, stop=length(w_list))
                    dec_batch[cur_index][i][j+1] = w_list[j]
                end
                # add <S>, <E>
                if cur_index == 1
                    dec_batch[cur_index][i][1] = 1
                else
                    dec_batch[cur_index][i][1] = get_symbol_idx(form_manager, '(')
                end
                dec_batch[cur_index][i][length(w_list) + 2] = 2
            end
        end

        for j in range(1, stop=2)
            #dec_s[cur_index][0][j] = torch.zeros((opt.batch_size, opt.rnn_size), dtype=torch.float, requires_grad=True)
            dec_s[cur_index][0][j] = param(opt["batch_size"], opt["rnn_size"] ; init=zeros)
            if using_gpu
                #dec_s[cur_index][0][j] = dec_s[cur_index][0][j].cuda()
                dec_s[cur_index][0][j] = Knet.KnetArray(dec_s[cur_index][0][j])
            end
        end

        if cur_index == 1
            for i in range(1, stop=opt["batch_size"])
                dec_s[1][0][1][i, :] = enc_s[enc_len_batch[i]][1][i, :]
                dec_s[1][0][2][i, :] = enc_s[enc_len_batch[i]][2][i, :]
            end

        else
            for i in range(1, stop=opt["batch_size"])
                if (cur_index <= size(queue_tree[i],1))
                    par_index = queue_tree[i][cur_index ]["parent"]
                    child_index = queue_tree[i][cur_index ]["child_index"]
                    dec_s[cur_index][0][1][i-1,:] = dec_s[par_index][child_index][1][i-1,:]
                    dec_s[cur_index][0][2][i-1,:] = dec_s[par_index][child_index][2][i-1,:]
                end
            end
        end
        gold_string = " "
        parent_h = dec_s[cur_index][0][2]
        for i in range(1, stop=size(dec_batch[cur_index], 2) - 1)

            dec_s[cur_index][i][1], dec_s[cur_index][i+1][2] = decode(l, dec_batch[cur_index][:,i],  parent_h)
            pred = attention_decode(l, enc_outputs, dec_s[cur_index][i][2])
            loss += nll(pred, dec_batch[cur_index][:,i+1]; dims=1, average=false)

        end
        cur_index = cur_index + 1
    end

    loss = loss / opt["batch_size"]
    return loss
end

function main(opt)
    Random.seed!(opt["seed"])
    @load  (string(opt["data_dir"]) * "/map.jld2") managers
    #------------
    word_manager, form_manager = managers
    using_gpu = false

    #L2L(src_input_size, tgt_input_size, opt, output_size)
    model = L2L(word_manager.vocab_size, form_manager.vocab_size, opt)

    ##-- load data
    train_loader = MinibatchLoader(opt, "train", using_gpu)

    if !isdir(opt["checkpoint_dir"])
        mkdir(opt["checkpoint_dir"])
    end

    ##-- start training
    step = 0
    epoch = 0
    optim_state = Dict("learningRate" => opt["learning_rate"], "alpha" =>  opt["decay_rate"])
    # default to rmsprop

    bestloss, bestmodel = eval_training(opt, train_loader,model,using_gpu, word_manager, form_manager), deepcopy(model)
    #if opt["opt_method"] == 0
    #progress!(adam(model, traindata), seconds=seconds) do y
    progress!(rmsprop(model, traindata; lr=optim_state["learningRate"], rho=optim_state["alpha"]), seconds=seconds) do y
        #devloss = loss(model, cdev)
        devloss = eval_training(opt, train_loader,model,using_gpu, word_manager, form_manager)
        if devloss < bestloss
            bestloss, bestmodel = devloss, deepcopy(model)
        end
        #println(stderr)
        (dev=devloss, tst=tstloss, mem=Float32(CUDA.usage[]))
    end

    iterations = opt["max_epochs"] * train_loader.num_batch
    #start_time = time.time()
    restarted = false
    # TODO revert back after tests
    #iterations = 2
    for i in range(1,stop=iterations)
        #epoch = i // train_loader.num_batch
        epoch = fld(i,train_loader.num_batch)
        train_loss = eval_training(opt, train_loader, encoder, decoder, attention_decoder, encoder_optimizer, decoder_optimizer, attention_decoder_optimizer, criterion, using_gpu, word_manager, form_manager)

        progress!(rmsprop(model, traindata; lr=optim_state["learningRate"], rho=optim_state["alpha"]), seconds=seconds) do y
            #devloss = loss(model, cdev)
            devloss = eval_training(opt, train_loader,model,using_gpu, word_manager, form_manager)
            if devloss < bestloss
                bestloss, bestmodel = devloss, deepcopy(model)
            end
        end
        #exponential learning rate decay
        if opt["opt_method"] == 0
            if mod(i , train_loader.num_batch) == 0 & opt["learning_rate_decay"] < 1
                if epoch >= opt["learning_rate_decay_after"]
                    decay_factor = opt["learning_rate_decay"]
                    optim_state["learningRate"] = optim_state["learningRate"] * decay_factor #decay it

                end
            end
        end
        if (epoch == opt["restart"]) &  !restarted
            restarted = True
            optim_state["learningRate"] = opt["learning_rate"]

        end

        #on last iteration
        if i == iterations
            checkpoint = Dict()

            checkpoint["lang2logic_model"] = model
            checkpoint["opt"] = opt
            checkpoint["i"] = i
            checkpoint["epoch"] = epoch

            Knet.save(opt["checkpoint_dir"] *"/model_seq2seq.jld2", "checkpoint", checkpoint)
        end
        if train_loss != train_loss
            print("loss is NaN.  This usually indicates a bug.")
            break
        end
    end
    save && Knet.save("attn-$(Int(time_ns())).jld2", "model", bestmodel)
    #bleu && Main.bleu(bestmodel,dev)
end

start = time()
s = ArgParseSettings()
@add_arg_table s begin
    "--gpuid"
        arg_type = Int
        help = "which gpu to use. -1 = use CPU"
        default = 0
    "--data_dir"
        help = "data path"
        arg_type = String
        default = "../data/"
    "--seed"
        help = "torch manual random number generator seed"
        arg_type = Int
        default = 123
    "--checkpoint_dir"
        help = "filename to autosave the checkpont to. Will be inside checkpoint_dir/"
        arg_type = String
        default = "checkpoint_dir"
    "--savefile"
        help = "max vocab size"
        arg_type = String
        default = "save"
    "--print_every"
        help = "how many steps/minibatches between printing out the loss"
        arg_type = Int
        default = 2000
    "--rnn_size"
        help = "size of LSTM internal state"
        arg_type = Int
        default = 200
    "--num_layers"
        help = "number of layers in the LSTM"
        arg_type = Int
        default = 1
    "--dropout"
        help = "dropout for regularization, used after each RNN hidden layer. 0 = no dropout"
        arg_type = Float64
        default = 0.3
    "--dropoutrec"
        help = "dropout for regularization, used after each c_i. 0 = no dropout"
        arg_type = Float64
        default = 0.3
    "--enc_seq_length"
        help = "number of timesteps to unroll for"
        arg_type = Int
        default = 60
    "--dec_seq_length"
        help = "number of timesteps to unroll for"
        arg_type = Int
        default = 220
    "--batch_size"
        help = "number of sequences to train on in parallel"
        arg_type = Int
        default = 20
    "--max_epochs"
        help = "number of full passes through the training data"
        arg_type = Int
        default = 130
    "--opt_method"
        help = "optimization method: 0-rmsprop 1-sgd"
        arg_type = Int
        default = 0
    "--learning_rate"
        help = "learning rate"
        arg_type = Float64
        default = 0.007
    "--init_weight"
        help = "initailization weight"
        arg_type = Float64
        default = 0.08
    "--learning_rate_decay"
        help = "learning rate decay"
        arg_type = Float64
        default = 0.98
    "--learning_rate_decay_after"
        help = "in number of epochs, when to start decaying the learning rate"
        arg_type = Int
        default = 5
    "--restart"
        help = "in number of epochs, when to restart the optimization"
        arg_type = Int
        default = -1
    "--decay_rate"
        help = "decay rate for rmsprop"
        arg_type = Float64
        default = 0.95
    "--grad_clip"
        help = "clip gradients at this value"
        arg_type = Int
        default = 5
end

args = parse_args(s)
main(args)
endtime = time()
time_passed = (endtime - start) / 60
print("total time: "* string(time_passed) * "minutes")


# TODO: This runs but is outdated and isn't a good reference anymore.

module Wager

using Dates
using Discord

# Guild ID -> username -> token count.
const TOKENS = Dict{Discord.Snowflake, Dict{String, UInt}}()
const TOKEN_START = 100
const TOKEN_INTERVAL = Minute(30)
const TOKEN_INCREMENT = 100

"""
Add users to the token cache via a `GuildCreate` event.
"""
function add_users(c::Client, e::GuildCreate)
    if !haskey(TOKENS, e.guild.id)
        TOKENS[e.guild.id] = Dict()
    end

    guild = TOKENS[e.guild.id]

    for m in e.guild.members
        if !haskey(guild, m.user.username)
            guild[m.user.username] = TOKEN_START
        end
    end
end

"""
Give out tokens to all users on an interval.
"""
function distribute_tokens(c::Client)
    while isopen(c)
        for g in keys(TOKENS)
            for u in keys(g)
                g[u] += TOKEN_INCREMENT
            end
        end
        sleep(TOKEN_INTERVAL)
    end
end

"""
Insert a guild and/or user from a message into the token cache if they don't exist.
"""
function ensure_updated(m::Message)
    if !haskey(TOKENS, m.guild_id)
        TOKENS[m.guild_id] = Dict()
    end
    if !haskey(TOKENS[m.guild_id], m.author.username)
        TOKENS[m.guild_id][m.author.username] = TOKEN_START
    end
end

"""
Get the token count for the user who sent a message.
"""
token_count(m::Message) = get(get(TOKENS, m.guild_id, Dict()), m.author.username, 0)

"""
Reply to a message with the author's token count via a `MessageCreate` event.
"""
function reply_token_count(c::Client, m::Message)
    ensure_updated(m)
    reply(c, m, "You have $(token_count(m)) tokens.")
end

"""
Reply to a message with the guild's token leaderboard via a `MessageCreate` event.
"""
function reply_token_leaderboard(c::Client, m::Message)
    ensure_updated(m)

    # Get user => token count pairs by token count in descending order.
    sorted = sort(collect(TOKENS[m.guild_id]); by=p -> p.second, rev=true)

    entries = String[]
    for i in 1:min(10, length(sorted))  # Display at most 10.
        user, tokens = sorted[i]
        push!(entries, "$user: $tokens")
    end

    reply(c, m, join(entries, "\n"))
end

"""
Transfer tokens from one user to another via a `MessageCreate` event.
"""
function send_tokens(c::Client, m::Message)
    ensure_updated(m)

    words = split(m.content)
    if length(words) < 3
        return reply(c, m, "Invalid !send command.")
    end

    tokens = tryparse(UInt, words[2])
    tokens === nothing && return reply(c, m, "'$(words[2])' is not a valid number.")

    recipient = words[3]
    if !haskey(TOKENS[m.guild_id], recipient)
        return reply(c, m, "Couldn't find user '$recipient' in this guild.")
    end
    if token_count(m) < tokens
        return reply(c, m, "You don't have enough tokens to give.")
    end

    TOKENS[m.guild_id][m.author.username] -= tokens
    TOKENS[m.guild_id][recipient] += tokens
    reply(c, m, "You sent $tokens tokens to $recipient.")
end

"""
Wager a user's tokens via a `MessageCreate` event. The user either doubles their bet or
loses it.
"""
function wager_tokens(c::Client, m::Message)
    ensure_updated(m)

    words = split(m.content)
    if length(words) < 2
        return reply(c, m, "Invalid !wager command.")
    end

    tokens = try
        parse(UInt, words[2])
    catch
        return reply(c, m, "'$(words[2])' is not a valid number of tokens.")
    end
    if token_count(m) < tokens
        return reply(c, m, "You don't have enough tokens to wager.")
    end

    if rand() > 0.5
        TOKENS[m.guild_id][m.author.username] += tokens
        reply(c, m, "You win!")
    else
        TOKENS[m.guild_id][m.author.username] -= tokens
        reply(c, m, "You lose.")
    end
end

function main()
    c = Client(ENV["DISCORD_TOKEN"]; prefix='!')
    add_handler!(c, GuildCreate, add_users)
    add_command!(c, :count, reply_token_count)
    add_command!(c, :leaderboard, reply_token_leaderboard)
    add_command!(c, :send, send_tokens)
    add_command!(c, :wager, wager_tokens)
    open(c)
    @async distribute_tokens(c)
    return c
end

end

if abspath(PROGRAM_FILE) == @__FILE__
    c = Wager.main()
    wait(c)
end


module ColorTypes

using Color, FixedPointNumbers, Base.Cartesian
import Color: Fractional, _convert

import Base: ==, abs, abs2, clamp, convert, div, isfinite, isinf,
    isnan, isless, length, one, promote_array_type, promote_rule, zero,
    trunc, floor, round, ceil, itrunc, ifloor, iceil, iround, bswap,
    mod, rem

export ARGB, BGR, RGB1, RGB4, BGRA, AbstractGray, Gray, GrayAlpha, Gray24, AGray32, YIQ, AlphaColor, ColorType, noeltype

typealias ColorType Union(ColorValue, AbstractAlphaColorValue)

# An alpha-channel-first memory layout
immutable AlphaColor{C<:ColorValue, T<:Fractional} <: AbstractAlphaColorValue{C,T}
    alpha::T
    c::C

    AlphaColor(x1::Real, x2::Real, x3::Real, alpha::Real = 1.0) = new(alpha, C(x1, x2, x3))
    AlphaColor(c::ColorValue, alpha::Real) = new(alpha, c)
end
AlphaColor{T<:Fractional}(c::ColorValue{T}, alpha::T = one(T)) = AlphaColor{typeof(c),T}(c, alpha)

typealias ARGB{T} AlphaColor{RGB{T}, T}

# Little-endian RGB (useful for BGRA & Cairo)
immutable BGR{T<:Fractional} <: AbstractRGB{T}
    b::T
    g::T
    r::T

    BGR(r::Real, g::Real, b::Real) = new(b, g, r)
end
BGR(r::Integer, g::Integer, b::Integer) = BGR{Float64}(r, g, b)
BGR(r::Fractional, g::Fractional, b::Fractional) = (T = promote_type(typeof(r), typeof(g), typeof(b)); BGR{T}(r, g, b))

typealias BGRA{T} AlphaColorValue{BGR{T}, T}

# Some readers return a byte for an alpha channel even if it's not meaningful
immutable RGB1{T<:Fractional} <: AbstractRGB{T}
    alphadummy::T
    r::T
    g::T
    b::T

    RGB1(r::Real, g::Real, b::Real) = new(one(T), r, g, b)
end
RGB1(r::Integer, g::Integer, b::Integer) = RGB1{Float64}(r, g, b)
RGB1(r::Fractional, g::Fractional, b::Fractional) = (T = promote_type(typeof(r), typeof(g), typeof(b)); RGB1{T}(r, g, b))

immutable RGB4{T<:Fractional} <: AbstractRGB{T}
    r::T
    g::T
    b::T
    alphadummy::T

    RGB4(r::Real, g::Real, b::Real) = new(r, g, b, one(T))
end
RGB4(r::Integer, g::Integer, b::Integer) = RGB4{Float64}(r, g, b)
RGB4(r::Fractional, g::Fractional, b::Fractional) = (T = promote_type(typeof(r), typeof(g), typeof(b)); RGB4{T}(r, g, b))

for CV in subtypes(AbstractRGB)
    CV == RGB && continue
    @eval begin
        convert{T}(::Type{$CV{T}}, c::$CV{T}) = c
        convert(::Type{$CV}, c::$CV) = c
        convert{T}(::Type{$CV{T}}, c::$CV) = $CV{T}(convert(T, c.r), convert(T, c.g), convert(T, c.b))
        convert{T<:Fractional}(::Type{$CV}, c::ColorValue{T}) = convert($CV{T}, c)
        convert(::Type{$CV}, c::AbstractRGB) = $CV(c.r, c.g, c.b)
        convert{T}(::Type{$CV{T}}, c::AbstractRGB) = $CV{T}(c.r, c.g, c.b)
    end
end
convert(::Type{RGB}, c::AbstractRGB) = RGB(c.r, c.g, c.b)
convert{T}(::Type{RGB{T}}, c::AbstractRGB) = RGB{T}(c.r, c.g, c.b)

# Sometimes you want to be explicit about grayscale. Also needed for GrayAlpha.
abstract AbstractGray{T} <: ColorValue{T}
immutable Gray{T<:Fractional} <: AbstractGray{T}
    val::T
end
convert{T}(::Type{Gray{T}}, x::Gray{T}) = x
convert{T,S}(::Type{Gray{T}}, x::Gray{S}) = Gray{T}(x.val)
convert{T<:Real}(::Type{T}, x::Gray) = convert(T, x.val)
convert{T}(::Type{Gray{T}}, x::Real) = Gray{T}(x)

convert{T}(::Type{Gray{T}}, x::AbstractRGB) = convert(Gray{T}, 0.299*x.r + 0.587*x.g + 0.114*x.b)  # Rec 601 luma conversion

zero{T}(::Type{Gray{T}}) = Gray{T}(zero(T))
 one{T}(::Type{Gray{T}}) = Gray{T}(one(T))

immutable Gray24 <: ColorValue{Uint8}
    color::Uint32
end
Gray24() = Gray24(0)
Gray24(val::Uint8) = (g = uint32(val); g<<16 | g<<8 | g)
Gray24(val::Ufixed8) = Gray24(reinterpret(val))

convert(::Type{Uint32}, g::Gray24) = g.color

typealias GrayAlpha{T} AlphaColorValue{Gray{T}, T}

immutable AGray32 <: AbstractAlphaColorValue{Gray24, Uint8}
    color::Uint32
end
AGray32() = AGray32(0)
AGray32(val::Uint8, alpha::Uint8) = (g = uint32(val); uint32(alpha)<<24 | g<<16 | g<<8 | g)
AGray32(val::Ufixed8, alpha::Ufixed8) = AGray32(reinterpret(val), reinterpret(alpha))

convert(::Type{Uint32}, g::AGray32) = g.color

convert(::Type{RGB}, x::Gray) = RGB(x.val, x.val, x.val)
convert{T}(::Type{RGB{T}}, x::Gray) = (g = convert(T, x.val); RGB{T}(g, g, g))

# YIQ (NTSC)
immutable YIQ{T<:FloatingPoint} <: ColorValue{T}
    y::T
    i::T
    q::T

    YIQ(y::Real, i::Real, q::Real) = new(y, i, q)
end
YIQ(y::FloatingPoint, i::FloatingPoint, q::FloatingPoint) = (T = promote_type(typeof(y), typeof(i), typeof(q)); YIQ{T}(y, i, q))

clamp{T}(c::YIQ{T}) = YIQ{T}(clamp(c.y, zero(T), one(T)),
                             clamp(c.i, convert(T,-0.5957), convert(T,0.5957)),
                             clamp(c.q, convert(T,-0.5226), convert(T,0.5226)))

function convert{T}(::Type{YIQ{T}}, c::AbstractRGB)
    rgb = clamp(c)
    YIQ{T}(0.299*rgb.r+0.587*rgb.g+0.114*rgb.b,
           0.595716*rgb.r-0.274453*rgb.g-0.321263*rgb.b,
           0.211456*rgb.r-0.522591*rgb.g+0.311135*rgb.b)
end
convert{T}(::Type{YIQ}, c::AbstractRGB{T}) = convert(YIQ{T}, c)

function _convert{T}(::Type{RGB{T}}, c::YIQ)
    cc = clamp(c)
    RGB{T}(cc.y+0.9563*cc.i+0.6210*cc.q,
           cc.y-0.2721*cc.i-0.6474*cc.q,
           cc.y-1.1070*cc.i+1.7046*cc.q)
end

# Y'CbCr
immutable YCbCr{T<:FloatingPoint} <: ColorValue{T}
    y::T
    cb::T
    cr::T

    YCbCr(y::Real, cb::Real, cr::Real) = new(y, cb, cr)
end
function YCbCr(y::FloatingPoint, cb::FloatingPoint, cr::FloatingPoint)
    T = promote_type(typeof(y), typeof(cb), typeof(cr))
    YCbCr{T}(y, cb, cr)
end

clamp{T}(c::YCbCr{T}) = YCbCr{T}(clamp(c.y, convert(T,16), convert(T,235)),
                                 clamp(c.cb, convert(T,16), convert(T,240)),
                                 clamp(c.cr, convert(T,16), convert(T,240)))

function convert{T}(::Type{YCbCr{T}}, c::AbstractRGB)
    rgb = clamp(c)
    YCbCr{T}(16+65.481*rgb.r+128.553*rgb.g+24.966*rgb.b,
             128-37.797*rgb.r-74.203*rgb.g+112*rgb.b,
             128+112*rgb.r-93.786*rgb.g-18.214*rgb.b)
end
convert{T}(::Type{YCbCr}, c::AbstractRGB{T}) = convert(YCbCr{T}, c)

function _convert{T}(::Type{RGB{T}}, c::YCbCr)
    cc = clamp(c)
    ny = cc.y - 16
    ncb = cc.cb - 128
    ncr = cc.cr - 128
    RGB{T}(0.004567ny - 1.39135e-7ncb + 0.0062586ncr,
           0.004567ny - 0.00153646ncb - 0.0031884ncr,
           0.004567ny + 0.00791058ncb - 2.79201e-7ncr)
end

# HSI
immutable HSI{T<:FloatingPoint} <: ColorValue{T}
    h::T
    s::T
    i::T

    HSI(h::Real, s::Real, i::Real) = new(h, s, i)
end
function HSI(h::FloatingPoint, s::FloatingPoint, i::FloatingPoint)
    T = promote_type(typeof(h), typeof(s), typeof(i))
    HSI{T}(h, s, i)
end

function convert{T}(::Type{HSI{T}}, c::AbstractRGB)
    rgb = clamp(c)
    α = (2rgb.r - rgb.g - rgb.b)/2
    β = 0.8660254*(rgb.g - rgb.b)
    h = atan2(β, α)
    i = (rgb.r + rgb.g + rgb.b)/3
    s = 1-min(rgb.r, rgb.g, rgb.b)/i
    s = ifelse(i > 0, s, zero(s))
    HSI{T}(h, s, i)
end
convert{T}(::Type{HSI}, c::AbstractRGB{T}) = convert(HSI{T}, c)

# TODO: HSI->RGB

## Generic algorithms

length(cv::ColorType) = div(sizeof(cv), sizeof(eltype(cv)))
# Because this can be called as `length(RGB)`, we might need to fill in a default element type.
# But the compiler chokes if we ask it to create RGB{Float64}{Float64}, even if that's inside
# the non-evaluated branch of a ternary expression, so we have to be sneaky about this.
length{CV<:ColorValue}(::Type{CV}) = _length(CV, eltype(CV))
_length{CV<:ColorValue}(::Type{CV}, ::Type{Any}) = length(CV{Float64})
_length{CV<:ColorValue}(::Type{CV}, ::DataType)  = div(sizeof(CV), sizeof(eltype(CV)))
length{CV,T}(::Type{AlphaColorValue{CV,T}}) = length(CV)+1
length{CV,T}(::Type{AlphaColor{CV,T}}) = length(CV)+1

for ACV in (ColorValue, AbstractRGB, AbstractGray, AbstractAlphaColorValue)
    for CV in subtypes(ACV)
        length(CV.parameters) == 1 || continue
        @eval noeltype{T}(::Type{$CV{T}}) = $CV
    end
end

for f in (:trunc, :floor, :round, :ceil, :itrunc, :ifloor, :iceil, :iround, :abs, :abs2, :isfinite, :isnan, :isinf, :bswap)
    @eval $f{T}(g::Gray{T}) = Gray{T}($f(g.val))
    @eval @vectorize_1arg Gray $f
end

for f in (:mod, :rem, :mod1)
    @eval $f(x::Gray, m::Gray) = Gray($f(x.val, m.val))
    @eval @vectorize_2arg Gray $f
end

# Return types for arithmetic operations
multype(a::Type,b::Type) = typeof(one(a)*one(b))
sumtype(a::Type,b::Type) = typeof(one(a)+one(b))
divtype(a::Type,b::Type) = typeof(one(a)/one(b))

# Math on ColorValues. These implementations encourage inlining and,
# for the case of Ufixed types, nearly halve the number of multiplications.
for CV in subtypes(AbstractRGB)
    @eval begin
        (*){R<:Real,T}(f::R, c::$CV{T}) = $CV{multype(R,T)}(f*c.r, f*c.g, f*c.b)
        (*){R<:Real,T}(f::R, c::AlphaColorValue{$CV{T},T}) =
            AlphaColorValue{$CV{multype(R,T)},multype(R,T)}(f*c.c.r, f*c.c.g, f*c.c.b, f*c.alpha)
        (*){R<:Real,T}(f::R, c::AlphaColor{$CV{T},T}) =
            AlphaColor{$CV{multype(R,T)},multype(R,T)}(f*c.c.r, f*c.c.g, f*c.c.b, f*c.alpha)
        function (*){R<:FloatingPoint,T<:Ufixed}(f::R, c::$CV{T})
            fs = f/reinterpret(one(T))
            $CV{multype(R,T)}(fs*reinterpret(c.r), fs*reinterpret(c.g), fs*reinterpret(c.b))
        end
        function (*){R<:Ufixed,T<:Ufixed}(f::R, c::$CV{T})
            fs = reinterpret(f)/widen(reinterpret(one(T)))^2
            $CV{multype(R,T)}(fs*reinterpret(c.r), fs*reinterpret(c.g), fs*reinterpret(c.b))
        end
        function (/){R<:FloatingPoint,T<:Ufixed}(c::$CV{T}, f::R)
            fs = one(R)/(f*reinterpret(one(T)))
            $CV{divtype(R,T)}(fs*reinterpret(c.r), fs*reinterpret(c.g), fs*reinterpret(c.b))
        end
        (+){S,T}(a::$CV{S}, b::$CV{T}) = $CV{sumtype(S,T)}(a.r+b.r, a.g+b.g, a.b+b.b)
        (-){S,T}(a::$CV{S}, b::$CV{T}) = $CV{sumtype(S,T)}(a.r-b.r, a.g-b.g, a.b-b.b)
        (+){S,T}(a::AlphaColorValue{$CV{S},S}, b::AlphaColorValue{$CV{T},T}) =
            AlphaColorValue{$CV{sumtype(S,T)},sumtype(S,T)}(a.c.r+b.c.r, a.c.g+b.c.g, a.c.b+b.c.b, a.alpha+b.alpha)
        (+){S,T}(a::AlphaColor{$CV{S},S}, b::AlphaColor{$CV{T},T}) =
            AlphaColor{$CV{sumtype(S,T)},sumtype(S,T)}(a.c.r+b.c.r, a.c.g+b.c.g, a.c.b+b.c.b, a.alpha+b.alpha)
        (-){S,T}(a::AlphaColorValue{$CV{S},S}, b::AlphaColorValue{$CV{T},T}) =
            AlphaColorValue{$CV{sumtype(S,T)},sumtype(S,T)}(a.c.r-b.c.r, a.c.g-b.c.g, a.c.b-b.c.b, a.alpha-b.alpha)
        (-){S,T}(a::AlphaColor{$CV{S},S}, b::AlphaColor{$CV{T},T}) =
            AlphaColor{$CV{sumtype(S,T)},sumtype(S,T)}(a.c.r-b.c.r, a.c.g-b.c.g, a.c.b-b.c.b, a.alpha-b.alpha)
        function (.+){T}(A::AbstractArray{$CV{T}}, b::AbstractRGB)
            bT = convert($CV{T}, b)
            out = similar(A)
            add!(out, A, bT)
        end
        function (.+){T}(A::Union(AbstractArray{AlphaColorValue{$CV{T},T}}, AbstractArray{AlphaColor{$CV{T},T}}), b::AbstractAlphaColorValue)
            bT = convert($CV{T}, b)
            out = similar(A)
            add!(out, A, bT)
        end
        (.+){T}(b::AbstractRGB, A::AbstractArray{$CV{T}}) = (.+)(A, b)
        (.+){T}(b::AbstractAlphaColorValue, A::Union(AbstractArray{AlphaColorValue{$CV{T},T}}, AbstractArray{AlphaColor{$CV{T},T}})) =
            (.+)(A, b)
        function (.-){T}(A::AbstractArray{$CV{T}}, b::AbstractRGB)
            bT = convert($CV{T}, b)
            out = similar(A)
            sub!(out, A, bT)
        end
        function (.-){T}(A::Union(AbstractArray{AlphaColorValue{$CV{T},T}}, AbstractArray{AlphaColor{$CV{T},T}}), b::AbstractAlphaColorValue)
            bT = convert($CV{T}, b)
            out = similar(A)
            sub!(out, A, bT)
        end
        function (.-){T}(b::AbstractRGB, A::AbstractArray{$CV{T}})
            bT = convert($CV{T}, b)
            out = similar(A)
            sub!(out, bT, A)
        end
        function (.-){T}(b::AbstractAlphaColorValue, A::Union(AbstractArray{AlphaColorValue{$CV{T},T}}, AbstractArray{AlphaColor{$CV{T},T}}))
            bT = convert($CV{T}, b)
            out = similar(A)
            sub!(out, bT, A)
        end
        isfinite{T<:Ufixed}(c::$CV{T}) = true
        isfinite{T<:Ufixed}(c::AbstractAlphaColorValue{$CV{T},T}) = true
        isfinite{T<:FloatingPoint}(c::$CV{T}) = isfinite(c.r) && isfinite(c.g) && isfinite(c.b)
        isfinite{T<:FloatingPoint}(c::AbstractAlphaColorValue{$CV{T},T}) = isfinite(c.c.r) && isfinite(c.c.g) && isfinite(c.c.b) && isfinite(c.alpha)
        isnan{T<:Ufixed}(c::$CV{T}) = false
        isnan{T<:Ufixed}(c::AbstractAlphaColorValue{$CV{T},T}) = false
        isnan{T<:FloatingPoint}(c::$CV{T}) = isnan(c.r) || isnan(c.g) || isnan(c.b)
        isnan{T<:FloatingPoint}(c::AbstractAlphaColorValue{$CV{T},T}) = isnan(c.c.r) || isnan(c.c.g) || isnan(c.c.b) || isnan(c.alpha)
        isinf{T<:Ufixed}(c::$CV{T}) = false
        isinf{T<:Ufixed}(c::AbstractAlphaColorValue{$CV{T},T}) = false
        isinf{T<:FloatingPoint}(c::$CV{T}) = isinf(c.r) || isinf(c.g) || isinf(c.b)
        isinf{T<:FloatingPoint}(c::AbstractAlphaColorValue{$CV{T},T}) = isinf(c.c.r) || isinf(c.c.g) || isinf(c.c.b) || isinf(c.alpha)
        abs(c::$CV) = abs(c.r)+abs(c.g)+abs(c.b) # should this have a different name?
        abs{T<:Ufixed}(c::$CV{T}) = float32(c.r)+float32(c.g)+float32(c.b) # should this have a different name?
        one{T<:ColorType}(::T) = one(T)
        one{T}(::Type{$CV{T}}) = $CV{T}(one(T),one(T),one(T))
        one{T}(::Type{AlphaColorValue{$CV{T},T}}) = AlphaColorValue{$CV{T},T}(one(T),one(T),one(T),one(T))
        one{T}(::Type{AlphaColor{$CV{T},T}}) = AlphaColorValue{$CV{T},T}(one(T),one(T),one(T),one(T))
        zero{T<:ColorType}(::T) = zero(T)
        zero{T}(::Type{$CV{T}}) = $CV{T}(zero(T),zero(T),zero(T))
        zero{T}(::Type{AlphaColorValue{$CV{T},T}}) = AlphaColorValue{$CV{T},T}(zero(T),zero(T),zero(T),zero(T))
        zero{T}(::Type{AlphaColor{$CV{T},T}}) = AlphaColorValue{$CV{T},T}(zero(T),zero(T),zero(T),zero(T))
    end
end
(*)(c::AbstractRGB, f::Real) = (*)(f, c)
(*){CV<:AbstractRGB}(c::AlphaColorValue{CV}, f::Real) = (*)(f, c)
(*){CV<:AbstractRGB}(c::AlphaColor{CV}, f::Real) = (*)(f, c)
(.*)(f::Real, c::AbstractRGB) = (*)(f, c)
(.*){CV<:AbstractRGB}(f::Real, c::AlphaColorValue{CV}) = (*)(f, c)
(.*){CV<:AbstractRGB}(f::Real, c::AlphaColor{CV}) = (*)(f, c)
(.*)(c::AbstractRGB, f::Real) = (*)(f, c)
(.*){CV<:AbstractRGB}(c::AlphaColorValue{CV}, f::Real) = (*)(f, c)
(.*){CV<:AbstractRGB}(c::AlphaColor{CV}, f::Real) = (*)(f, c)
(/)(c::AbstractRGB, f::Real) = (one(f)/f)*c
(/){CV<:AbstractRGB}(c::AlphaColorValue{CV}, f::Real) = (one(f)/f)*c
(/){CV<:AbstractRGB}(c::AlphaColor{CV}, f::Real) = (one(f)/f)*c
(/)(c::AbstractRGB, f::Integer) = (one(eltype(c))/f)*c
(/){CV<:AbstractRGB}(c::AlphaColorValue{CV}, f::Integer) = (one(eltype(c))/f)*c
(/){CV<:AbstractRGB}(c::AlphaColor{CV}, f::Integer) = (one(eltype(c))/f)*c
(./)(c::AbstractRGB, f::Real) = (/)(c, f)
(./){CV<:AbstractRGB}(c::AlphaColorValue{CV}, f::Real) = (/)(c, f)
(./){CV<:AbstractRGB}(c::AlphaColor{CV}, f::Real) = (/)(c, f)
(+){CV<:AbstractRGB}(A::AbstractArray{CV}, b::AbstractRGB) = (.+)(A, b)
(+){CV<:AbstractRGB,T,CV2<:AbstractRGB}(A::AbstractArray{AlphaColorValue{CV,T}}, b::AbstractAlphaColorValue{CV2}) = (.+)(A, b)
(+){CV<:AbstractRGB,T,CV2<:AbstractRGB}(A::AbstractArray{AlphaColor{CV,T}}, b::AbstractAlphaColorValue{CV2}) = (.+)(A, b)
(-){CV<:AbstractRGB}(A::AbstractArray{CV}, b::AbstractRGB) = (.-)(A, b)
(-){CV<:AbstractRGB,T,CV2<:AbstractRGB}(A::AbstractArray{AlphaColorValue{CV,T}}, b::AbstractAlphaColorValue{CV2}) = (.-)(A, b)
(-){CV<:AbstractRGB,T,CV2<:AbstractRGB}(A::AbstractArray{AlphaColor{CV,T}}, b::AbstractAlphaColorValue{CV2}) = (.-)(A, b)
(+){CV<:AbstractRGB}(b::AbstractRGB, A::AbstractArray{CV}) = (.+)(b, A)
(+){CV<:AbstractRGB,T,CV2<:AbstractRGB}(b::AbstractAlphaColorValue{CV2}, A::AbstractArray{AlphaColorValue{CV,T}}) = (.+)(A, b)
(+){CV<:AbstractRGB,T,CV2<:AbstractRGB}(b::AbstractAlphaColorValue{CV2}, A::AbstractArray{AlphaColor{CV,T}}) = (.+)(A, b)
(-){CV<:AbstractRGB}(b::AbstractRGB, A::AbstractArray{CV}) = (.-)(b, A)
(-){CV<:AbstractRGB,T,CV2<:AbstractRGB}(b::AbstractAlphaColorValue{CV2}, A::AbstractArray{AlphaColorValue{CV,T}}) = (.-)(A, b)
(-){CV<:AbstractRGB,T,CV2<:AbstractRGB}(b::AbstractAlphaColorValue{CV2}, A::AbstractArray{AlphaColor{CV,T}}) = (.-)(A, b)

# Math on Gray
for CV in subtypes(AbstractGray)
    @eval begin
        (*){R<:Real,T}(f::R, c::$CV{T}) = $CV{multype(R,T)}(f*c.val)
        (*)(c::$CV, f::Real) = (*)(f, c)
        (.*)(f::Real, c::$CV) = (*)(f, c)
        (.*)(c::$CV, f::Real) = (*)(f, c)
        (/)(c::$CV, f::Real) = (one(f)/f)*c
        (/)(c::$CV, f::Integer) = (one(eltype(c))/f)*c
        (./)(c::$CV, f::Real) = (/)(c, f)
        (+){S,T}(a::$CV{S}, b::$CV{T}) = $CV{sumtype(S,T)}(a.val+b.val)
        (-){S,T}(a::$CV{S}, b::$CV{T}) = $CV{sumtype(S,T)}(a.val-b.val)
        (*){S,T}(a::$CV{S}, b::$CV{T}) = $CV{multype(S,T)}(a.val*b.val)
        (+)(A::AbstractArray{$CV}, b::AbstractGray) = (.+)(A, b)
        (-)(A::AbstractArray{$CV}, b::AbstractGray) = (.-)(A, b)
        (+)(b::AbstractGray, A::AbstractArray{$CV}) = (.+)(b, A)
        (-)(b::AbstractGray, A::AbstractArray{$CV}) = (.-)(b, A)
        function (.+){T}(A::AbstractArray{$CV{T}}, b::AbstractGray)
            bT = convert($CV{T}, b)
            out = similar(A)
            add!(out, A, bT)
        end
        (.+){T}(b::AbstractGray, A::AbstractArray{$CV{T}}) = (.+)(A, b)
        function (.-){T}(A::AbstractArray{$CV{T}}, b::AbstractGray)
            bT = convert($CV{T}, b)
            out = similar(A)
            sub!(out, A, bT)
        end
        function (.-){T}(A::AbstractArray{$CV{T}}, b::AbstractGray)
            bT = convert($CV{T}, b)
            out = similar(A)
            sub!(out, A, bT)
        end
        function (.-){T}(b::AbstractGray, A::AbstractArray{$CV{T}})
            bT = convert($CV{T}, b)
            out = similar(A)
            sub!(out, bT, A)
        end
        (*)(A::AbstractArray, b::AbstractGray) = A .* b
        (*)(b::AbstractGray, A::AbstractArray) = b .* A
        (/)(A::AbstractArray, b::AbstractGray) = A ./ b
        function (.*){T,S}(A::AbstractArray{$CV{T}}, b::AbstractGray{S})
            Tout = multype(T,S)
            out = similar(A, $CV{Tout})
            mul!(out, A, b)
        end
        (.*){T,S}(b::AbstractGray{S}, A::AbstractArray{$CV{T}}) = A .* b
        function (./){T,S}(A::AbstractArray{$CV{T}}, b::AbstractGray{S})
            Tout = divtype(T,S)
            out = similar(A, $CV{Tout})
            div!(out, A, b)
        end
        isfinite{T<:Ufixed}(c::$CV{T}) = true
        isfinite{T<:FloatingPoint}(c::$CV{T}) = isfinite(c.val)
        isnan{T<:Ufixed}(c::$CV{T}) = false
        isnan{T<:FloatingPoint}(c::$CV{T}) = isnan(c.val)
        isinf{T<:Ufixed}(c::$CV{T}) = false
        isinf{T<:FloatingPoint}(c::$CV{T}) = isinf(c.val)
        abs(c::$CV) = abs(c.val) # should this have a different name?
        abs{T<:Ufixed}(c::$CV{T}) = float32(c.val) # should this have a different name?

        (<)(c::$CV, r::Real) = c.val < r
        (<)(r::Real, c::$CV) = r < c.val
        isless(c::$CV, r::Real) = c.val < r
        isless(r::Real, c::$CV) = r < c.val
        (<)(a::$CV, b::AbstractGray) = a.val < b.val
    end
end
(/)(a::AbstractGray, b::AbstractGray) = a.val/b.val
div(a::AbstractGray, b::AbstractGray) = div(a.val, b.val)
(+)(a::AbstractGray, b::Number) = a.val+b
(-)(a::AbstractGray, b::Number) = a.val-b
(+)(a::Number, b::AbstractGray) = a+b.val
(-)(a::Number, b::AbstractGray) = a-b.val
(.+)(a::AbstractGray, b::Number) = a.val+b
(.-)(a::AbstractGray, b::Number) = a.val-b
(.+)(a::Number, b::AbstractGray) = a+b.val
(.-)(a::Number, b::AbstractGray) = a-b.val

@ngenerate N typeof(out) function add!{T,N}(out, A::AbstractArray{T,N}, b)
    @inbounds begin
        @nloops N i A begin
            @nref(N, out, i) = @nref(N, A, i) + b
        end
    end
    out
end
# need a separate sub! because of unsigned types
@ngenerate N typeof(out) function sub!{T,N}(out, A::AbstractArray{T,N}, b::ColorType)  # TODO: change to b::T when julia #8045 fixed
    @inbounds begin
        @nloops N i A begin
            @nref(N, out, i) = @nref(N, A, i) - b
        end
    end
    out
end
@ngenerate N typeof(out) function sub!{T,N}(out, b::ColorType, A::AbstractArray{T,N})
    @inbounds begin
        @nloops N i A begin
            @nref(N, out, i) = b - @nref(N, A, i)
        end
    end
    out
end
@ngenerate N typeof(out) function mul!{T,N}(out, A::AbstractArray{T,N}, b)
    @inbounds begin
        @nloops N i A begin
            @nref(N, out, i) = @nref(N, A, i) * b
        end
    end
    out
end
@ngenerate N typeof(out) function div!{T,N}(out, A::AbstractArray{T,N}, b)
    @inbounds begin
        @nloops N i A begin
            @nref(N, out, i) = @nref(N, A, i) / b
        end
    end
    out
end

# To help type inference
for ACV in (ColorValue, AbstractRGB, AbstractGray)
    for CV in subtypes(ACV)
        (length(CV.parameters) == 1 && !(CV.abstract)) || continue
        @eval promote_array_type{T<:Real,S<:Real}(::Type{T}, ::Type{$CV{S}}) = $CV{promote_type(T, S)}
        @eval promote_rule{T<:Fractional,S<:Fractional}(::Type{$CV{T}}, ::Type{$CV{S}}) = $CV{promote_type(T, S)}
        for AC in subtypes(AbstractAlphaColorValue)
            (length(AC.parameters) == 2 && !(AC.abstract)) || continue
            @eval promote_array_type{T<:Real,S<:Real}(::Type{T}, ::Type{$AC{$CV{S},S}}) = (TS = promote_type(T, S); $AC{$CV{TS}, TS})
            @eval promote_rule{T<:Fractional,S<:Fractional}(::Type{$CV{T}}, ::Type{$CV{S}}) = $CV{promote_type(T, S)}
            @eval promote_rule{T<:Fractional,S<:Integer}(::Type{$CV{T}}, ::Type{S}) = $CV{promote_type(T, S)} # for Array{RGB}./Array{Int}
        end
    end
end

for (CV, CVstr, fields) in ((BGR,  "BGR",  (:(c.r),:(c.g),:(c.b))),
                            (RGB1, "RGB1", (:(c.r),:(c.g),:(c.b))),
                            (RGB4, "RGB4", (:(c.r),:(c.g),:(c.b))),
                            (ARGB, "ARGB", (:(c.c.r),:(c.c.g),:(c.c.b),:(c.alpha))),
                            (BGRA, "BGRA", (:(c.c.r),:(c.c.g),:(c.c.b),:(c.alpha))),
                            (Gray, "Gray", (:(c.val),)),
                            (GrayAlpha, "GrayAlpha", (:(c.c.val),:(c.alpha))))
    Color.makeshow(CV, CVstr, fields)
end

for T in (RGB24, ARGB32, Gray24, AGray32)
    @eval begin
        ==(x::Uint32, y::$T) = x == convert(Uint32, y)
        ==(x::$T, y::Uint32) = ==(y, x)
    end
end
=={T}(x::Gray{T}, y::Gray{T}) = x.val == y.val
=={T}(x::T, y::Gray{T}) = x == convert(T, y)
=={T}(x::Gray{T}, y::T) = ==(y, x)

end


module Compiler

using Dates
import LinearAlgebra

import ..OptimizingIR
import ..BusinessDays
import ..BS
import ..CAL_BRL
import ..InterestRates
import ..Observable
import ..Konst
import ..RiskFactor
import ..ContractAttributes
import ..PricingModel
import ..DiscountFactor
import ..DiscountFactorForward
import ..ObservableAt
import ..BufferedObservable
import ..LiftObs
import ..LiftObs2
import ..ReduceObs
import ..PricingDate
import ..AdjustedDate
import ..HistoricalValue
import ..Scale
import ..Worthless
import ..Give
import ..WhenAt
import ..Both
import ..Either
import ..Unit
import ..Contract
import ..get_carry_type
import ..CARRY_TYPE_NONE
import ..CARRY_TYPE_CURVE
import ..has_carry_curve
import ..get_carry_curve_symbol
import ..Currencies
import ..StaticHedgingModel
import ..FutureValueModel
import ..SpotCurrency
import ..AbstractPricer
import ..AbstractCashflowPricer
import ..Scenario
import ..ExposureResult
import ..ForwardDiff
import ..riskfactors
import ..FixedIncomeEvent
import ..FixedIncomeContract
import ..AbstractFixedIncomeContract
import ..SwapContract
import ..price
import ..exposures
import ..MarketData
import ..Stock
import ..BlackScholesModel
import ..BinomialModelDaily
import ..FixedCashRiskFactor
import ..FixedNonCashRiskFactor
import ..exch
import ..risk_factor_symbol
import ..market_data_symbol
import ..get_functional_currency
import ..foreachcashflow
import ..event_symbol
import ..get_horizon
import ..Cashflow
import ..CashRiskFactor
import ..NonCashRiskFactor
import ..Volatility
import ..Anytime
import ..SecondOrderRiskFactor
import ..DeltaNormalExposuresMethod
import ..DeltaGammaApproxExposuresMethod
import ..EuropeanOption
import ..is_riskfree_discountfactor

include("types.jl")
include("opcodes.jl")
include("compilation.jl")
include("pricer_impl.jl")
include("model_general.jl")
include("model_statichedging.jl")
include("model_futurevalue.jl")
include("model_binomial_daily.jl")
include("model_black_scholes.jl")

end # module Compiler


module Chords # Musical

export Chord, Major, Minor
export notes, root, third, minorthird, fifth # , pitches
export Am
import ..Musical.Notations: Notation, A, B, C, D, E, F, G

abstract type Quality end

# M
struct Major <: Quality
end

# m
struct Minor <: Quality
end

# aug
struct Augmented <: Quality
end

# dim
struct Diminished{N<:Int} <: Quality
end

struct HalfDiminished <: Quality
end

struct Dominant <: Quality
end

struct Chord
    notation::Notation
    q::Type{Q} where {Q<:Quality}
end

function notes(chord::Chord)::Tuple
    notes(chord.notation, chord.q)
end

function notes(notation::Notation, q::Type{Major})::Tuple
    (C, E, G)
end

function notes(notation::Notation, q::Type{Minor})::Tuple
    (A, C, E)
end

root(chord::Chord) = notes(chord)[1]
third(chord::Chord) = notes(chord)[2]
minorthird(chord::Chord) = notes(chord)[2]
fifth(chord::Chord) = notes(chord)[3]
# pitches(chord::Chord) =

const Am = Chord(A, Minor)

end # module Musical.Chords