const ALL_COLS = [:name, :charitytype, :ein, :fisyr, :fileyear, :invrealized,
  :invrealizeddirect, :totrev, :totexp,
  :invrealizedindirect, :name, :netassets,
  :pfexpbook, :pfexpcharity, :totinv, :category,
  :returnamt, :returncheckamt,:netincome, :naics]

const CATEGORY_CONSOLIDATED_THRESHOLD = 100_000

#=const :Arts, :Education, :Environmental, :AnimalWelfare, :Health, :MentalHealth,
:MedicalDiseases, :MedicalResearch, :CrimeAndLegal, :Employment, :FoodAndAgriculture,
:Housing, :PublicSafety, :Recreation, :YouthDevelopment, :HumanServices, :ForeignAffairs,
:SocialAction, :CommunityImprovement, :GrantmakingFoundations, :ScienceandTechResearch,
:SocialScienceResearch, :SocietyBenefitAndMultipurpose, :ReligionRelated,
:MutualBenefitOrg, :Unknown, :Other=#

function processReturns!(data::NCCSData)

  local N::Int = size(data.df,1) #length of dataframe
  local assetCutoff::Float64 = ASSET_CUTOFF #minimum allowed maximum assets
  local largeDifferenceCutoff::Float64 = LARGE_DIFFERENCE_CUTOFF #max discrepenacy between methods

  println("data.df N1: $N")
  println("data.df pf N1: $(sum(data.df[:charitytype] .== :pf))")
  #filter out organizations which are too small
  data.df[:tokeep] = trues(size(data.df,1))
  dfByein::GroupedDataFrame = groupby(data.df, :ein)
  for subdf::SubDataFrame ∈ dfByein
    subN::Int = size(subdf,1)
    maxAssets::Float64 = maximum(collect(skipmissing(subdf[:netassets])))
    if (maxAssets < assetCutoff)
      subdf[:tokeep] = falses(subN)
    end

    if subN ≠ length(unique(subdf[:fisyr])) # check if we have duplicate filings
      deleteOrg::Bool = false
      subdfByfisyr = groupby(subdf, :fisyr)
      for ssubdf::SubDataFrame ∈ subdfByfisyr
        nrows::Int = size(ssubdf, 1)
        if nrows > 1 # if we are at the org-year with the dup
          fileYears::Vector{Int} = unique(ssubdf[:fileyear])
          if length(fileYears) == nrows #if we can reconcile by fileYear
            maxFileYear::Int = maximum(fileYears) # keep the newest version
            ssubdf[ssubdf[:fileyear] .≠ maxFileYear, :tokeep] = false
          else
            deleteOrg = true #If we cannot correct, throw out the org
          end
        end
      end
      if deleteOrg #delete the org if we need to
        subdf[:tokeep] = false
      end
    end
  end

  #delete the invalid rows
  data.df = data.df[data.df[:tokeep] .== true, :]
  N = size(data.df,1)

  println("data.df N2: $N")

  #println("N2-pf: $(sum(data.df[:charitytype] .== :pf)) \nN2-pf-Return Amt: $(
  #  sum((data.df[:charitytype] .== :pf) .& ((!ismissing).(data.df[:returnamt]))))")

  #data.df[:returnamt] = Vector{Union{Float64,Missing}}(missings(N)) #don't need this since its done during mapping
  data.df[:return] = Vector{Union{Float64,Missing}}(missings(N))

  #data.df[:returncheckamt] = Vector{Union{Float64,Missing}}(missings(N))
  data.df[:returncheck] = Vector{Union{Float64,Missing}}(missings(N))

  data.df[:largeDifference] = Vector{Union{Bool,Missing}}(missings(N))

  sort!(data.df, [:ein, :fisyr])
  #ctr::Int = 0
  dfByein = groupby(data.df, :ein)
  for subdf::SubDataFrame ∈ dfByein
    subN = size(subdf,1)

    if (subdf[1,:charitytype] == :co) || (subdf[1,:charitytype] == :pc)

      for i ∈ 2:subN
        if subdf[i-1, :fisyr] == subdf[i, :fisyr] - 1

          #NOTE: returns = unrealized + inv income s.t. unrealized = Δassets - netincome
          returnCandidateAmt::MFloat64 = (subdf[i, :netassets] .- subdf[i-1, :netassets]
            ) .+ subdf[i, :invrealizeddirect] .- subdf[i, :netincome]
          returnCandidate::MFloat64 = returnCandidateAmt / subdf[i-1, :netassets]

          returnCheckCandidateAmt::MFloat64 = (subdf[i, :netassets] .- subdf[i-1, :netassets]
            ) .+ subdf[i, :invrealizedindirect] .- subdf[i, :netincome]
          returnCheckCandidate::MFloat64 = returnCheckCandidateAmt / subdf[i-1, :netassets]

          if ((!ismissing(returnCandidate)) && returnCandidate < 2. && returnCandidate > -0.9 &&
              abs(returnCandidate - returnCheckCandidate) ≤ largeDifferenceCutoff)# && abs(subdf[i, :totinv]) ≥ 1. #make sure we have valid numbers here
            subdf[i, :returnamt] = returnCandidateAmt
            subdf[i, :return] = returnCandidate

            subdf[i, :returncheckamt] = returnCheckCandidateAmt
            subdf[i, :returncheck] = returnCheckCandidate
          end
        end
      end
    elseif subdf[1,:charitytype] == :pf
      for i ∈ 2:subN
        if subdf[i-1, :fisyr] == subdf[i, :fisyr] - 1

          #NOTE: returns = unrealized + inv income s.t. unrealized = Δassets - netincome
          returnCandidateAmt::MFloat64 = subdf[i, :returnamt]
          returnCandidate::MFloat64 = returnCandidateAmt / subdf[i-1, :netassets]

          returnCheckCandidateAmt::MFloat64 = subdf[i, :returncheckamt]
          returnCheckCandidate::MFloat64 = returnCheckCandidateAmt / subdf[i-1, :netassets]

          if ( (!ismissing(returnCandidate)) && (returnCandidate < 2.) && (returnCandidate > -0.9) &&
              (!ismissing(returnCheckCandidate)) &&
              abs(returnCandidate - returnCheckCandidate) ≤ largeDifferenceCutoff)# && abs(subdf[i, :totinv]) ≥ 1. #make sure we have valid numbers here
            subdf[i, :returnamt] = returnCandidateAmt
            subdf[i, :return] = returnCandidate

            subdf[i, :returncheckamt] = returnCheckCandidateAmt
            subdf[i, :returncheck] = returnCheckCandidate
          #=elseif ctr < 50
            ctr+=1
            println("###\nreturn: $(returnCandidate)")
            println("returnchk: $(returnCheckCandidate)")
            println("returnamt: $(returnCandidateAmt)")
            println("returnchk: $((returnCheckCandidateAmt))")=#
          end
        end
      end
    end
  end

  sort!(data.df, [:ein, :fisyr])

  return data
end

function processWealth!(data::NCCSData)

  data.df[:lnetassets] = (log).(data.df[:netassets])
  data.df[:ladjnetassets] = (log).(data.df[:adjnetassets])

  return data
end

#data processing related to the categorization
function processCategories!(data::NCCSData;
  minPointsPerCategory::Int = MIN_POINTS_PER_CATEGORY,
  categoryConsolidatedThreshold::Int = CATEGORY_CONSOLIDATED_THRESHOLD)::Nothing

  data.df[:categoryname] = (s::MSymbol->((ismissing(s) || (!haskey(category, s))) ?
    missing : category[s])).(Vector{MSymbol}(data.df[:category]))

  data.df[:tokeep] .= true
  data.df[:categoryfiltered] = deepcopy(data.df[:category])
  data.df[:categorynamefiltered] = deepcopy(data.df[:categoryname])

  dfBycategoryfisyr::GroupedDataFrame = groupby(data.df, [:category, :fisyr])
  for subdf::SubDataFrame ∈ dfBycategoryfisyr
    if size(subdf,1) < minPointsPerCategory
      subdf[:categoryfiltered] = :ZZ
      subdf[:categorynamefiltered] = category[:ZZ]
    end
  end

  data.df[:categoryconsolidated] = deepcopy(data.df[:categoryfiltered])
  data.df[:categorynameconsolidated] = deepcopy(data.df[:categorynamefiltered])

  dfBycategorynamefiltered::GroupedDataFrame = groupby(data.df, :categorynamefiltered)
  for subdf::SubDataFrame ∈ dfBycategorynamefiltered
    if size(subdf,1) < categoryConsolidatedThreshold
      subdf[:categoryconsolidated] = :ZZ
      subdf[:categorynameconsolidated] = category[:ZZ]
    end
  end

  data.df[:naics2] = (s::MString->(!ismissing(s) ? Symbol(s[1:2]) : missing)).(data.df[:naics])
  data.df[:naics2name] = (s::MSymbol->(naicsCodes[s])).(data.df[:naics2])

  return nothing
end

#adjusts for inflation
#currently just does wealth, eventually can use this for return fields
function adjustInflation!(data::NCCSData; fredPath::String = FRED_PATH,
  fredInflationFile::String = FRED_INFLATION_FILE, inflationSeries::Symbol = :CPI,
  baseYear::Int = BASE_YEAR, fredDateFormat::String = FRED_DATE_FORMAT,
  adjustWealthCutoff::Bool = true)::Nothing

  #read in the inflation info
  freddf::DataFrame = CSV.read("$fredPath\\$fredInflationFile.csv")

  #make a dictionary
  cpiIndex::Dict =
    Dict(Dates.year(Date(freddf[i,:DATE], fredDateFormat))=>freddf[i, inflationSeries]
      for i ∈ 1:size(freddf,1))

  #get vectors for performance purpuses
  nrows::Int = size(data.df, 1)
  baseValue::Float64 = cpiIndex[baseYear]
  netassets::Vector{Int} = data.df[:netassets]
  adjnetassets = Vector{MFloat64}(undef, nrows)
  fisyr::Vector{Int} = data.df[:fisyr]
  @inbounds @simd for i ∈ 1:nrows
    if haskey(cpiIndex, fisyr[i])
      adjnetassets[i] = netassets[i] / (cpiIndex[fisyr[i]] / baseValue)
    end
  end

  data.df[:adjnetassets] = adjnetassets
  if adjustWealthCutoff #this is to account for the cutoff inflating forward
    inflatedCutoff::Float64 = ADJ_NET_ASSETS_CUTOFF * baseValue/cpiIndex[minimum(data.df[:fisyr])]
    println("Inflated cutoff of $inflatedCutoff")
    data.df = data.df[
      (f::MFloat64-> (!ismissing(f)) && (f ≥ inflatedCutoff)).(data.df[:adjnetassets]), :]
  end

  return nothing
end

#does an initial filter to get down the file size
function processAll!(data::NCCSData; allcols::Vector{Symbol} = ALL_COLS,
  adjNetAssetsCutoff::Float64 = ADJ_NET_ASSETS_CUTOFF)

  data = removeFields!(data, setdiff(names(data.df), allcols))
  data.df[:rows] = collect(1:size(data.df,1))
  println("N0: $(size(data.df,1))")
  deleterows!(data.df, data.df[(!).(completecases(data.df[[:fisyr, :netassets, :totrev, :totexp]])),:rows])

  adjustInflation!(data)
  processReturns!(data)
  processWealth!(data)
  processCategories!(data)

  println("N3: $(size(data.df,1)) N3-Return: $(sum((!ismissing).(data.df[:return])))")
  println("N3-pf: $(sum(data.df[:charitytype] .== :pf)) \nN3-pf-Return: $(
    sum((data.df[:charitytype] .== :pf) .& ((!ismissing).(data.df[:return]))))\nN3 check return: $(
    sum((data.df[:charitytype] .== :pf) .& ((!ismissing).(data.df[:returncheckamt]))))\nN3 net income: $(
    sum((data.df[:charitytype] .== :pf) .& ((!ismissing).(data.df[:netincome]))))")

  data.df = data.df[((!ismissing).(data.df[:adjnetassets])) .&
    (data.df[:adjnetassets] .≥ adjNetAssetsCutoff), :]

  println("N4: $(size(data.df,1)) N4-Return: $(sum((!ismissing).(data.df[:return])))")

  return data
end

function constructPrimaryOutput(; refreshFilter::Bool = true,
    primaryOutputName::String = PRIMARY_OUTPUT_NAME,
    writePrimaryCSV::Bool = false,
    dataPath::String = DATA_PATH)::NCCSData

  local data::NCCSData
  local outPath = "$dataPath\\$(primaryOutputName)_long.csv"
  if refreshFilter
    data = deserializeNCCSData()
    data = processAll!(data)
    serializeNCCSData(data, dataName = primaryOutputName)
  else
    data = deserializeNCCSData(dataName = primaryOutputName)
  end

  #for debugging
  #uCSV.write(outPath, data.df[400_000:450_000,:])

  writePrimaryCSV && uCSV.write(outPath, data.df) #write to a csv if desired

  return data

end


using SimpleDirectMediaLayer

# import SimpleDirectMediaLayer:
#     Window, Renderer

export World
export gen_id

const DISPLAY_RATIO = 16.0 / 9.0
const WIDTH = 1024 + 512
# Larget number causes the view to encompass more of the world
# which makes objects appear smaller.
const VIEW_SCALE = 1.5

const WINDOW_POSITION = (1000, 100)

mutable struct World
    title::String

    # Device properties
    window_position_x::UInt32
    window_position_y::UInt32
    window_width::UInt32
    window_height::UInt32

    window_centered::Bool

    view_width::Float64
    view_height::Float64
    view_centered::Bool

    # view space to device-space projection
    view_space::Math.AffineTransform
    inv_view_space::Math.AffineTransform

    ids::UInt32

    window #::Ptr{Window}
    renderer #::Ptr{Renderer}

    function World(title::String)
        o = new()
        o.title = title
        o.window_position_x = WINDOW_POSITION[1]
        o.window_position_y = WINDOW_POSITION[2]
        o.window_width = WIDTH
        o.window_height = UInt32(Float64(WIDTH) / DISPLAY_RATIO)
        o.window_centered = true
        o.view_centered = true
        o.view_width = Float64(o.window_width) * VIEW_SCALE
        o.view_height = Float64(o.window_height) * VIEW_SCALE
        o.ids = 0
        o.window = o    # Not assigned yet
        o.renderer = o  # Not assigned yet

        o.view_space = Math.AffineTransform{Float64}()
        o.inv_view_space = Math.AffineTransform{Float64}()

        set_view_space!(o)

        println("Display dimensions: [", o.window_width, " x ", o.window_height, "]")
        println("View dimensions: [", o.view_width, " x ", o.view_height, "]")

        o
    end
end

function gen_id(world::World)
    world.ids += 1
    world.ids
end

function set_view_space!(world::Ranger.World)
    center = Math.AffineTransform{Float64}()

    # What separates world from view is the ratio between the device (aka window)
    # and an optional centering translation.
    width_ratio = Float64(world.window_width) / world.view_width
    height_ratio = Float64(world.window_height) / world.view_height

    if world.view_centered
        Math.make_translate!(center, Float64(world.window_width) / 2.0, Float64(world.window_height) / 2.0)
    end

    Math.scale!(center, width_ratio, height_ratio)
    Math.set!(world.view_space, center)

    Math.invert!(world.view_space, world.inv_view_space)
end


using DrWatson
@quickactivate
using ArgParse
using GenerativeAD
import StatsBase: fit!, predict
using StatsBase
using BSON, FileIO
using Flux
using GenerativeModels
using DistributionsAD
using ValueHistories

s = ArgParseSettings()
@add_arg_table! s begin
	"modelname"
		default = "vae"
		arg_type = String
		help = "model name"
	"datatype"
		default = "tabular"
		arg_type = String
		help = "tabular or image"
	"dataset"
		default = "iris"
		arg_type = String
		help = "dataset"
	"--seed"
		default = nothing
		help = "if specified, only results for a given seed will be recomputed"
	"--anomaly_class"
		default = nothing
		help = "if specified, only results for a given anomaly class will be recomputed"
end
parsed_args = parse_args(ARGS, s)
@unpack dataset, datatype, modelname, seed, anomaly_class = parsed_args

masterpath = datadir("experiments/$(datatype)/$(modelname)/$(dataset)")
files = GenerativeAD.Evaluation.collect_files(masterpath)
mfiles = filter(f->occursin("model", f), files)
if seed != nothing
	filter!(x->occursin("/seed=$seed/", x), mfiles)
end
if anomaly_class != nothing
	filter!(x->occursin("/ac=$(anomaly_class)/", x), mfiles)
end

kld_batched(m,x,batchsize) =
	vcat(map(y->vec(Base.invokelatest(GenerativeModels.kl_divergence,
			condition(m.encoder,y), m.prior)), Flux.Data.DataLoader(x, batchsize=batchsize))...)
kld_batched_gpu(m,x,batchsize) =
	vcat(map(y-> cpu(vec(Base.invokelatest(GenerativeModels.kl_divergence,
			Base.invokelatest(condition, m.encoder, gpu(Array(y))), m.prior))), Flux.Data.DataLoader(x, batchsize=batchsize))...)

function save_elbo_score(f::String, data, seed::Int, ac=nothing)
	# get model
	savepath = dirname(f)
	mdata = load(f)
	model = mdata["model"]

	# setup entries to be saved
	save_entries = (
		modelname = modelname,
		fit_t = mdata["fit_t"],
		history = mdata["history"],
		dataset = dataset,
		npars = sum(map(p->length(p), Flux.params(model))),
		model = nothing,
		seed = seed
		)
	save_entries = (ac == nothing) ? save_entries : merge(save_entries, (anomaly_class=ac,))

	# test if the file alread exists
	parameters = merge(mdata["parameters"], (L=100, score = "elbo",))
	savef = joinpath(savepath, savename(parameters, "bson", digits=5))
	if !isfile(savef)
		@info "computing sample score for $f"

		# now load the sampled reconstruction probability score
		namepars = DrWatson.parse_savename(f)[2]
		sf = filter(x->
			occursin("init_seed=$(namepars["init_seed"])", x) &&
			occursin("score=reconstruction-sampled", x),
			readdir(savepath))
		if length(sf)==0
			@info "sampled reconstruction score for $f not found!"
			return
		end
		sf = joinpath(savepath, sf[1])
		score_data = load(sf)
		rec_scores = (score_data[:tr_scores], score_data[:val_scores], score_data[:tst_scores])

		# compute the full scores
		score_fun(x) = (ac == nothing) ?
			kld_batched(model, x, 512) : kld_batched_gpu(gpu(model), x, 512)
		tr_data, val_data, tst_data = data

		# extract scores
		tr_scores, tr_eval_t, _, _, _ = @timed score_fun(tr_data[1])
		val_scores, val_eval_t, _, _, _ = @timed score_fun(val_data[1])
		tst_scores, tst_eval_t, _, _, _ = @timed score_fun(tst_data[1])

		result = (
			parameters = parameters,
			tr_scores = tr_scores .+ rec_scores[1],
			tr_labels = tr_data[2],
			tr_eval_t = tr_eval_t + score_data[:tr_eval_t],
			val_scores = val_scores .+ rec_scores[2],
			val_labels = val_data[2],
			val_eval_t = val_eval_t + score_data[:val_eval_t],
			tst_scores = tst_scores .+ rec_scores[3],
			tst_labels = tst_data[2],
			tst_eval_t = tst_eval_t + score_data[:tst_eval_t]
			)

		result = Dict{Symbol, Any}([sym=>val for (sym,val) in pairs(merge(result, save_entries))]) # this has to be a Dict
		tagsave(savef, result, safe = true)
		(@info "Results saved to $savef")
	end
end

for f in mfiles
	# get data
	savepath = dirname(f)
	local seed = parse(Int, replace(basename(savepath), "seed=" => ""))
	ac = occursin("ac=", savepath) ? parse(Int, replace(basename(dirname(savepath)), "ac=" => "")) : nothing
	data = (ac == nothing) ?
		GenerativeAD.load_data(dataset, seed=seed) :
		GenerativeAD.load_data(dataset, seed=seed, anomaly_class_ind=ac)

	# compute and save the score
	save_elbo_score(f, data, seed, ac)
end
@info "DONE"


module GNSSBenchmarks
    using
        CUDA,
        CSV,
        StructArrays,
        BenchmarkTools,
        LoopVectorization,
        Tracking,
        TrackingLoopFilters,
        GNSSSignals,
        DataFrames,
        PGFPlotsX,
        LinearAlgebra,
        StaticArrays

    using Unitful: upreferred, Hz, dBHz, ms, kHz, MHz

    import
        LinearAlgebra.dot,
        Base.length,
        Tracking.TrackingState,
        Tracking.NumAnts,
        Tracking.MomentsCN0Estimator,
        Tracking.AbstractCN0Estimator,
        Tracking.AbstractCorrelator,
        Tracking.EarlyPromptLateCorrelator,
        Tracking.SecondaryCodeOrBitDetector,
        Tracking.GainControlledSignal,
        Tracking.found,
        Tracking.TrackingResults,
        Tracking.BitBuffer,
        Tracking.get_default_post_corr_filter,
        Tracking.get_num_samples_left_to_integrate,
        Tracking.get_num_samples,
        Tracking.get_num_ants,
        Tracking.get_current_carrier_frequency,
        Tracking.get_current_code_frequency,
        TrackingLoopFilters.AbstractLoopFilter,
        GNSSSignals.AbstractGNSSSystem

    export
        main,
        plotall,
        benchmark_downconvert,
        benchmark_carrier_replica,
        benchmark_code_replica,
        benchmark_correlate,
        benchmark_tracking_loop,
        plot_carrier_replica,
        gpu_correlate,
        plot_correlate_all,
        sgpu_correlate,
        plot_code_replica_all

    include("main.jl")
    include("plot.jl")
    include("gpu_downconvert.jl")
    include("gpu_carrier_replica.jl")
    include("gpu_code_replica.jl")
    include("gpu_correlate.jl")
    include("gpu_tracking_state.jl")
    include("gpu_tracking_results.jl")
    include("gpu_tracking_loop.jl")
    include("benchmark_carrier_replica.jl")
    include("benchmark_code_replica.jl")
    include("benchmark_downconvert.jl")
    include("benchmark_correlate.jl")
    include("benchmark_gpu_tracking_loop.jl")

    const MAX_NUM_SAMPLES = 50000
    const SAMPLES = StepRange(2500,2500,MAX_NUM_SAMPLES)
    const ANTENNA = [1, 4, 16]
end


set_control!(z::Algames.KnotPoint, u) = z.z = [Algames.state(z); u]

function mpc_loop(prob::Algames.GameProblem, x0::SVx, sce::Scenario, vis::Vector{Visualizer},
    mpc_opts::MPCOptions{T}, mpc_stats::MPCStatistics{T}, traj_stats::TrajStatistics{T};
    seed::Vector{Int}=[1,6], display_traj=false, display_player=true) where {T,SVx}
    # We select the seed to make sure both players start with a different local nash equilibrium.
    probsize = prob.probsize
    p = probsize.p
    N = probsize.N
    pz = probsize.pz
    pu = probsize.pu

    # Initialization
    reset!(mpc_stats)
    reset!(traj_stats)
    probs = [deepcopy(prob) for i=1:p]

    prob.opts.shift = 2^10
    prob.opts.f_init = x -> (rand(x) .- 0.5)
    prob.opts.amplitude_init = 5.0

    # Set tolerances
    prob.opts.ϵ_opt = 3e-2
    prob.opts.ϵ_dyn = 3e-3
    prob.opts.ϵ_con = 3e-3
    prob.opts.ϵ_sta = 3e-3

    for i = 1:p
        probs[i].opts.inner_print = false
        probs[i].opts.outer_print = false
        probs[i].opts.inner_iter = 10
        probs[i].opts.outer_iter = 7
        probs[i].opts.seed = seed[1+(i-1)%p]
        probs[i].x0 = x0

        newton_solve!(probs[i])
        probs[i].opts.shift = 0
        probs[i].opts.amplitude_init = 1e-8
        plot!(probs[i].model, probs[i].pdtraj.pr)
    end

    dt = zeros(p)
    for j = 1:mpc_opts.M
        println("$(round(mpc_stats.t,digits=2))/$(round(mpc_opts.h, digits=1))s")
        trajs = deepcopy([probs[i].pdtraj.pr for i=1:p])
        # Solve
        for i = 1:p
            display_player ? set_state!(vis[i], probs[i].model, sce, probs[i].x0) : nothing
            display_traj ? set_line_traj!(vis[i], probs[i].model, sce.player, probs[i].pdtraj.pr) : nothing
            dt[i] = max(0.02, @elapsed newton_solve!(probs[i]))
        end
        # Record
        z0 = probs[1].pdtraj.pr[1]
        z = Algames.KnotPoint(Algames.state(z0), Algames.control(z0), maximum(dt))
        record!(traj_stats, z.dt, trajs)
        record!(mpc_stats, z)
        if mpc_stats.t > mpc_opts.h
            break
        end

        # Update solver
        X = []
        S = []
        for i = 1:p
            x,s = first_order_rollout(probs[i].pdtraj.pr, maximum(dt))
            push!(X, x)
            push!(S, s)
        end
        x = zeros(n)
        for i = 1:p
            x[pz[i]] = X[i][pz[i]]
        end
        x = SVector{n,T}(x)
        for i = 1:p
            probs[i].x0 = x
            probs[i].opts.shift = maximum(S)
        end
    end
    return nothing
end

function run_experiment(prob::Algames.GameProblem, x0::SVx, sce::Scenario, vis::Vector{Visualizer},
    mpc_opts::MPCOptions{T}; seed::Vector{Int}=[1,6], display_traj=false,
    display_player=false, n_sample::Int=5) where {T,SVx}
    mpc_stats = Vector{MPCStatistics}([])
    traj_stats = Vector{TrajStatistics}([])
    for l = 1:n_sample
        println("sample $l/$n_sample")
        m_stats = MPCStatistics()
        t_stats = TrajStatistics(probsize)
        xinit = random_initial_state(prob.model, x0, seed=l)
        mpc_loop(prob, xinit, sce, vis, mpc_opts, m_stats, t_stats,
            seed=[l,n_sample+l], display_traj=display_traj, display_player=display_player)
        push!(mpc_stats, m_stats)
        push!(traj_stats, t_stats)
    end
    return mpc_stats, traj_stats
end

function process_data(model::AbstractGameModel, mpc_stats::Vector{<:MPCStatistics},
    traj_stats::Vector{<:TrajStatistics}; upsampling::Int=3, α::T=1.0, atol::T=0.5) where {T}

    N = upsampling*Int(floor(mean([t_stats.iter for t_stats in traj_stats])))
    tf = α*mean([sum(t_stats.dt) for t_stats in traj_stats])
    t = zeros(N)
    X = []

    n_sample = length(mpc_stats)
    probsize = traj_stats[1].probsize
    p = probsize.p
    plt = plot()
    for l = 1:n_sample
        t_stats = traj_stats[l]
        for i = 1:p
            for j = i+1:p
                dist = [distance(model, trajs[i], trajs[j]) for trajs in t_stats.traj]
                plot!(cumsum(t_stats.dt), dist, linewidth=2.0, label="")
                scatter!(cumsum(t_stats.dt), dist, markersize=6.0, linewidth=2.0, label="$l",
                    markershapes=marker[1+(l-1)%length(marker)])
                x, t = resample(dist, cumsum(t_stats.dt), N, tf)
                push!(X,x)
            end
        end
    end

    x = mean(X)
    equa = [sum([X[l][i] .<= atol for l=1:n_sample]) for i=1:N]
    diff = [n_sample-equa[i] for i = 1:N]
    plot!(t, equa, linewidth=5.0, label="equa")
    plot!(t, diff, linewidth=5.0, label="diff")
    plot!(t, x, linewidth=5.0)
    display(plt)
    return x, t, equa, diff
end

function get_figure(x, t, equa, diff)
    N = length(t)
    plt = plot(
        xlim=(0,Inf),
        ylim=(0,Inf),
        xlabel="time (s)",
        ylabel="Nash equilibrium mismatch")

    plot!(t,zeros(N), label="Equilibrum match", ribbon=(zeros(N),ones(N)), color=:blue, fillalpha=0.7)
    plot!(t,zeros(N), label="Equilibrum mismatch", ribbon=(zeros(N),diff/(equa[1]+diff[1])), color=:orange, fillalpha=1.0)
    plot!(t,x./maximum(x),linewidth=3.0, label="Normalized mean discrepancy")
    for k = 1:N
        println("($(t[k]), $(diff[k]))")
    end
    for k = 1:N
        println("($(t[k]), $(equa[k]))")
    end
    display(plt)
    return nothing
end

function resample(x::AbstractVector, t::AbstractVector, N::Int, tf::T) where {T}
    t_ = zeros(N)
    x_ = zeros(N)
    for i = 1:N
        τ = (i-1)*tf/N
        t_[i] = τ
        it = findfirst(x -> x>=τ, t)
        it == nothing ? it = length(x) : nothing
        x_[i] = x[it]
    end
    return x_, t_
end

function distance(model::AbstractGameModel, t1::Algames.Traj, t2::Algames.Traj)
    p = model.p
    d = 0.0
    N = length(t1)
    @assert N == length(t2)
    for k = 1:N
        x1 = Algames.state(t1[k])
        x2 = Algames.state(t2[k])
        vs1 = standardize(model, x1)
        vs2 = standardize(model, x2)
        for i = 1:p
            Δ = [vs1[i].x - vs2[i].x, vs1[i].y - vs2[i].y, vs1[i].θ - vs2[i].θ, vs1[i].v - vs2[i].v]
            d += norm(Δ)
        end
    end
    return d
end

function random_initial_state(model::AbstractGameModel, x0::SVx; amplitude::T=1.0, seed::Int=rand(1:100)) where {SVx}
    Random.seed!(seed)
    p = model.p
    vs = standardize(model, x0)
    x = amplitude*[0.10, .65]
    y = amplitude*[0.02, 0.03]
    θ = amplitude*[0.10, 0.05]
    v = amplitude*[0.02, 0.02]
    for i = 1:p
        vs[i].x += x[i]*(rand()-0.5)
        vs[i].y += y[i]*(rand()-0.5)
        vs[i].θ += θ[i]*(rand()-0.5)
        vs[i].v += v[i]*(rand()-0.5)
    end

    xinit = specialize(model, vs)
    return xinit
end


# *********************************************************************************
# REopt, Copyright (c) 2019-2020, Alliance for Sustainable Energy, LLC.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
# Redistributions of source code must retain the above copyright notice, this list
# of conditions and the following disclaimer.
#
# Redistributions in binary form must reproduce the above copyright notice, this
# list of conditions and the following disclaimer in the documentation and/or other
# materials provided with the distribution.
#
# Neither the name of the copyright holder nor the names of its contributors may be
# used to endorse or promote products derived from this software without specific
# prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
# OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
# OF THE POSSIBILITY OF SUCH DAMAGE.
# *********************************************************************************
"""
    add_cost_curve_vars_and_constraints(m, p; _n="")

There are two situations under which we add binary constraints to the model in order to create a segmented cost curve
for a technology:
    1. When a technology has tax or investment incentives with upper capacity limits < tech.max_kw
        - first segment(s) have lower slope than last segment
    2. When a technology has multiple cost/size pairs (not implemented yet, used for CHP in v1)
        - we interpolate the slope between the cost/size points, typically with economies of scale pricing
We used to use cost curve segments for when a technology has a non-zero existing_kw by setting the first segment to a
zero cost (and slope) from zero kw to the existing_kw. Instead, we now have dvPurchaseSize >= dvSize - existing_kw.

To avoid unnecessary decision variables (and constraints) we use anonymous variables and register them in the model
manually. With this method the decision variable containers do not have to be rectangular.
For example, (as implemented in v1 of the API) the decision variable dvSegmentSystemSize is indexed on
[p.techs.all, p.Subdivision, p.Seg], where:
    - p.Subdivision is hardcoded to ["CapCost"] - and so it is unnecessary,
    - p.Seg is the maximum number of segments in any cost curve
    - and p.techs.all includes all technologies
We can instead construct dvSegmentSystemSize as follows:
```julia
    for t in p.SegmentedTechs
        dv = "dvSegmentSystemSize" * t
        m[Symbol(dv)] = @variable(m, [1:p.SegSize[t]], base_name=dv)
    end
```
Then, for example, the dvSegmentSystemSize for Wind at segment 2 is accessed via:
```julia
    m[:dvSegmentSystemSizeWind][2]
```
In this fashion only technologies with more than one cost curve segment, and only the necessary amount of segments, are
modeled using the following binary variables and constraints.
"""
function add_cost_curve_vars_and_constraints(m, p; _n="")

    for t in p.techs.segmented
        dv = "dvSegmentSystemSize" * t
        m[Symbol(dv)] = @variable(m, [1:p.n_segs_by_tech[t]], base_name=dv, lower_bound=0)

        dv = "binSegment" * t
        m[Symbol(dv)] = @variable(m, [1:p.n_segs_by_tech[t]], base_name=dv, binary=true)
    end

    ##Constraint (7f)-1: Minimum segment size
        @constraint(m, SegmentSizeMinCon[t in p.techs.segmented, s in 1:p.n_segs_by_tech[t]],
        m[Symbol("dvSegmentSystemSize"*t)][s] >= p.seg_min_size[t][s] * m[Symbol("binSegment"*t)][s]
    )

    ##Constraint (7f)-2: Maximum segment size
    @constraint(m, SegmentSizeMaxCon[t in p.techs.segmented, s in 1:p.n_segs_by_tech[t]],
        m[Symbol("dvSegmentSystemSize"*t)][s] <= p.seg_max_size[t][s] * m[Symbol("binSegment"*t)][s]
    )

    ##Constraint (7g):  Segments add up to system size
    @constraint(m, SegmentSizeAddCon[t in p.techs.segmented],
        sum(m[Symbol("dvSegmentSystemSize"*t)][s] for s in 1:p.n_segs_by_tech[t]) == m[:dvPurchaseSize][t]
    )

    ##Constraint (7h): At most one segment allowed
    @constraint(m, SegmentSelectCon[t in p.techs.segmented],
        sum(m[Symbol("binSegment"*t)][s] for s in 1:p.n_segs_by_tech[t]) <= 1
    )
end


include("../rgi-setup.jl")
using BSON, DataStructures
function load_validation(region_str)
    #region_str ="03"
    region = parse(Int, region_str)

    # dir_output = "../output/validation/production-no-dhdt/"
    # dir_output = "../output/validation/production-dhdt/"
    #dir_output = "../output/validation-alps/production-dhdt/"

    if region in [4, 11]
        sub = "apr-20"
        dir_output = "../output/vali/validation-$(region_str)_fit-sigma/$sub"
    else
        sub = "apr-21"
        dir_output = "../output/vali/validation-$(region_str)_fit-sigma/$sub"
    end

    # calibration glaciers
    fl_fit = fit_target -> joinpath(dir_output, "reg$(region_str)_FIT_$(fit_target)")
    # validation glaciers with prior transfer
    fl_test1 = prior_choice -> joinpath(dir_output, "reg$(region_str)_PRIOR_$(prior_choice)-testgls")
    # test glaciers with prior transfer
    fl_test1_fitgls = prior_choice -> joinpath(dir_output, "reg$(region_str)_PRIOR_$(prior_choice)-fitgls")
    # validation glaciers with prior transfer and fit to IV
    fl_test2 = prior_choice -> joinpath(dir_output, "reg$(region_str)_PRIOR_$(prior_choice)_FIT_iv")

    fit_targets = [BM.FitTarget.h, BM.FitTarget.h_iv, BM.FitTarget.iv, BM.FitTarget.length]
    prior_choice = [:base, :h, :iv, :h_iv] # base means no prior transfer

    out_fit = OrderedDict()
    for ft in fit_targets
        tmp = BSON.load(fl_fit(ft)*".bson")[:out_fit]
        out_fit[Symbol(ft)] = tmp
    end

    out_test = OrderedDict()
    for p in prior_choice
        tmp = BSON.load(fl_test1(p)*".bson")[:out_test]
        out_test[(:length, p)] = tmp
    end
    for p in [:base, :h_iv]
        tmp = BSON.load(fl_test2(p)*".bson")[:out_test]
        out_test[(:iv, p)] = tmp
    end

    out_test_fitgls = OrderedDict()
    for p in prior_choice
        tmp = BSON.load(fl_test1_fitgls(p)*".bson")[:out_test]
        out_test_fitgls[(:length, p)] = tmp
    end

    return out_fit, out_test, out_test_fitgls
end


"""
    Panda <: Environment

    6-degree-of-freedom robotic arm designed by Franka Emika
"""
struct Panda end

function panda(;
    representation=:minimal,
    timestep=0.01,
    gravity=[0.0; 0.0; -9.81],
    friction_coefficient=0.8,
    damper=10.0,
    spring=0.0,
    info=nothing,
    model_type=:end_effector,
    seed=1,
    contact=true,
    limits=true,
    vis=Visualizer(),
    name=:robot,
    # infeasible_control=false,
    opts_step=SolverOptions(rtol=3.0e-4, btol=3.0e-4, undercut=1.5),
    opts_grad=SolverOptions(rtol=3.0e-4, btol=3.0e-4, undercut=1.5),
    T=Float64)

    mechanism = get_mechanism(:panda,
        timestep=timestep,
        gravity=gravity,
        friction_coefficient=friction_coefficient,
        damper=damper,
        spring=spring,
        contact=contact,
        limits=limits,
        model_type=model_type)

    initialize!(mechanism, :panda)

    if representation == :minimal
        nx = minimal_dimension(mechanism)
    elseif representation == :maximal
        nx = maximal_dimension(mechanism)
    end
    nu = input_dimension(mechanism)
    no = nx

    aspace = BoxSpace(nu,
        low=(-1.0e8 * ones(nu)),
        high=(1.0e8 * ones(nu)))
    ospace = BoxSpace(no,
        low=(-Inf * ones(no)),
        high=(Inf * ones(no)))

    rng = MersenneTwister(seed)
    z = get_maximal_state(mechanism)
    x = representation == :minimal ? maximal_to_minimal(mechanism, z) : z
    fx = zeros(nx, nx)
    fu = zeros(nx, nu)

    u_prev = zeros(nu)
    control_mask = I(nu)
    control_scaling = Diagonal(ones(nu))

    build_robot(mechanism, vis=vis, name=name)

    TYPES = [Panda, T, typeof(mechanism), typeof(aspace), typeof(ospace), typeof(info)]
    Environment{TYPES...}(mechanism, representation, aspace, ospace,
        x, fx, fu,
        u_prev, control_mask' * control_scaling,
        nx, nu, no,
        info,
        [rng], vis,
        opts_step, opts_grad)
end


# Test by creating a network, writing it, then reading it back in and make sure that all weights and biases match
import NeuralVerification: Layer, ReLU, Id

@testset "Write + Read nnet test" begin

    # 3 --> 3 --> 2 --> 5
    l1 = Layer([3.0 2.0 1.0; 5.0 6.0 7.0; 8.0 9.0 10.0], [0.8; 1.0; 1.2], ReLU())
    l2 = Layer([1.5 2.5 3.5; 4.5 6.5 7.5], [-1.0; -3.0], ReLU())
    # The .nnet file doesn't store an activation at each layer,
    l3 = Layer([10.0 -1.0; -2.0 3.0; 4.0 5.0; 10.0 7.0; -3.5 -4.5], [0.0; -1.0; 0.0; 10.0; -10.0], Id())

    # Write out the network
    network_file = string(tempname(), ".nnet")
    println("File: ", network_file)
    nnet = Network([l1, l2, l3])
    write_nnet(network_file, nnet; header_text="test_header")

    # Read back in the network
    new_nnet = read_nnet(network_file)

    # Test that all weights, biases, and activations are the same
    @test new_nnet.layers[1].weights == l1.weights;
    @test new_nnet.layers[1].bias == l1.bias;
    @test new_nnet.layers[1].activation == l1.activation;

    @test new_nnet.layers[2].weights == l2.weights;
    @test new_nnet.layers[2].bias == l2.bias;
    @test new_nnet.layers[2].activation == l2.activation;

    @test new_nnet.layers[3].weights == l3.weights;
    @test new_nnet.layers[3].bias == l3.bias;
    @test new_nnet.layers[3].activation == l3.activation;
end


function KalmanEstimatorBuilder()
    return KalmanEstimatorBuilder(())
end

function add_propagation_configuration(obj::KalmanEstimatorBuilder, arg0::IntegratedPropagatorBuilder, arg1::CovarianceMatrixProvider)
    return jcall(obj, "addPropagationConfiguration", KalmanEstimatorBuilder, (IntegratedPropagatorBuilder, CovarianceMatrixProvider), arg0, arg1)
end

function build(obj::KalmanEstimatorBuilder)
    return jcall(obj, "build", KalmanEstimator, ())
end

function decomposer(obj::KalmanEstimatorBuilder, arg0::MatrixDecomposer)
    return jcall(obj, "decomposer", KalmanEstimatorBuilder, (MatrixDecomposer,), arg0)
end

function estimated_measurements_parameters(obj::KalmanEstimatorBuilder, arg0::ParameterDriversList)
    return jcall(obj, "estimatedMeasurementsParameters", KalmanEstimatorBuilder, (ParameterDriversList,), arg0)
end


module MGEO

import Base: convert

using DelimitedFiles
using Distributed
using LinearAlgebra
using StaticArrays
using Random

################################################################################
#                                    Files
################################################################################

include("types.jl")
include("configuration.jl")
include("misc.jl")
include("mgeo_canonical.jl")
include("mgeo_var.jl")
include("mgeo_real.jl")

end # module


# see examples.jl first

# Type Definition
@aml mutable struct Person "person", check_course
    age::UInt64, "~"
    field, "study-field"
    GPA::Float64 = 4.5, "~", GPAcheck
    courses::Vector{String}, "taken-courses"
    professors::UN{DataFrame} = nothing, "table"
    id::Int64, att"~"
    comment::UN{String} = nothing, txt"end"
end

@aml mutable struct University doc"university"
    name, att"university-name"
    people::Vector{Person}, "person"
end

# Value Checking Functions
GPAcheck(x) = x <= 4.5 && x >= 0

function check_course(age, field, GPA, courses, professors, id, comment)

    if field == "Mechanical Engineering"
        relevant = ["Artificial Intelligence", "Robotics", "Machine Design"]
    elseif field == "Computer Engineering"
        relevant = ["Julia", "Algorithms"]
    else
        error("study field is not known")
    end

    return any(in.(courses, Ref(relevant)))
end

################################################################

@aml mutable struct MyGeneralXML{T} "my-general-xml"
    myfield::T, "~"
end
pxml_string = MyGeneralXML{String}(myfield = "a")
pprint(pxml_string)
pxml_vector = MyGeneralXML{Vector{String}}(myfield = ["b","c"])
pprint(pxml_vector)


function test_init(rw, ϵ, pos)
    @test rw.ϵ == ϵ
    @test rw.pos == pos
    @test length(rw) == length(ϵ)
    @test eltype(rw) == eltype(ϵ)
end

@testset "random walk" begin
    @testset "initialisation" begin
        @test_throws AssertionError BSI.RandomWalk((1.0, 2.0), (false,))
        test_init(BSI.RandomWalk(1.0, false), (1.0,), (false,))
        test_init(BSI.RandomWalk([1.0], false), (1.0,), (false,))
        test_init(BSI.RandomWalk(1.0, (false,)), (1.0,), (false,))
        test_init(BSI.RandomWalk(1.0, 1), (1.0,), (true,))
        test_init(BSI.RandomWalk(1.0), (1.0,), (false,))
        test_init(BSI.RandomWalk(1.0, []), (1.0,), (false,))

        test_init(BSI.RandomWalk((1.0, 2.0)), (1.0,2.0), (false,false))
        test_init(BSI.RandomWalk([1.0, 2.0], []), (1.0,2.0), (false,false))
        test_init(BSI.RandomWalk([1.0, 2.0], 2), (1.0,2.0), (false,true))
        test_init(BSI.RandomWalk([1.0, 2.0], (1,2)), (1.0,2.0), (true,true))
    end
end


# gll_library
function zwgljd!(z::Vector{Float64}, w::Vector{Float64}, np::Int64, alpha::Float64, beta::Float64)
    # convert to references
    np_ref = Ref{Int32}(Int32(np))
    alpha_ref = Ref{Float64}(alpha)
    beta_ref = Ref{Float64}(beta)

    ccall((:zwgljd_, "../../fortran/lib/gll_library.so"), Cvoid, (Ptr{Float64}, Ptr{Float64}, Ref{Int32}, Ref{Float64}, Ref{Float64}), z, w, np_ref, alpha_ref, beta_ref)
end

# sem_mesh_mod
function anchor_point_index!(iax::Vector{Int32}, iay::Vector{Int32}, iaz::Vector{Int32})
    iax_32 = Int32.(iax)
    iay_32 = Int32.(iay)
    iaz_32 = Int32.(iaz)
    @assert length(iax) == 27 && length(iay) == 27 && length(iaz) == 27
    ccall((:anchor_point_index_, "../../fortran/lib/sem_mesh_mod.so"), Cvoid, (Ptr{Int32}, Ptr{Int32}, Ptr{Int32}), iax_32, iay_32, iaz_32)
    iax .= Int32.(iax_32)
    iay .= Int32.(iay_32)
    iaz .= Int32.(iaz_32)
end

# sem_mesh_mod
function xyz2cube_bounded!(xyz_anchor::Array{Float64,2}, xyz::Array{Float64,1}, uvw::Array{Float64,1}, misloc::Float64, flag_inside::Bool)
    misloc_ref = Ref{Float64}(misloc)
    flag_inside_ref = Ref{Bool}(flag_inside)
    ccall((:xyz2cube_bounded_, "../../fortran/lib/sem_mesh_mod.so"), Cvoid, (Ptr{Float64}, Ptr{Float64}, Ptr{Float64}, Ref{Float64}, Ref{Bool}), xyz_anchor, xyz, uvw, misloc_ref, flag_inside_ref)
    return misloc_ref.x, flag_inside_ref.x
end

# sem_mesh_mod
function lagrange_poly!(x::Float64, ngll::Int64, xgll::Vector{Float64}, lagrange::Vector{Float64})
    x_ref = Ref{Float64}(x)
    ngll_ref = Ref{Int32}(Int32(ngll))
    ccall((:lagrange_poly_, "../../fortran/lib/sem_mesh_mod.so"), Cvoid, (Ref{Float64}, Ref{Int32}, Ptr{Float64}, Ptr{Float64}), x_ref, ngll_ref, xgll, lagrange)
end


"""

`unique(arr, dims...)`

Return unique elements of an array `arr` of type `LabeledArray`/`DictArray`/`Nullable AbstractArrayWrapper`.

##### Arguments

* `arr` : an array
* `dims...` : either an integer or, if an array is a DictArray or a LabeledArray, a list of integers. It specifies the directions along which to traverse. Any duplicate elements will be replaced by Nullable{T}(). If all components along some direction are missing, those components will be removed and the whole array size will shrink.
If `dims...` is missing, unique elements along the whole directions will be found. It is equivalent to `unique(arr, 1, 2, ..., ndims(arr))`.
Note that it compares each slice spanned by directions orthogonal to `dims...`.

##### Examples

```julia
julia> unique(nalift([1 2 3;3 4 1]))
2x2 MultidimensionalTables.AbstractArrayWrapper{Nullable{Int64},2,Array{Nullable{Int64},2}}:
 Nullable(1)  Nullable(2)
 Nullable(3)  Nullable(4)

julia> unique(nalift([1 2 3;3 4 1]), 1)
2x3 MultidimensionalTables.AbstractArrayWrapper{Nullable{Int64},2,Array{Nullable{Int64},2}}:
 Nullable(1)  Nullable(2)  Nullable(3)
 Nullable(3)  Nullable(4)  Nullable(1)

julia> unique(nalift([1 2 3;3 4 1]), 2)
2x3 MultidimensionalTables.AbstractArrayWrapper{Nullable{Int64},2,Array{Nullable{Int64},2}}:
 Nullable(1)  Nullable(2)  Nullable(3)
 Nullable(3)  Nullable(4)  Nullable(1)

julia> unique(nalift([1 2 3;1 2 3;4 5 6]), 1)
2x3 MultidimensionalTables.AbstractArrayWrapper{Nullable{Int64},2,Array{Nullable{Int64},2}}:
 Nullable(1)  Nullable(2)  Nullable(3)
 Nullable(4)  Nullable(5)  Nullable(6)

julia> t = darr(a=[1 2 1;1 5 1;1 2 1], b=[:a :b :a;:a :c :a;:a :b :a])
3 x 3 DictArray

a b |a b |a b
----+----+----
1 a |2 b |1 a
1 a |5 c |1 a
1 a |2 b |1 a

julia> unique(t, 1)
2 x 3 DictArray

a b |a b |a b
----+----+----
1 a |2 b |1 a
1 a |5 c |1 a

julia> unique(t, 2)
3 x 2 DictArray

a b |a b
----+----
1 a |2 b
1 a |5 c
1 a |2 b

julia> m = larr(a=[1 2 1;1 5 1;1 2 1], b=[:a :b :a;:a :c :a;:a :b :a], axis1=["X","Y","Z"])
3 x 3 LabeledArray

  |1   |2   |3
--+----+----+----
  |a b |a b |a b
--+----+----+----
X |1 a |2 b |1 a
Y |1 a |5 c |1 a
Z |1 a |2 b |1 a

julia> unique(m, 1)
2 x 3 LabeledArray

  |1   |2   |3
--+----+----+----
  |a b |a b |a b
--+----+----+----
X |1 a |2 b |1 a
Y |1 a |5 c |1 a

julia> unique(m, 2)
3 x 2 LabeledArray

  |1   |2
--+----+----
  |a b |a b
--+----+----
X |1 a |2 b
Y |1 a |5 c
Z |1 a |2 b
```

"""
Base.unique{T}(arr::AbstractArrayWrapper{Nullable{T}}) = unique(arr, 1:ndims(arr)...)
Base.unique(arr::DictArray, dim::Int, dims::Int...) = unique(arr, 1:ndims(arr)...)
Base.unique(arr::LabeledArray, dim::Int, dims::Int...) = unique(arr, 1:ndims(arr)...)
Base.unique{T}(arr::AbstractArrayWrapper{Nullable{T}}, dim::Int, dims::Int...) = unique_inner(arr, (dim, dims...))[2]
Base.unique(arr::DictArray, dim::Int, dims::Int...) = unique_inner(arr, (dim, dims...))[2]
Base.unique(arr::LabeledArray, dim::Int, dims::Int...) = begin
  dimall = (dim, dims...)
  (coords_toshow, newdata) = unique_inner(arr.data, dimall)
  newaxes = ntuple(length(arr.axes)) do d
    axis = arr.axes[d]
    if d in dimall
      coords = coords_toshow[findfirst(dimall, d)]
      if isa(axis, DefaultAxis)
        DefaultAxis(length(coords))
      else
        arr.axes[d][coords]
      end
    else
      arr.axes[d]
    end
  end
  LabeledArray(newdata, newaxes)
end

@generated unique_inner{N}(arr::DictArray, dims::NTuple{N,Int}) = quote
  # first rearrange the axes so that the calculatioin can be more convenient.
  # this might be later optimized.
  ndimsarr = ndims(arr)
  sizearr = size(arr)
  remainingdirs = filter(x->!(x in dims), 1:ndimsarr)
  remainingcoords = ntuple(d->Colon(), length(remainingdirs))
  newdirs = (remainingdirs...,dims...)
  permuted_arr = permutedims_if_necessary(arr, newdirs)
  elems_sofar = Set()
  axiselems_toshow = [falses(sizearr[dims[i]]) for i in 1:$N]
  result = similar(permuted_arr, eltype(permuted_arr))
  @nloops $N i d->1:sizearr[dims[d]] begin
    coords = @ntuple($N,i)
    full_coords = (remainingcoords...,coords...)
    oneslice = getindexvalue(permuted_arr, full_coords...)
    if oneslice in elems_sofar
      setna!(result, full_coords...)
    else
      push!(elems_sofar, oneslice)
      for (index,coord) in enumerate(coords)
        axiselems_toshow[index][coord] = true
      end
      result[full_coords...] = oneslice
    end
  end
  coords_toshow = [find(axiselems_toshow[i]) for i in 1:$N]
  (coords_toshow, ipermutedims_if_necessary(result[remainingcoords...,coords_toshow...], newdirs))
end

@generated unique_inner{T,N}(arr::AbstractArray{Nullable{T}}, dims::NTuple{N,Int}) = quote
  # first rearrange the axes so that the calculatioin can be more convenient.
  # this might be later optimized.
  ndimsarr = ndims(arr)
  sizearr = size(arr)
  remainingdirs = filter(x->!(x in dims), 1:ndimsarr)
  remainingcoords = ntuple(d->Colon(), length(remainingdirs))
  newdirs = (remainingdirs...,dims...)
  permuted_arr = permutedims_if_necessary(arr, newdirs)
  elems_sofar = Set()
  axiselems_toshow = [falses(sizearr[dims[i]]) for i in 1:$N]
  result = similar(permuted_arr, eltype(permuted_arr))
  @nloops $N i d->1:sizearr[dims[d]] begin
    coords = @ntuple($N,i)
    full_coords = (remainingcoords...,coords...)
    oneslice = permuted_arr[full_coords...]
    if oneslice in elems_sofar
      setna!(result, full_coords...)
    else
      push!(elems_sofar, oneslice)
      for (index,coord) in enumerate(coords)
        axiselems_toshow[index][coord] = true
      end
      result[full_coords...] = oneslice
    end
  end
  coords_toshow = [find(axiselems_toshow[i]) for i in 1:$N]
  (coords_toshow, ipermutedims_if_necessary(result[remainingcoords...,coords_toshow...], newdirs))
end


# This file is for functions that just need simple standard overloading.

## Helpers:

function nameddimsarray_result(original_nda, reduced_data, reduction_dims)
    L = names(original_nda)
    return NamedDimsArray{L}(reduced_data)
end

# if reducing over `:` then results is a scalar
function nameddimsarray_result(original_nda, reduced_data, reduction_dims::Colon)
    return reduced_data
end

###################################################################################
# Overloads

# 1 Arg
for (mod, funs) in (
    (:Base, (
        :sum, :prod, :count, :maximum, :minimum, :extrema, :cumsum, :cumprod,
        :sort, :sort!,)
    ),
    (:Statistics, (:mean, :std, :var, :median, :cov, :cor)),
)
    for fun in funs
        @eval function $mod.$fun(a::NamedDimsArray; dims=:, kwargs...)
            numerical_dims = dim(a, dims)
            data = $mod.$fun(parent(a); dims=numerical_dims, kwargs...)
            return nameddimsarray_result(a, data, numerical_dims)
        end
    end
end

# 1 arg before
for (mod, funs) in (
    (:Base, (:mapslices,)),
)
    for fun in funs
        @eval function $mod.$fun(f, a::NamedDimsArray; dims=:, kwargs...)
            numerical_dims = dim(a, dims)
            data = $mod.$fun(f, parent(a); dims=numerical_dims, kwargs...)
            return nameddimsarray_result(a, data, numerical_dims)
        end
    end
end

# 2 arg before
for (mod, funs) in (
    (:Base, (:mapreduce,)),
)
    for fun in funs
        @eval function $mod.$fun(f1, f2, a::NamedDimsArray; dims=:, kwargs...)
            numerical_dims = dim(a, dims)
            data = $mod.$fun(f1, f2, parent(a); dims=numerical_dims, kwargs...)
            return nameddimsarray_result(a, data, numerical_dims)
        end
    end
end

################################################
# Non-dim Overloads

# 1 Arg
for (mod, funs) in (
    (:Base, (:zero, :one, :copy,)),
)
    for fun in funs
        @eval function $mod.$fun(a::NamedDimsArray{L}) where L
            data = $mod.$fun(parent(a))
            return NamedDimsArray{L}(data)
        end
    end
end


@noinline function dostuff(A; B = I)
  StrideArrays.@avx for i in eachindex(A)
    A[i] += 1
  end
  C = A * B
  s = zero(eltype(C))
  StrideArrays.@avx for i in eachindex(C)
    s += C[i]
  end
  s
end
function gc_preserve_test()
  A = @StrideArray rand(8, $(2 << 2))
  B = @StrideArray rand($(1 << 3), 8)
  @gc_preserve dostuff(A, B = B)
end

@testset "Miscellaneous" begin
  x = @StrideArray rand(127)
  @test maximum(abs, x) == maximum(abs, Array(x))
  y = @StrideArray rand(3)
  @test maximum(abs, y) == maximum(abs, Array(y))
  @test iszero(@allocated gc_preserve_test())

  A = @StrideArray rand($(5 << 1), $(1 << 3))
  @test StrideArrays.size(A) === (StaticInt(10), StaticInt(8))
  A_u = view(A, StaticInt(1):StaticInt(6), :)
  A_l = view(A, StaticInt(7):StaticInt(10), :)
  @test A == @inferred(vcat(A_u, A_l))
  @test StrideArrays.size(A) === StrideArrays.size(vcat(A_u, A_l))

  # On 1.5 tests fail if you don't do this first.
  @test pointer(view(A, 1:StaticInt(6), :)) == pointer(A)
  A_u = view(A, 1:StaticInt(6), :)
  A_l = view(A, StaticInt(7):StaticInt(10), :)
  @test A == @inferred(vcat(A_u, A_l))
  @test StrideArrays.size(A) == StrideArrays.size(vcat(A_u, A_l))
  @test StrideArrays.size(A) !== StrideArrays.size(vcat(A_u, A_l))

  Aa = Array(A)
  @test sum(A) ≈ sum(Aa)
  @test maximum(A) == maximum(Aa) == maximum(abs, @. -A)
  @test mapreduce(abs2, +, A) ≈ mapreduce(abs2, +, Aa)

  @test A[1, 1] == A[1] == vec(A)[1]
  A[1, 1] = 3
  @test A[1, 1] == A[1] == vec(A)[1] == 3
  A[2] = 4
  @test A[2, 1] == A[2] == vec(A)[2] == 4
  vec(A)[3] = 5
  @test A[3, 1] == A[3] == vec(A)[3] == 5

  @test ArrayInterface.stride_rank(StrideArrays.similar_layout(A')) ===
        ArrayInterface.stride_rank(StrideArrays.similar_layout(Aa')) ===
        (StaticInt(2), StaticInt(1))
  @test ArrayInterface.contiguous_axis(StrideArrays.similar_layout(A')) ===
        ArrayInterface.contiguous_axis(StrideArrays.similar_layout(Aa')) ===
        StrideArrays.StaticInt{2}()

  let B = @StrideArray rand(10, 10)
    @test exp(B) ≈ exp(Array(B))
  end
end


module ML
abstract Regressor
abstract Classifier

include("classifiers/linear_model.jl")
# package code goes here

end # module


abstract type AbstractCFDateTime <: Dates.TimeType
end

const RegTime = Union{Dates.Millisecond,Dates.Second,Dates.Minute,Dates.Hour,Dates.Day}

for (CFDateTime,calendar) in [(:DateTimeStandard,"standard"),
                              (:DateTimeJulian,"julian"),
                              (:DateTimeProlepticGregorian,"prolepticgregorian"),
                              (:DateTimeAllLeap,"allleap"),
                              (:DateTimeNoLeap,"noleap"),
                              (:DateTime360Day,"360day")]
    @eval begin
        struct $CFDateTime <: AbstractCFDateTime
            instant::UTInstant{Millisecond}
            $CFDateTime(instant::UTInstant{Millisecond}) = new(instant)
        end
    end
end


using Printf

function getAccord_Mut(reader)

    mutation_dict = Dict{String, String}()

    while !eof(reader)
        line = readline(reader)
        if !( line[1:6] == "T_Freq" )
            s = split(line, "\t")
            freq = parse(Float64, s[1])
            if freq > 0.02
                mutation_dict["$(s[29])\t$(s[30])"] = line
            end
        end
    end

    return mutation_dict

end

function writeData(dict_t, outf)
    af = basename(outf)
    if !(isfile(".\\test_02\\$af"))
        output_file = open(  ".\\test_02\\$af" ,"w")
        for (mut, v) in dict_t
            write(output_file, @sprintf("%s\n", v))

        end
        close(output_file)
    end
    return 0
end

md = open(getAccord_Mut, ARGS[1])

#for (mut, v) in md
#    println(@sprintf("%s\t%s", mut, v))
        #if haskey(md, mut)
        #    accor_N += 1
        #end
#end

a = writeData(md, ARGS[1])

#open("/usr/share/dict/words") do io
    #       while !eof(io)
    #           word = readline(io)
    #           if length(word) >= 24
    #               println(word)
    #           end
    #       end
    #   end


using RCall

function ggplot2_distribution(values::AbstractVector, filename;
    title = "Distribution",
    xaxistitle = "Cases",
    types = [:histogram, :density],
    yaxistitle = "Count",
    xlog10 = false)
    values = convert(Vector{Float64}, values)
    R"library(ggplot2)"
    R"library(ggthemes)"
    @rput values
    R"df <- data.frame(dummytoensurerightsize = values)"
    @rput xaxistitle
    R"df[, xaxistitle] <- values"
    @rput yaxistitle
    @rput title
    R"p <- ggplot(df, aes_string(x = xaxistitle)) + theme_tufte() + xlab(xaxistitle) + ggtitle(title)"
    if xlog10
        R"p <- p + scale_x_log10()"
    end
    basefilename, ext = splitext(filename)
    if in(:histogram, types)
        R"phist <- p + geom_histogram(binwidth=.5, color='black', fill='blue', alpha=0.5) + ylab(yaxistitle)"
        fname = basefilename * "_histogram" * ext
        @rput fname
        R"ggsave(fname, phist)"
    end
    if in(:density, types)
        R"pdens <- p + geom_density(fill='blue', alpha=0.5)"
        fname = basefilename * "_density" * ext
        @rput fname
        R"ggsave(fname, pdens)"
    end
end


using DataDeps

register(DataDep(
  "statlog-project-landsat-satellite",
  "http://archive.ics.uci.edu/ml/datasets/Statlog+Project",
  "http://archive.ics.uci.edu/ml/machine-learning-databases/statlog/satimage/sat.trn",
  "e896dc88a960fa2404160fc4c3cb3dc53fcf4afd80ba920bf2d261bd42d12613",
  post_fetch_method=(path -> begin
    UCIData.process_dataset(path,
        target_index=37,
        feature_indices=1:36,
        delim=' ',
    )
  end),
))


create_udp_entry(packageid::Integer) = create_package(rdb_pkg_id_to_type(packageid))
function create_udp_entry(::Type{RDB_SYNC_t},
    mask::UInt32=0x00000000,
    cmdMask::UInt32=RDB_SYNC_CMD_RENDER_SINGLE_FRAME, #RDB_SYNC_CMD_RENDER_CONTINUOUS,
    systemTime::Cdouble=time()
    )

    RDB_SYNC_t(mask, cmdMask, systemTime)
end
function create_udp_entry(::Type{RDB_TRIGGER_t},
    deltaT::Cfloat,
    frameNo::UInt32,
    spare::Int32=convert(Int32, 0)
    )

    RDB_TRIGGER_t(deltaT, frameNo, spare)
end

function create_udp_entry_header{T<:RDB_PACKAGE_ELEMENT}(elements::Vector{T}, flags::UInt16=0x0000)
    headerSize = sizeof(RDB_MSG_ENTRY_HDR_t)
    #dataSize = sizeof(elements)
    #elementSize = div(dataSize, length(elements))
    dataSize = sizeof(elements[1])*length(elements)
    elementSize = sizeof(elements[1])
    #pkgId = rdb_type_to_pkg_id(T)
    pkgId = rdb_type_to_pkg_id(elements[1])
    RDB_MSG_ENTRY_HDR_t(headerSize, dataSize, elementSize, pkgId, flags)
end

function create_udp_entry(::Type{RDB_DRIVER_CTRL_t},
    playerId::UInt32;
    steeringWheel::Cfloat=convert(Cfloat,0),
    steeringSpeed::Cfloat=convert(Cfloat,0),
    throttlePedal::Cfloat=convert(Cfloat,0),
    brakePedal::Cfloat=convert(Cfloat,0),
    clutchPedal::Cfloat=convert(Cfloat,0),
    accelTgt::Cfloat=convert(Cfloat,0),
    steeringTgt::Cfloat=convert(Cfloat,0),
    curvatureTgt::Cdouble=convert(Cdouble,0),
    steeringTorque::Cfloat=convert(Cfloat,0),
    engineTorqueTgt::Cfloat=convert(Cfloat,0),
    speedTgt::Cfloat=convert(Cfloat,0),
    gear::UInt8=convert(UInt8,0),
    sourceId::UInt8=convert(UInt8,0),
    spare0::Tuple{UInt8, UInt8}=(convert(UInt8,0),convert(UInt8,0)),
    validityFlags::UInt32=convert(UInt32,0),
    flags::UInt32=convert(UInt32,0),
    mockupInput0::UInt32=convert(UInt32,0),
    mockupInput1::UInt32=convert(UInt32,0),
    mockupInput2::UInt32=convert(UInt32,0),
    spare::UInt32=convert(UInt32,0)
    )

    RDB_DRIVER_CTRL_t(    playerId,
    steeringWheel,
    steeringSpeed,
    throttlePedal,
    brakePedal,
    clutchPedal,
    accelTgt,
    steeringTgt,
    curvatureTgt,
    steeringTorque,
    engineTorqueTgt,
    speedTgt,
    gear,
    sourceId,
    spare0,
    validityFlags,
    flags,
    mockupInput0,
    mockupInput1,
    mockupInput2,
    spare)

end


export exportImage, exportMovie, exportMovies

exportImage(filename, im::ImageMeta; kargs...) = exportImage(filename, arraydata(im); kargs...)

#old syntax
function exportImage(filename, im::AbstractMatrix{T};
                    colormap::Union{Array, String}="grays", kargs...) where {T<:Real}
  exportImage(filename, im, colormap; kargs...)
end

function exportImage(filename, im::AbstractMatrix{T}, colorm::String;kargs...) where {T<:Real}
  return exportImage(filename, im, cmap(colorm); kargs...)
end

function exportImage(filename, im::AbstractMatrix{T}, colorm::Vector{C}; vmin=0.0, vmax=1.0,
                    normalize=true, pixelResizeFactor=1) where {T<:Real, C<:Colorant}
  imC = colorize( im, vmin, vmax, colorm, normalize=normalize )
  exportImage(filename, imC; pixelResizeFactor=pixelResizeFactor)
end

function exportImage(filename, im::AbstractMatrix{T}; pixelResizeFactor=1) where {T<:Colorant}
  file, ext = splitext(filename)

  imR = repeat(im, inner=[pixelResizeFactor,pixelResizeFactor])

  minPxSpacing = minimum(pixelspacing(im))
  newSize = ceil.(Int64, collect(pixelspacing(im)) / minPxSpacing .* collect(size(imR)) )

  dataResized = Images.imresize(imR,(newSize[1],newSize[2]))
  rgbdata = RGB.(dataResized)
  ImageMagick.save(filename, rgbdata)
end

function exportImage(filename, data::Vector{T}; kargs...) where {T<:AbstractMatrix}
    file, ext = splitext(filename)
    exportImage(file*"_xy.png", data[1]; kargs...)
    exportImage(file*"_xz.png", data[2]; kargs...)
    exportImage(file*"_yz.png", data[3]; kargs...)
end

### export movies ###

exportMovie(filename, data::ImageMeta; kargs...) = exportMovie(filename, data.data; kargs...)

#old syntax
function exportMovie(filename, data::AbstractArray{T,3};
                    colormap::Union{Array, String}="grays", kargs...) where {T<:Real}
  exportMovie(filename, data, colormap; kargs...)
end

function exportMovie(filename, data::AbstractArray{T,3}, colorm::String; kargs...) where {T<:Real}
  return exportMovie(filename, data, cmap(colorm); kargs...)
end

function exportMovie(filename, data::AbstractArray{T,3}, colorm::Vector{C}; vmin=0.0, vmax=1.0,
                    normalize=true, pixelResizeFactor=1) where {T<:Real, C<:Colorant}
    imC = colorize( data, vmin, vmax, colorm, normalize=normalize )
    exportMovie(filename, imC; pixelResizeFactor=pixelResizeFactor)
end

function exportMovie(filename, data::AbstractArray{T,3}; pixelResizeFactor=1) where {T<:Colorant}
  file, ext = splitext(filename)

  if pixelResizeFactor > 1
    data = repeat(data, inner=[pixelResizeFactor,pixelResizeFactor,1])
  end

  minPxSpacing = minimum(pixelspacing(data))
  newSize = ceil.(Int64, collect(pixelspacing(data)[1:2]) / minPxSpacing .* collect(size(data)[1:2]) )

  datai = similar(data, (newSize[1],newSize[2],size(data,3)))

  for l=1:size(data,3)
    datai[:,:,l] = Images.imresize(data[:,:,l],(newSize[1],newSize[2]))
  end

  rgbdata = RGB.(datai)
  @debug "saving $filename"
  ImageMagick.save(file*".gif", rgbdata)
end

function exportMovies(filename, data::Vector; kargs...)
  file, ext = splitext(filename)

  exportMovie(file*"_xy.gif", data[1]; kargs...)
  exportMovie(file*"_xz.gif", data[2]; kargs...)
  exportMovie(file*"_yz.gif", data[3]; kargs...)
end


function parse_disjuncts(head)
    ret = MacroTools.postwalk(head) do ex
        if @capture(ex, num_::call_(args__))
            quote $call($(args...), p = $num) end
        else
            ex
        end
    end
    ret
end

function parse_call(call, names)
    names = map(names) do name
        if name isa QuoteNode
            quote Constant(String($name)) end
        else
            name
        end
    end
    return quote sp.add_fact($call($(names...))) end
end

function parse_body(body)
    if @capture(body, head_ :- tail_)
        disjuncts = parse_disjuncts(head)
        return quote sp.add_clause(AnnotatedDisjunction($disjuncts, $tail)) end

    elseif @capture(body, head_ << tail_)
        return quote sp.add_clause($head << $tail) end

    elseif @capture(body, call_(names__))
        return parse_call(call, names)

    else
        body
    end
end

function _problox(expr)
    if @capture(expr, function name_(args__) body__ end)
        trans = map(body) do ex
            if @capture(ex, begin body__; end)
                length(body) == 1 || error("Parsing error at $body.")
                parse_body(body[1])
            else
                error("Parsing error at $body.")
            end
        end
        expr = quote
            function $name($(args...))
                sp = SimpleProgram()
                $(trans...)
                sp
            end
        end

    elseif @capture(expr, begin body__ end)
        trans = map(body) do ex
            if @capture(ex, begin body__; end)
                length(body) == 1 || error("Parsing error at $body.")
                parse_body(body[1])
            else
                error("Parsing error at $body.")
            end
        end
        expr = quote
            sp = SimpleProgram()
            $(trans...)
            sp
        end
    end
    expr
end

macro problox(expr)
    new = _problox(expr)
    new = MacroTools.postwalk(rmlines ∘ unblock, new)
    esc(new)
end


type ODESolution{uType,uType2,DType,tType,rateType,P,A,IType} <: AbstractODESolution
  u::uType
  u_analytic::uType2
  errors::DType
  t::tType
  k::rateType
  prob::P
  alg::A
  interp::IType
  dense::Bool
  tslocation::Int
  retcode::Symbol
end
(sol::ODESolution)(t,deriv::Type=Val{0};idxs=nothing) = sol.interp(t,idxs,deriv)
(sol::ODESolution)(v,t,deriv::Type=Val{0};idxs=nothing) = sol.interp(v,t,idxs,deriv)

function build_solution{uType,tType,isinplace}(
        prob::Union{AbstractODEProblem{uType,tType,isinplace},AbstractDDEProblem{uType,tType,isinplace}},
        alg,t,u;dense=false,timeseries_errors=true,dense_errors=true,
        calculate_error = true,
        k=[],interp = (tvals) -> nothing, retcode = :Default, kwargs...)

  if has_analytic(prob.f)
    u_analytic = Vector{uType}(0)
    errors = Dict{Symbol,eltype(prob.u0)}()
    sol = ODESolution(u,u_analytic,errors,t,k,prob,alg,interp,dense,0,retcode)
    if calculate_error
      calculate_solution_errors!(sol;timeseries_errors=timeseries_errors,dense_errors=dense_errors)
    end
    return sol
  else
    return ODESolution(u,nothing,nothing,t,k,prob,alg,interp,dense,0,retcode)
  end
end

function calculate_solution_errors!(sol::AbstractODESolution;fill_uanalytic=true,timeseries_errors=true,dense_errors=true)
  if fill_uanalytic
    for i in 1:size(sol.u,1)
      push!(sol.u_analytic,sol.prob.f(Val{:analytic},sol.t[i],sol.prob.u0))
    end
  end

  save_everystep = length(sol.u) > 2
  if !isempty(sol.u_analytic)
    sol.errors[:final] = mean(abs.(sol.u[end]-sol.u_analytic[end]))

    if save_everystep && timeseries_errors
      sol.errors[:l∞] = maximum(vecvecapply((x)->abs.(x),sol.u-sol.u_analytic))
      sol.errors[:l2] = sqrt(mean(vecvecapply((x)->float.(x).^2,sol.u-sol.u_analytic)))
      if sol.dense && dense_errors
        densetimes = collect(linspace(sol.t[1],sol.t[end],100))
        interp_u = sol(densetimes)
        interp_analytic = [sol.prob.f(Val{:analytic},t,sol.u[1]) for t in densetimes]
        sol.errors[:L∞] = maximum(vecvecapply((x)->abs.(x),interp_u-interp_analytic))
        sol.errors[:L2] = sqrt(mean(vecvecapply((x)->float.(x).^2,interp_u-interp_analytic)))
      end
    end
  end
end

function build_solution(sol::AbstractODESolution,u_analytic,errors)
  ODESolution(sol.u,u_analytic,errors,sol.t,sol.k,sol.prob,sol.alg,sol.interp,sol.dense,sol.tslocation,sol.retcode)
end


__precompile__()

module AstroTime

using EarthOrientation
using Reexport
import RemoteFiles: path, isfile

export @timescale

include("utils.jl")

include("LeapSeconds.jl")
include("TimeScales.jl")
include("Periods.jl")
include("Epochs.jl")

@reexport using .TimeScales
@reexport using .Periods
@reexport using .LeapSeconds
@reexport using .Epochs

function __init__()
    isfile(LSK_FILE) && push!(LSK_DATA, path(LSK_FILE))
end

"""
    @timescale scale

Define a new timescale and the corresponding `Epoch` type alias.

# Example

```jldoctest
julia> @timescale Custom

julia> Custom <: TimeScale
true
julia> CustomEpoch == Epoch{Custom, T} where T <: Number
true
```
"""
macro timescale(scale)
    if !(scale isa Symbol)
        error("Invalid time scale name.")
    end
    epoch = Symbol(scale, "Epoch")
    return quote
        struct $(esc(scale)) <: TimeScale end
        const $(esc(epoch)) = Epoch{$(esc(scale))}
        nothing
    end
end

function update()
    EarthOrientation.update()
    download(LSK_FILE)
    push!(LSK_DATA, path(LSK_FILE))
    nothing
end

end # module


using Compat
using Test
using MPI

MPI.Init()

function scatter_array(A, root)
    comm = MPI.COMM_WORLD
    T = eltype(A)
    if MPI.Comm_rank(comm) == root
        global B = copy(A)
    else
        global B = Array{T}(undef,1)
    end
    C = MPI.Scatter(B, 1, root, comm)
end

comm = MPI.COMM_WORLD
root = 0

A = collect(1:MPI.Comm_size(comm))
B = scatter_array(A, root)
@test B[1] == MPI.Comm_rank(comm) + 1
for typ in Base.uniontypes(MPI.MPIDatatype)
    global A = convert(Vector{typ},collect(1:MPI.Comm_size(comm)))
    global B = scatter_array(A, root)
    @test B[1] == convert(typ,MPI.Comm_rank(comm) + 1)
end

MPI.Finalize()


module LightGraphConvert

using CircuitViz
using LightGraphs

g = SimpleDiGraph()
LightGraphs.add_vertex!(g)
LightGraphs.add_vertex!(g)
LightGraphs.add_vertex!(g)
LightGraphs.add_edge!(g, 1, 2)
LightGraphs.add_edge!(g, 1, 3)

new = CircuitViz.convert_to_catlab_graph(g)
compile!(new)

end # module


# Modified from Graphs.jl.
# Workaround for edge_index not being O(1) on SimpleGraph.
# edge_index was only needed to test for cycles, so
# this implementation skips that check.

# Graphs.jl is licensed under the MIT License:
#
# Copyright (c) 2012: John Myles White and other contributors.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

mutable struct TopologicalSortVisitor
    vertices::Vector{Int}
    parents::Vector{Int}

    function TopologicalSortVisitor(n::Int)
        vs = Int[]
        sizehint!(vs, n)
        return new(vs, zeros(Int, n))
    end
end

function depth_first_visit_impl!(
    adjlist,
    offsets,
    vertex_stack,                   # an (initialized) stack of vertex
    index_stack,                    # stack with out edge indices
    vertexcolormap::Vector{Int},    # an (initialized) color-map to indicate status of vertices
    visitor::TopologicalSortVisitor,
)
    while !isempty(vertex_stack)
        u = pop!(vertex_stack)
        out_idx = pop!(index_stack)
        #uegs = out_edges(u,graph)
        len_uegs = offsets[u+1] - offsets[u]
        found_new_vertex = false

        while out_idx <= len_uegs && !found_new_vertex
            v = adjlist[offsets[u]+out_idx-1]
            out_idx += 1
            v_color = vertexcolormap[v]

            if v_color == 0
                found_new_vertex = true
                vertexcolormap[v] = 1
                push!(vertex_stack, u)
                push!(index_stack, out_idx)
                visitor.parents[v] = u

                push!(vertex_stack, v)
                push!(index_stack, 1)
            end
        end

        if !found_new_vertex
            close_vertex!(visitor, u)
            vertexcolormap[u] = 2
        end
    end
end

function traverse_graph(
    adjlist,
    offsets,
    s,
    visitor::TopologicalSortVisitor,
    vertexcolormap,
    vertex_stack,
    index_stack,
)
    vertexcolormap[s] = 1

    resize!(vertex_stack, 1)
    vertex_stack[1] = s
    resize!(index_stack, 1)
    index_stack[1] = 1

    return depth_first_visit_impl!(
        adjlist,
        offsets,
        vertex_stack,
        index_stack,
        vertexcolormap,
        visitor,
    )
end

function close_vertex!(visitor::TopologicalSortVisitor, v::Int)
    return push!(visitor.vertices, v)
end

function reverse_topological_sort_by_dfs(
    adjlist,
    offsets,
    num_vertices,
    cmap::Vector{Int},
    vertex_stack::Vector{Int} = Int[],
    index_stack::Vector{Int} = Int[],
)
    @assert length(cmap) == num_vertices
    fill!(cmap, 0)
    visitor = TopologicalSortVisitor(num_vertices)

    for s in 1:num_vertices
        if cmap[s] == 0
            traverse_graph(
                adjlist,
                offsets,
                s,
                visitor,
                cmap,
                vertex_stack,
                index_stack,
            )
        end
    end

    return visitor.vertices, visitor.parents
end


using Test
using MLDatasets
using MLDatasets.FastAIDatasets
using MLDatasets.FastAIDatasets: datasetpath, loaddataclassification,
    mapobs, filterobs, groupobs, joinobs, getobs, nobs

@testset "datasetpath" begin
    @test_nowarn FastAIDatasets.datasetpath("mnist_var_size_tiny")
end

@testset "loaddataset" begin
    @test_nowarn loaddataclassification(datasetpath("mnist_var_size_tiny"))
end

@testset "Data container transformations" begin
    @testset "mapobs" begin
        data = 1:10
        mdata = mapobs(-, data)
        @test getobs(mdata, 8) == -8

        mdata2 = mapobs((-, x -> 2x), data)
        @test getobs(mdata2, 8) == (-8, 16)

        nameddata = mapobs((x = sqrt, y = log), data)
        @test getobs(nameddata, 10) == (x = sqrt(10), y = log(10))
        @test getobs(nameddata.x, 10) == sqrt(10)
    end

    @testset "filterobs" begin
        data = 1:10
        fdata = filterobs(>(5), data)
        @test nobs(fdata) == 5
    end

    @testset "groupobs" begin
        data = -10:10
        datas = groupobs(>(0), data)
        @test length(datas) == 2
    end

    @testset "joinobs" begin
        data1, data2 = 1:10, 11:20
        jdata = joinobs(data1, data2)
        @test getobs(jdata, 15) == 15
    end
end


### A Pluto.jl notebook ###
# v0.14.2

using Markdown
using InteractiveUtils

# ╔═╡ a7871567-172a-42ec-b889-56c5440c99fa
begin
	using Revise
	using ProgressMeter
end

# ╔═╡ 7fa178d6-9f1c-11eb-3604-9777d8da228e
using Curiosity, Plots, PlutoUI, Statistics, Random

# ╔═╡ 60fa4e34-de7a-43c3-91d6-e4f60f1ca272
include("../experiments/tabular_tmaze.jl")

# ╔═╡ 3d63f4ea-a20a-438d-a359-35a7f84b789e
TTMU = Curiosity.TabularTMazeUtils

# ╔═╡ 6f96b22d-1bbe-4711-b4bd-b9b89130fead
default_args(ϵ) =
    Dict(
        # Behaviour Items
        "behaviour_eta" => 0.04,
        "behaviour_gamma" => 0.9,
        "behaviour_learner" => "GPI",
        "behaviour_update" => "TB",
        "behaviour_trace" => "AccumulatingTraces",
        "behaviour_opt" => "Auto",
        "behaviour_lambda" => 0.9,
        "behaviour_alpha_init" => 0.1,
        "exploration_param" => ϵ,
        "exploration_strategy" => "epsilon_greedy",

        # Demon Attributes
        "demon_alpha_init" => 0.1,
        "demon_eta" => 0.5,
        "demon_discounts" => 0.9,
        "demon_learner" => "SR",
        "demon_update" => "TB",
        "demon_policy_type" => "greedy_to_cumulant",
        "demon_opt" => "Auto",
        "demon_lambda" => 0.9,
        "demon_trace"=> "AccumulatingTraces",
        "demon_beta_m" => 0.9,
        "demon_beta_v" => 0.99,

        # Environment Config
        "constant_target"=> 1.0,
        "cumulant_schedule" => "DrifterDistractor",
        "distractor" => (1.0, 5.0),
        "drifter" => (1.0, sqrt(0.01)),
        "exploring_starts"=>false,

        # Agent and Logger
        "horde_type" => "regular",
        "intrinsic_reward" => "weight_change",
        "logger_keys" => [LoggerKey.TTMAZE_ERROR, LoggerKey.TTMAZE_UNIFORM_ERROR, LoggerKey.TTMAZE_OLD_ERROR],
        "save_dir" => "TabularTMazeExperiment",
        "seed" => 1,
        "steps" => 10000,
        "use_external_reward" => true,
        "logger_interval" => 100,
    )

# ╔═╡ b1573f0a-6622-4a40-b171-24f8afae3505
env = TabularTMaze(true, Curiosity.TMazeCumulantSchedules.Constant(1))

# ╔═╡ c655ab97-c20c-4e62-8435-533d50f47ade
begin
	start!(env)
	plot(env)
end

# ╔═╡ 037b8feb-c1de-4c5e-a8f4-fbd6b762673e
function get_policy_arrows(env,agent)
	#HARDCODED CAUSE IDK WHAT ELSE TO DO :(
	s_x = 7
	s_y = 9
	SIZE = 20
	OFFSET = 1*SIZE
	arrow_x = []
	arrow_y = []
	arrow_u = []
	arrow_v = []
	function generate_obs(state::Array{Int64})
		obs = zeros(Int64, 9, 7)
		obs[state[1],state[2]] = 1
		toReturn = findfirst(!iszero, obs[Curiosity.valid_state_mask()])
		return toReturn
	end
	for j ∈ 1:s_x
		for i ∈ 1:s_y
			sqr_i = ((i)*SIZE + 1):((i+1)*SIZE)
			sqr_j = ((j)*SIZE + 1):((j+1)*SIZE)

			k = generate_obs([i,j])
			if k != nothing # If it is not part of the 21 valid cells
				# @show k,i,j
				obs = zeros(5)
				obs[1] = k
				b_π = Curiosity.μ_π(agent, obs)
				# val, ACTION = findmax(b_π)

				# drawarray is bugged. Need to have it off. This is the correct transformation
				# when yflip=true
				# https://github.com/JuliaPlots/Plots.jl/issues/1498
				x_coord = OFFSET + (j-1) * SIZE + OFFSET/2
				y_coord = s_y * SIZE + 2*OFFSET - i * SIZE - OFFSET/2
				# yflip = false
				# x_coord = s_x * SIZE + 2 * OFFSET - (j) * SIZE - OFFSET/2
				# y_coord = OFFSET + (i-1) * SIZE + OFFSET/2

				max_prob,_ = findmax(b_π)
				for (ACTION,π) in enumerate(b_π)
					if π != max_prob
						continue
					end

					if ACTION == 1
						u,v = 0,1
					elseif ACTION == 2
						u,v, = 1,0
					elseif ACTION == 3
						u,v = 0,-1
					elseif ACTION == 4
						u,v = -1,0
					end
					push!(arrow_x,x_coord)
					push!(arrow_y,y_coord)
					push!(arrow_u, u*SIZE/2)
					push!(arrow_v,v*SIZE/2)
				end
			end
		end
	end
	return arrow_x,arrow_y,arrow_u,arrow_v
end

# ╔═╡ 77dae42d-3be5-43b0-93b0-611fa8c9f123
function main_experiment(parsed=default_args(0.1); progress=false, working=false, show_policy = false)

    num_steps = parsed["steps"]
    Random.seed!(parsed["seed"])

    cumulant_schedule = TTMU.get_cumulant_schedule(parsed)
    exploring_starts = parsed["exploring_starts"]
    env = TabularTMaze(exploring_starts, cumulant_schedule)

    agent = TabularTMazeExperiment.construct_agent(parsed)

    eps = 1
    max_num_steps = num_steps
    steps = Int[]

    if progress
        p = Progress(max_num_steps)
    end
	anim = Animation()
    while sum(steps) < max_num_steps
        cur_step = 0
        max_episode_steps = max_num_steps - sum(steps)
        tr, stp =
            run_episode!(env, agent, max_episode_steps) do (s, a, s_next, r, t)
                #This is a callback for every timestep where logger can go
                # agent is accesible in this scope
				plot(env, title=string("Eps: ", parsed["exploration_param"],
					" Step: ",cur_step,
					" Ep: ", eps,
					" DL: ", parsed["demon_learner"],
					" BL: ", parsed["behaviour_learner"]))
				if show_policy
					arrow_x, arrow_y, arrow_u, arrow_v = get_policy_arrows(env,agent)
					quiver!(arrow_x,arrow_y,quiver=(arrow_u,arrow_v))
				end
				frame(anim)
                cur_step+=1
            end
        push!(steps, stp)
        eps += 1

        if progress
            ProgressMeter.update!(p, sum(steps))
        end
    end
	anim
end

# ╔═╡ 465bd0cd-3204-41a2-a68c-f3e38cdd0b5f
anim = main_experiment(default_args(0.2), show_policy=true, progress=true)

# ╔═╡ 1bacedb7-8373-4d52-99b8-54e1e5460346
# let
# 	for ϵ ∈ [0.1, 0.2, 0.3, 0.4]
# 		anim = main_experiment(default_args(ϵ))
# 		mp4(anim, "GPI_SR_epsilon_$(ϵ).mp4")
# 	end
# end

# ╔═╡ Cell order:
# ╠═a7871567-172a-42ec-b889-56c5440c99fa
# ╠═7fa178d6-9f1c-11eb-3604-9777d8da228e
# ╠═3d63f4ea-a20a-438d-a359-35a7f84b789e
# ╠═60fa4e34-de7a-43c3-91d6-e4f60f1ca272
# ╠═6f96b22d-1bbe-4711-b4bd-b9b89130fead
# ╟─b1573f0a-6622-4a40-b171-24f8afae3505
# ╠═c655ab97-c20c-4e62-8435-533d50f47ade
# ╠═037b8feb-c1de-4c5e-a8f4-fbd6b762673e
# ╟─77dae42d-3be5-43b0-93b0-611fa8c9f123
# ╠═465bd0cd-3204-41a2-a68c-f3e38cdd0b5f
# ╠═1bacedb7-8373-4d52-99b8-54e1e5460346


module NSGAII

import YAML
using Cambrian
import JSON

include("evolution.jl")

end


# adjusters.jl

module AdjustersTest

using Test
using Dates
using QDates

#trunc
@testset "trunc" begin
    qdt = QDates.QDate(2012,12,21)
    @test trunc(qdt, Dates.Year) == QDates.QDate(2012)
    @test trunc(qdt, Dates.Month) == QDates.QDate(2012,12)
    @test trunc(qdt, Dates.Day) == QDates.QDate(2012,12,21)
end

# Date functions
@testset "Date functions" begin
    # Name functions
    mtk = QDates.QDate(2017,1,1)
    ksrg = QDates.QDate(2017,2,2)
    yyi = QDates.QDate(2017,3,3)
    udk = QDates.QDate(2017,4,4)
    stk = QDates.QDate(2017,5,5)
    lstk = QDates.QDate(2017,5,true,6)
    mndk = QDates.QDate(2017,6,7)
    fdk = QDates.QDate(2017,7,8)
    hdk = QDates.QDate(2017,8,9)
    ngtk = QDates.QDate(2017,9,10)
    kmndk = QDates.QDate(2017,10,11)
    smtk = QDates.QDate(2017,11,12)
    sws = QDates.QDate(2017,12,13)

    @testset "firstdayofyear" begin
        @test QDates.firstdayofyear(sws) == QDates.QDate(2017,1,1)
        @test Dates.firstdayofyear(sws) == QDates.QDate(2017,1,1)
    end
    @testset "lastdayofyear" begin
        @test QDates.lastdayofyear(mtk) == QDates.QDate(2017,12,30)
        @test Dates.lastdayofyear(mtk) == QDates.QDate(2017,12,30)
    end
    @testset "lastdayofmonth" begin
        @test QDates.lastdayofmonth(mtk) == Dates.lastdayofmonth(mtk) == QDates.QDate(2017,1,29)
        @test QDates.lastdayofmonth(ksrg) == QDates.QDate(2017,2,30)
        @test QDates.lastdayofmonth(yyi) == QDates.QDate(2017,3,29)
        @test QDates.lastdayofmonth(udk) == QDates.QDate(2017,4,30)
        @test QDates.lastdayofmonth(stk) == QDates.QDate(2017,5,29)
        @test QDates.lastdayofmonth(lstk) == QDates.QDate(2017,5,true,29)
        @test QDates.lastdayofmonth(mndk) == QDates.QDate(2017,6,30)
        @test QDates.lastdayofmonth(fdk) == QDates.QDate(2017,7,29)
        @test QDates.lastdayofmonth(hdk) == QDates.QDate(2017,8,30)
        @test QDates.lastdayofmonth(ngtk) == QDates.QDate(2017,9,29)
        @test QDates.lastdayofmonth(kmndk) == QDates.QDate(2017,10,30)
        @test QDates.lastdayofmonth(smtk) == QDates.QDate(2017,11,30)
        @test QDates.lastdayofmonth(sws) == QDates.lastdayofyear(mtk) == QDates.QDate(2017,12,30)
    end
    @testset "firstdayofmonth" begin
        @test QDates.firstdayofmonth(mtk) == QDates.firstdayofyear(sws) == QDates.QDate(2017,1,1)
        @test QDates.firstdayofmonth(ksrg) == QDates.QDate(2017,2,1)
        @test QDates.firstdayofmonth(yyi) == QDates.QDate(2017,3,1)
        @test QDates.firstdayofmonth(udk) == QDates.QDate(2017,4,1)
        @test QDates.firstdayofmonth(stk) == QDates.QDate(2017,5,1)
        @test QDates.firstdayofmonth(lstk) == QDates.QDate(2017,5,true,1)
        @test QDates.firstdayofmonth(mndk) == QDates.QDate(2017,6,1)
        @test QDates.firstdayofmonth(fdk) == QDates.QDate(2017,7,1)
        @test QDates.firstdayofmonth(hdk) == QDates.QDate(2017,8,1)
        @test QDates.firstdayofmonth(ngtk) == QDates.QDate(2017,9,1)
        @test QDates.firstdayofmonth(kmndk) == QDates.QDate(2017,10,1)
        @test QDates.firstdayofmonth(smtk) == QDates.QDate(2017,11,1)
        @test QDates.firstdayofmonth(sws) == Dates.firstdayofmonth(sws) == QDates.QDate(2017,12,1)
    end
end

# # Adjusters
# # Adjuster Constructors # Not implemented
# @test QDates.QDate(Dates.isXXXX,2014) == QDates.QDate(2014,XX,XX)
# @test QDates.QDate(Dates.isYYYY,2014,5) == QDates.QDate(2014,5,YY)

# tonext, toprev, tofirst, tolast
@testset "tonext, toprev, tofirst, tolast" begin
    qdt = QDates.QDate(2017,5,3)
    @test Dates.tonext(qdt, QDates.先勝) == QDates.QDate(2017,5,9)
    @test Dates.tonext(qdt, QDates.先勝; same=true) == qdt
    @test Dates.tonext(qdt, QDates.友引) == QDates.QDate(2017,5,4)
    @test Dates.tonext(qdt, QDates.先負) == QDates.QDate(2017,5,5)
    @test Dates.tonext(qdt, QDates.仏滅) == QDates.QDate(2017,5,6)
    @test Dates.tonext(qdt, QDates.大安) == QDates.QDate(2017,5,7)
    @test Dates.tonext(qdt, QDates.赤口) == QDates.QDate(2017,5,8)
    # No dayofweek function for out of range values
    @test_throws KeyError Dates.tonext(qdt, 7)

    #test func, diff steps, negate, same
    @test Dates.tonext(QDates.is先勝, qdt) == QDates.QDate(2017,5,9)
    @test Dates.tonext(QDates.is先勝, qdt; same=true) == qdt
    @test Dates.tonext(QDates.is友引, qdt) == QDates.QDate(2017,5,4)
    @test Dates.tonext(QDates.is先負, qdt) == QDates.QDate(2017,5,5)
    @test Dates.tonext(QDates.is仏滅, qdt) == QDates.QDate(2017,5,6)
    @test Dates.tonext(QDates.is大安, qdt) == QDates.QDate(2017,5,7)
    @test Dates.tonext(QDates.is赤口, qdt) == QDates.QDate(2017,5,8)
    @test Dates.tonext(QDates.isleapmonth, qdt) == QDates.QDate(2017,5,true,1)
    @test Dates.tonext(QDates.isleapmonth, qdt; step=Dates.Month(1)) == QDates.QDate(2017,5,true,3)

    # @test Dates.tonext(x->!QDates.is大安(x), qdt; negate=true) == QDates.QDate(2017,5,7)
    @test Dates.tonext(!(x->!QDates.is大安(x)), qdt) == QDates.QDate(2017,5,7)
    # Reach adjust limit
    @test_throws ArgumentError Dates.tonext(QDates.is大安, qdt; limit=3)

    #toprev
    @test Dates.toprev(qdt, QDates.先勝) == QDates.QDate(2017,4,28)
    @test Dates.toprev(qdt, QDates.先勝; same=true) == qdt
    @test Dates.toprev(qdt, QDates.友引) == QDates.QDate(2017,4,29)
    @test Dates.toprev(qdt, QDates.先負) == QDates.QDate(2017,4,30)
    @test Dates.toprev(qdt, QDates.仏滅) == QDates.QDate(2017,4,25)
    @test Dates.toprev(qdt, QDates.大安) == QDates.QDate(2017,5,1)
    @test Dates.toprev(qdt, QDates.赤口) == QDates.QDate(2017,5,2)
    # No dayofweek function for out of range values
    @test_throws KeyError Dates.toprev(qdt, 8)

    #tofirst
    @test Dates.tofirst(qdt, QDates.先勝) == QDates.QDate(2017,5,3)
    @test Dates.tofirst(qdt, QDates.友引) == QDates.QDate(2017,5,4)
    @test Dates.tofirst(qdt, QDates.先負) == QDates.QDate(2017,5,5)
    @test Dates.tofirst(qdt, QDates.仏滅) == QDates.QDate(2017,5,6)
    @test Dates.tofirst(qdt, QDates.大安) == QDates.QDate(2017,5,1)
    @test Dates.tofirst(qdt, QDates.赤口) == QDates.QDate(2017,5,2)

    @test Dates.tofirst(qdt, QDates.先勝, of=Dates.Year) == QDates.QDate(2017,1,1)
    @test Dates.tofirst(qdt, QDates.友引, of=Dates.Year) == QDates.QDate(2017,1,2)
    @test Dates.tofirst(qdt, QDates.先負, of=Dates.Year) == QDates.QDate(2017,1,3)
    @test Dates.tofirst(qdt, QDates.仏滅, of=Dates.Year) == QDates.QDate(2017,1,4)
    @test Dates.tofirst(qdt, QDates.大安, of=Dates.Year) == QDates.QDate(2017,1,5)
    @test Dates.tofirst(qdt, QDates.赤口, of=Dates.Year) == QDates.QDate(2017,1,6)

    #tolast
    @test Dates.tolast(qdt, QDates.先勝) == QDates.QDate(2017,5,27)
    @test Dates.tolast(qdt, QDates.友引) == QDates.QDate(2017,5,28)
    @test Dates.tolast(qdt, QDates.先負) == QDates.QDate(2017,5,29)
    @test Dates.tolast(qdt, QDates.仏滅) == QDates.QDate(2017,5,24)
    @test Dates.tolast(qdt, QDates.大安) == QDates.QDate(2017,5,25)
    @test Dates.tolast(qdt, QDates.赤口) == QDates.QDate(2017,5,26)

    @test Dates.tolast(qdt, QDates.先勝, of=Dates.Year) == QDates.QDate(2017,12,26)
    @test Dates.tolast(qdt, QDates.友引, of=Dates.Year) == QDates.QDate(2017,12,27)
    @test Dates.tolast(qdt, QDates.先負, of=Dates.Year) == QDates.QDate(2017,12,28)
    @test Dates.tolast(qdt, QDates.仏滅, of=Dates.Year) == QDates.QDate(2017,12,29)
    @test Dates.tolast(qdt, QDates.大安, of=Dates.Year) == QDates.QDate(2017,12,30)
    @test Dates.tolast(qdt, QDates.赤口, of=Dates.Year) == QDates.QDate(2017,12,25)
end

end


unary_operators = [:(+), :(-), :(!)]

numeric_unary_operators = [:(+), :(-)]

logical_unary_operators = [:(!)]

elementary_functions = [:abs, :sign, :acos, :acosh, :asin,
                        :asinh, :atan, :atanh, :sin, :sinh,
                        :cos, :cosh, :tan, :tanh, :ceil, :floor,
                        :round, :trunc, :exp, :exp2, :expm1, :log, :log10, :log1p,
                        :log2, :exponent, :sqrt, :gamma, :lgamma, :digamma,
                        :erf, :erfc, :square]

special_comparison_operators = [:isless]

comparison_operators = [:(==), :(.==), :(!=), :(.!=),
                        :(>), :(.>), :(>=), :(.>=), :(<), :(.<),
                        :(<=), :(.<=)]

scalar_comparison_operators = [:(==), :(!=), :(>), :(>=),
                               :(<), :(<=)]

array_comparison_operators = [:(.==), :(.!=), :(.>), :(.>=), :(.<), :(.<=)]

binary_operators = [:(+), :(.+), :(-), :(.-), :(*), :(.*), :(/), :(./),
                    :(.^), :(div), :(mod), :(fld), :(rem),
                    :(&), :(|), :($)]

induced_binary_operators = [:(^)]

arithmetic_operators = [:(+), :(.+), :(-), :(.-), :(*), :(.*), :(/), :(./),
                        :(.^), :(div), :(mod), :(fld), :(rem)]

induced_arithmetic_operators = [:(^)]

biscalar_operators = [:(max), :(min)]

scalar_arithmetic_operators = [:(+), :(-), :(*), :(/),
                               :(div), :(mod), :(fld), :(rem)]

induced_scalar_arithmetic_operators = [:(^)]

array_arithmetic_operators = [:(+), :(.+), :(-), :(.-), :(.*), :(./), :(.^)]

bit_operators = [:(&), :(|), :($)]

unary_vector_operators = [:min, :max, :prod, :sum, :mean, :median, :std,
                          :var, :mad, :norm, :skewness, :kurtosis]

pairwise_vector_operators = [:diff, :percent_change]

cumulative_vector_operators = [:cumprod, :cumsum, :cumsum_kbn, :cummin, :cummax]

ffts = [:fft]

binary_vector_operators = [:dot, :cor, :cov,
                           :cor_spearman, :cov_spearman]

columnar_operators = [:colmins, :colmaxs, :colprods, :colsums,
                      :colmeans, :colmedians, :colstds, :colvars,
                      :colffts, :colnorms]

boolean_operators = [:any, :all]

# All unary operators return NA when evaluating NA
for f in unary_operators
  @eval begin
    @assert isna(($f)(NA))
  end
end

# All elementary functions return NA when evaluating NA
for f in elementary_functions
  @eval begin
    @assert isna(($f)(NA))
  end
end

# All comparison operators return NA when comparing NA with NA
# All comparison operators return NA when comparing scalars with NA
# All comparison operators return NA when comparing NA with scalars
for f in comparison_operators
  @eval begin
    @assert isna(($f)(NA, NA))
    @assert isna(($f)(NA, 1))
    @assert isna(($f)(1, NA))
  end
end

# All arithmetic operators return NA when operating on two NA's
# All arithmetic operators return NA when operating on a scalar and an NA
# All arithmetic operators return NA when operating on an NA and a scalar
for f in arithmetic_operators
  @eval begin
    @assert isna(($f)(NA, NA))
    @assert isna(($f)(1, NA))
    @assert isna(($f)(NA, 1))
  end
end

# All bit operators return NA when operating on two NA's
# All bit operators return NA when operating on a scalar and an NA
# All bit operators return NA when operating on an NA and a scalar
for f in bit_operators
  @eval begin
    @assert isna(($f)(NA, NA))
    @assert isna(($f)(1, NA))
    @assert isna(($f)(NA, 1))
  end
end

# Unary operators on DataVector's should be equivalent to elementwise
# application of those same operators
N = 5
dv = dataones(N)
for f in numeric_unary_operators
  @eval begin
    for i in 1:length(dv)
      @assert (($f)(dv))[i] == ($f)(dv[i])
    end
  end
end
dv = datatrues(N)
for f in logical_unary_operators
  @eval begin
    for i in 1:length(dv)
      @assert (($f)(dv))[i] == ($f)(dv[i])
    end
  end
end

# Unary operators on DataFrame's should be equivalent to elementwise
# application of those same operators
df = DataFrame(quote
                 A = [1, 2, 3, 4]
                 B = [1.0, pi, pi, e]
               end)
for f in numeric_unary_operators
  @eval begin
    for i in 1:nrow(df)
      for j in 1:ncol(df)
        @assert (($f)(df))[i, j] == ($f)(df[i, j])
      end
    end
  end
end
df = DataFrame(quote
                 A = [true, false, true, false]
               end)
for f in logical_unary_operators
  @eval begin
    for i in 1:nrow(df)
      for j in 1:ncol(df)
        @assert (($f)(df))[i, j] == ($f)(df[i, j])
      end
    end
  end
end

# Elementary functions on DataVector's
N = 5
dv = dataones(N)
for f in elementary_functions
  @eval begin
    for i in 1:length(dv)
      @assert (($f)(dv))[i] == ($f)(dv[i])
    end
  end
end

# Elementary functions on DataFrames's
N = 5
df = DataFrame(quote
                 A = dataones($(N))
                 B = dataones($(N))
               end)
for f in elementary_functions
  @eval begin
    for i in 1:nrow(df)
      for j in 1:ncol(df)
        @assert (($f)(df))[i, j] == ($f)(df[i, j])
      end
    end
  end
end

# Broadcasting operations between NA's and DataVector's
N = 5
dv = dataones(N)
for f in arithmetic_operators
  @eval begin
    for i in 1:length(dv)
      @assert isna((($f)(dv, NA))[i])
      @assert isna((($f)(NA, dv))[i])
    end
  end
end

# Broadcasting operations between NA's and DataVector's
N = 5
dv = dataones(N)
for f in arithmetic_operators
  @eval begin
    for i in 1:length(dv)
      @assert isna((($f)(dv, NA))[i])
      @assert isna((($f)(NA, dv))[i])
    end
  end
end

# Broadcasting operations between scalars and DataVector's
N = 5
dv = dataones(N)
for f in arithmetic_operators
  @eval begin
    for i in 1:length(dv)
      @assert (($f)(dv, 1))[i] == ($f)(dv[i], 1)
      @assert (($f)(1, dv))[i] == ($f)(1, dv[i])
    end
  end
end

# Broadcasting operations between NA's and DataFrames's
N = 5
df = DataFrame(quote
                 A = dataones($(N))
                 B = dataones($(N))
               end)
for f in arithmetic_operators
  @eval begin
    for i in 1:nrow(df)
      for j in 1:ncol(df)
        @assert isna((($f)(df, NA))[i, j])
        @assert isna((($f)(NA, df))[i, j])
      end
    end
  end
end

# Broadcasting operations between scalars and DataFrames's
N = 5
df = DataFrame(quote
                 A = dataones($(N))
                 B = dataones($(N))
               end)
for f in arithmetic_operators
  @eval begin
    for i in 1:nrow(df)
      for j in 1:ncol(df)
        @assert (($f)(df, 1))[i, j] == ($f)(df[i, j], 1)
        @assert (($f)(1, df))[i, j] == ($f)(1, df[i, j])
      end
    end
  end
end

# Binary operations on (DataVector, Vector) or (Vector, DataVector)
N = 5
v = ones(N)
dv = dataones(N)
dv[1] = NA
for f in array_arithmetic_operators
  @eval begin
    for i in 1:length(dv)
      @assert isna(($f)(v, dv)[i]) && isna(dv[i]) ||
              (($f)(v, dv))[i] == ($f)(v[i], dv[i])
      @assert isna(($f)(dv, v)[i]) && isna(dv[i]) ||
              (($f)(dv, v))[i] == ($f)(dv[i], v[i])
    end
  end
end

# Binary operations on pairs of DataVector's
N = 5
dv = dataones(N)
dv[1] = NA
for f in array_arithmetic_operators
  @eval begin
    for i in 1:length(dv)
      @assert isna(($f)(dv, dv)[i]) && isna(dv[i]) ||
              (($f)(dv, dv))[i] == ($f)(dv[i], dv[i])
    end
  end
end

# Binary operations on pairs of DataFrame's
# TODO: Test in the presence of in-operable types like Strings
N = 5
df = DataFrame(quote
                 A = dataones($(N))
                 B = dataones($(N))
               end)
for f in array_arithmetic_operators
  @eval begin
    for i in 1:nrow(df)
      for j in 1:ncol(df)
        @assert isna(($f)(df, df)[i, j]) && isna(df[i, j]) ||
                (($f)(df, df))[i, j] == ($f)(df[i, j], df[i, j])
      end
    end
  end
end

# Unary vector operators on DataVector's
N = 5
dv = dataones(5)
for f in unary_vector_operators
  @eval begin
    if isnan(($f)(dv.data))
      @assert isnan(($f)(dv))
    else
      @assert ($f)(dv) == ($f)(dv.data)
    end
  end
end
dv[1] = NA
for f in unary_vector_operators
  @eval begin
    @assert isna(($f)(dv))
  end
end

# TODO: Pairwise vector operators on DataVector's

# Cumulative vector operators on DataVector's
N = 5
dv = dataones(N)
for f in cumulative_vector_operators
  @eval begin
    for i in 1:length(dv)
      @assert (($f)(dv))[i] == ($f)(dv.data)[i]
    end
  end
end
dv[4] = NA
for f in cumulative_vector_operators
  @eval begin
    for i in 1:3
      @assert (($f)(dv))[i] == ($f)(dv.data)[i]
    end
    for i in 4:N
      @assert isna((($f)(dv))[i])
    end
  end
end

# FFT's on DataVector's
N = 5
dv = dataones(5)
for f in ffts
  @eval begin
    @assert ($f)(dv) == ($f)(dv.data)
  end
end
dv[1] = NA
for f in ffts
  @eval begin
    @assert isna(($f)(dv))
  end
end

# Binary vector operators on DataVector's
N = 5
dv = dataones(5)
for f in binary_vector_operators
  @eval begin
    @assert (($f)(dv, dv) == ($f)(dv.data, dv.data)) ||
            (isnan(($f)(dv, dv)) && isnan(($f)(dv.data, dv.data)))
  end
end
dv[1] = NA
for f in binary_vector_operators
  @eval begin
    @assert isna(($f)(dv, dv))
  end
end

# TODO: Columnar operators on DataFrame's

# Boolean operators on DataVector's
N = 5
@assert any(datafalses(N)) == false
@assert any(datatrues(N)) == true
@assert all(datafalses(N)) == false
@assert all(datatrues(N)) == true

dv = datafalses(N)
dv[3] = true
@assert any(dv) == true
@assert all(dv) == false

dv = datafalses(N)
dv[2] = NA
dv[3] = true
@assert any(dv) == true
@assert all(dv) == false

dv = datafalses(N)
dv[2] = NA
@assert isna(any(dv))
@assert all(dv) == false

dv = datafalses(1)
dv[1] = NA
@assert isna(any(dv))
@assert isna(all(dv))

# Boolean operators on DataFrames's
N = 5
df = DataFrame(quote
                 A = datafalses($(N))
               end)
@assert any(df) == false
@assert any(!df) == true
@assert all(df) == false
@assert all(!df) == true

df = DataFrame(quote
                 A = datafalses($(N))
               end)
df[3, 1] = true
@assert any(df) == true
@assert all(df) == false

df = DataFrame(quote
                 A = datafalses($(N))
               end)
df[2, 1] = NA
df[3, 1] = true
@assert any(df) == true
@assert all(df) == false

df = DataFrame(quote
                 A = datafalses($(N))
               end)
df[2, 1] = NA
@assert isna(any(df))
@assert all(df) == false

df = DataFrame(quote
                 A = datafalses($(N))
               end)
df[1, 1] = NA
@assert isna(any(dv))
@assert isna(all(dv))

# Is this a genuine special case?
@assert isna(NA ^ 2.0)

#
# Equality tests
#

dv = DataVector[1, NA]
alt_dv = DataVector[2, NA]
df = DataFrame({dv})
alt_df = DataFrame({alt_dv})

@assert isna(NA == NA)
@assert isna(NA != NA)

# @assert isna(dv == dv) # SHOULD RAISE ERROR
# @assert isna(dv != dv) # SHOULD RAISE ERROR

# @assert isna(df == df) # SHOULD RAISE ERROR
# @assert isna(df != df) # SHOULD RAISE ERROR

@assert isequal(dv, dv)
@assert isequal(df, df)

@assert !isequal(dv, alt_dv)
@assert !isequal(df, alt_df)

@assert isequal(DataVector[1, NA] .== DataVector[1, NA], DataVector[true, NA])
@assert isequal(PooledDataVector[1, NA] .== PooledDataVector[1, NA], DataVector[true, NA])
@assert isequal(DataFrame({dv}) .== DataFrame({dv}), DataFrame({DataVector[true, NA]}))

@assert all(isna(NA .== dataones(5)))
@assert all(isna(dataones(5) .== NA))

@assert all(isna(NA .== df))
@assert all(isna(df .== NA))

# Run length encoding
dv = dataones(5)
dv[3] = NA

v, l = rle(dv)
@assert isequal(v, DataVector[1.0, NA, 1.0])
@assert isequal(l, [2, 1, 2])

rdv = inverse_rle(v, l)
@assert isequal(dv, rdv)