# insert functions

"""
    insert_boundary!(tier, time; split_at = 0)
    insert_boundary!(tg, num, time; split_at = 0)

Insert a boundary at `time` in an interval `tier`, which can also be specified
by its number in a `TextGrid`. This action splits an existing interval and
increases the size of the tier by 1.

The keyword argument `split_at` indicates the starting position in the label of
the interval being split that belongs to the right interval. The remaining part
of the label belongs to the left interval. A value outside any valid index for
the label (i.e. the default value 0, any negative integer, or any positive
integer greater than its length) means that the whole original label belongs to
the left interval.
"""
function insert_boundary!(tier::Tier, time::Real; split_at::Int = 0)
    tier.class == "interval" || error("Cannot insert boundaries in point tiers.")
    index = findfirst(x -> x.xmin ≤ time ≤ x.xmax, tier.contents)
    isnothing(index) && error("Time out of bounds of tier.")
    if tier.contents[index].xmin == time || tier.contents[index].xmax == time
        error("Boundary already exists in Tier $tier at $(time)s.")
    end

    label_length = length(tier.contents[index].label)
    split_at ∈ 1:label_length || (split_at = label_length + 1)
    insert!(tier.contents, index + 1, Interval(index + 1, time, tier.contents[index].xmax, tier.contents[index].label[split_at:end]))
    tier.contents[index].xmax = time
    tier.contents[index].label = tier.contents[index].label[1:(split_at - 1)]
    tier.size += 1
    reindex_intervals!(tier, from = index + 1)
end

insert_boundary!(tg::TextGrid, num::Int, time::Real; split_at::Int = 0) = insert_boundary!(tg[num], time, split_at = split_at)

"""
    insert_interval!(tier, start, stop, label; split_at = 0)
    insert_interval!(tg, num, start, stop, label; split_at = 0)

Insert an interval from `start` to `stop` in an interval `tier`. Inserted
intervals must not straddle an existing boundary, but if either `start` or
`stop` coincides with an existing edge of an interval, then the other boundary
is added to split the original interval.

The keyword argument `split_at` indicates the starting position in the label of
the interval being split that belongs to the interval immediately to the right
of the inserted interval. Note that if either `start` or `stop` coincides with
an existing edge, `split_at` is automatically overridden and the whole original
label goes to the split interval. See also `insert_boundary!`.
"""
function insert_interval!(tier::Tier, start::Real, stop::Real, label::AbstractString; split_at::Int = 0)
    tier.class == "interval" || error("Cannot insert intervals in point tiers.")

    start_index, stop_index = is_interval_insertable(tier, start, stop)

    if start == tier.contents[start_index].xmin
        # If only left edge already exists, insert new right edge and force original label to the right
        insert_boundary!(tier, stop, split_at = 1)
        relabel!(tier, start_index, label)
    elseif stop == tier.contents[stop_index].xmax
        # If only right edge already exists, insert new left edge and force original label to the left
        insert_boundary!(tier, start)
        relabel!(tier, start_index + 1, label)
    else
        insert_boundary!(tier, stop, split_at = split_at)
        insert_boundary!(tier, start)
        relabel!(tier, start_index + 1, label)
    end
    return tier
end

insert_interval!(tg::TextGrid, num::Int, start::Real, stop::Real, label::AbstractString; split_at::Int = 0) = insert_interval!(tg[num], start, stop, label, split_at = split_at)

"""
    copy_interval!(tier, source_tier, index; split_at = 0)
    copy_interval!(tg, num, source_num, index; split_at = 0)

Copy the `index`-th interval from `source_tier` to `tier`.

See also `insert_interval!`.
"""
copy_interval!(tier::Tier, source_tier::Tier, index::Int; split_at::Int = 0) = insert_interval!(tier, source_tier.contents[index].xmin, source_tier.contents[index].xmax, source_tier.contents[index].label, split_at = split_at)
copy_interval!(tg::TextGrid, num::Int, source_num::Int, index::Int; split_at::Int = 0) = copy_interval!(tg[num], tg[source_num], index, split_at = split_at)

"""
Insert multiple intervals to an interval `tier`, with boundaries and labels
defined by `starts`, `stops` and `labels`. The keyword argument `split_at` can
be either a single integer if constant across inserted intervals or a vector of
integers if custom per interval.

Intervals are inserted in the specified order, and intervals that cannot be
inserted are simply discarded rather than throw an error.

See also `insert_interval!`
"""
function insert_intervals!(tier::Tier, starts::Vector{<:Real}, stops::Vector{<:Real}, labels::Vector{<:AbstractString}; split_at::Union{Int, Vector{Int}} = 0)
    num_added = minimum(length, [starts, stops, labels])
    if split_at isa Vector
        length(split_at) < num_added && error("`split_at must not be shorter than `starts`, `stops` and `labels`.")
        for (start, stop, label, split_at_i) in zip(starts, stops, labels, split_at)
            try
                insert_interval!(tier, start, stop, label, split_at = split_at_i)
            catch
                continue
            end
        end
    else
        for (start, stop, label) in zip(starts, stops, labels)
            try
                insert_interval!(tier, start, stop, label, split_at = split_at)
            catch
                continue
            end
        end
    end
    return tier
end

insert_intervals!(tg::TextGrid, num::Int, starts::Vector{<:Real}, stops::Vector{<:Real}, labels::Vector{<:AbstractString}; split_at::Union{Int, Vector{Int}} = 0) = insert_intervals!(tg[num], starts, stops, labels, split_at = split_at)

function is_interval_insertable(tier::Tier, start::Real, stop::Real)
    start ≥ stop && error("Start time must come before stop time.")
    start_index = find_interval(tier, start)
    stop_index = find_low_interval(tier, stop)
    start_index * stop_index == 0 && error("Time out of bounds of tier.")
    start_index != stop_index && error("Cannot insert interval that straddles a boundary.")
    start == tier.contents[start_index].xmin && stop == tier.contents[stop_index].xmax && error("Interval already exists with the same boundaries.")
    return start_index, stop_index
end

# remove functions

"""
    remove_left_boundary!(tier, index; delim = "")
    remove_left_boundary!(tg, num, index; delim = "")

Remove the left boundary of the `index`-th interval in a `tier`.

The keyword argument `delim` defines the string used to join the labels from
the two intervals being combined.
"""
function remove_left_boundary!(tier::Tier, index::Int; delim::AbstractString = "")
    tier.class == "interval" || error("Cannot remove boundaries from point tiers.")
    index == 1 && error("Cannot remove left edge of tier.")

    tier.contents[index - 1].xmax = tier.contents[index].xmax
    relabel!(tier, index - 1, tier.contents[index - 1].label * delim * tier.contents[index].label)
    deleteat!(tier.contents, index)
    tier.size -= 1
    reindex_intervals!(tier, from = index)
end

remove_left_boundary!(tg::TextGrid, num::Int, index::Int; delim::AbstractString = "") = remove_left_boundary!(tg[num], index, delim = delim)

"""
    remove_right_boundary!(tier, index; delim = "")
    remove_right_boundary!(tg, num, index; delim = "")

Remove the right boundary of the `index`-th interval in an interval `tier`.

The keyword argument `delim` defines the string used to join the labels from
the two intervals being combined.
"""
function remove_right_boundary!(tier::Tier, index::Int; delim::AbstractString = "")
    tier.class == "interval" || error("Cannot remove boundaries from point tiers.")
    index == tier.size && error("Cannot remove right edge of tier.")

    remove_left_boundary!(tier, index + 1, delim = delim)
end

remove_right_boundary!(tg::TextGrid, num::Int, index::Int; delim::AbstractString = "") = remove_right_boundary!(tg[num], index, delim = delim)

"""
    remove_boundary!(tier, time; delim = "", tolerance = 0.0001)
    remove_boundary!(tier, num, time; delim = "", tolerance = 0.0001)

Remove the boundary at `time` (± `tolerance`) from an interval `tier`.

The keyword argument `delim` defines the string used to join the labels from
the two intervals being combined.
"""
function remove_boundary!(tier::Tier, time::AbstractFloat; delim::AbstractString = "", tolerance::Real = 0.0001)
    index = findlast(x -> abs(x.xmin - time) ≤ tolerance, tier.contents)
    if !isnothing(index)
        remove_left_boundary!(tier, index, delim)
    end
end

remove_boundary!(tg::TextGrid, num::Int, time::AbstractFloat; delim::AbstractString = "", tolerance::Real = 0.0001) = remove_boundary!(tg[num], time, delim = delim, tolerance = tolerance)

"""
    remove_boundary!(tier, index, edge; delim = "")
    remove_boundary!(tier, num, index, edge; delim = "")

Remove left and/or right boundaries of the `index`-th interval in `tier`.
Depending on value of `edge`, equivalent to:
- `"left"`: `remove_left_boundary!`
- `"right"`: `remove_right_boundary!`
- `"both"`: a combination of the two

The keyword argument `delim` defines the string used to join the labels from
the two intervals being combined.
"""
function remove_boundary!(tier::Tier, index::Int, edge::AbstractString; delim::AbstractString = "")
    edge ∈ ("left", "right", "both") || error("`edge` must be `left`, `right` or `both`.")
    edge != "left" && remove_right_boundary!(tier, index, delim = delim)
    edge != "right" && remove_left_boundary!(tier, index, delim = delim)
    return tier
end

remove_boundary!(tg::TextGrid, num::Int, index::Int, edge::AbstractString; delim::AbstractString = "") = remove_boundary!(tg[num], index, edge, delim = delim)

function remove_interval!(tier::Tier, index::Int)
    tier.xmin == tier.contents[index].xmin && tier.xmax == tier.contents[index].xmax && error("Cannot remove only interval in tier $tier.")

    relabel!(tier, index, "")
    edge = if tier.contents[index].xmin == tier.xmin
        "right"
    elseif tier.contents[index].xmax == tier.xmax
        "left"
    else
        "both"
    end

    remove_boundary!(tier, index, edge)
end

remove_interval!(tg::TextGrid, num::Int, index::Int) = remove_interval!(tg[num], index)

# move functions

"""
    move_left_boundary!(tier, index; by = 0, to = -1)
    move_left_boundary!(tg, num, index; by = 0, to = -1)

Move the left boundary of the `index`-th interval in an interval `tier`, either
`by` a specified shift or `to` a specified destination. Boundary movement is
not allowed to cross or overlap with other boundaries.

See also `move_right_boundary!`, `move_boundary!` and `move_interval!`.
"""
function move_left_boundary!(tier::Tier, index::Int; by::Real = 0, to::Real = -1)
    index == 1 && error("Cannot move left edge of tier.")
    by == 0 && to == -1 && error("Boundary movement not specified.")
    by != 0 && to != -1 && error("Both `by` and `to` specified.")

    by != 0 && (to = tier.contents[index].xmin + by)

    (to < tier.xmin || to > tier.xmax) && error("Cannot move boundary out of bounds.")
    (to ≤ tier.contents[index - 1].xmin || to ≥ tier.contents[index].xmax) && error("Cannot move boundary beyond interval duration.")

    tier.contents[index].xmin = to
    tier.contents[index - 1].xmax = to
    return tier
end

move_left_boundary!(tg::TextGrid, num::Int, index::Int; by::Real = 0, to::Real = -1) = move_left_boundary!(tg[num], index, by = by, to = to)

"""
    move_right_boundary!(tier, index; by = 0, to = -1)
    move_right_boundary!(tg, num, index; by = 0, to = -1)

Move the right boundary of the `index`-th interval in an interval `tier`, either
`by` a specified shift or `to` a specified destination. Boundary movement is
not allowed to cross or overlap with other boundaries.

See also `move_left_boundary!`, `move_boundary!` and `move_interval!`.
"""
function move_right_boundary!(tier::Tier, index::Int; by::Real = 0, to::Real = -1)
    index == tier.size && error("Cannot move right edge of tier.")
    move_left_boundary!(tier, index + 1, by = by, to = to)
end

move_right_boundary!(tg::TextGrid, num::Int, index::Int; by::Real = 0, to::Real = -1) = move_right_boundary!(tg[num], index, by = by, to = to)

"""
    move_interval!(tier, index; by = 0, to = -1)
    move_interval!(tg, num, index; by = 0, to = -1)

Move both left and right boundaries of the `index`-th interval in an interval
`tier`, either `by` a specified shift or `to` a specified destination. `to`
specifies the destination of the left boundary. Boundary movement is not allowed
to cross or overlap with other boundaries.

See also `move_left_boundary!`, `move_right_boundary!` and `move_boundary!`.
"""
function move_interval!(tier::Tier, index::Int; by::Real = 0, to::Real = -1)
    index == 1 && error("Cannot move left edge of tier.")
    index == tier.size && error("Cannot move right edge of tier.")
    by == 0 && to == -1 && error("Boundary movement not specified.")
    by != 0 && to != -1 && error("Both `by` and `to` specified.")

    if by != 0
        left_to = tier.contents[index].xmin + by
        right_to = tier.contents[index].xmax + by
    else
        left_to = to
        right_to = to + tier.contents[index].xmax - tier.contents[index].xmin
    end

    (left_to < tier.xmin || right_to > tier.xmax) && error("Cannot move boundary out of bounds.")
    (left_to ≤ tier.contents[index - 1].xmin || right_to ≥ tier.contents[index + 1].xmax) && error("Cannot move boundary beyond interval duration.")

    tier.contents[index].xmin = left_to
    tier.contents[index - 1].xmax = left_to
    tier.contents[index].xmax = right_to
    tier.contents[index + 1].xmin = right_to

    return tier
end

move_interval!(tg::TextGrid, num::Int, index::Int; by::Real = 0, to::Real = -1) = move_interval!(tg[num], index, by = by, to = to)

"""
    move_boundary!(tier, index, edge; by = 0, to = -1)
    move_boundary!(tg, num, index, edge; by = 0, to = -1)

Move one or both boundaries of the `index`-th interval in an interval `tier`,
either `by` a specified shift or `to` a specified destination. Equivalent to
the following depending on value of `edge`:
- "left": `move_left_boundary!`
- "right": `move_right_boundary!`
- "both": `move_interval!`
"""
function move_boundary!(tier::Tier, index::Int, edge::AbstractString; by::Real = 0, to::Real = -1)
    if edge == "left"
        return move_left_boundary!(tier, index, by = by, to = to)
    elseif edge == "right"
        return move_right_boundary!(tier, index, by = by, to = to)
    elseif edge == "both"
        return move_interval!(tier, index, by = by, to = to)
    else
        error("`edge` must be `left`, `right` or `both`.")
    end
end

move_boundary!(tg::TextGrid, num::Int, index::Int, edge::AbstractString; by::Real = 0, to::Real = -1) = move_boundary!(tg[num], index, edge, by = by, to = to)

"""
    move_boundary!(tier, time; by = 0, to = -1, tolerance = 0.0001)
    move_boundary!(tg, num, time; by = 0, to = -1, tolerance = 0.0001)

Move an interval boundary located at `time` (± `tolerance`), either `by` a
specified shift or `to` a specified destination. Boundary movement is not
allowed to cross or overlap with other boundaries. Nothing happens if no
boundary is found within the tolerance limit.
"""
function move_boundary!(tier::Tier, time::AbstractFloat; by::Real = 0, to::Real = -1, tolerance::Real = 0.0001)
    index = find_interval(tier, time)
    if time - tier.contents[index].xmin ≤ tolerance
        return move_left_boundary!(tier, index, by = by, to = to)
    elseif tier.contents[index].xmax - time ≤ tolerance
        return move_right_boundary!(tier, index, by = by, to = to)
    end
end

move_boundary!(tg::TextGrid, num::Int, time::AbstractFloat; by::Real = 0, to::Real = -1, tolerance::Real = 0.0001) = move_boundary!(tg[num], time, by = by, to = to, tolerance = tolerance)


#!/usr/bin/env julia
# vim:set ft=julia ts=4 sw=4 sts=4 autoindent:

# Profiler driver script.
#
# Author:   Pontus Stenetorp    <pontus stenetorp se>
# Version:  2014-05-11

include("hdr.jl")

require("conllx.jl")
require("dep.jl")
require("hybrid.jl")
require("parse.jl")

using CoNLLX
using DepGraph
using Hybrid
using Parse

reportpath = "$repodir/wrk/profile.txt"

open(talpath, "r") do data_f
    sents = collect(Sentence, conllxparse(data_f, useproj=true))
    Profile.init(2^24, 0.001)

    # "Warm up" the code by performing an initial pass.
    next(train(sents, epochs=1))

    @profile begin
        for _ in train(sents, epochs=2)
            nothing
        end
    end

    open(reportpath, "w") do report_f
        Profile.print(report_f)
    end
end


# MIT License
#
# Copyright (c) 2018 Martin Biel
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

@with_kw mutable struct ConstantStepParameters{T <: AbstractFloat}
    γ::T = 0.01
end

"""
    ConstantStep

Functor object for using a constant step size in a quasigradient algorithm. Create by supplying a [`Constant`](@ref) object through `step` to `QuasiGradient.Optimizer` or by setting the [`StepSize`](@ref) attribute.

...
# Parameters
- `γ::AbstractFloat = 0.01`: Step length
...
"""
struct ConstantStep{T <: AbstractFloat} <: AbstractStep
    parameters::ConstantStepParameters{T}

    function ConstantStep(::Type{T}; kw...) where T <: AbstractFloat
        return new{T}(ConstantStepParameters{T}(; kw...))
    end
end
step(step::ConstantStep, ::Integer, ::Float64, ::AbstractVector, ::AbstractVector) = step.parameters.γ

# API
# ------------------------------------------------------------
"""
    Constant

Factory object for [`ConstantStep`](@ref). Pass to `step` in `Quasigradient.Optimizer` or set the [`StepSize`](@ref) attribute. See ?ConstantStep for parameter descriptions.

"""
struct Constant <: AbstractStepSize
    parameters::ConstantStepParameters{Float64}
end
Constant(γ::AbstractFloat) = Constant(ConstantStepParameters(; γ = Float64(γ)))
Constant(; kw...) = Constant(ConstantStepParameters(; kw...))

function (stepsize::Constant)(::Type{T}) where T <: AbstractFloat
    return ConstantStep(T; type2dict(stepsize.parameters)...)
end

function str(::Constant)
    return "constant step size"
end


function build_robot!(vis::Visualizer, model::PushBot; name::Symbol=:PushBot,
		r=0.05, n_leg::Int=30, α=1.0)
	r = convert(Float32, r)
	r_contact = convert(Float32, r * 1.5)
	r_arm = convert(Float32, r * 0.3)
	wall_thickness = 0.15
	l_joint = 0.16
	body_mat = MeshPhongMaterial(color = RGBA(0.0, 0.0, 0.0, α))
	contact_mat = MeshPhongMaterial(color = RGBA(1.0, 165.0 / 255.0, 0.0, α))
	wall_mat = MeshPhongMaterial(color = RGBA(0.7, 0.7, 0.7, 1.0))

	default_background!(vis)

	link = Cylinder(Point3f0(0.0), Point3f0(0.0, 0.0, model.l-3r_arm), r)
	setobject!(vis[name][:robot]["link"], link, body_mat)
	setobject!(vis[name][:robot]["linkbase"], Sphere(Point3f0(0.0), r), body_mat)

	setobject!(vis[name][:robot]["contact1"], Sphere(Point3f0(0.0), r_contact), contact_mat)
	setobject!(vis[name][:robot]["joint1"],
		Rect(Vec(-l_joint/2, -r/3, -4r_arm),Vec(l_joint, 2r/3, 2r_arm)), body_mat)
	setobject!(vis[name][:robot]["joint2"],
		Rect(Vec(-l_joint/2, -r/3, 2r_arm),Vec(l_joint, 2r/3, 2r_arm)), body_mat)

	for i = 1:n_leg
		setobject!(vis[name][:robot]["arm$i"], Sphere(Point3f0(0), r_arm), body_mat)
	end

	setobject!(vis[name][:env]["wall1"],
		Rect(Vec(0, 0, 0),Vec(wall_thickness, wall_thickness, 1.5)), wall_mat)

	setobject!(vis[name][:env]["wall2"],
		Rect(Vec(0, 0, 0),Vec(wall_thickness, wall_thickness, 1.5)), wall_mat)

	settransform!(vis[name][:env]["wall1"], Translation([0.5 + 1.5 * r;                 -wall_thickness/2; 0.0]))
	settransform!(vis[name][:env]["wall2"], Translation([-wall_thickness-0.5 - 1.5 * r; -wall_thickness/2; 0.0]))

	# settransform!(vis["/Cameras/default"],
	# 	compose(Translation(0.0, 0.5, -1.0), LinearMap(RotZ(-pi / 2.0))))
	return nothing
end

function set_robot!(vis::Visualizer, model::PushBot, q::AbstractVector;
		name::Symbol=:PushBot, n_leg::Int=30)

	pee = [_kinematics(model, q, mode = :ee)[1], 0.0, _kinematics(model, q, mode = :ee)[2]]
    p1 = [_kinematics(model, q, mode = :d)[1], 0.0, _kinematics(model, q, mode = :d)[2]]

	settransform!(vis[name][:robot]["joint1"], compose(cable_transform(zeros(3), pee), Translation(0,0,0)))
	settransform!(vis[name][:robot]["joint2"], cable_transform(zeros(3), pee))
	settransform!(vis[name][:robot]["link"], cable_transform(pee, zeros(3)))
	settransform!(vis[name][:robot]["contact1"], Translation(p1))

	r_range = range(0, stop = q[2], length = n_leg)
	for i = 1:n_leg
		q_tmp = Array(copy(q))
		q_tmp[2] = r_range[i]
		p_leg = [_kinematics(model, q_tmp, mode = :d)[1], 0.0, _kinematics(model, q_tmp, mode = :d)[2]]
        settransform!(vis[name][:robot]["arm$i"], Translation(p_leg))
    end

	return nothing
end

function pusher_arc(center::AbstractVector, radius::Real, α::Real; sense::Symbol=:trig)
	p = center + radius*[cos(α), 0, sin(α)]
	if sense == :trig
		θ = α + π/2
	elseif sense == :antitrig
		θ = α - π/2
	else
		error("Invalid sense.")
	end
	pθ = [p; θ]
	return pθ
end

function pusher_arc_traj(center::AbstractVector, radius::Real, α_start::Real, α_end::Real,
	 	N::Int; sense::Symbol=:trig)
	pθ = []
	for α in range(α_start, stop=α_end, length=N)
		pθ_ = pusher_arc(center, radius, α, sense=sense)
		push!(pθ, pθ_)
	end
	return pθ
end

function generate_pusher_traj(d::ImpulseDisturbance, traj::ContactTraj; side::Symbol=:right,
		center=zeros(3), radius::Real=0.8, N_arc::Int=17)
	H = traj.H
	if side == :right
		sense = :trig
		θ_rest = 0.0
	elseif side == :left
		sense = :antitrig
		θ_rest = π
	else
		error("Invalid side.")
	end

	pθ = [pusher_arc(center, radius, θ_rest; sense=sense) for t=1:H_sim]

	for (i,idx) in enumerate(d.idx)
		w = d.w[i][1]
		θ_impact = traj.q[idx+1][1] + π/2
		θ_next = traj.q[idx+2][1] + π/2

		if ((w > 0.0) && (side == :right)) || ((w < 0.0) && (side == :left))
			forward_arc = pusher_arc_traj(center, radius, θ_rest, θ_impact, N_arc; sense=sense)
			backward_arc = pusher_arc_traj(center, radius, θ_next, θ_rest, N_arc; sense=sense)
			arc = [forward_arc; backward_arc] # 2N
			pθ[idx-N_arc-0:idx+N_arc-1] .= arc
		end
	end
	return pθ
end

function build_disturbance!(vis::Visualizer, model::PushBot; name::Symbol=:Pusher, r=0.025, α=1.0)
	r1 = convert(Float32, r)
	r2 = convert(Float32, 4r)
	pusher_mat = MeshPhongMaterial(color = RGBA(1.0, 0.0, 0.0, α))

	# build pusher
	p1 = Cylinder(Point3f0(0.0), Point3f0(0.0, 0.0, 0.250), r1)
	p2 = Cylinder(Point3f0(0.0), Point3f0(0.0, 0.0, 0.075), r2)
	setobject!(vis[name, "pusher1"], p1, pusher_mat)
	setobject!(vis[name, "pusher2"], p2, pusher_mat)
	return anim
end

function set_disturbance!(vis::Visualizer, model::PushBot, pθ::AbstractVector; name::Symbol=:Pusher, offset::Real=0.0)
	p = pθ[1:3]
	θ = pθ[4]
	settransform!(vis[name],
		compose(Translation(p...),
		compose(LinearMap(RotY(-θ-pi/2)),
				Translation(0,0,offset))))
	return nothing
end

function animate_disturbance!(vis::Visualizer, anim::MeshCat.Animation, model::PushBot,
		pθ::AbstractVector; name::Symbol=:Pusher, offset::Real=0.0)
	H = length(pθ)
	for t in 1:H
		MeshCat.atframe(anim, t) do
			set_disturbance!(vis, model, pθ[t], name=name, offset=offset)
		end
	end
	setanimation!(vis, anim)
	return anim
end

function visualize_disturbance!(vis::Visualizer, model::ContactModel, pθ::AbstractVector;
		h=0.01, α=1.0,
		sample=max(1, Int(floor(length(pθ) / 100))),
		anim::MeshCat.Animation=MeshCat.Animation(Int(floor(1/h))),
		name::Symbol=:Pusher, offset::Real=0.0)

	build_disturbance!(vis, model, name=name, α=α)
	animate_disturbance!(vis, anim, model, pθ[1:sample:end], name=name, offset=offset)
	return anim
end


# make sure to include this file in src/GeophysOperator.jl
module FooPhysics
# completely independent of transD_GP
# you can use whatever packages you want here

# this is probably where you'll store all variables needed for your physics
# e.g., wavenumbers and so on
mutable struct Foo
    x         :: Int
    somefield :: Vector{Float64}
end

# this is what will return the computation from our physics engine
function returnphysics!(F::Foo, m::AbstractArray)
    n = length(F.somefield)
    F.somefield .= F.x*rand(n) .+ sum(m[:]) # pre-allocated assignment
end

end


mutable struct ImgItpLast{TO,N,TI,NC,A<:AbstractArray{TI,N}} <: AbstractCachedArray{TO,N,TI,NC,A}
    parent::A
    coefs::Vector{Float64}
    cached::Array{TO,NC}
    current_I::Tuple
end

function ImgItpLast(img::Array34{T}, coefs, nd_cache=ndims(img)-1, out_type=Float64) where {T}
    nd = ndims(img)
    map(check01, coefs)
    if size(img,3) !== length(coefs)
        error("Input image size in the Z-slice dimension (3) should equal the number of interpolation coefficients provided")
    end
    if nd_cache > nd
        error("nd_cache must be less than or equal to ndims(img)")
    end
    ncache_rngs = axes(img)[nd_cache+1:nd]
    ci = map(first, ncache_rngs)
    za = ImgItpLast{out_type,nd,T,nd_cache,typeof(img)}(img, coefs, zeros(out_type, size(img)[1:nd_cache]...), ci)
    update_cache!(za, ci)
    return za
end

ImgItpLast(img::Array34{T}, coef::Float64, nd_cache=ndims(img)-1, out_type=Float64) where {T} =
    ImgItpLast(img, fill(coef, size(img, 3)), nd_cache, out_type)

function update_cache!(A::ImgItpLast{TO,N,TI,2}, inds::NTuple{NQ, Int}) where {TO,N,TI,NQ}
    #sometimes indexing works better than views with memory-mapped arrays
    pslice = A.parent[:, :, inds...]
    pslice_next = A.parent[:, :, Base.front(inds)...,last(inds)+1]
    coef = A.coefs[first(inds)] #get coef for this slice
    interp!(A.cached, pslice, pslice_next, coef)
    A.current_I = inds
end

function update_cache!(A::ImgItpLast{TO,N,TI,3}, inds::NTuple{NQ,Int}) where {TO,N,TI,NQ}
    #sometimes indexing works better than views with memory-mapped arrays
    pslice = A.parent[:, :, :, inds...]
    pslice_next = A.parent[:, :, :, Base.front(inds)...,last(inds)+1]
    interp!(A.cached, pslice, pslice_next, A.coefs)
    A.current_I = inds
end

size(A::ImgItpLast) = (Base.front(size(A.parent))...,last(size(A.parent))-1,)
Base.axes(A::ImgItpLast) = (Base.front(axes(parent(A)))..., first(last(axes(parent(A)))):(last(last(axes(parent(A))))-1),)
show(io::IO, A::ImgItpLast{TO}) where {TO} = print(io, "ImgItpLast of size $(size(A)) mapped to element type $TO\n")
show(io::IO, ::MIME"text/plain", A::ImgItpLast{TO}) where {TO} = show(io, A)

ImgItpLast(img::ImageMeta, coefs, out_type=Float64; kwargs...) = ImageMeta(ImgItpLast(data(img), coefs, out_type; kwargs...), properties(img))
ImgItpLast(img::AxisArray, coefs, out_type=Float64; kwargs...) = match_axisspacing(ImgItpLast(data(img),coefs,out_type; kwargs...), img)

#user interface
interp_last(A::AbstractArray, coef::Float64; kwargs...) = interp_last(A, fill(coef, size(A,3)); kwargs...)

function interp_last(A::AbstractArray{T,N}, coefs::Vector{Float64}; cache3d=false, out_type=Float64) where {T,N}
    if !in(N, (3,4))
        warn("$N-dimensional input was provided.  Only 3D and 4D inputs are tested; this will probably fail")
    end
    if N ==3 && cache3d
        cache3d = false
        warn("The cache3d kwarg was set for a 3D input matrix. This is only useful for 4D timeseries; ignoring kwarg")
    end
    return cache3d ? ImgItpLast(A, coefs, 3, out_type) : ImgItpLast(A, coefs, 2, out_type)
end


using Distributions
using Cytof5, Random, RCall
using JLD2, FileIO
import Printf
include("../sim_study/util.jl")
include("compress_data.jl")
Random.seed!(0)

function loadSingleObj(objPath)
  data = load(objPath)
  return data[collect(keys(data))[1]]
end

# TODO:
# Write a macro to reconstruct types

# Turn reconstructed type into type State
function convertState(s)
  println("Trying to convert reconstructed state to Cytof5.Model.State ...")
  return Cytof5.Model.State(Z=s.Z, delta=s.delta, alpha=s.alpha, v=s.v, W=s.W,
                            sig2=s.sig2, eta=s.eta, lam=s.lam, gam=s.gam,
                            y_imputed=s.y_imputed)
end

# FIXME: similarity_Z...
function convertConstants(c)
  println("Trying to convert reconstructed state to Cytof5.Model.Constants ...")
  return Cytof5.Model.Constants(alpha_prior=c.alpha_prior,
                                delta_prior=c.delta_prior, W_prior=c.W_prior,
                                eta_prior=c.eta_prior, sig2_prior=c.sig2_prior,
                                beta=c.beta, K=c.K, L=c.L,
                                sig2_range=c.sig2_range,
                                # there should be new stuff here ...
                                probFlip_Z=c.probFlip_Z, similarity_Z=x -> x)

end

# TODO: Write somrthing generic here.
function convertType(s, T)
  if T == Cytof5.Model.State
    return convertState(s)
  elseif T == Cytof5.Model.Constants
    return convertConstants(s)
  else
    println("Don't know how to convert to type $(T)")
  end
end

function post_process(path_to_output, thresh=0.9, min_presences=[0, .01, .03, .05])
  jld_filename = split(path_to_output, "/")[end]
  outdir = join(split(path_to_output, "/")[1:end-1], "/")
  # datapath = "data/reduced_cb.jld2"
  datapath = "$outdir/reduced_data/reduced_cb.jld2"
  println("datapath: $datapath")
  IMGDIR = "$outdir/img/"

  run(`mkdir -p $(IMGDIR)/dden/`)

  println("Loading Data ...")
  cbData = loadSingleObj(datapath)
  cbData = decompress_data(cbData)
  println("Loading Results ...")

  # FIXME: Not efficient!
  init_is_defined = "init" in keys(load(path_to_output))
  println("init_is_defined = $init_is_defined ...")

  if init_is_defined
    @load path_to_output out ll lastState c init dden
  else
    @load path_to_output out ll lastState c
  end

  I, K_MCMC = size(lastState.W)
  J = size(lastState.Z, 1)
  N = size.(lastState.y_imputed, 1)

  # Set png resolution settings
  s_png = 10
  fy(clus) = util.addCut(clus, s_png)
  fZ(Z) = util.addGridLines(Z, s_png)

  println("Making eta")
  open("$IMGDIR/eta.txt", "w") do file
    # TODO
    eta = util.getPosterior(:eta, out[1])
    eta0_mean = mean([ei[0] for ei in eta])
    eta1_mean = mean([ei[1] for ei in eta])
    eta_mean = Dict(0 => eta0_mean, 1 => eta1_mean)

    header = "i", "j", "z", "l", "eta"
    write(file, "$(join(header, "     "))\n")
    for i in 1:I
      for j in 1:J
        for z in 0:1
          for l in 1:c.L[z]
            line = (i, j, z, l, eta_mean[z][i, j, l])
            line = Printf.@sprintf("%d %5d %5d %5d     %.5f", line...);
            write(file, "$line\n")
          end
        end
      end
    end
  end

  println("Making eta_obs.txt")
  try
    open("$IMGDIR/eta_obs.txt", "w") do file
      header = "i", "j", "z", "l", "p"
      write(file, "$(join(header, "     "))\n")

      for i in 1:I
        for j in 1:J
          for z in 0:1
            for l in 1:c.L[z]
              idx_observed = util.idx_observed_ij(cbData, i, j)
              z_ij = out[1][end][:Z][j, out[1][end][:lam][i][idx_observed]]
              gam_ij = out[2][end][:gam][i][idx_observed, j]
              mij_sum = length(idx_observed)
              p = sum((gam_ij .== l) .& (z_ij .== z)) / mij_sum
              line = (i, j, z, l, p)
              line = Printf.@sprintf("%d %5d %5d %5d     %.5f", line...);
              write(file, "$line\n")
            end
          end
        end
      end
    end
  catch
    open("$IMGDIR/eta_obs.log", "w") do file
      write(file, "failed ... \n")
    end
  end

  # Plot loglikelihood
  util.plotPdf("$(IMGDIR)/ll_complete_history.pdf")
  util.plot(ll, ylab="log-likelihood", xlab="MCMC iteration", typ="l");
  util.abline(v=length(ll) - length(out[1]))
  util.devOff()

  util.plotPdf("$(IMGDIR)/ll_postburn.pdf")
  util.plot(ll[end-length(out[1]):end], ylab="log-likelihood", xlab="MCMC iteration", typ="l");
  util.devOff()

  function addGridLines(J::Int, K::Int, col="grey", lwd=1)
    util.abline(v=(1:K) .+ .5, h=(1:J) .+ .5, col=col, lwd=lwd);
  end

  # Plot Z
  Zpost = util.getPosterior(:Z, out[1])
  Zmean = mean(Zpost)

  println("Making Z ...")
  try
    util.plotPdf("$IMGDIR/Z_mean.pdf")
    util.myImage(Zmean, xlab="Features", ylab="Markers", addL=true, col=util.greys(11),
                 f=Z->addGridLines(J, K_MCMC));
    util.devOff()

    util.plotPdf("$IMGDIR/Z_mean_est_leftordered.pdf")
    util.myImage(Cytof5.Model.leftOrder((Zmean .> .5)*1),
                 xlab="Features", ylab="Markers", addL=true, col=util.greys(11),
                 f=Z->addGridLines(J, K_MCMC));
    util.devOff()
  catch
    println("Failed to make complete making Z ...")
  end

  println("Plot Z diffs")
  if init_is_defined
    util.plotPdf("$IMGDIR/Z_minus_init.pdf")
    for i in 1:length(Zpost)
      z = Zpost[i]
      util.myImage(z - init.Z, xlab="Features", ylab="Markers", addL=true, col=util.blueToRed(3),
                   zlim=[-1, 1], f=Z->addGridLines(J, K_MCMC), main="Z_$i - Z_0");
    end
    util.devOff()
  else
    println("Can't make Z-diffs because init isn't defined ...")
    run(`rm -f $IMGDIR/Z_minus_init.pdf`)
  end

  # Plot Posterior Density for i, j observed
  for i in 1:I
    for j in 1:J
      println("Plot posterior density for (i: $i, j: $j) observed ...")
      util.plotPdf("$(IMGDIR)/dden/dden_i$(i)_j$(j).pdf")
      dyij = util.density(filter(yij -> !isnan(yij), cbData[i][:, j]))
      dd_ij = hcat([dd[i, j] for dd in dden]...)
      pdyij_mean = R"rowMeans($dd_ij)" .+ 0
      pdyij_lower = R"apply($dd_ij, 1, quantile, .025)" .+ 0
      pdyij_upper = R"apply($dd_ij, 1, quantile, .975)" .+ 0
      h = maximum([dyij[:y] .+ 0; pdyij_upper])
      util.plot(c.y_grid, pdyij_mean, xlab="y", ylab="density", ylim=[0, h],
                main="i: $i, j: $j", col="blue", lwd=2, typ="l")
      util.colorBtwn(c.y_grid, pdyij_lower, pdyij_upper, from=-10, to=10,
                     col=util.rgba("blue", .3))
      util.lines(dyij, col="grey", lwd=2)
      util.devOff()
    end
  end

  for i in 1:I
    println("Separate graphs for yZ $(i)...")
    idx_best = R"estimate_ZWi_index($(out[1]), $i)"[1]
    Zi = out[1][idx_best][:Z]
    Wi = out[1][idx_best][:W][i,:]

    # Point Est for Wi and Zi. txt and pdf.
    open("$IMGDIR/W_$(i)_hat.txt", "w") do file
      write(file, "$(join(Wi, "\n"))\n")
    end

    open("$IMGDIR/W$(i)_hat_ordered_cumsum.txt", "w") do file
      ord = sortperm(Wi, rev=true)
      cs_wi_sorted = cumsum(Wi[ord])
      write(file, "num_features,k,wi,cumprop\n")
      for k in 1:K_MCMC
        write(file, "$(k),$(ord[k]),$(Wi[ord][k]),$(cs_wi_sorted[k])\n")
      end
    end

    open("$IMGDIR/Z$(i)_hat.txt", "w") do file
      for j in 1:J
        zj = join(Int.(Zi[j, :]), ",")
        write(file, "$(zj)\n")
      end
    end

    lami = out[1][idx_best][:lam][i]
    ord = sortperm(Wi, rev=true)
    lami = util.reorder_lami(ord, lami)

    for min_presence in min_presences
      common_celltypes = util.get_common_celltypes(Wi, thresh=min_presence,
                                                   filter_by_min_presence=true)
      println("common celltypes (min_presence > $min_presence): $common_celltypes")
      K_trunc = length(common_celltypes)

      util.plotPng("$IMGDIR/y_dat$(i)_only_minpresence$(min_presence).png", s=s_png)
      ord_yi = sortperm(lami)
      function f(y)
        util.addCut(lami, s_png)
        R"""
        axis(1, at=1:$J, label=1:$J, las=2, fg="grey", cex.axis=1.5)
        """
      end
      util.myImage(cbData[i][ord_yi[1 .<= lami[ord_yi] .<= K_trunc], :],
                   addL=true, f=f, xaxt="n",
                   zlim=[-4,4], col=util.blueToRed(9), na="black", xlab="markers",
                   var"cex.lab"=1.5, ylab="cells", var"cex.y.leg"=1.5);
      # util.myImage(cbData[i][sortperm(lami), :], addL=true, f=yi->util.addCut(lami),
      #              zlim=[-4,4], col=util.blueToRed(9), na="black", xlab="markers",
      #              ylab="cells");
      util.devOff()

      util.plotPdf("$IMGDIR/Z_hat$(i)_minpresence$(min_presence).pdf", w=5, h=10)
      util.myImage(Zi[:, common_celltypes], addL=false, ylab="markers", yaxt="n",
                   f=Z->addGridLines(J, K_trunc), xaxt="n", xlab="celltypes",
                   var"cex.lab"=1.5);

      perc = string.(round.(Wi[common_celltypes] * 100, digits=2), "%")
      R"""
      axis(3, at=1:$K_trunc, label=$(perc), las=1, fg="grey", cex.axis=1)
      axis(1, at=1:$K_trunc, label=$(common_celltypes), las=1,
           fg="grey", cex.axis=1)
      axis(2, at=1:$J, label=1:$J, las=2, fg="grey", cex.axis=1)
      """
      util.devOff()

      util.plotPdf("$IMGDIR/ZT_hat$(i)_minpresence$(min_presence).pdf")
      perc = string.(round.(Wi[common_celltypes] * 100, digits=2), "%")
      function fz(z)
        addGridLines(K_trunc, J)
        R"""
        axis(4, at=1:$K_trunc, label=$(perc), las=1, fg="grey", cex.axis=1.3)
        axis(2, at=1:$K_trunc, label=$(common_celltypes), las=1,
             fg="grey", cex.axis=1.3)
        axis(1, at=1:$J, label=1:$J, las=2, fg="grey", cex.axis=1.3)
        """
      end
      R"par(mar=c(5.1, 4, 2.1, 5.1))"
      util.myImage(Zi[:, common_celltypes]', addL=false, ylab="celltypes", yaxt="n",
                   f=fz, xaxt="n", xlab="markers", var"cex.lab"=1.5);
      R"par(mar=c(5, 4, 4, 2) + .1)"
      util.devOff()
    end
  end

  # Plot alpha
  alphaPost = util.getPosterior(:alpha, out[1])
  println("Making alpha ...")
  util.plotPdf("$(IMGDIR)/alpha.pdf")
  util.plotPost(alphaPost, ylab="density", xlab="alpha", main="");
  util.devOff()

  # Plot W
  Wpost = util.getPosterior(:W, out[1])
  W_mean = mean(Wpost)

  println("Making W...")
  util.plotPdf("$IMGDIR/W.pdf")

  cols = vcat([k .+ c.K .* (0:I-1) for k in 1:c.K]...)
  colnames = vec(["$i:$k" for i in 1:I, k in 1:c.K])
  W_post = hcat([hcat([w[i, :] for w in Wpost]...)' for i in 1:I]...)
  util.boxplot(W_post[:, cols], col=R"rep(2:$(I+1), $(c.K))", names=colnames, las=2, cex=.2);
  util.abline(v=(0:c.K) * I .+ .5, lty=2, col="grey")

  R"par(mfrow=c($I, 1), mar=c(5, 5.1, 0.5, 2.1))"
  for i in 1:I
    util.boxplot(hcat([w[i, :] for w in Wpost]...)', ylab="Posterior: W$i",
                 xlab=i<I ? "" : "Features",
                 col="steelblue", pch=20, cex=0);
  end
  R"par(mfrow=c(1, 1), mar=rcommon::mar.default())"
  util.devOff()

  # Missing Mechanism
  println("Making beta  ...")
  open("$IMGDIR/beta.txt", "w") do file
    for i in 1:I
      bi = join(c.beta[:, i], ", ")
      write(file, "for i=$i, beta = $bi \n")
    end
  end

  # y_imputed cache
  y_imputed = [ o[:y_imputed] for o in out[2] ]
  B = length(y_imputed)
  y_imputed_min = minimum(vcat([minimum.(yb) for yb in y_imputed]...))
  y_imputed_max = maximum(vcat([maximum.(yb) for yb in y_imputed]...))
  y_imputed_range = [y_imputed_min, y_imputed_max]

  # Plot missing mechanism
  println("Making prob miss  ...")
  util.plotPdf("$IMGDIR/prob_miss.pdf")
  R"par(mfrow=c($I, 1))"
  for i in 1:I
    util.plotProbMiss(c.beta, i, xlim=y_imputed_range)
  end
  R"par(mfrow=c(1,1))"
  util.devOff()

  # Get mus
  delta0Post = hcat([m[:delta][0] for m in out[1]]...)'
  delta1Post = hcat([m[:delta][1] for m in out[1]]...)'
  mus0Post = -cumsum(delta0Post, dims=2)
  mus1Post =  cumsum(delta1Post, dims=2)
  musPost = [ mus0Post mus1Post ]

  println("Making mus...")
  util.plotPdf("$IMGDIR/mus.pdf")
  util.boxplot(musPost, ylab="mu*", xlab="", xaxt="n", col="steelblue", pch=20, cex=0);
  util.abline(h=0, v=size(mus0Post, 2) + .5, col="grey30", lty=1);
  util.devOff()

  util.plotPdf("$IMGDIR/delta0.pdf")
  util.plotPosts(delta0Post);
  util.devOff()

  util.plotPdf("$IMGDIR/delta1.pdf")
  util.plotPosts(delta1Post);
  util.devOff()

  # Get sig2
  sig2Post = hcat(util.getPosterior(:sig2, out[1])...)'
  sig2Mean = mean(sig2Post, dims=1)
  sig2Sd = std(sig2Post, dims=1)

  println("Making sig2...")
  util.plotPdf("$IMGDIR/sig2.pdf")
  util.plotPosts(sig2Post);
  util.boxplot(sig2Post, ylab="sig2", xlab="sample",
               col="steelblue", pch=20, cex=0,
               var"cex.lab"=1.5, var"cex.axis"=1.5);
  util.devOff()

  println("Making v...")
  vPost = hcat(util.getPosterior(:v, out[1])...)'
  util.plotPdf("$IMGDIR/v.pdf")
  util.boxplot(vPost, ylab="v", xlab="", xaxt="n", col="steelblue", pch=20, cex=0);
  util.devOff()

  util.plotPdf("$IMGDIR/v_each.pdf")
  for k in 1:c.K
    util.plotPost(vPost[:, k], main="v_$k")
  end
  util.devOff()

  println("Making v_cumprod...")
  util.plotPdf("$IMGDIR/v_cumprod.pdf")
  util.boxplot(cumprod(vPost, dims=2),
               ylab="v_cumprod", xlab="", xaxt="n", col="steelblue", pch=20, cex=0);
  util.devOff()

  # Posterior of eps
  println("Making eps...")
  epsPost = hcat(util.getPosterior(:eps, out[1])...)'
  util.plotPdf("$IMGDIR/eps.pdf")
  util.plotPosts(epsPost);
  util.devOff()

  # Posterior of y_imputed
  println("Making ydatPost.pdf ...")
  util.plotPdf("$(IMGDIR)/ydatPost.pdf")
  R"par(mfrow=c(4,2))"
  for i in 1:I
    for j in 1:J
      print("\r i: $i, j: $j")
      numMissing = sum(cbData[i][:, j] .=== NaN)
      h = maximum(util.density(y_imputed[1][i][:, j])[:y])
      util.plot(util.density([cbData[i][:, j]; fill(-10, numMissing)], na=true, bw=.3),
                col="red", xlim=[-8,8], main="Y sample: $(i), marker: $(j),
                missing: $numMissing", bty="n", fg="grey", ylim=[0,h])
      for iter in 1:B
        yimp = y_imputed[iter]
        util.lines(util.density(yimp[i][:, j]), col=util.rgba("blue", .5))
      end
    end
  end
  R"par(mfrow=c(1,1))"
  println()
  util.devOff()

  idx_missing = [ findall(isnan.(cbData[i])) for i in 1:I ]
  num_missing_per_sample= 3
  idx_missing = [shuffle!(idx_missing[i])[1:num_missing_per_sample] for i in 1:I]
  idx_missing = [map(idx -> (i, idx[1], idx[2]), idx_missing[i]) for i in 1:I]
  idx_missing = vcat(idx_missing...)
  for idx in idx_missing
    i, n, j = idx
    y_inj = [y_imputed[b][i][n, j] for b in 1:B]
    util.plotPdf("$(IMGDIR)/y_trace_i$(i)_n$(n)_j$(j).pdf")
    util.hist(y_inj, col="blue", border="transparent",
              main="", xlab="y (i: 2, n: $n, j: $j)", ylab="counts")
    util.plot(y_inj, typ="l", xlab="Index", ylab="y (i: $i, n: $n, j: $j)", main="")
    util.devOff()
  end

  for i in 1:I
    idx_missing = findall(isnan.(cbData[i]))
    y_i = [y_imputed[b][i][idx_missing] for b in 1:B]
    util.plotPdf("$IMGDIR/y_imputed_hist_i$(i).pdf")
    R"par(mfrow=c($I, 1), mar=c(5, 5.1, 0.5, 2.1))"
    util.hist(mean(y_i),    xlim=y_imputed_range, xlab="means of imputed y for sample $i", ylab="counts", main="");
    util.hist(maximum(y_i), xlim=y_imputed_range, xlab="max of imputed y for sample $i", ylab="counts", main="");
    util.hist(minimum(y_i), xlim=y_imputed_range, xlab="min of imputed y for sample $i", ylab="counts", main="");
    R"par(mfrow=c(1,1), mar=mar.default())"
    util.devOff()
  end

  for i in 1:I
    util.plotPng("$IMGDIR/y_imputed$(i).png", s=s_png)
    util.yZ_inspect(out[1], i=i, lastState.y_imputed, zlim=[-4,4], using_zero_index=false,
                    thresh=thresh, col=util.blueToRed(9), fy=fy, fZ=fZ)
    util.devOff()
  end

  for i in 1:I
    util.plotPng("$IMGDIR/y_dat$(i).png", s=s_png)
    util.yZ_inspect(out[1], i=i, cbData, zlim=[-4,4], using_zero_index=false, na="black",
                    thresh=thresh, col=util.blueToRed(9), fy=fy, fZ=fZ)
    util.devOff()
  end

  println("Making yZ with Z mean...")
  if any(lastState.eps .== 0)
    for i in 1:I
      util.plotPng("$IMGDIR/y_dat$(i)_with_zmean.png", s=s_png)

      idx_best = R"estimate_ZWi_index($(out[1]), $i)"[1]
      Zi = out[1][idx_best][:Z]
      Wi = out[1][idx_best][:W][i,:]
      lami = out[1][idx_best][:lam][i]

      open("$IMGDIR/lam$(i)_best.txt", "w") do file
        for lam_in in lami
          write(file, "$(lam_in)\n")
        end
      end

      S = [findall(lami .== k) for k in 1:c.K]
      Zi_bar = mean([[mean(o[:Z][j, o[:lam][i][S[k]]]) for j in 1:J, k in 1:c.K] for o in out[1]])

      util.yZ(cbData[i], Zi_bar, Wi, lami, zlim=[-4,4], thresh=thresh, col=util.blueToRed(9),
              na="black", using_zero_index=false, col_Z=R"grey(seq(1, 0, len=11))",
              colorbar_Z=true, cex_z_leg=0.001, fy=fy, fZ=fZ)

      util.devOff()
    end
  else
    # Get lam
    println("Making lam...")
    lamPost = util.getPosterior(:lam, out[1])
    unique(lamPost)

    prop_lam0 = hcat([[mean(lam[i] .== 0) for lam in lamPost] for i in 1:I]...)
    sum_lam0 = hcat([[sum(lam[i] .== 0) for lam in lamPost] for i in 1:I]...)
    println("prop0 dim: $(size(prop_lam0))")
    util.plotPdf("$IMGDIR/lam0.pdf")
    util.boxplot(prop_lam0, ylab="P(lambda_i = 0 | data)",
                 xlab="sample", col="steelblue", pch=20, cex=0);
    util.devOff()

    util.plotPdf("$IMGDIR/lam0_counts.pdf")
    util.boxplot(sum_lam0, ylab="sum(lambda_i = 0 | data)",
                 xlab="sample", col="steelblue", pch=20, cex=0);
    util.devOff()

    # Plot lambda = 0 group
    println("Plot noisy group")
    for i in 1:I
      idx_best = R"estimate_ZWi_index($(out[1]), $i)"[1]
      lami = out[1][idx_best][:lam][i]

      open("$IMGDIR/lam$(i)_best.txt", "w") do file
        for lam_in in lami
          write(file, "$(lam_in)\n")
        end
      end

      idx0 = findall(lami .== 0)
      if length(idx0) > 0
        println("making y_dat$(i)_lam0.png")
        util.plotPng("$IMGDIR/y_dat$(i)_lam0.png", s=s_png)
        util.myImage(cbData[i][idx0, :], xlab="markers", ylab="cells",
                     na="black", col=util.blueToRed(9), addL=true, zlim=[-4, 4], nticks=11)
        util.devOff()

        util.plotPng("$IMGDIR/y_imputed$(i)_lam0.png", s=s_png)
        util.myImage(lastState.y_imputed[i][idx0, :], xlab="markers", ylab="cells",
                     na="black", col=util.blueToRed(9), addL=true, zlim=[-4, 4])
        util.devOff()
      else
        println("length(lam0) == 0. Not making y_dat$(i)_lam0.png!")
      end
    end
  end

  for i in 1:I
    println("Printing number of predominant celltypes for i = $i ...")
    idx_best = R"estimate_ZWi_index($(out[1]), $i)"[1]
    Wi = out[1][idx_best][:W][i,:]

    open("$IMGDIR/K_P$(round(Int, thresh * 100))_i$(i).txt", "w") do file
      cswi = cumsum(sort(Wi, rev=true))
      K_TOP = findfirst(cswi .> thresh)
      write(file, "K_TOP: $K_TOP \n")
    end
  end

  if init_is_defined
    println("Making y_dat init ...")
    for i in 1:I
      util.plotPng("$IMGDIR/y_dat$(i)_init.png", s=s_png)
      Zi = init.Z
      Wi = init.W[i, :]
      lami = init.lam[i]

      util.yZ(cbData[i], Zi, Wi, lami, zlim=[-4,4], thresh=thresh, col=util.blueToRed(9),
              na="black", using_zero_index=false, col_Z=R"grey(seq(1, 0, len=11))",
              colorbar_Z=true, cex_z_leg=0.001, fy=fy, fZ=fZ)
      util.devOff()
    end
  else
    println("Skipping this plot, init is not defined ...")
  end
end


using Plots
gr()

Random.seed!(2)

colors = [RGB(.22,.596,.149), RGB(.8,.361,.361), RGB(.702, .322, .8)]
border = [RGB(.133, .541, .133), RGB(.8, .2, .2), RGB(.584, .345, .698)]

y = rand(50)
for i in eachindex(y)
    if rand() < .8
        y[i] = 0.0
    end
end

scatter(y, c = colors, ms = 10, markerstrokewidth = 5, markerstrokecolor = border,
    legend=false, grid=false, axis=false)

savefig(joinpath(@__DIR__(), "logo.png"))


__precompile__()

module DiffEqFinancial
  using DiffEqBase, DiffEqNoiseProcess, Markdown, LinearAlgebra

  import RandomNumbers: Xorshifts

  include("problems.jl")

  export HestonProblem, BlackScholesProblem, GeneralizedBlackScholesProblem,
  ExtendedOrnsteinUhlenbeckProblem, OrnsteinUhlenbeckProblem, GeometricBrownianMotionProblem,
  MfStateProblem

end # module


using ADatasets
using EvalCurves

"""
    gridSearch(f, parameters...)
Maps `f` to product of `parameters`.
"""
gridsearch(f, parameters...) = map(p -> f(p), Base.product(parameters...))

# function printandrun(f, p)
#     println(p)
#     (p, f(p))
# end

global mainfolder = "D:/dev/julia/"
if !isdir(mainfolder)
    mainfolder = "/home/bimjan/dev/julia/"
    if !isdir(mainfolder)
        error("The main folder is unknown")
    end
end
const datafolder = mainfolder * "data/loda/public/datasets/numerical"

loaddata(dataset, difficulty) =  ADatasets.makeset(ADatasets.loaddataset(dataset, difficulty, datafolder)..., 0.8, "low")

computeauc(score, labels) = EvalCurves.auc(EvalCurves.roccurve(score, labels)...)


#
# Empty storage
#

struct Empty{ElT, StoreT <: TensorStorage} <: TensorStorage{ElT}
end

# Get the Empty version of the TensorStorage
function empty(::Type{StoreT}) where {StoreT <: TensorStorage{ElT}} where {ElT}
  return Empty{ElT, StoreT}
end

# Defaults to Dense
function Empty(::Type{ElT}) where {ElT}
  return empty(Dense{ElT, Vector{ElT}})()
end

Empty() = Empty(Float64)

Base.copy(S::Empty) = S

Base.isempty(::Empty) = true

nnzblocks(::Empty) = 0

nnz(::Empty) = 0

function Base.complex(::Type{<: Empty{ElT, StoreT}}) where {ElT,
                                                            StoreT}
  return Empty{complex(ElT), complex(StoreT)}
end

function Base.complex(S::Empty)
  return complex(typeof(S))()
end

#Base.size(::Empty) = 0

function Base.show(io::IO,
                   mime::MIME"text/plain",
                   S::Empty)
  println(io, typeof(S))
end

#
# EmptyTensor (Tensor using Empty storage)
#

const EmptyTensor{ElT,
                  N,
                  StoreT,
                  IndsT} = Tensor{ElT,
                                  N,
                                  StoreT,
                                  IndsT} where {StoreT <: Empty}

Base.isempty(::EmptyTensor) = true

function EmptyTensor(::Type{ElT}, inds) where {ElT <: Number}
  return tensor(Empty(ElT), inds)
end

function EmptyTensor(::Type{StoreT}, inds) where {StoreT <: TensorStorage}
  return tensor(empty(StoreT)(), inds)
end

function EmptyBlockSparseTensor(::Type{ElT}, inds) where {ElT <: Number}
  StoreT = BlockSparse{ElT, Vector{ElT}, length(inds)}
  return EmptyTensor(StoreT, inds)
end

# From an EmptyTensor, return the closest Tensor type
function Base.fill(::Type{<: Tensor{ElT,
                                    N,
                                    EStoreT,
                                    IndsT}}) where {ElT <: Number,
                                                    N,
                                                    EStoreT <: Empty{ElT,
                                                                     StoreT},
                                                    IndsT} where {StoreT}
  return Tensor{ElT, N, StoreT, IndsT}
end

function Base.zeros(T::TensorT) where {TensorT <: EmptyTensor}
  TensorR = fill(TensorT)
  return zeros(TensorR, inds(T))
end

function addblock(T::EmptyTensor{<: Number, N},
                  block::Block{N}) where {N}
  R = zeros(T)
  addblock!(R, block)
  return R
end

addblock!!(T::EmptyTensor{<: Number, N},
           block::Block{N}) where {N} = addblock(T, block)

Base.@propagate_inbounds function Base.setindex(T::EmptyTensor{<: Number, N},
                                                x::Number,
                                                I::Vararg{Int, N}) where {N}
  R = zeros(T)
  R[I...] = x
  return R
end

function Base.setindex(T::EmptyTensor{<: Number, Any},
                                      x::Number,
                                      I::Int...)
  error("Setting element of EmptyTensor with Any number of dimensions not defined")
end

setindex!!(T::EmptyTensor,
           x::Number,
           I::Int...) = setindex(T, x, I...)

function Base.show(io::IO,
                   mime::MIME"text/plain",
                   T::EmptyTensor)
  summary(io, T)
  println(io)
end


using GeoData, Test, Statistics, Dates, Plots, NCDatasets, ArchGDAL
using GeoData: name, mode, window, DiskStack
testpath = joinpath(dirname(pathof(GeoData)), "../test/")
include(joinpath(testpath, "test_utils.jl"))

maybedownload("https://raw.githubusercontent.com/rspatial/raster/master/inst/external/rlogo.grd", "rlogo.grd")
maybedownload("https://github.com/rspatial/raster/raw/master/inst/external/rlogo.gri", "rlogo.gri")
path = joinpath(testpath, "data/rlogo")
@test isfile(path * ".grd")
@test isfile(path * ".gri")

@testset "Grd array" begin
    grdarray = GRDarray(path);

    @testset "array properties" begin
        @test grdarray isa GRDarray{Float32,3}
    end

    @testset "dimensions" begin
        @test length(val(dims(dims(grdarray), Lon))) == 101
        @test ndims(grdarray) == 3
        @test dims(grdarray) isa Tuple{<:Lon,<:Lat,<:Band}
        @test refdims(grdarray) == ()
        @test bounds(grdarray) == ((0.0, 101.0), (0.0, 77.0), (1, 3))
    end

    @testset "other fields" begin
        @test missingval(grdarray) == -3.4f38
        @test metadata(grdarray) isa GRDarrayMetadata
        @test name(grdarray) == Symbol("red:green:blue")
        @test label(grdarray) == "red:green:blue"
        @test units(grdarray) == nothing
        customgrdarray = GRDarray(path; name=:test, usercrs=EPSG(4326));
        @test name(customgrdarray) == :test
        @test label(customgrdarray) == "test"
        @test usercrs(dims(customgrdarray, Lat)) == EPSG(4326)
        @test usercrs(dims(customgrdarray, Lon)) == EPSG(4326)
        @test usercrs(customgrdarray) == EPSG(4326)
        proj = ProjString("+proj=merc +datum=WGS84")
        @test crs(dims(customgrdarray, Lat)) == proj
        @test crs(dims(customgrdarray, Lon)) == proj
        @test crs(customgrdarray) == proj
    end

    @testset "getindex" begin
        @test grdarray[Band(1)] isa GeoArray{Float32,2}
        @test grdarray[Lat(1), Band(1)] isa GeoArray{Float32,1}
        @test grdarray[Lon(1), Band(1)] isa GeoArray{Float32,1}
        @test grdarray[Lon(50), Lat(30), Band(1)] == 115.0f0
        @test grdarray[1, 1, 1] == 255.0f0
        @test grdarray[Lat(At(20)), Lon(At(20)), Band(3)] == 255.0f0
        @test grdarray[Lat(Contains(60)), Lon(Contains(20)), Band(1)] == 255.0f0
    end

    # @testset "setindex" begin
    #     A = grdarray[:, :, :]
    #     temp = grdarray[1, 1, 1]
    #     println(temp)
    #     @test temp != 100.0f0
    #     grdarray[1, 1, 1] = 100.0f0
    #     grdarray[:, :, :] = 100.0f0
    #     @test grdarray[1, 1, 1] == 100.0f0
    #     grdarray[1, 1, 1] = temp
    #     @test grdarray[1, 1, 1] == temp
    #     println("sum: ", sum(A .- grdarray[:, :, :]))
    #     temp = grdarray[Lon(20), Lat(10), Band(3)]
    #     println(temp)
    #     @test temp != 200.0f0
    #     grdarray[Lon(20), Lat(10), Band(3)] = 200.0f0
    #     @test grdarray[20, 10, 3] == 200.0f0
    #     grdarray[Lon(20), Lat(10), Band(3)] = temp
    # end

    @testset "selectors" begin
        geoarray = grdarray[Lat(Contains(3)), Lon(:), Band(1)]
        @test geoarray isa GeoArray{Float32,1}
        @test grdarray[Lon(Contains(20)), Lat(Contains(10)), Band(1)] isa Float32
    end

    @testset "conversion to GeoArray" begin
        geoarray = grdarray[Lon(1:50), Lat(1:1), Band(1)]
        @test size(geoarray) == (50, 1)
        @test eltype(geoarray) <: Float32
        @time geoarray isa GeoArray{Float32,1}
        @test dims(geoarray) isa Tuple{<:Lon,Lat}
        @test refdims(geoarray) isa Tuple{<:Band}
        @test metadata(geoarray) == metadata(grdarray)
        @test missingval(geoarray) == -3.4f38
        @test name(geoarray) == Symbol("red:green:blue")
    end

    @testset "save" begin
        @testset "2d" begin
            filename2 = tempname()
            write(filename2, GRDarray, grdarray[Band(1)])
            saved = GeoArray(GRDarray(filename2))
            # 1 band is added again on save
            @test size(saved) == size(grdarray[Band(1:1)])
            @test data(saved) == data(grdarray[Band(1:1)])
        end

        @testset "3d with subset" begin
            geoarray = GeoArray(grdarray)[1:100, 1:50, 1:2]
            filename = tempname()
            write(filename, GRDarray, geoarray)
            saved = GeoArray(GRDarray(filename))
            @test size(saved) == size(geoarray)
            @test refdims(saved) == ()
            @test bounds(saved) == bounds(geoarray)
            @test size(saved) == size(geoarray)
            @test missingval(saved) === missingval(geoarray)
            @test metadata(saved) != metadata(geoarray)
            @test metadata(saved)["creator"] == "GeoData.jl"
            @test all(metadata.(dims(saved)) .== metadata.(dims(geoarray)))
            @test name(saved) == name(geoarray)
            @test all(mode.(dims(saved)) .== mode.(dims(geoarray)))
            @test dims(saved) isa typeof(dims(geoarray))
            @test all(val.(dims(saved)) .== val.(dims(geoarray)))
            @test all(mode.(dims(saved)) .== mode.(dims(geoarray)))
            @test all(metadata.(dims(saved)) .== metadata.(dims(geoarray)))
            @test dims(saved) == dims(geoarray)
            @test all(data(saved) .=== data(geoarray))
            @test saved isa typeof(geoarray)
            @test data(saved) == data(geoarray)
        end

        @testset "to netcdf" begin
            filename2 = tempname()
            write(filename2, NCDarray, grdarray[Band(1)])
            saved = GeoArray(NCDarray(filename2))
            @test size(saved) == size(grdarray[Band(1)])
            @test replace_missing(saved, missingval(grdarray)) ≈ reverse(grdarray[Band(1)]; dims=Lat)
            @test replace_missing(saved, missingval(grdarray)) ≈ reverse(grdarray[Band(1)]; dims=Lat)
            @test_broken val(dims(saved, Lon)) ≈ val(dims(grdarray, Lon)) .+ 0.5
            @test_broken val(dims(saved, Lat)) ≈ val(dims(grdarray, Lat)) .+ 0.5
            @test_broken bounds(saved, Lat) == bounds(grdarray, Lat)
            @test_broken bounds(saved, Lon) == bounds(grdarray, Lon)
        end

        @testset "to gdal" begin
            # No Band
            gdalfilename = tempname() * ".tif"
            write(gdalfilename, GDALarray, grdarray[Band(1)])
            size(grdarray)
            gdalarray = GDALarray(gdalfilename)
            # @test convert(ProjString, crs(gdalarray)) == convert(ProjString, EPSG(4326))
            @test val(dims(gdalarray, Lon)) ≈ val(dims(grdarray, Lon))
            @test reverse(val(dims(gdalarray, Lat))) ≈ val(dims(grdarray, Lat))
            @test GeoArray(gdalarray) ≈ permutedims(grdarray[Band(1)], [Lon(), Lat()])
            # 3 Bands
            gdalfilename2 = tempname() * ".tif"
            write(gdalfilename2, GDALarray, grdarray)
            gdalarray2 = GDALarray(gdalfilename2)
            @test all(GeoArray(gdalarray2) .== GeoArray(grdarray))
            @test val(dims(gdalarray2, Band)) == 1:3
        end

    end

    @testset "show" begin
        sh = sprint(show, grdarray)
        # Test but don't lock this down too much
        @test contains(sh, "GRDarray")
        @test contains(sh, "Latitude")
        @test contains(sh, "Longitude")
        @test contains(sh, "Band")
    end

    @testset "plot" begin
        p = grdarray |> plot
    end

end

@testset "Grd stack" begin
    grdstack = GRDstack((a=path, b=path))

    @testset "indexing" begin
        @test grdstack[:a][Lat(20), Lon(20), Band(3)] == 70.0f0
        @test grdstack[:a][Lat([2,3]), Lon(40), Band(2)] == [240.0f0, 246.0f0]
    end

    @testset "child array properties" begin
        @test size(grdstack[:a]) == size(GeoArray(grdstack[:a])) == (101, 77, 3)
        @test grdstack[:a] isa GeoArray{Float32,3}
    end

    @testset "window" begin
        windowedstack = GRDstack((a=path, b=path); window=(Lat(1:5), Lon(1:5), Band(1)))
        @test window(windowedstack) == (Lat(1:5), Lon(1:5), Band(1))
        windowedarray = windowedstack[:a]
        @test windowedarray isa GeoArray{Float32,2}
        @test length.(dims(windowedarray)) == (5, 5)
        @test size(windowedarray) == (5, 5)
        @test windowedarray[1:3, 2:2] == reshape([255.0f0, 255.0f0, 255.0f0], 3, 1)
        @test windowedarray[1:3, 2] == [255.0f0, 255.0f0, 255.0f0]
        @test windowedarray[1, 2] == 255.0f0
        windowedstack = GRDstack((a=path, b=path); window=(Lat(1:5), Lon(1:5), Band(1:1)))
        windowedarray = windowedstack[:b]
        @test windowedarray[1:3, 2:2, 1:1] == reshape([255.0f0, 255.0f0, 255.0f0], 3, 1, 1)
        @test windowedarray[1:3, 2:2, 1] == reshape([255.0f0, 255.0f0, 255.0f0], 3, 1)
        @test windowedarray[1:3, 2, 1] == [255.0f0, 255.0f0, 255.0f0]
        @test windowedarray[1, 2, 1] == 255.0f0
        windowedstack = GRDstack((a=path, b=path); window=(Band(1),));
        windowedarray = windowedstack[:b]
        @test windowedarray[1:3, 2:2] == reshape([255.0f0, 255.0f0, 255.0f0], 3, 1)
        @test windowedarray[1:3, 2] == [255.0f0, 255.0f0, 255.0f0]
        @test windowedarray[30, 30] == 185.0f0
        windowedarray |> plot
    end

    # Stack Constructors
    @testset "conversion to GeoStack" begin
        stack = GeoStack(grdstack)
        @test Symbol.(Tuple(keys(grdstack))) == keys(stack)
        smallstack = GeoStack(grdstack; keys=(:a,))
        @test keys(smallstack) == (:a,)
    end

    if VERSION > v"1.1-"
        @testset "copy" begin
            geoarray = zero(GeoArray(grdstack[:a]))
            copy!(geoarray, grdstack, :a)
            # First wrap with GeoArray() here or == loads from disk for each cell.
            # we need a general way of avoiding this in all disk-based sources
            @test geoarray == GeoArray(grdstack[:a])
        end
    end

    @testset "save" begin
        geoarray = GeoArray(grdstack[:b])
        filename = tempname()
        write(filename, GRDarray, grdstack)
        base, ext = splitext(filename)
        filename_b = string(base, "_b", ext)
        saved = GeoArray(GRDarray(filename_b))
        @test typeof(saved) == typeof(geoarray)
        @test data(saved) == data(geoarray)
    end

end

@testset "Grd series" begin
    series = GeoSeries([path, path], (Ti,); childtype=GRDarray, childkwargs=(usercrs=EPSG(4326), name=:test))
    @test GeoArray(series[Ti(1)]) ==
        GeoArray(GRDarray(path; usercrs=EPSG(4326), name=:test))
    stacks = [DiskStack((a=path, b=path); childtype=GRDarray, childkwargs=(usercrs=EPSG(4326), name=:test))]
    series = GeoSeries(stacks, (Ti,))
    @test series[Ti(1)][:a] ==
        GeoArray(GRDarray(path; usercrs=EPSG(4326), name=:test))
    modified_series = modify(Array, series)
    @test typeof(modified_series) <: GeoSeries{<:GeoStack{<:NamedTuple{(:a,:b),<:Tuple{<:GeoArray{Float32,3,<:Tuple,<:Tuple,<:Array{Float32,3}},Vararg}}}}
end

nothing


"""
    Classic(;method=median)
    Classic(method)

Classic PSF subtraction using the median of entire data cube. If another statistic is desired, like the `mean`, it can be passed as an argument as long as it supports slicing multi-dimensional arrays.

# References
1. [Marois et al. 2006](http://adsabs.harvard.edu/abs/2006ApJ...641..556M) Angular Differential Imaging: A Powerful High-Contrast Imaging Technique
"""
struct Classic <: ADIAlgorithm
    method
end
Classic(;method=median) = Classic(method)

"""
    ADI.ClassicDesign(n, frame)

Output for the [`Classic`](@ref) algorithm which contains the static frame unrolled into a vector (with size `(1, Npx)`). [`reconstruct`](@ref) will tile this vector in a non-allocating way `n` times to appear like a flattened cube. [`ADI.design`](@ref) will return the static frame .
"""
struct ClassicDesign{FT} <: ADIDesign
    n::Int # number of frames in original data
    frame::FT
end

design(des::ClassicDesign) = des.frame

reconstruct(des::ClassicDesign) = TiledArray(des.frame, des.n)

function fit(alg::Classic, data::AbstractMatrix; ref=data, kwargs...)
    n = size(data, 1)
    return ClassicDesign(n, alg.method(ref, dims=1))
end

#######################################

struct TiledArray{T,N,AT<:AbstractArray{T,N},RT} <: AbstractArray{T,N}
    parent::AT
    tilerange::RT
end

Base.size(t::TiledArray) = map(length, axes(t))
Base.axes(t::TiledArray) = (t.tilerange, Base.tail(axes(t.parent))...)
Base.parent(t::TiledArray) = t.parent

TiledArray(parent::AbstractArray, n::Int) = TiledArray(parent, Base.OneTo(n))

Base.@propagate_inbounds function Base.getindex(t::TiledArray{T,N}, idxs::Vararg{Int,N}) where {T,N}
    @boundscheck checkbounds(t, idxs...)
    return t.parent[firstindex(t.parent, 1), Base.tail(idxs)...]
end


using ITensors, Test

@testset "sweepnext function" begin
  @testset "one site" begin
    N = 6
    count = 1
    output = [
      (1, 1),
      (2, 1),
      (3, 1),
      (4, 1),
      (5, 1),
      (6, 1),
      (6, 2),
      (5, 2),
      (4, 2),
      (3, 2),
      (2, 2),
      (1, 2),
    ]
    for (b, ha) in sweepnext(N; ncenter=1)
      @test (b, ha) == output[count]
      count += 1
    end
    @test count == 2 * N + 1
  end

  @testset "two site" begin
    N = 6
    count = 1
    output = [
      (1, 1), (2, 1), (3, 1), (4, 1), (5, 1), (5, 2), (4, 2), (3, 2), (2, 2), (1, 2)
    ]
    for (b, ha) in sweepnext(N)
      @test (b, ha) == output[count]
      count += 1
    end
    @test count == 2 * (N - 1) + 1
  end

  @testset "three site" begin
    N = 6
    count = 1
    output = [(1, 1), (2, 1), (3, 1), (4, 1), (4, 2), (3, 2), (2, 2), (1, 2)]
    for (b, ha) in sweepnext(N; ncenter=3)
      @test (b, ha) == output[count]
      count += 1
    end
    @test count == 2 * (N - 2) + 1
  end
end

nothing


# This file contains code that was formerly a part of Julia. License is MIT: http://julialang.org/license

# slightly optimized version of DataStructure.jl priority queue
# swapped index field from a dict to an array
# removed checking for presence of keys when inserting

# Binary heap indexing
heapleft(i::Int)   = i << 1         #2i
heapright(i::Int)  = (i << 1) + 1   #2i + 1
heapparent(i::Int) = i >> 1         #div(i, 2)

using StaticArrays
mutable struct ArrayPQ{K,V,O<:Ordering} <: AbstractDict{K,V}
    # Binary heap of (element, priority) pairs.
    xs::Vector{Pair{K,V}}
    o::O

    # Map elements to their index in xs
    index::Vector{Int}

    function ArrayPQ{K,V,O}(o::O, itr) where {K,V,O<:Ordering}
        xs    = Vector{Pair{K,V}}(undef, length(itr))
        index = Vector{Int}(length(itr))
        @inbounds for (i,p) in enumerate(itr)
            xs[i]    = p
            index[i] = i
        end
        pq = new{K,V,O}(xs, o, index)

        # heapify
        for i in heapparent(length(pq.xs)):-1:1
            percolate_down!(pq, i)
        end

        pq
    end
end
ArrayPQ(kv::Vector{T}, o::Ordering=Forward) where {K,V,T <: Pair{K,V}} = ArrayPQ{K,V,typeof(o)}(o, kv)
ArrayPQ(kv, o::Ordering=Forward) = ArrayPQ(o, kv)

length(pq::ArrayPQ)  = length(pq.xs)
isempty(pq::ArrayPQ) = isempty(pq.xs)
peek(pq::ArrayPQ)    = (@inbounds return pq.xs[1])

function percolate_down!(pq::ArrayPQ, i::Integer)
    @inbounds x = pq.xs[i]
    @inbounds while (l = heapleft(i)) <= length(pq)
        r = heapright(i)
        j = r > length(pq) || lt(pq.o, pq.xs[l].second, pq.xs[r].second) ? l : r
        if lt(pq.o, pq.xs[j].second, x.second)
            pq.index[pq.xs[j].first] = i
            pq.xs[i] = pq.xs[j]
            i = j
        else
            break
        end
    end
    @inbounds pq.index[x.first] = i
    @inbounds pq.xs[i] = x
    nothing
end

function percolate_up!(pq::ArrayPQ, i::Integer)
    @inbounds x = pq.xs[i]
    @inbounds while i > 1
        j = heapparent(i)
        if lt(pq.o, x.second, pq.xs[j].second)
            pq.index[pq.xs[j].first] = i
            pq.xs[i] = pq.xs[j]
            i = j
        else
            break
        end
    end
    @inbounds pq.index[x.first] = i
    @inbounds pq.xs[i] = x
    nothing
end

function getindex(pq::ArrayPQ{K,V}, key) where {K,V}
    @inbounds return pq.xs[pq.index[key]].second
end

# Change the priority of an existing element
# does not check if the element is present
function setindex!(pq::ArrayPQ{K, V}, value, key) where {K,V}
    @inbounds i = pq.index[key]
    @inbounds oldvalue = pq.xs[i].second
    @inbounds pq.xs[i] = Pair{K,V}(key, value)
    if lt(pq.o, oldvalue, value)
        percolate_down!(pq, i)
    else
        percolate_up!(pq, i)
    end
    value
end

# Unordered iteration through key value pairs in a ArrayPQ
start(pq::ArrayPQ) = start(pq.index)
done(pq::ArrayPQ, i) = done(pq.index, i)
function next(pq::ArrayPQ{K,V}, i) where {K,V}
    (k, idx), i = next(pq.index, i)
    return (pq.xs[idx], i)
end


## modeled after sort tests in python's sortperf.py
##
## Kevin Squire
##

module SortPerf

export sortperf, sort_plots, view_sort_plots, save_sort_plots, std_sort_tests, sort_median, sort_scale

import Base.Sort: Algorithm, Forward, ReverseOrdering, ord
import Base.Order.Ordering

using SortingAlgorithms
using DataFrames
using Winston
using Color

# rand functions for testing
randstr(n::Int) = [randstring() for i = 1:n]
randint(n::Int) = rand(1:n,n)

randfns = Dict(Int => randint,
               Int32 => x->round(Int32,randint(x)),
               Int64 => x->round(Int64,randint(x)),
               Int128 => x->round(Int128,randint(x)),
               Float32 => x->round(Float32,rand(x)),
               Float64 => rand,
               String => randstr)

std_types = [Int, Float64, String]
sort_algs = [InsertionSort, HeapSort, MergeSort, QuickSort, RadixSort, TimSort] #, TimSortUnstable]

# DataFrame labels
labels = [symbol("test_type"), symbol("sort_alg"), symbol("log_size"), symbol("size"), symbol("*sort"),
          symbol("\\sort"), symbol("/sort"), symbol("3sort"), symbol("+sort"), symbol("~sort"), symbol("=sort"), symbol("!sort")]

# Corresponding descriptions
sort_descr = Dict( '*' => "random",
                   '\\' => "reversed",
                   '/' => "sorted",
                   '3' => "3 exchanges",
                   '+' => "10 appended",
                   '~' => "4 unique items",
                   '=' => "all equal",
                   '!' => "qsort median killer")

# Test algorithm performance on a data vector
function _sortperf(alg::Algorithm, origdata::Vector, order::Ordering=Forward; replicates=3, skip_median_killer=false)
    srand(1)
    times = Dict[]   # Array of Dicts!
    n = length(origdata)
    logn = log2(length(origdata))

    rec = Dict(symbol("test_type") => string(eltype(origdata)),
               symbol("sort_alg") => string(alg)[1:end-5],
               symbol("log_size") => isinteger(logn) ? round(Int, logn) : logn,
               symbol("size") => length(origdata))

    if alg==RadixSort && !isbits(origdata[1])
        println("Skipping RadixSort for non bitstype $(eltype(origdata))")
        reptimes = Dict{Symbol, Float64}()
        for label in labels[5:end]
            reptimes[label] = 0.0
        end
        return times
    end

    for rep = 1:replicates
        print(" $rep")
        reptimes = Dict{Symbol, Float64}()

        ## Random
        data = copy(origdata)
        gc()
        if !issorted(data, order)
            reptimes[symbol("*sort")] = @elapsed sort!(data, alg, order)
            @assert issorted(data, order)
        end

        ## Sorted
        gc()
        reptimes[symbol("/sort")] = @elapsed sort!(data, alg, order)
        @assert issorted(data, order)

        ## Reverse sorted
        reverse!(data)
        @assert issorted(data, ReverseOrdering(order))
        gc()
        reptimes[symbol("\\sort")] = @elapsed sort!(data, alg, order)
        @assert issorted(data, order)

        ## Sorted with 3 exchanges
        for i = 1:3
            n1 = rand(1:n)
            n2 = rand(1:n)
            data[n1], data[n2] = data[n2], data[n1]
        end
        gc()
        reptimes[symbol("3sort")] = @elapsed sort!(data, alg, order)
        @assert issorted(data, order)

        ## Sorted with 10 unsorted values at end
        if length(data) >= 20
            for i = 1:10
                val = splice!(data, i:i)
                append!(data, val)
            end
            gc()
            reptimes[symbol("+sort")] = @elapsed sort!(data, alg, order)
            @assert issorted(data, order)
        end

        ## Random data with 4 unique values
        idxs = Int[]
        for i = 1:4
            idx = rand(1:n)
            while idx in idxs
                idx = rand(1:n)
            end
            push!(idxs, idx)
        end
        vals = data[idxs]
        data4 = vals[rand(1:4, n)]
        gc()
        reptimes[symbol("~sort")] = @elapsed sort!(data4, alg, order)
        @assert issorted(data, order)

        ## All values equal
        data1 = data[fill(rand(1:n), n)]
        gc()
        reptimes[symbol("=sort")] = @elapsed sort!(data1, alg, order)
        @assert issorted(data1, order)

        ## quicksort median killer: first half descending, second half ascending
        if skip_median_killer || alg==QuickSort && !(eltype(data) <: Integer) && length(data) >= 2^18
            print("(median killer skipped) ")
        else
            last = (length(data)>>1)<<1 # make sure data length is even
            qdata = vcat(data[last:-2:2], data[1:2:last])
            gc()
            reptimes[symbol("!sort")] = @elapsed sort!(qdata, alg, order)
            @assert issorted(qdata, order)
        end

        push!(times, merge(rec, reptimes))
    end
    println()

    times
end

# run sortperf over a range of data lengths
function _sortperf(alg::Algorithm, data::Vector, log2range::Range, order::Ordering=Forward; named...)
    lg2range = filter(x -> 2^x <= length(data), collect(log2range))
    times = Dict[]
    for logsize in lg2range
        println("  $logsize")
        size = 2^logsize
        if (alg == InsertionSort && logsize >= 14)
            println("Skipped")
            continue
        end

        append!(times, _sortperf(alg, data[1:size], order; named...))
    end

    times
end

# Generate a random array and run sortperf
function _sortperf(alg::Algorithm, T::Type, size::Int, args...; named...)
    println("Testing $T...")
    _sortperf(alg, randfns[T](size), args...; named...)
end

# Generate a random array and run sortperf over different ranges of that array
function _sortperf(alg::Algorithm, T::Type, log2range::Range, args...; named...)
    println("Testing $T...")
    _sortperf(alg, randfns[T](2^last(log2range)), log2range, args...; named...)
end

# Run sortperf for a number of types
function _sortperf(alg::Algorithm, types::Vector{DataType}, args...; named...)
    times = Dict[]
    for T in types
        append!(times, _sortperf(alg, T, args...; named...))
    end
    times
end

# Run sortperf on a number of algorithms
function _sortperf(algs::Vector{Algorithm}, args...; named...)
    times = Dict[]
    for alg in algs
        println("\n$alg\n")
        append!(times, _sortperf(alg, args...; named...))
    end
    times
end

# Returns a DataFrame version of sortperf output
sortperf(args...; named...) = DataFrame(_sortperf(args...; named...), labels)

# Get median sort timings
sort_median(df::DataFrame) = groupby(df, [:log_size, :size, :sort_alg, :test_type]) |> median

sort_scale(df::DataFrame, base_sort) = by(df, ["log_size", "size", "test_type"],
                                          x->(row = find(x["sort_alg"] .== base_sort);
                                              ht = size(x,1);
                                              hcat(x[:,3:3],x[:,5:end]./vcat(rep(x[row,5:end],ht)...))))

# Create sort plots
function sort_plots(df, base_sort, cols = ["*sort_median", "\\sort_median", "/sort_median", "3sort_median",
                                           "+sort_median", "~sort_median", "=sort_median", "!sort_median"])
    plots = PlotContainer[]
    sort_algs = unique(df["sort_alg"])
    dc = distinguishable_colors(12)[[1,3,4,5,6,7,8,9,10,11,12]]
    colors = Dict([symbol(alg) for alg in sort_algs], Uint32[convert(RGB24, r) for r in dc])
    linestyles = repmat(["solid", "dotted", "dotdashed"]', 6, 1)[:]
    for test_type_df in groupby(df, "test_type")
        test = test_type_df[1, "test_type"]

        for col in cols
            if length(DataFrames.removeNA(df[col])) == 0; continue; end

            seq = split(col, '_')[1]
            seq_name = sort_descr[seq[1]]

            ts = "$(test)_$(seq_name)" |> x -> replace(x, r"[ \[]", "_") |> x -> replace(x, '\]', "")

            println("Plotting $ts")
            plt = FramedPlot()
            setattr(plt, "xlabel", "log n")
            setattr(plt, "ylabel", "time (relative to $base_sort")
            setattr(plt, "title", "Sort Comparison ($test, $seq_name)")
            #setattr(plt.y1, "log", true )
            setattr(plt, "yrange", (0, 1.35))
            setattr(plt, "width", 768)

            curves = Any[]
            for (i, sort_alg_df_all) in enumerate(groupby(test_type_df, "sort_alg"))
                sort_alg_df = sort_alg_df_all[complete_cases(sort_alg_df_all[:,[col]]),:]
                sort_alg = sort_alg_df[1, "sort_alg"]
                c = colors[sort_alg]
                s = linestyles[i]
                pts = Points(sort_alg_df["log_size"], sort_alg_df[col], "color", c, "symboltype", "circle")
                cv = Curve(sort_alg_df["log_size"], sort_alg_df[col], "color", c, "linestyle", s)
                setattr(cv, "label", replace(sort_alg, "_", "\\_"))
                push!(curves, cv)
                add(plt, pts)
                if length(sort_alg_df["log_size"]) > 1
                    add(plt, cv)
                end
            end

            add(plt, Legend(.1, .9, curves))

            push!(plots, plt)
        end
    end

    plots
end

# Save plots
save_sort_plots(plots, pdffile="sortperf.pdf") = file(plots, pdffile)
#save_sort_plots(df::DataFrame, pdffile="sortperf.pdf") =
#    save_sort_plots(sort_plots(sort_median(df), ["*sort_median", "\\sort_median", "/sort_median", "3sort_median",
#                                                 "+sort_median", "~sort_median", "=sort_median", "!sort_median"]),
#                               pdffile)
save_sort_plots(df::DataFrame, base_sort, pdffile="sortperf.pdf") =
    save_sort_plots(sort_plots(sort_scale(sort_median(df), base_sort), base_sort), pdffile)

function view_sort_plots(plots)
    for p in plots
        Winston.display(p)
    end
end

# Test standard sort functions
function std_sort_tests(;sort_algs=SortPerf.sort_algs, types=SortPerf.std_types, range=6:20, replicates=3,
                        lt::Function=isless, by::Function=identity, rev::Bool=false, order::Ordering=Forward,
                        save::Bool=false, prefix="sortperf", skip_median_killer=false, base_sort="QuickSort")
    sort_times = sortperf(sort_algs, types, range, ord(lt,by,rev,order);
                           replicates=replicates, skip_median_killer=skip_median_killer)

    if save
        pdffile = prefix*".pdf"
        tsvfile = prefix*".tsv"
        save_sort_plots(sort_times, base_sort, pdffile)
        writetable(tsvfile, sort_times)
    end

    sort_times
end

end # module SortPerf


############################################################################################
"""
$(TYPEDEF)

Sampling Container that determines tuning updates.

# Fields
$(TYPEDFIELDS)
"""
abstract type SamplingPhase end
" Find/converges to the typical set (estimators suffer from initial but ultimately transient biases)"
struct Warmup <: SamplingPhase end
"Explore typical set and tune MCMC parameter(initial bias rapidly vanishes and the estimators become much more accurate.)"
struct Adaptionˢˡᵒʷ <: SamplingPhase end
"Tune final stepsize for fixed Covariance Matrix."
struct Adaptionᶠᵃˢᵗ <: SamplingPhase end
"Sample parameter with tuned MCMC (gradually reducing the precision error of the MCMC estimators towards zero.)"
struct Exploration <: SamplingPhase end

############################################################################################
"""
$(TYPEDEF)

Default configuration for tuning iterations in each cycle.

# Fields
$(TYPEDFIELDS)
"""
struct ConfigTuningWindow
    "MCMC Phase tuning window lengths, i.e: 5 = 5 repeats of second window in phasenames"
    window::Vector{Int64}
    "(Increasing) Length of windows, i.e.: if window=3, buffer=10 -> total window length: 10-20-30"
    buffer::Vector{Int64}
    "Name of phasenames, currently supported: Warmup(), Adaptionˢˡᵒʷ(), Adaptionᶠᵃˢᵗ(), Exploration()"
    phasenames::Vector{SamplingPhase}
    function ConfigTuningWindow(
        window::Vector{Int64},
        buffer::Vector{Int64},
        phasenames::Vector{SamplingPhase}
        )
        @argcheck size(window, 1) == size(buffer, 1) "Window and Buffer size different"
        @argcheck (size(window, 1) + 1) == size(phasenames, 1) "Window and phasename (without Exploration) size different"
        return new(
            window, buffer, phasenames
            )
    end
end

############################################################################################
"""
$(TYPEDEF)

Information about current SamplingPhase.

# Fields
$(TYPEDFIELDS)
"""
struct PhaseTune{T<:Tuple}
    "Boolean if current iteration needs update."
    update::Updater
    "Counts current iteration in phase."
    iter::Iterator
    "MCMC Phases ~ counter = current cyle in slices/name/iterations."
    counter::Iterator
    "Vector of MCMC iterations at each phase."
    slices::Vector{Int64}
    "Name of Sampling phases."
    name::T
    "Counts total iterations."
    iterations::Vector{Int64}
    ## MCMC Windows
    "Counts cycles in adaption phases Warmup(), Adaptionˢˡᵒʷ(), Adaptionᶠᵃˢᵗ(),
    i.e., 1-5-1 means 1 window init, 5 adapt, 1 exploration."
    window::Vector{Int64}
    "Counts iteration in each cycle for adaption phases Warmup(), Adaptionˢˡᵒʷ(), Adaptionᶠᵃˢᵗ(),
    i.e., 50-25-50 means 1 time 50, 5 times 25*i, i in 1:5, 1 time 50."
    buffer::Vector{Int64}
    function PhaseTune(;
        window=[1, 5, 1],
        buffer=[50, 25, 50],
        phasenames=[Warmup(), Adaptionˢˡᵒʷ(), Adaptionᶠᵃˢᵗ(), Exploration()],
    )
        ArgCheck.@argcheck length(window) == length(buffer) == (length(phasenames) - 1) "Size of window, buffer, or phasenames are not matching."
        ## Initiate mutable update container
        update = Updater(false)
        #!NOTE: start with 0, so first propose!() call will update iter.current to 1, and proposal will be first filled with 1st column.
        iter = Iterator(0)
        #!NOTE: start with 1 as Warmup matrix is initiated before propose! steps
        counter = Iterator(1)
        ## Initiate Warmup and Adaption Cycles
        phase_name = SamplingPhase[]
        phase_iterations = Int64[]
        for i in eachindex(window)
            for j in Base.OneTo(window[i])
                push!(phase_iterations, convert(Int64, j * buffer[i]))
                push!(phase_name, phasenames[i])
            end
        end
        ## Include Exploration Cycle
        push!(phase_name, phasenames[end])
        push!(phase_iterations, 1)
        ## Calculate cumsum of cylces for chainₜ iterations
        phase_slices = cumsum(phase_iterations)
        ## Change type of phase_name for concrete struct type
        phase_name_tuple = Tuple(phase_name)
        return new{typeof(phase_name_tuple)}(
            update,
            iter,
            counter,
            phase_slices,
            phase_name_tuple,
            phase_iterations,
            window,
            buffer,
        )
    end
end

############################################################################################
"""
$(SIGNATURES)
Update current MCMC Phase.

# Examples
```julia
```

"""
function update!(phase::PhaseTune, iter::Int64)
    @unpack slices = phase
    ## Check if updated last iteration
    if phase.update.current
        ## If update == true, set updating to false if updated last iteration
        update!(phase.update)
        ## Set back to 0 and add 1 afterwards
        phase.iter.current = 0
    end
    ## If update now, change Sampling phase
    if phase.counter.current < length(slices) && slices[phase.counter.current] == iter
        update!(phase.update)
        update!(phase.counter)
    end
    ## Update current iteration
    update!(phase.iter)
    return nothing
end

############################################################################################
# Export
export SamplingPhase, ConfigTuningWindow, Warmup, Adaptionˢˡᵒʷ, Adaptionᶠᵃˢᵗ, Exploration, PhaseTune


"""
`FWF.parsefield{T}(source::FWF.Source, ::Type{T}, row::Int, col::Int)` => `Nullable{T}`
`FWF.parsecol{T}(source::FWF.Source, ::Type{T}, col::Int)` => `Nullable{T}`

`source` is the source to read from
`T` is the type of the column / field
`row` is the row that is being read, if applicable
`col` is the column that is being read.

whitespace is ignored for numerial types, string trimming is configurable by options.
If `checkfornulls` is set in `opt` fields will be compared to the null list in `opt`, any values found will result in a missing value
Parsing happens for Integrer, Float64, Date
The parameter list without a row results in a whole column being streamed from the file.

"""
function parsefield end

missingon(source::FWF.Source) = (source.options.usemissings)
checkmissing(key::String, d::Set{String}) = key in d

function get_format(source::FWF.Source, col::Int)
    !haskey(source.options.dateformats, col) && return nothing
    return source.options.dateformats[col]
end

# This main dispatch will get called for field based parsing and dispatch to other methods
# We are going to pre-load whole lines and cache the results in the Source.
# Assume that every col==1 means load the next line
function parsefield(source::FWF.Source, ::Type{T}, row::Int, col::Int) where {T}
    # Assume that every col==1 means load the next line
    if !((col == source.lastcol+1) || (col == 1 && Data.size(source.schema)[2] == source.lastcol))
        throw(FWF.ParsingException("Out of order access col=$col lastcol=$(source.lastcol)"))
    end
    source.lastcol = col
    if col == 1
        readsplitline!(source.currentline, source)
    end

    tmp = source.currentline[col]
    ismissing(tmp) && return missing
    # We know ref is a string
    ref::String = tmp
    if missingon(source) && checkmissing(ref, source.options.missingvals)
        return missing
    end

    return parsefield(T, source.options.usemissings, ref, get_format(source, col))
end

# Batch of simple parsers to convert strings
usemissing_or_val(b, v) = b ? missing : v
if VERSION >= v"0.7.0-DEV"
    null_to_missing(x, b, v) = x == nothing ? usemissing_or_val(b, v) : x
else
    null_to_missing(x, b, v) = isnull(x) ? usemissing_or_val(b, v)  : unsafe_get(x)
end

parsefield(::Type{Int}, usemissing::Bool, string::String, format) =
    null_to_missing(tryparse(Int, string), usemissing, 0)
parsefield(::Type{Float64}, usemissing::Bool, string::String, format) =
    null_to_missing(tryparse(Float64, string), usemissing, 0.)
parsefield(::Type{String}, usemissing::Bool, string::String, format) = string
parsefield(::Type{Date}, usemissing::Bool, string::String, format) =
    null_to_missing(tryparse(Date, string, format), usemissing, Date())
parsefield(::Type{Union{Missing, T}}, usemissing::Bool, string::String, format) where {T} =
    (parsefield(T, usemissing, string, format))
parsefield(::Type{Missing}, usemissing::Bool, string::String, format) = (missing)

# Generic fallback
function parsefield(T, usemissing::Bool, string::String, format)
    return null_to_missing(tryparse(T, string), usemissing, nothing)
end


function get_partition_dirs(table_name)
  meta_path = joinpath("data", table_name, "_meta")
  meta_file = open(meta_path, "r")
  meta_data = read(meta_file, String)
  # Could parse JSON but this is faster + easier
  dir_regex = r"\"dir\":\s*\"(.*)\""
  sort([m.captures[1] for m in eachmatch(dir_regex, meta_data)])
end

function get_column_type(col_ext)
  if col_ext == ".u64"
    UInt64
  else
    println("Unknown column type ", col_ext)
    exit(2)
  end
end

function get_type_column(col_type)
  if col_type == UInt32
    ".u32"
  else
    println("Unknown column type ", col_type)
    exit(3)
  end
end

function convert_column(partition_dirs, col_name, to_type)
  col_files = readdir(partition_dirs[1])
  col_basenames = filter(d -> startswith(d, col_name * "."), col_files)
  if length(col_basenames) > 1
    println("Found multiple columns ", col_basenames)
    exit(1)
  end
  (col_basename, col_ext) = splitext(col_basenames[1])
  for p in partition_dirs
    col_path = joinpath(p, col_basename * col_ext)
    println(col_path)
    io_in = open(col_path, "r")
    data = UInt8[]
    readbytes!(io_in, data, Inf)
    close(io_in)
    data = reinterpret(get_column_type(col_ext), data)
    data = convert(Array{to_type}, data)
    println(length(data))
    mv(col_path, col_path * ".bak")
    new_path = joinpath(p, col_basename * get_type_column(to_type))
    println(new_path)
    io_out = open(new_path, "w")
    write(io_out, data)
    close(io_out)
  end
end

function convert_table_column(table_name, col_name, to_type)
  dirs = get_partition_dirs(table_name)
  convert_column(dirs, col_name, to_type)
end

convert_table_column("agg1m", "volume", UInt32)


isdefined(:AbstractTrees) && workspace()
using AbstractTrees
using Base.Test
import Base: ==

AbstractTrees.children(x::Array) = x
tree = Any[1,Any[2,3]]

AbstractTrees.print_tree(STDOUT, tree)
@test collect(Leaves(tree)) == [1,2,3]
@test collect(PostOrderDFS(tree)) == Any[1,2,3,Any[2,3],Any[1,Any[2,3]]]
@test collect(StatelessBFS(tree)) == Any[Any[1,Any[2,3]],1,Any[2,3],2,3]

tree2 = Any[Any[1,2],Any[3,4]]
@test collect(PreOrderDFS(tree2)) == Any[tree2,Any[1,2],1,2,Any[3,4],3,4]

"""
    A tree in which every node has 0 or 1 children
"""
immutable OneTree
    nodes::Vector{Int}
end
AbstractTrees.treekind(::Type{OneTree}) = AbstractTrees.IndexedTree()
AbstractTrees.siblinglinks(::Type{OneTree}) = AbstractTrees.StoredSiblings()
AbstractTrees.relative_state(t::OneTree, _, __::Int) = 1
Base.getindex(t::OneTree, idx) = t.nodes[idx]
AbstractTrees.childindices(tree::OneTree, node::Int) =
    (ret = (node == 0 || tree[node] == 0) ? () : (tree[node],))
AbstractTrees.children(tree::OneTree) = AbstractTrees.children(tree, tree)
AbstractTrees.rootstate(tree::OneTree) = 1
AbstractTrees.printnode(io::IO, t::OneTree) =
    AbstractTrees.printnode(io::IO, t[AbstractTrees.rootstate(t)])

ot = OneTree([2,3,4,0])
AbstractTrees.print_tree(STDOUT, ot)
@test collect(AbstractTrees.Leaves(ot)) == [0]
@test collect(AbstractTrees.PreOrderDFS(ot)) == [2,3,4,0]
@test collect(AbstractTrees.PostOrderDFS(ot)) == [0,4,3,2]

"""
    Stores an explicit parent for some other kind of tree
"""
immutable ParentTree{T}
    tree::T
    parents::Vector{Int}
end
AbstractTrees.treekind{T}(::Type{ParentTree{T}}) = AbstractTrees.treekind(T)
AbstractTrees.parentlinks{T}(::Type{ParentTree{T}}) = AbstractTrees.StoredParents()
AbstractTrees.siblinglinks{T}(::Type{ParentTree{T}}) = AbstractTrees.siblinglinks(T)
AbstractTrees.relative_state(t::ParentTree, _, __::Int) =
    AbstractTrees.relative_state(t.tree, _, __)
Base.getindex(t::ParentTree, idx) = t.tree[idx]
AbstractTrees.childindices(tree::ParentTree, node::Int) = AbstractTrees.childindices(tree.tree, node)
AbstractTrees.children(tree::ParentTree) = AbstractTrees.children(tree, tree)
AbstractTrees.rootstate(tree::ParentTree) = AbstractTrees.rootstate(tree.tree)
AbstractTrees.parentind(tree::ParentTree, node::Int) = tree.parents[node]
AbstractTrees.printnode(io::IO, t::ParentTree) =
    AbstractTrees.printnode(io::IO, t[AbstractTrees.rootstate(t)])

pt = ParentTree(ot,[0,1,2,3])
AbstractTrees.print_tree(STDOUT, pt)
@test collect(AbstractTrees.Leaves(pt)) == [0]
@test collect(AbstractTrees.PreOrderDFS(pt)) == [2,3,4,0]
@test collect(AbstractTrees.PostOrderDFS(pt)) == [0,4,3,2]

#=
immutable IntTree
    num::Int
    children::Vector{IntTree}
end
==(x::IntTree,y::IntTree) = x.num == y.num && x.children == y.children

@test treemap(PostOrderDFS(tree)) do ind, x, children
    IntTree(isa(x,Int) ? x : mapreduce(x->x.num,+,0,children),
        isempty(children) ? IntTree[] : children)
end == IntTree(6,[IntTree(1,IntTree[]),IntTree(5,[IntTree(2,IntTree[]),IntTree(3,IntTree[])])])

@test collect(PostOrderDFS([])) == Any[[]]
=#


"""

--------------------------------------------------------------------------------------\n
Author: Huckleberry Febbo, Graduate Student, University of Michigan
Date Create: 6/28/2017, Last Modified: 6/28/2017 \n
-------------------------------------------------------------------------------------\n
"""
macro controlVariable(n,name,num)
  n=esc(n)
  return quote
      $name=$n.r.ocp.u[:,num]
    end
end

"""

--------------------------------------------------------------------------------------\n
Author: Huckleberry Febbo, Graduate Student, University of Michigan
Date Create: 5/31/2017, Last Modified: 6/14/2017 \n
-------------------------------------------------------------------------------------\n
"""
macro DiffEq(n,expr)
  n=esc(n)
  return quote
      $n.stateEquations=$(Expr(:quote,expr))
    end
end

function dx_EQ{T<:Any}(n::NLOpt,x::Array{T,2},u::Array{T,2})
  dx_expr=parse_DiffEq(n,x,u)
  for st in 1:n.ocp.state.num
    for j in 1:n.ocp.state.pts
      dx[j,st]=create_DiffEq(n,dx_expr[j,st])
    end
  end
  return dx
end

"""
#TODO consider appending a zero so it is not just a jump variable
--------------------------------------------------------------------------------------\n
Author: Huckleberry Febbo, Graduate Student, University of Michigan
Date Create: 6/11/2017, Last Modified: 12/06/2019 \n
-------------------------------------------------------------------------------------\n
"""

function parse_DiffEq{T<:Any}(n::NLOpt,x::Array{T,2},u::Array{T,2})
  if n.s.ocp.integrationMethod==:tm; L=size(x)[1]; else; L=size(x)[1]-1; end

  dx=Array{Expr}(L,n.ocp.state.num);
  for qq in 1:L
    dx[qq,:]=deepcopy(n.ocp.DXexpr)
  end

  #TODO use the statenames here! -> currently running too often!
  state=Array{Symbol}(undef, n.ocp.state.num,1)
  for st in 1:n.ocp.state.num
    state[st]=Symbol(string("x",:($(st))))
  end
  control=Array{Symbol}(undef, n.ocp.control.num,1)
  for ctr in 1:n.ocp.control.num
    control[ctr]=Symbol(string("u",:($(ctr))))
  end

  for st_high in 1:n.ocp.state.num
    for qq in 1:L #TODO this will not start at 1 for the :ps method...unless you go bac to x and u
      # states
      for st in 1:n.ocp.state.num
        dx[qq,st_high]=rePlace(dx[qq,st_high],state[st],:($(n.r.ocp.x[qq,st])))
      end

      # controls
      for ctr in 1:n.ocp.control.num
        dx[qq,st_high]=rePlace(dx[qq,st_high],control[ctr],:($(n.r.ocp.u[qq,ctr])))
      end
    end
  end
  return dx
end
#dx=parse_DiffEq(n,n.r.ocp.x,n.r.ocp.u)

"""
--------------------------------------------------------------------------------------\n
Author: Huckleberry Febbo, Graduate Student, University of Michigan
Date Create: 6/12/2017, Last Modified: 6/12/2017 \n
Citations: \n
----------\n
Based off roughly of:
Source: [located here](https://github.com/JuliaLang/julia/blob/master/base/cartesian.jl#L278-L353)
-------------------------------------------------------------------------------------\n
"""
type Replace
    sym::Symbol
    val::JuMP.Variable
end

function rePlaceEX(ex::Expr,r::Replace)
  for i in 1:length(ex.args)
    if ex.args[i]==r.sym
      ex.args[i]=r.val
    end
  end
  return ex
end

function dig(ex)
  branch=falses(length(ex.args));
  for i in 1:length(ex.args)
    if typeof(ex.args[i])==Expr;
      branch[i]=true;
    end
  end
  return branch
end

function rePlace(ex::Expr,sym::Symbol,val::JuMP.Variable)
  r=Replace(sym,val)
  ex_tmp=[deepcopy(ex)]
  unwrap=true
  level=1
  while unwrap
    expr_arr=[];startCT=true;
    for j in 1:length(ex_tmp)
      ex_tmp[j]=rePlaceEX(ex_tmp[j],r);   # replace any on this branch
      branch=dig(ex_tmp[j]);
      if any(branch) && startCT
        expr_arr=ex_tmp[j].args[branch];
        tree=[any(branch)];
        startCT=false;
      elseif any(branch)
        expr_arr=[expr_arr,ex_tmp[j].args[branch]];
        tree=[tree,any(branch)];
      end
    end

    if level==1; ex=ex_tmp; end

    if isempty(expr_arr)
      unwrap=false
    else
      ex_tmp=expr_arr
      level+=1;
      if level>100
        error("\n Are the variables in the expression nested > 100 levels deep? If not report a bug, if so, consider increasing this limit \n ")
      end
    end
  end
  return ex[1]
end

#= TODO make test function
using JuMP
m=Model()
@variable(m,x[1:2,1:4])
expr=:($:x3*3*sin($:x3)*cos(cos($:x3)))
expr=rePlace(expr,:x3,:($(x[2,3])))
=#


module TestDistance
using ParallelKMeans: colwise!
using Test

@testset "naive singlethread colwise" begin
    X = [1.0 3.0 4.0; 2.0 5.0 6.0]
    y = [1.0, 2.0]
    r = Vector{Float64}(undef, 3)

    colwise!(r, X, y, 1)
    @test all(r .≈ [0.0, 13.0, 25.0])
end

@testset "multithread colwise" begin
    X = [1.0 3.0 4.0; 2.0 5.0 6.0]
    y = [1.0, 2.0]
    r = Vector{Float64}(undef, 3)

    colwise!(r, X, y, 2)
    @test all(r .≈ [0.0, 13.0, 25.0])
end

end # module