const USE_GPU  = false  # Use GPU? If this is set false, then the CUDA packages do not need to be installed! :)
const GPU_ID   = 0
using ParallelStencil
using ParallelStencil.FiniteDifferences2D
@static if USE_GPU
    @init_parallel_stencil(CUDA, Float64, 2)
    CUDA.device!(GPU_ID) # select GPU
else
    @init_parallel_stencil(Threads, Float64, 2)
end
using Test, Plots, Printf, Statistics, LinearAlgebra

################################
@parallel function def_err!(errD::Data.Array, D::Data.Array)

    @all(errD) = @all(D)

    return
end

@parallel function chk_err!(errD::Data.Array, D::Data.Array)

    @all(errD) = @all(D) - @all(errD)

    return
end

@parallel function compute_ϕ!(ϕ::Data.Array, rdh::Data.Number, rr::Data.Number, Zb::Data.Array, D::Data.Array, H::Data.Array)

    @all(ϕ) = @all(Zb) + rdh*@all(D) + rr*@all(H)

    return
end

@parallel_indices (ix,iy) function compute_flux!(Ux::Data.Array, Uy::Data.Array, qDx::Data.Array, qDy::Data.Array, k::Data.Number, _dx::Data.Number, _dy::Data.Number, ϕ::Data.Array, D::Data.Array)

    if (ix<=size(Ux,1)  && iy<=size(Ux,2))    Ux[ix,iy] = -k*_dx*(ϕ[ix+1,iy]-ϕ[ix,iy]) end
    if (ix<=size(Uy,1)  && iy<=size(Uy,2))    Uy[ix,iy] = -k*_dy*(ϕ[ix,iy+1]-ϕ[ix,iy]) end
    if (ix<=size(qDx,1) && iy<=size(qDx,2))  qDx[ix,iy] = Ux[ix,iy]*(Ux[ix,iy]>0.0)*D[ix,iy] + Ux[ix,iy]*(Ux[ix,iy]<0.0)*D[ix+1,iy  ] end
    if (ix<=size(qDy,1) && iy<=size(qDy,2))  qDy[ix,iy] = Uy[ix,iy]*(Uy[ix,iy]>0.0)*D[ix,iy] + Uy[ix,iy]*(Uy[ix,iy]<0.0)*D[ix  ,iy+1] end

    return
end

@parallel function compute_∂D!(∂D::Data.Array, _dx::Data.Number, _dy::Data.Number, M::Data.Number, qDx::Data.Array, qDy::Data.Array)

    @inn(∂D) = -(_dx*@d_xi(qDx) + _dy*@d_yi(qDy)) + 1.0/M

    return
end

@parallel function update_D!(∂Ddτ::Data.Array, D::Data.Array, damp::Data.Number, dτ::Data.Number, ∂D::Data.Array)

    @all(∂Ddτ) =  damp*@all(∂Ddτ) + @all(∂D)
    @all(D)    =  @all(D) + dτ*@all(∂Ddτ)

    return
end

@parallel_indices (ix,iy) function set_BCx!(D::Data.Array, val_BC::Data.Number)

    if (ix==1         && iy<=size(D,2)) D[ix,iy] = val_BC end
    if (ix==size(D,1) && iy<=size(D,2)) D[ix,iy] = val_BC end

    return
end

@parallel_indices (ix,iy) function set_BCy!(D::Data.Array, val_BC::Data.Number)
    # fixed height
    if (ix<=size(D,1) && iy==1        ) D[ix,iy] = val_BC end
    if (ix<=size(D,1) && iy==size(D,2)) D[ix,iy] = val_BC end
    # no flux
    # if (ix<=size(D,1) && iy==1        ) D[ix,iy] = D[ix,iy+1] end
    # if (ix<=size(D,1) && iy==size(D,2)) D[ix,iy] = D[ix,iy-1] end
    return
end

# physics scales
const ρ̂i  = 910
const ρ̂w  = 1000
const ĝ   = 9.81
const ϕ̂   = 1e7
const x̂   = 1e6
const M̂   = 1e-6
const D̂   = 1
# physics dependent
const t̂   = D̂/M̂
const Ĥ   = ϕ̂/ρ̂w/ĝ
const Ψ̂   = ϕ̂/x̂
const û   = x̂/t̂
const k̂   = û/Ψ̂
const rr  = ρ̂i/ρ̂w
const rdh = D̂/Ĥ    # fixed ice thikness routing
const s2d = 3600*24

@views function flow_routing2D()
    # nondim
    Lx, Ly = 10e3/x̂, 10e3/x̂
    k      = 0.1
    M      = M̂*s2d
    # numerics
    res    = 2
    nx     = res*128 # fastest if multiple of 16 (as no overlength here)
    ny     = res*128 # fastest if multiple of 16 (as no overlength here)
    nt     = 1
    itrMax = 1e8
    nout   = 1000
    nmax   = 100
    ε      = 1e-8     # aboslute tolerance
    damp   = 0.94  #0.9 for nx=100
    dτ_sc  = res*30.0 #60 for nx,ny=256 #30 for nx,ny=128 #24 for nx,ny=96 #12.0 for nx,ny=64
    dx, dy = Lx/nx, Ly/ny
    _dx, _dy = 1.0/dx, 1.0/dy
    xc     = LinRange(dx/2, Lx-dx/2, nx)
    yc     = LinRange(dy/2, Ly-dy/2, ny)
    # initial
    Zb     = (Lx.-xc)./Lx./100.0 .- 30.0./Ĥ.*exp.(-(xc.-Lx/2.0).^2/(Lx/8.0)^2 .-(yc'.-Ly/2.0).^2/(Ly/8.0)^2)
    H      = 100.0./Ĥ .- xc.*3.0 .+ 0*yc'
    Zb     = Data.Array(Zb)
    H      = Data.Array(H)
    # H      = (Lx-xc)/Lx/100 + 30/H_s*exp(-(xc-Lx/2).^2/(Lx/8)^2) + (xc + 50)/H_s;
    D      = 1.0.*@ones(nx,ny)
    # D      = max(Zb)-Zb + 10;
    ∂D     = @zeros(nx  ,ny  )
    ϕ      = @zeros(nx  ,ny  )
    Ux     = @zeros(nx-1,ny  )
    Uy     = @zeros(nx  ,ny-1)
    qDx    = @zeros(nx-1,ny  )
    qDy    = @zeros(nx  ,ny-1)
    ∂Ddτ   = @zeros(nx  ,ny  )
    errD   = @zeros(nx  ,ny  )
    err1=[]; err2=[]
    # action
    iter = 1; err = 2*ε;  max_U = 1.0
    while err > ε && iter < itrMax
    	@parallel def_err!(errD, D)
    	# flow routing physics
    	@parallel compute_ϕ!(ϕ, rdh, rr, Zb, D, H)
    	@parallel compute_flux!(Ux, Uy, qDx, qDy, k, _dx, _dy, ϕ, D)
    	@parallel compute_∂D!(∂D, _dx, _dy, M, qDx, qDy)
    	if mod(iter,nmax)==0 || iter==1
    		max_U = max(maximum(abs.(Ux)), maximum(abs.(Uy)))
    	end
    	dτ = min(dx,dy)/max_U/dτ_sc
    	@parallel update_D!(∂Ddτ, D, damp, dτ, ∂D)
    	@parallel set_BCx!(D, 0.0)
    	@parallel set_BCy!(D, 0.0)
    	# Check errs
    	if mod(iter,nout)==0
    		@parallel chk_err!(errD, D)
    		ermb  = 1.0./M*dx*dy*(nx-2)*(ny-2) - sum(abs.(qDx[[1, end],2:end-1])) - sum(abs.(qDy[2:end-1,[1, end]]))
    		push!(err1, maximum(abs.(errD[:]))); push!(err2, abs(ermb))
    		err   = err1[end]
    		@printf("iter=%d  errD=%1.3e, errMB=%1.3e \n", iter, err1[end], err2[end])
    	end
    	iter+=1
    end
    # ploting
    xcp = xc*x̂/1e3; ycp = yc*x̂/1e3
    p1 = heatmap(xcp, ycp, Zb'*Ĥ, aspect_ratio=1, xlims=(xcp[1], xcp[end]), ylims=(ycp[1], ycp[end]), c=:inferno, title="Zb")
    # p1 = heatmap(transpose(qDx), aspect_ratio=1)
    p2 = heatmap(xcp, ycp, D'*D̂ + Zb'*Ĥ, aspect_ratio=1, xlims=(xcp[1], xcp[end]), ylims=(ycp[1], ycp[end]), c=:inferno, title="Zb+D")
    p3 = plot(xcp[2:end-1], D[2:end-1,Int(round(ny/2))]*D̂, ylabel="D [m]", yscale=:log10, linewidth=2, framestyle=:box, legend=false)
    l  = @layout [a b; c]
    # display(plot(p1, p2, p3, layout = l))
    savefig(plot(p1, p2, p3, layout = l), joinpath(@__DIR__, "output/o$nx.png"))
    return
end

@time flow_routing2D()


module OperationsTest

import Random: randstring
import LibGit2
using Test
using UUIDs

using Pkg
using Pkg.Types

import Random: randstring
import LibGit2

include("utils.jl")

const TEST_PKG = (name = "Example", uuid = UUID("7876af07-990d-54b4-ab0e-23690620f79a"))

import Pkg.Types: semver_spec, VersionSpec
@testset "semver notation" begin
    @test semver_spec("^1.2.3") == VersionSpec("1.2.3-1")
    @test semver_spec("^1.2")   == VersionSpec("1.2.0-1")
    @test semver_spec("^1")     == VersionSpec("1.0.0-1")
    @test semver_spec("^0.2.3") == VersionSpec("0.2.3-0.2")
    @test semver_spec("^0.0.3") == VersionSpec("0.0.3-0.0.3")
    @test semver_spec("^0.0")   == VersionSpec("0.0.0-0.0")
    @test semver_spec("^0")     == VersionSpec("0.0.0-0")
    @test semver_spec("~1.2.3") == VersionSpec("1.2.3-1.2")
    @test semver_spec("~1.2")   == VersionSpec("1.2.0-1.2")
    @test semver_spec("~1")     == VersionSpec("1.0.0-1")
    @test semver_spec("1.2.3")  == semver_spec("^1.2.3")
    @test semver_spec("1.2")    == semver_spec("^1.2")
    @test semver_spec("1")      == semver_spec("^1")
    @test semver_spec("0.0.3")  == semver_spec("^0.0.3")
    @test semver_spec("0")      == semver_spec("^0")

    @test semver_spec("0.0.3, 1.2") == VersionSpec(["0.0.3-0.0.3", "1.2.0-1"])
    @test semver_spec("~1.2.3, ~v1") == VersionSpec(["1.2.3-1.2", "1.0.0-1"])

    @test   v"1.5.2"  in semver_spec("1.2.3")
    @test   v"1.2.3"  in semver_spec("1.2.3")
    @test !(v"2.0.0"  in semver_spec("1.2.3"))
    @test !(v"1.2.2"  in semver_spec("1.2.3"))
    @test   v"1.2.99" in semver_spec("~1.2.3")
    @test   v"1.2.3"  in semver_spec("~1.2.3")
    @test !(v"1.3"    in semver_spec("~1.2.3"))
    @test  v"1.2.0"   in semver_spec("1.2")
    @test  v"1.9.9"   in semver_spec("1.2")
    @test !(v"2.0.0"  in semver_spec("1.2"))
    @test !(v"1.1.9"  in semver_spec("1.2"))
    @test   v"0.2.3"  in semver_spec("0.2.3")
    @test !(v"0.3.0"  in semver_spec("0.2.3"))
    @test !(v"0.2.2"  in semver_spec("0.2.3"))
    @test   v"0.0.0"  in semver_spec("0")
    @test  v"0.99.0"  in semver_spec("0")
    @test !(v"1.0.0"  in semver_spec("0"))
    @test  v"0.0.0"   in semver_spec("0.0")
    @test  v"0.0.99"  in semver_spec("0.0")
    @test !(v"0.1.0"  in semver_spec("0.0"))

    @test semver_spec("<1.2.3") == VersionSpec("0.0.0 - 1.2.2")
    @test semver_spec("<1.2") == VersionSpec("0.0.0 - 1.1")
    @test semver_spec("<1") == VersionSpec("0.0.0 - 0")
    @test semver_spec("<2") == VersionSpec("0.0.0 - 1")
    @test semver_spec("<0.2.3") == VersionSpec("0.0.0 - 0.2.2")
    @test semver_spec("<2.0.3") == VersionSpec("0.0.0 - 2.0.2")
    @test   v"0.2.3" in semver_spec("<0.2.4")
    @test !(v"0.2.4" in semver_spec("<0.2.4"))

    @test semver_spec("=1.2.3") == VersionSpec("1.2.3")
    @test semver_spec("=1.2") == VersionSpec("1.2.0")
    @test semver_spec("  =1") == VersionSpec("1.0.0")
    @test   v"1.2.3" in semver_spec("=1.2.3")
    @test !(v"1.2.4" in semver_spec("=1.2.3"))
    @test !(v"1.2.2" in semver_spec("=1.2.3"))

    @test semver_spec("≥1.3.0") == semver_spec(">=1.3.0")

    @test semver_spec(">=   1.2.3") == VersionSpec("1.2.3-*")
    @test semver_spec(">=1.2  ") == VersionSpec("1.2.0-*")
    @test semver_spec("  >=  1") == VersionSpec("1.0.0-*")
    @test   v"1.0.0" in semver_spec(">=1")
    @test   v"0.0.1" in semver_spec(">=0")
    @test   v"1.2.3" in semver_spec(">=1.2.3")
    @test !(v"1.2.2" in semver_spec(">=1.2.3"))

    @test_throws ErrorException semver_spec("^^0.2.3")
    @test_throws ErrorException semver_spec("^^0.2.3.4")
    @test_throws ErrorException semver_spec("0.0.0")
end

# TODO: Should rewrite these tests not to rely on internals like field names
@testset "union, isjoinable" begin
    @test sprint(print, VersionRange("0-0.3.2")) == "0-0.3.2"
    # test missing paths on union! and isjoinable
    # there's no == for VersionBound or VersionRange
    unified_vr = union!([VersionRange("1.5-2.8"), VersionRange("2.5-3")])[1]
    @test unified_vr.lower.t == (UInt32(1), UInt32(5), UInt32(0))
    @test unified_vr.upper.t == (UInt32(3), UInt32(0), UInt32(0))
    unified_vr = union!([VersionRange("2.5-3"), VersionRange("1.5-2.8")])[1]
    @test unified_vr.lower.t == (UInt32(1), UInt32(5), UInt32(0))
    @test unified_vr.upper.t == (UInt32(3), UInt32(0), UInt32(0))
    unified_vr = union!([VersionRange("1.5-2.2"), VersionRange("2.5-3")])[1]
    @test unified_vr.lower.t == (UInt32(1), UInt32(5), UInt32(0))
    @test unified_vr.upper.t == (UInt32(2), UInt32(2), UInt32(0))
    unified_vr = union!([VersionRange("1.5-2.2"), VersionRange("2.5-3")])[2]
    @test unified_vr.lower.t == (UInt32(2), UInt32(5), UInt32(0))
    @test unified_vr.upper.t == (UInt32(3), UInt32(0), UInt32(0))
    unified_vb = Types.VersionBound(union!([v"1.5", v"1.6"])[1])
    @test unified_vb.t == (UInt32(1), UInt32(5), UInt32(0))
    unified_vb = Types.VersionBound(union!([v"1.5", v"1.6"])[2])
    @test unified_vb.t == (UInt32(1), UInt32(6), UInt32(0))
    unified_vb = Types.VersionBound(union!([v"1.5", v"1.5"])[1])
    @test unified_vb.t == (UInt32(1), UInt32(5), UInt32(0))
end

temp_pkg_dir() do project_path
    @testset "simple add and remove with preview" begin
        Pkg.init(project_path)
        Pkg.add(TEST_PKG.name; preview = true)
        @test !isinstalled(TEST_PKG)
        Pkg.add(TEST_PKG.name)
        @test isinstalled(TEST_PKG)
        @eval import $(Symbol(TEST_PKG.name))
        Pkg.rm(TEST_PKG.name; preview = true)
        @test isinstalled(TEST_PKG)
        Pkg.rm(TEST_PKG.name)
        @test !isinstalled(TEST_PKG)
    end

    @testset "package with wrong UUID" begin
        @test_throws ResolverError Pkg.add(PackageSpec(TEST_PKG.name, UUID(UInt128(1))))
    end

    @testset "adding and upgrading different versions" begin
        # VersionNumber
        Pkg.add(PackageSpec(TEST_PKG.name, v"0.3"))
        @test Pkg.installed()[TEST_PKG.name] == v"0.3"
        Pkg.add(PackageSpec(TEST_PKG.name, v"0.3.1"))
        @test Pkg.installed()[TEST_PKG.name] == v"0.3.1"
        Pkg.rm(TEST_PKG.name)

        # VersionRange
        Pkg.add(PackageSpec(TEST_PKG.name, VersionSpec(VersionRange("0.3.0-0.3.2"))))
        @test Pkg.installed()[TEST_PKG.name] == v"0.3.2"
        Pkg.up(; level = UpgradeLevel(:patch))
        @test Pkg.installed()[TEST_PKG.name] == v"0.3.3"
        Pkg.up(; level = UpgradeLevel(:minor))
        @test Pkg.installed()[TEST_PKG.name].minor != 3
        Pkg.rm(TEST_PKG.name)
    end

    @testset "testing" begin
        # TODO: Check that preview = true doesn't actually execute the test
        # TODO: Test-only dependencies
        Pkg.add(TEST_PKG.name)
        Pkg.test(TEST_PKG.name; coverage=true)
        pkgdir = Base.locate_package(Base.PkgId(TEST_PKG.uuid, TEST_PKG.name))
        # No coverage files being generated?
        @test_broken TEST_PKG.name * ".cov" in readdir(pkgdir)
        Pkg.rm(TEST_PKG.name)
    end

    @testset "pinning / freeing" begin
        Pkg.add(TEST_PKG.name)
        old_v = Pkg.installed()[TEST_PKG.name]
        Pkg.pin(PackageSpec(TEST_PKG.name, v"0.2"))
        @test Pkg.installed()[TEST_PKG.name].minor == 2
        Pkg.up(TEST_PKG.name)
        @test Pkg.installed()[TEST_PKG.name].minor == 2
        Pkg.free(TEST_PKG.name)
        Pkg.up()
        @test Pkg.installed()[TEST_PKG.name] == old_v
        Pkg.rm(TEST_PKG.name)
    end

    @testset "develop / freeing" begin
        Pkg.add(TEST_PKG.name)
        old_v = Pkg.installed()[TEST_PKG.name]
        Pkg.rm(TEST_PKG.name)
        mktempdir() do devdir
            withenv("JULIA_PKG_DEVDIR" => devdir) do
                Pkg.REPLMode.pkgstr("develop $(TEST_PKG.name)")
                @test isinstalled(TEST_PKG)
                @test Pkg.installed()[TEST_PKG.name] > old_v
                test_pkg_main_file = joinpath(devdir, TEST_PKG.name, "src", TEST_PKG.name * ".jl")
                @test isfile(test_pkg_main_file)
                # Pkg #152
                write(test_pkg_main_file,
                    """
                    module Example
                        export hello, domath
                        const example2path = joinpath(@__DIR__, "..", "deps", "deps.jl")
                        if !isfile(example2path)
                            error("Example is not installed correctly")
                        end
                        hello(who::String) = "Hello, \$who"
                        domath(x::Number) = x + 5
                    end
                    """)
                mkpath(joinpath(devdir, TEST_PKG.name, "deps"))
                write(joinpath(devdir, TEST_PKG.name, "deps", "build.jl"),
                    """
                    touch("deps.jl")
                    """
                )
                Pkg.build(TEST_PKG.name)
                @test isfile(joinpath(devdir, TEST_PKG.name, "deps", "deps.jl"))
                Pkg.test(TEST_PKG.name)
                Pkg.free(TEST_PKG.name)
                @test Pkg.installed()[TEST_PKG.name] == old_v
            end
        end
    end

    @testset "stdlibs as direct dependency" begin
        uuid_pkg = (name = "CRC32c", uuid = UUID("8bf52ea8-c179-5cab-976a-9e18b702a9bc"))
        Pkg.add("CRC32c")
        @test haskey(Pkg.installed(), uuid_pkg.name)
        Pkg.up()
        Pkg.test("CRC32c")
        Pkg.rm("CRC32c")
    end

    @testset "package name in resolver errors" begin
        try
            Pkg.add([PackageSpec(TEST_PKG.name, VersionSpec(v"55"))])
        catch e
            @test occursin(TEST_PKG.name, sprint(showerror, e))
        end
    end

    @testset "protocols" begin
        mktempdir() do devdir
            withenv("JULIA_PKG_DEVDIR" => devdir) do
                try
                    Pkg.setprotocol!("notarealprotocol")
                    # Pkg.develop is broken, update to use when fixed
                    @test_throws CommandError pkg"develop Example"
                    Pkg.setprotocol!()
                    pkg"develop Example"
                    @test isinstalled(TEST_PKG)
                finally
                    Pkg.setprotocol!()
                end
            end
        end
    end

    @testset "check logging" begin
        usage = Pkg.TOML.parse(String(read(joinpath(Pkg.logdir(), "manifest_usage.toml"))))
        @test any(x -> startswith(x, joinpath(project_path, "Manifest.toml")), keys(usage))
    end

    @testset "adding nonexisting packages" begin
        nonexisting_pkg = randstring(14)
        @test_throws CommandError Pkg.add(nonexisting_pkg)
        @test_throws CommandError Pkg.up(nonexisting_pkg)
    end

    Pkg.rm(TEST_PKG.name)

    @testset "legacy CI script" begin
        mktempdir() do dir
            LibGit2.with(LibGit2.clone("https://github.com/JuliaLang/Example.jl", joinpath(dir, "Example.jl"))) do r
                cd(joinpath(dir, "Example.jl")) do
                    let Pkg = Pkg
                        Pkg.clone(pwd())
                        Pkg.build("Example")
                        Pkg.test("Example"; coverage=true)
                        @test isfile(Pkg.dir("Example", "src", "Example.jl"))
                    end
                end
            end
        end
    end

    @testset "add julia" begin
        @test_throws CommandError Pkg.add("julia")
    end

    @testset "up in Project without manifest" begin
        mktempdir() do dir
            cp(joinpath(@__DIR__, "test_packages", "UnregisteredWithProject"), joinpath(dir, "UnregisteredWithProject"))
            cd(joinpath(dir, "UnregisteredWithProject")) do
               with_current_env() do
                    Pkg.up()
                    @test haskey(Pkg.installed(), "Example")
                end
            end
        end
    end

    @testset "failing building a package should throw" begin
        mktempdir() do path
            cd(path) do
                Pkg.generate("FailBuildPkg")
                cd("FailBuildPkg")
                with_current_env() do
                    write_build(pwd(), "error()")
                    @test_throws CommandError Pkg.build()
                end
            end
        end
    end
end

temp_pkg_dir() do project_path
    @testset "libgit2 downloads" begin
        Pkg.add(TEST_PKG.name; use_libgit2_for_all_downloads=true)
        @test haskey(Pkg.installed(), TEST_PKG.name)
        Pkg.rm(TEST_PKG.name)
    end
end

@testset "preview generate" begin
    mktempdir() do tmp
        cd(tmp) do
            Pkg.generate("Foo"; preview=true)
            @test !isdir(joinpath(tmp, "Foo"))
        end
    end
end

temp_pkg_dir() do project_path
    @testset "test should instantiate" begin
        mktempdir() do dir
            cp(joinpath(@__DIR__, "test_packages", "UnregisteredWithProject"), joinpath(dir, "UnregisteredWithProject"))
            cd(joinpath(dir, "UnregisteredWithProject")) do
                with_current_env() do
                    Pkg.add("Test") # test https://github.com/JuliaLang/Pkg.jl/issues/324
                    Pkg.test()
                end
            end
        end
    end
end

temp_pkg_dir() do project_path
    @testset "valid project file names" begin
        extract_uuid(toml_path) = begin
            uuid = ""
            for line in eachline(toml_path)
                m = match(r"uuid = \"(.+)\"", line)
                if m !== nothing
                    uuid = m.captures[1]
                    break
                end
            end
            return uuid
        end

        cd(project_path) do
            mktempdir() do tmp; cd(tmp) do
                pkg_name = "FooBar"
                # create a project and grab its uuid
                Pkg.generate(pkg_name)
                uuid = extract_uuid(joinpath(pkg_name, "Project.toml"))
                # activate project env
                Pkg.activate(abspath(pkg_name))
                # add an example project to populate manifest file
                Pkg.add("Example")
                Pkg.activate()
                # change away from default names
                mv(joinpath(pkg_name, "Project.toml"), joinpath(pkg_name, "JuliaProject.toml"))
                mv(joinpath(pkg_name, "Manifest.toml"), joinpath(pkg_name, "JuliaManifest.toml"))
                # make sure things still work
                Pkg.develop(abspath(pkg_name))
                @test isinstalled((name=pkg_name, uuid=UUID(uuid)))
                Pkg.rm(pkg_name)
                @test !isinstalled((name=pkg_name, uuid=UUID(uuid)))
            end end
        end # cd project_path
    end # @testset
end

include("repl.jl")

end # module


r = PH.ScalarPenaltyParameter(25.0)
atol = 1e-8
rtol = 1e-12
max_iter = 500
obj_val = 178.3537498401004
var_vals = Dict([
    "x[1]_{0,1,2,3}" => 7.5625,
    "x[2]_{0,1,2,3}" => 0.0,
    "x[3]_{0,1,2,3}" => 1.0,
    "y_{0,1}" => 1.75,
    "y_{2,3}" => 0.0,
    "z[1]_{0}" => -0.65625,
    "z[2]_{0}" => -0.65625,
    "z[1]_{1}" => 2.84375,
    "z[2]_{1}" => 2.84375,
    "z[1]_{2}" => -1.78125,
    "z[2]_{2}" => -1.78125,
    "z[1]_{3}" => 4.71875,
    "z[2]_{3}" => 4.71875,
])

# Setup distributed stuff
nw = 2
diff = nw - nworkers()
diff > 0 && Distributed.addprocs(nw)
@assert Distributed.nworkers() == nw
@everywhere using Pkg
@everywhere Pkg.activate(@__DIR__)
@everywhere using ProgressiveHedging
@everywhere const PH = ProgressiveHedging
@everywhere using Ipopt
@everywhere using JuMP
include("common.jl")

@testset "Error Handling" begin
    ch_size = 10
    worker_inf = PH._launch_workers(ch_size, ch_size)

    @everywhere struct MakeWorkerError <: PH.Message end

    PH._send_message(worker_inf, first(workers()), MakeWorkerError())

    my_task = @async begin
        PH._wait_for_shutdown(worker_inf)
    end
    if timeout_wait(my_task)
        @test istaskfailed(my_task)
        @test typeof(my_task.exception) <: RemoteException
    else
        error("Timed out on test")
    end

    my_task = @async begin
        PH._wait_for_shutdown(worker_inf)
        return PH._is_running(worker_inf)
    end
    if timeout_wait(my_task)
        @test !fetch(my_task)
    else
        error("Timed out on test")
    end
end

@testset "Solve" begin

    # Solve the problem
    (n, err, rerr, obj, soln, phd) = PH.solve(build_scen_tree(),
                                              create_model,
                                              r,
                                              atol=atol,
                                              rtol=rtol,
                                              opt=Ipopt.Optimizer,
                                              opt_args=(print_level=0,tol=1e-12),
                                              max_iter=max_iter,
                                              report=0,
                                              timing=false,
                                              warm_start=true)

    @test err < atol
    @test isapprox(obj, obj_val)
    @test n < max_iter

    for row in eachrow(soln)
        var = row[:variable] * "_{" * row[:scenarios] * "}"
        @test isapprox(row[:value], var_vals[var], atol=1e-7)
    end

end

@testset "User Specified Workers" begin

    st = build_scen_tree()
    worker_assignments = Dict(k=>Set{PH.ScenarioID}() for k in 2:3)
    for s in PH.scenarios(st)
        if PH.value(s) < 3
            push!(worker_assignments[2], s)
        else
            push!(worker_assignments[3], s)
        end
    end

    # Solve the problem
    (n, err, rerr, obj, soln, phd) = PH.solve(build_scen_tree(),
                                              create_model,
                                              r;
                                              atol=atol,
                                              rtol=rtol,
                                              opt=Ipopt.Optimizer,
                                              opt_args=(print_level=0,tol=1e-12),
                                              max_iter=max_iter,
                                              report=0,
                                              worker_assignments=worker_assignments,
                                              timing=false,
                                              warm_start=true)

    @test err < atol
    @test isapprox(obj, obj_val)
    @test n < max_iter

    for row in eachrow(soln)
        var = row[:variable] * "_{" * row[:scenarios] * "}"
        @test isapprox(row[:value], var_vals[var], atol=1e-7)
    end

    for s in PH.scenarios(phd)
        if PH.value(s) < 3
            @test phd.scenario_map[s].pid == 2
        else
            @test phd.scenario_map[s].pid == 3
        end
    end

end

@testset "Lower Bound Algorithm" begin
    (n, err, rerr, obj, soln, phd) = PH.solve(PH.two_stage_tree(2),
                                              two_stage_model,
                                              PH.ScalarPenaltyParameter(2.0),
                                              atol=atol,
                                              rtol=rtol,
                                              max_iter=max_iter,
                                              report=0,
                                              lower_bound=5,
                                              timing=false,
                                              warm_start=false
                                              )

    @test err < atol
    @test isapprox(obj, 8.25, atol=1e-6)
    @test n < max_iter

    lb_df = PH.lower_bounds(phd)
    for row in eachrow(lb_df)
        @test row[:bound] <= 8.25
    end

    @test size(lb_df,1) == 12
    @test isapprox(lb_df[size(lb_df,1), "absolute gap"], 0.0, atol=1e-7)
    @test isapprox(lb_df[size(lb_df,1), "relative gap"], 0.0, atol=(1e-7/8.25))
end

# Tear down distrbuted stuff
Distributed.rmprocs(Distributed.workers())
@assert Distributed.nprocs() == 1


using Cbc
using QuadraticToBinary

const CBC_OPTIMIZER = Cbc.Optimizer()
MOI.set(CBC_OPTIMIZER, MOI.Silent(), true)
const CBC_CACHE = MOIU.UniversalFallback(MOIU.Model{Float64}())
const CBC_CACHED = MOIU.CachingOptimizer(CBC_CACHE, CBC_OPTIMIZER)
const CBC_BRIDGED = MOI.Bridges.full_bridge_optimizer(CBC_CACHED, Float64)

push!(solvers, (opt = CBC_BRIDGED, mode = BilevelJuMP.SOS1Mode()))
push!(solvers_sos, (opt = CBC_BRIDGED, mode = BilevelJuMP.SOS1Mode()))
push!(solvers_indicator, (opt = CBC_BRIDGED, mode = BilevelJuMP.IndicatorMode()))

push!(solvers_bin_exp, (
    opt = QuadraticToBinary.Optimizer{Float64}(CBC_BRIDGED),
    mode = BilevelJuMP.StrongDualityInequalityMode{Float64}(1e-9)))
MOI.set(solvers_bin_exp[end].opt, QuadraticToBinary.GlobalVariablePrecision(), 1e-5)

push!(solvers_bin_exp, (
    opt = QuadraticToBinary.Optimizer{Float64}(CBC_BRIDGED),
    mode = BilevelJuMP.StrongDualityEqualityMode()))
MOI.set(solvers_bin_exp[end].opt, QuadraticToBinary.GlobalVariablePrecision(), 1e-5)

# push!(solvers_sos_quad, (
#     opt = QuadraticToBinary.Optimizer{Float64}(CBC_BRIDGED),
#     mode = BilevelJuMP.SOS1Mode()))
# MOI.set(solvers_bin_exp[end].opt, QuadraticToBinary.GlobalVariablePrecision(), 1e-5)


function graphloadertest()

	# create a graph
	# with no edges or vertices loaded
	g = Graph()
	# create a graph loader for the Graph
	gl = TestGraphLoader(g)

	# use another test method to obtain graph data for the test
	sourcegraph = buildsocialtestgraph()
	# copy this data to the test graph loader
	# this is to act in place of a persistent store from which to load
	gl.sourceedges = sourcegraph.edges
	gl.sourcevertices = sourcegraph.vertices

	# count the countries
	@test query(gl, (vertices, v->v.typelabel == "Country"), count) == 30

	@test query(gl, (vertices, v->v.typelabel == "Country" && get(v,"Name") == "Germany"), count) == 1

	# start with the vertices for Germany and France
	# follow the edges of type "LivesIn" to find all the people within the graph who live in Germany or France
	#     only edges that have a ForYears property of >= 5 (indicating the person has lived in the associated country for 5 or more years) will be followed
	# the tail of each edge is a person, filter the people selected to those with an age property >= 30
	part1 = query(gl,
				(vertices, v->v.typelabel == "Country" && (get(v,"Name") == "Germany" || get(v,"Name") == "France")),
			 	(incoming, e->e.typelabel == "LivesIn" && get(e,"ForYears",0) >= 5),
			  	(tail, v->get(v,"Age",0) >= 30))
	# get a count of these results
	@test query(part1, count) == 10

	# start with the results of the first query
	# follow edges of type "Follows" (in an incoming direction)
	# the tail of each edge is a person who follows a person selected in the first query
	# select a distinct set of vertices (as there may have been multiple edges to each person..as a person could follow more than one person selected by the first query)
	part2 = query(part1,
				(incoming, e->e.typelabel == "Follows"),
			  	tail,
				distinct
			 )
	# get a count of the results
	testcount = query(part2, count)
	@test testcount == 78
end

@time graphloadertest()


function SimpleRatioAmbiguityAcceptance(arg0::jdouble)
    return SimpleRatioAmbiguityAcceptance((jdouble,), arg0)
end

function accept(obj::SimpleRatioAmbiguityAcceptance, arg0::Vector{IntegerLeastSquareSolution})
    return jcall(obj, "accept", IntegerLeastSquareSolution, (Vector{IntegerLeastSquareSolution},), arg0)
end

function number_of_candidates(obj::SimpleRatioAmbiguityAcceptance)
    return jcall(obj, "numberOfCandidates", jint, ())
end


function place!(trader::AbstractInvestor, market::AbstractMarket)
    order = get_order(trader, market)
    if ~is_valid_order(order, market, trader)
        if ~isnothing(order)
            #println("Invalid order $(order.price) * $(order.quantity) $(typeof(order))")
            #println("Has in position $(order.dealer.positions[order.from]) (reserved: $(get(trader.reserved, order.from, 0)) and active $(market ∉ keys(trader.active_orders) ? 0 : get_reserved(trader.active_orders[market])))")
        end
        return
    end

    # Cancel active order if has any and overwrite it with this
    #println("------")
    #println("$(trader.name): Making order $(order.price) * $(order.quantity) $(typeof(order))")
    cancel!(trader, market)
    # Store the active order
    trader.active_orders[market] = order
    reserve!(trader, order)

    trades = place!(market, order)

    return trades
end

function is_valid_order(order::BidLimitOrder, market::AbstractMarket, trader::AbstractInvestor)
    is_not_over_reserved = get_unreserved(order.dealer, order.from, exclude=market) >= get_amount_commited(order)
    is_not_none = order != nothing
    is_positive = order.quantity > 0
    return is_not_none && is_positive && is_not_over_reserved
end

function is_valid_order(order::AskLimitOrder, market::AbstractMarket, trader::AbstractInvestor)

    is_not_over_reserved = get_unreserved(order.dealer, order.from, exclude=market) >= get_amount_commited(order)
    is_not_none = order != nothing
    is_positive = order.quantity > 0
    return is_not_none && is_positive && is_not_over_reserved
end

function is_valid_order(order::Nothing, market::AbstractMarket, trader::AbstractInvestor)
    return false
end

function transfer!(from::AbstractInvestor, to::AbstractInvestor; asset, quantity)
    from[asset] -= quantity
    to[asset] += quantity
end

"Reserve asset (so that investor has the capital for future transaction)"
function reserve!(inv::AbstractInvestor, order::LimitOrder)

    #println("$order")
    # THIS WAS THE BUG:
    asset = order.from

    quantity = get_amount_commited(order)
    #println("Reserving $quantity of $(asset.name) for $(inv.name) ($(get(inv.reserved, asset, 0)))")
    #quantity = order isa BidLimitOrder ? order.quantity * order.price : order.quantity
    # ∉ is same as "not in" Python
    if asset ∉ keys(inv.reserved)
        inv.reserved[asset] = quantity
        return
    end

    # Check the reserved does not surpass the position
    if inv.positions[asset] < inv.reserved[asset]
        throw(DomainError(inv.reserved[asset], "The reserve is more ($(inv.reserved[asset])) than there is in the positions ($(inv.positions[asset]))"))
    elseif quantity > inv.positions[asset]
        #println("Invalid order: $(order.price) * $(order.quantity) $(typeof(order)) (asset $(asset.name))")
        throw(DomainError(quantity, "Cannot reserve more asset ($quantity) than there is in the positions (Position: $(inv.positions[asset]), reserved: $(inv.reserved[asset])) $(inv.name)"))
    end

    #println("Reserving asset $asset")
    inv.reserved[asset] += quantity
    #println(inv.reserved[asset])
end

"Release reserved asset"
function release!(inv::AbstractInvestor, quantity::Int64, asset::AbstractAsset)
    # ∉ is same as "not in" Python
    #println("Releasing $quantity of $(asset.name) for investor $(inv.name) ($(get(inv.reserved, asset, 0)))")
    if asset ∉ keys(inv.reserved)
        inv.reserved[asset] = 0
        return
    end
    # if inv.positions[asset] < inv.reserved[asset]
    #     throw(DomainError(inv.reserved[asset], "The reserve is more ($(inv.reserved[asset])) than there is in the positions ($(inv.positions[asset]))"))
    # elseif inv.positions[asset] < quantity
    #     throw(DomainError(quantity, "Cannot release more asset ($quantity) than there is in the positions ($(inv.positions[asset]))"))
    # elseif inv.reserved[asset] < quantity
    #     throw(DomainError(quantity, "Cannot release more asset ($quantity) than there is in the reserve ($(inv.reserved[asset]), Investor $(inv.name))"))
    # end
    inv.reserved[asset] -= quantity
    #println(inv.reserved[asset])
end

"Release all reserved asset"
function release_all!(inv::AbstractInvestor)
    inv.reserved = Dict{AbstractAsset, Int64}()
end

"Cancel active order"
function cancel!(trader::AbstractInvestor, market::AbstractMarket)
    # Cancelling order is conducted the following manner:
    #   1. Set the quantity of the active order to zero
    #   2. Release the asset from reserved
    #   3. Let the market clean the order away on next clearing
    active = get_active_order(trader, market)
    if isnothing(active)
        # No active orders
        return
    end

    active_quantity = get_amount_commited(active)
    #println("Active Quantity $active_quantity. Previous active: $active_order")

    release!(trader, active_quantity, active.from)
    # BUG: For some reason cash is not completely released here!
    #println("Asset $asset reserved $(trader.reserved[asset])")

    # There is no elegant way of deleting it from the market
    # completely so we set quantity as 0 and market will
    # clean it away
    active.quantity = 0

    # Check for debugging
    if trader.active_orders[market].quantity != 0
        throw(DomainError(trader.active_orders[market].quantity, "Active amount not zero"))
    end
end


# MIT License
#
# Copyright (c) 2018 Martin Biel
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

struct IndexMap
    n::Int
    m₁::Int
    m₂::Int
    map::Dict{RowCol, IdxGroup}

    function IndexMap(n::Integer, m₁::Integer, m₂::Integer)
        return new(n, m₁, m₂, Dict{RowCol, IdxGroup}())
    end
end
Base.setindex!(map::IndexMap, val::IdxGroup, key::RowCol) = setindex!(map.map, val, key)
Base.getindex(map::IndexMap, key::RowCol) = getindex(map.map, key)

struct LPData{T <: AbstractFloat, M <: AbstractMatrix}
    c₁::Vector{T}
    c₂::T
    A::M
    b::Vector{T}
    d₁::Vector{T}
    C::M
    d₂::Vector{T}
    lb::Vector{T}
    ub::Vector{T}
    is_binary::Vector{Bool}
    is_integer::Vector{Bool}
    indexmap::IndexMap

    function LPData(::Type{T}, ::Type{<:Matrix}, n::Int, m₁::Int, m₂::Int) where T <: AbstractFloat
        return new{T, Matrix{T}}(
            zeros(T, n),
            zero(T),
            zeros(T, m₁, n),
            zeros(T, m₁),
            fill(convert(T, -Inf), m₂),
            zeros(T, m₂, n),
            fill(convert(T, Inf), m₂),
            fill(convert(T, -Inf), n),
            fill(convert(T, Inf), n),
            fill(false, n),
            fill(false, n),
            IndexMap(n, m₁, m₂))
    end

    function LPData(::Type{T}, ::Type{<:SparseMatrixCSC}, n::Int, m₁::Int, m₂::Int) where T <: AbstractFloat
        return new{T, SparseMatrixCSC{T,Int}}(
            zeros(T, n),
            zero(T),
            spzeros(T, m₁, n),
            zeros(T, m₁),
            fill(convert(T, -Inf), m₂),
            spzeros(T, m₂, n),
            fill(convert(T, Inf), m₂),
            fill(convert(T, -Inf), n),
            fill(convert(T, Inf), n),
            fill(false, n),
            fill(false, n),
            IndexMap(n, m₁, m₂))
    end

    function LPData(::Type{T}, c₁::AbstractVector, c₂::Real,
                    A::AbstractMatrix, b::AbstractVector,
                    d₁::AbstractVector, C::AbstractMatrix, d₂::AbstractVector,
                    lb::AbstractVector, ub::AbstractVector,
                    is_binary::AbstractVector, is_integer::AbstractVector,
                    map::IndexMap) where T <: AbstractFloat
        A_ = convert(Matrix{T}, A)
        C_ = convert(Matrix{T}, C)
        # Sanity check
        _lpcheck(c₁, c₂, A, b, d₁, C, d₂, lb, ub, is_binary, is_integer)
        # Return data
        return new{T, Matrix{T}}(c₁, c₂, A_, b, d₁, C_, d₂, lb, ub, is_binary, is_integer, map)
    end

    function LPData(::Type{T}, c₁::AbstractVector, c₂::Real,
                    A::AbstractSparseMatrix, b::AbstractVector,
                    d₁::AbstractVector, C::AbstractSparseMatrix, d₂::AbstractVector,
                    lb::AbstractVector, ub::AbstractVector,
                    is_binary::AbstractVector, is_integer::AbstractVector,
                    map::IndexMap) where T <: AbstractFloat
        A_ = convert(SparseMatrixCSC{T}, A)
        C_ = convert(SparseMatrixCSC{T}, C)
        # Sanity check
        _lpcheck(c₁, c₂, A, b, d₁, C, d₂, lb, ub, is_binary, is_integer)
        # Return data
        return new{T, SparseMatrixCSC{T,Int}}(c₁, c₂, A_, b, d₁, C_, d₂, lb, ub, is_binary, is_integer, map)
    end
end

LPData(n::Int, m₁::Int, m₂::Int) =
    LPData(Float64, Matrix, n, m₁, m₂)
LPData(t::Type{T}) where T <: AbstractFloat =
    LPData(t, 0, 0, 0)
LPData() =
    LPData(Float64)

SparseLPData(n::Int, m₁::Int, m₂::Int) =
    LPData(Float64, SparseMatrixCSC, n, m₁, m₂)
SparseLPData(t::Type{T}) where T <: AbstractFloat =
    SparseLPData(t, 0, 0, 0)
SparseLPData() =
    SparseLPData(Float64)

function _lpcheck(c₁::AbstractVector, c₂::Real,
                  A::AbstractMatrix, b::AbstractVector,
                  d₁::AbstractVector, C::AbstractMatrix, d₂::AbstractVector,
                  lb::AbstractVector, ub::AbstractVector,
                  is_binary::AbstractVector, is_integer::AbstractVector)
    n₁ = length(c₁)
    m₁, n₂ = size(A)
    m₂, n₃ = size(C)

    if n₁ ≠ n₂
        throw(DomainError(A ,"LPData: `A` must have $(n₁) columns, has $(n₂)"))
    elseif m₁ ≠ length(b)
        throw(DomainError(b ,"LPData: `b` must have $(m₁) elements, , has $(length(b))"))
    elseif n₁ ≠ n₃
        throw(DomainError(D ,"LPData: `D` must have $(n₁) columns, , has $(n₃)"))
    elseif m₂ ≠ length(d₁) || m₂ ≠ length(d₂)
        throw(DomainError(d ,"LPData: `d₁` and `d₂` must have $(m₂) elements, has $(length(d₁)) and $(length(d₂))"))
    elseif n₁ ≠ length(lb) || n₁ ≠ length(ub)
        throw(DomainError((lb, ub) ,"LPData: Both `lb` and `ub` must have $(n₁) elements, , has $(length(lb)) and $(length(ub))"))
    elseif n₁ ≠ length(is_binary) || n₁ ≠ length(is_integer)
        throw(DomainError((lb, ub) ,"LPData: Both `is_binary` and `is_integer` must have $(n₁) elements, , has $(length(is_binary)) and $(length(is_integer))"))
    end
end

function LPData(c₁::AbstractVector, c₂::Real,
                A::AbstractMatrix, b::AbstractVector,
                d₁::AbstractVector, C::AbstractMatrix, d₂::AbstractVector,
                lb::AbstractVector, ub::AbstractVector,
                is_binary::AbstractVector, is_integer::AbstractVector,
                map::IndexMap)
    return LPData(Float64, c₁, c₂, A, b, d₁, C, d₂, lb, ub, is_binary, is_integer, map)
end

function LPData(cor::RawCor{T};
                colrange = collect(keys(cor.cols)),
                rowrange = collect(keys(cor.rows)),
                include_constant = false) where T <: AbstractFloat
    # Compute sizes
    n = length(colrange)
    m₁ = count(x -> x[3] == EQ, [cor.rows[row] for row in rowrange])
    m₂ = length(rowrange) - m₁
    # Prepare index map
    map = IndexMap(n, m₁, m₂)
    # Prepare matrices
    c₁ = zeros(T, n)
    c₂ = zero(T)
    A  = zeros(T, m₁, n)
    b  = zeros(T, m₁)
    d₁ = fill(convert(T, -Inf), m₂)
    C  = zeros(T, m₂, n)
    d₂ = fill(convert(T, Inf), m₂)
    lb = zeros(T, n)
    ub = fill(convert(T, Inf), n)
    is_binary = fill(false, n)
    is_integer = fill(false, n)
    # Aux counters
    ncols = 0
    neqrows = 0
    nineqrows = 0
    for (row, rowval) in cor.rows
        if row in rowrange
            break
        end
        if rowval[3] == EQ
            neqrows += 1
        elseif rowval[3] == LEQ || rowval[3] == GEQ
            nineqrows += 1
        end
    end
    # Fill (in)equalities with ranges
    for (col, val) in cor.cols
        col in colrange || (ncols += 1; continue)
        colidx = cor.vars[col]
        for (row, rowval) in val
            row in rowrange || continue
            idx, rowidx, rowsense = cor.rows[row]
            rowcol = (row, col)
            if rowidx == 0
                j = colidx - ncols
                c₁[j] = rowval
                map[rowcol] = (0,j,OBJ)
            elseif rowsense == EQ
                i = rowidx - neqrows
                j = colidx - ncols
                A[i, j] = rowval
                b[i] = get(cor.rhs, row, zero(T))
                map[rowcol] = (i,j,EQ)
                map[(row,RHS)] = (i,0,EQ)
            elseif rowsense == LEQ
                i = rowidx - nineqrows
                j = colidx - ncols
                C[i, j] = rowval
                d₂[i] = get(cor.rhs, row, zero(T))
                map[rowcol] = (i,j,LEQ)
                map[(row,RHS)] = (i,0,LEQ)
                if haskey(cor.ranges, row)
                    d₁[i] = d₂[i] - abs(cor.ranges[row])
                    map[rowcol] = (i,j,RANGE)
                    map[(row,RHS)] = (i,0,RANGE)
                end
            else
                i = rowidx - nineqrows
                j = colidx - ncols
                C[i, j] = rowval
                d₁[i] = get(cor.rhs, row, zero(T))
                map[rowcol] = (i,j,GEQ)
                map[(row,RHS)] = (i,0,GEQ)
                if haskey(cor.ranges, rowsymbol)
                    d₂[i] = d₁[i] + abs(cor.ranges[row])
                    map[rowcol] = (i,j,RANGE)
                    map[(row,RHS)] = (i,0,RANGE)
                end
            end
        end
    end
    # Get any objective constant
    if cor.objgiven && include_constant
        c₂ = -get(cor.rhs, cor.objname, zero(T))
    end
    # Fill bounds
    fill_bounds!(cor, lb, ub, is_binary, is_integer, colrange, map)
    # Return the problem
    return LPData(T, c₁, c₂, A, b, d₁, C, d₂, lb, ub, is_binary, is_integer, map)
end

function SparseLPData(cor::RawCor{T};
                      colrange = collect(keys(cor.cols)),
                      rowrange = collect(keys(cor.rows)),
                      include_constant = false) where T <: AbstractFloat
    # Compute sizes
    n = length(colrange)
    m₁ = count(x -> x[3] == EQ, [cor.rows[row] for row in rowrange])
    m₂ = length(rowrange) - m₁
    # Prepare index map
    map = IndexMap(n, m₁, m₂)
    # Prepare vectors
    c₁ = zeros(T, n)
    c₂ = zero(T)
    b  = zeros(T, m₁)
    d₁ = fill(convert(T, -Inf), m₂)
    d₂ = fill(convert(T, Inf), m₂)
    lb = zeros(T, n)
    ub = fill(convert(T, Inf), n)
    is_binary = fill(false, n)
    is_integer = fill(false, n)
    # Prepare matrices
    Aᵢ = Vector{Int}()
    Aⱼ = Vector{Int}()
    Aᵥ = Vector{T}()
    Cᵢ = Vector{Int}()
    Cⱼ = Vector{Int}()
    Cᵥ = Vector{T}()
    # Aux counters
    ncols = 0
    neqrows = 0
    nineqrows = 0
    for (row, rowval) in cor.rows
        if row in rowrange
            break
        end
        if rowval[3] == EQ
            neqrows += 1
        elseif rowval[3] == LEQ || rowval[3] == GEQ
            nineqrows += 1
        end
    end
    # Fill (in)equalities with ranges
    for (col, val) in cor.cols
        col in colrange || (ncols += 1; continue)
        colidx = cor.vars[col]
        for (row, rowval) in val
            row in rowrange || row == cor.objname || continue
            idx, rowidx, rowsense = cor.rows[row]
            rowcol = (row, col)
            if rowidx == 0
                j = colidx - ncols
                c₁[j] = rowval
                map[rowcol] = (0,j,OBJ)
            elseif rowsense == EQ
                i = rowidx - neqrows
                j = colidx - ncols
                push!(Aᵢ, i)
                push!(Aⱼ, j)
                push!(Aᵥ, rowval)
                b[i] = get(cor.rhs, row, zero(T))
                map[rowcol] = (i,j,EQ)
                map[(row,RHS)] = (i,0,EQ)
            elseif rowsense == LEQ
                i = rowidx - nineqrows
                j = colidx - ncols
                push!(Cᵢ, i)
                push!(Cⱼ, j)
                push!(Cᵥ, rowval)
                d₂[i] = get(cor.rhs, row, zero(T))
                map[rowcol] = (i,j,LEQ)
                map[(row,RHS)] = (i,0,LEQ)
                if haskey(cor.ranges, row)
                    d₁[i] = d₂[i] - abs(cor.ranges[row])
                    map[rowcol] = (i,j,RANGE)
                    map[(row,RHS)] = (i,0,RANGE)
                end
            else
                i = rowidx - nineqrows
                j = colidx - ncols
                push!(Cᵢ, i)
                push!(Cⱼ, j)
                push!(Cᵥ, rowval)
                d₁[i] = get(cor.rhs, row, zero(T))
                map[rowcol] = (i,j,GEQ)
                map[(row,RHS)] = (i,0,GEQ)
                if haskey(cor.ranges, row)
                    d₂[i] = d₁[i] - abs(cor.ranges[row])
                    map[rowcol] = (i,j,RANGE)
                    map[(row,RHS)] = (i,0,RANGE)
                end
            end
        end
    end
    # Create sparse structures
    A  = sparse(Aᵢ, Aⱼ, Aᵥ, m₁, n)
    C  = sparse(Cᵢ, Cⱼ, Cᵥ, m₂, n)
    # Get any objective constant
    if cor.objgiven && include_constant
        c₂ = -get(cor.rhs, cor.objname, zero(T))
    end
    # Fill bounds
    fill_bounds!(cor, lb, ub, is_binary, is_integer, colrange, map)
    # Return the problem
    return LPData(T, c₁, c₂, A, b, d₁, C, d₂, lb, ub, is_binary, is_integer, map)
end

function fill_bounds!(cor::RawCor{T}, lb, ub, is_binary, is_integer, colrange, map) where T <: AbstractFloat
    ncols = 0
    for (col, val) in cor.bounds
        col in colrange || (ncols += 1; continue)
        colidx = cor.vars[col]
        j = colidx - ncols
        for (bndtype, bndval) in val
            if bndtype == LOWER
                lb[j] = bndval
                map[(LOWER,col)] = (0,j,GEQ)
            elseif bndtype == UPPER
                ub[j] = bndval
                map[(UPPER,col)] = (0,j,LEQ)
            elseif bndtype == FIXED
                lb[j] = ub[j] = bndval
                map[(FIXED,col)] = (0,j,LEQ)
                map[(FIXED,col)] = (0,j,GEQ)
            elseif bndtype == BINARY
                is_binary[j] = true
                lb[j] = convert(T, -Inf)
                map[(BINARY,col)] = (0,j,BINARY)
            elseif bndtype == INTEGER
                is_integer[j] = true
                lb[j] = convert(T, -Inf)
                map[(INTEGER,col)] = (0,j,INTEGER)
            elseif bndtype == INTEGER_LOWER
                is_integer[j] = true
                lb[j] = bndval
                map[(INTEGER,col)] = (0,j,INTEGER_LOWER)
            elseif bndtype == INTEGER_UPPER
                is_integer[j] = true
                ub[j] = bndval
                map[(INTEGER,col)] = (0,j,INTEGER_UPPER)
            else
                lb[j] = convert(T, -Inf)
                map[(FREE,col)] = (0,j,FREE)
            end
        end
    end
    return nothing
end


const CONST_FIELD_NUM = 10

function return_type(f, ts::A) where {T <: Type, A <:AbstractArray{T, 1}}
    ts = Base.return_types(f, ts)
    if length(ts) === 1
        ts[1]
    else
        Union{ts...}
    end
end

function return_type(f, ts :: NTuple{N, T}) where {N, T <: Type}
    ts = Base.return_types(f, ts)
    if length(ts) === 1
        ts[1]
    else
        Union{ts...}
    end
end

function return_type(f, t :: Type)
    return_type(f, (t,))
end

for i = 1:CONST_FIELD_NUM
    types = [Symbol("T", j) for j = 1:i]
    inp = :(Type{Tuple{$(types...)}})
    out = Expr(:vect, types...)
    @eval @inline type_unpack(n::Int, ::$inp) where {$(types...)} = $out
end

function type_unpack(n :: Int, ::Type{T}) where T <: Tuple
    if T === Base.Bottom
        type_unpack(n, Any)
    else
        collect(T.parameters)
    end
end

function type_unpack(n::Int, :: Type{Tuple{}})
    throw("TypeError at query expression.")
end

function type_unpack(n::Int, :: Type{Any})
    fill(Any, n)
end


## Date-time type conversions for indexing
function _convert(::Type{Date}, str::String)
    Date(Dates.parse_components(str, Dates.dateformat"yyyy-mm-dd")...)
end

function _convert(::Type{String}, date::Date)
    Dates.format(date, "yyyy-mm-dd")
end

"""
# Indexing

`TS` can be indexed using row and column indices. The row selector
could be an integer, a range, an array or it could also be a `Date`
object or an ISO-formatted date string ("2007-04-10"). There are
methods to subset on year, year-month, and year-quarter. The latter
two subset `coredata` by matching on the index column.

Column selector could be an integer or any other selector which
`DataFrame` indexing supports. You can use a Symbols to fetch specific
columns (ex: `ts[[:x1, :x2]]`). For fetching column values
as `Vector`, use `Colon` with column name: `ts[:, :x1]`. For Matrix
output, use the constructor: `Matrix(ts)`.

For fetching the index column vector use the `index()` method.

# Examples

```jldoctest; setup = :(using TSx, DataFrames, Dates, Random, Statistics)
julia> using Random;

julia> random(x) = rand(MersenneTwister(123), x);

julia> ts = TS([random(10) random(10) random(10)])
julia> show(ts)

# first row
julia> ts[1]
(1 x 3) TS with Int64 Index

 Index  x1        x2        x3
 Int64  Float64   Float64   Float64
─────────────────────────────────────
     1  0.768448  0.768448  0.768448

# first five rows
julia> ts[1:5]
(5 x 3) TS with Int64 Index

 Index  x1        x2        x3
 Int64  Float64   Float64   Float64
─────────────────────────────────────
     1  0.768448  0.768448  0.768448
     2  0.940515  0.940515  0.940515
     3  0.673959  0.673959  0.673959
     4  0.395453  0.395453  0.395453
     5  0.313244  0.313244  0.313244

# first five rows, x2 column; returns a vector
julia> ts[1:5, :x2]
5-element Vector{Float64}:
 0.7684476751965699
 0.940515000715187
 0.6739586945680673
 0.3954531123351086
 0.3132439558075186

julia> ts[1:5, 2:3]
(5 x 2) TS with Int64 Index

 Index  x2        x3
 Int64  Float64   Float64
───────────────────────────
     1  0.768448  0.768448
     2  0.940515  0.940515
     3  0.673959  0.673959
     4  0.395453  0.395453
     5  0.313244  0.313244

# individual rows
julia> ts[[1, 9]]
(2 x 3) TS with Int64 Index

 Index  x1        x2        x3
 Int64  Float64   Float64   Float64
─────────────────────────────────────
     1  0.768448  0.768448  0.768448
     9  0.26864   0.26864   0.26864

julia> ts[:, :x1]            # returns a Vector
10-element Vector{Float64}:
 0.7684476751965699
 0.940515000715187
 0.6739586945680673
 0.3954531123351086
 0.3132439558075186
 0.6625548164736534
 0.5860221243068029
 0.05213316316865657
 0.26863956854495097
 0.10887074134844155

julia> ts[:, [:x1, :x2]]
(10 x 2) TS with Int64 Index

 Index  x1         x2
 Int64  Float64    Float64
─────────────────────────────
     1  0.768448   0.768448
     2  0.940515   0.940515
     3  0.673959   0.673959
     4  0.395453   0.395453
     5  0.313244   0.313244
     6  0.662555   0.662555
     7  0.586022   0.586022
     8  0.0521332  0.0521332
     9  0.26864    0.26864
    10  0.108871   0.108871

julia> dates = collect(Date(2007):Day(1):Date(2008, 2, 22));
julia> ts = TS(random(length(dates)), dates)
julia> show(ts[1:10])
(10 x 1) TS with Date Index

 Index       x1
 Date        Float64
───────────────────────
 2007-01-01  0.768448
 2007-01-02  0.940515
 2007-01-03  0.673959
 2007-01-04  0.395453
 2007-01-05  0.313244
 2007-01-06  0.662555
 2007-01-07  0.586022
 2007-01-08  0.0521332
 2007-01-09  0.26864
 2007-01-10  0.108871

julia> ts[Date(2007, 01, 01)]
(1 x 1) TS with Dates.Date Index

 Index       x1
 Date        Float64
──────────────────────
 2007-01-01  0.768448

julia> ts[[Date(2007, 1, 1), Date(2007, 1, 2)]]
(2 x 1) TS with Date Index

 Index       x1
 Date        Float64
──────────────────────
 2007-01-01  0.768448
 2007-01-02  0.940515

julia> ts[[Date(2007, 1, 1), Date(2007, 1, 2)], :x1]
2-element Vector{Float64}:
 0.7684476751965699
 0.940515000715187

julia> ts[Date(2007)]
(1 x 1) TS with Dates.Date Index

 Index       x1
 Date        Float64
──────────────────────
 2007-01-01  0.768448

julia> ts[Year(2007)]
(365 x 1) TS with Dates.Date Index

 Index       x1
 Date        Float64
───────────────────────
 2007-01-01  0.768448
 2007-01-02  0.940515
 2007-01-03  0.673959
 2007-01-04  0.395453
 2007-01-05  0.313244
 2007-01-06  0.662555
 2007-01-07  0.586022
 2007-01-08  0.0521332
     ⋮           ⋮
 2007-12-24  0.468421
 2007-12-25  0.0246652
 2007-12-26  0.171042
 2007-12-27  0.227369
 2007-12-28  0.695758
 2007-12-29  0.417124
 2007-12-30  0.603757
 2007-12-31  0.346659
       349 rows omitted

julia> ts[Year(2007), Month(11)]
(30 x 1) TS with Date Index

 Index       x1
 Date        Float64
───────────────────────
 2007-11-01  0.214132
 2007-11-02  0.672281
 2007-11-03  0.373938
 2007-11-04  0.317985
 2007-11-05  0.110226
 2007-11-06  0.797408
 2007-11-07  0.095699
 2007-11-08  0.186565
 2007-11-09  0.586859
 2007-11-10  0.623613
 2007-11-11  0.62035
 2007-11-12  0.830895
 2007-11-13  0.72423
 2007-11-14  0.493046
 2007-11-15  0.767975
 2007-11-16  0.462157
 2007-11-17  0.779754
 2007-11-18  0.398596
 2007-11-19  0.941196
 2007-11-20  0.578657
 2007-11-21  0.702451
 2007-11-22  0.746427
 2007-11-23  0.301046
 2007-11-24  0.619772
 2007-11-25  0.425161
 2007-11-26  0.410939
 2007-11-27  0.0883656
 2007-11-28  0.135477
 2007-11-29  0.693611
 2007-11-30  0.557009

julia> ts[Year(2007), Quarter(2)];

julia> ts["2007-01-01"]
(1 x 1) TS with Date Index

 Index       x1
 Date        Float64
──────────────────────
 2007-01-01  0.768448

julia> ts[1, :x1]  # returns a scalar
0.7684476751965699

julia> ts[1, "x1"]; # same as above

```
"""
###
## Row-Column interfaces
###

### Inputs: row scalar, column scalar; Output: scalar
function Base.getindex(ts::TS, i::Int, j::Int)
    ts.coredata[i, j+1]
end

function Base.getindex(ts::TS, i::Int, j::Union{Symbol, String})
    return ts.coredata[i, j]
end

function Base.getindex(ts::TS, dt::T, j::Int) where {T<:TimeType}
    idx = findfirst(x -> x == dt, index(ts))
    ts.coredata[idx, j+1]
end

function Base.getindex(ts::TS, dt::T, j::Union{String, Symbol}) where {T<:TimeType}
    idx = findfirst(x -> x == dt, index(ts))
    ts.coredata[idx, j]
end
###

### Inputs: row scalar, column vector; Output: TS
function Base.getindex(ts::TS, i::Int, j::AbstractVector{Int})
    TS(ts.coredata[[i], Cols(:Index, j.+1)]) # increment: account for Index
end

function Base.getindex(ts::TS, i::Int, j::AbstractVector{T}) where {T<:Union{String, Symbol}}
    TS(ts.coredata[[i], Cols(:Index, j)])
end

function Base.getindex(ts::TS, dt::T, j::AbstractVector{Int}) where {T<:TimeType}
    idx = findfirst(x -> x == dt, index(ts))
    ts[idx, j]
end

function Base.getindex(ts::TS, dt::D, j::AbstractVector{T}) where {D<:TimeType, T<:Union{String, Symbol}}
    idx = findfirst(x -> x == dt, index(ts))
    ts[idx, j]
end
###

### Inputs: row scalar, column range; Output: TS
function Base.getindex(ts::TS, i::Int, j::UnitRange)
    return TS(ts.coredata[[i], Cols(:Index, 1 .+(j))])
end
###

### Inputs: row vector, column scalar; Output: vector
function Base.getindex(ts::TS, i::AbstractVector{Int}, j::Int)
    ts.coredata[i, j+1] # increment: account for Index
end

function Base.getindex(ts::TS, i::AbstractVector{Int}, j::Union{String, Symbol})
    ts.coredata[i, j]
end

function Base.getindex(ts::TS, dt::AbstractVector{T}, j::Int) where {T<:TimeType}
    idx = map(d -> findfirst(x -> x == d, index(ts)), dt)
    if length(idx) == 0
        return nothing
    end
    ts[idx, j]
end

function Base.getindex(ts::TS, dt::AbstractVector{T}, j::Union{String, Symbol}) where {T<:TimeType}
    idx = map(d -> findfirst(x -> x == d, index(ts)), dt)
    if length(idx) == 0
        return nothing
    end
    ts[idx, j]
end
###

### Inputs: row vector, column vector; Output: TS
function Base.getindex(ts::TS, i::AbstractVector{Int}, j::AbstractVector{Int})
    TS(ts.coredata[i, Cols(:Index, j.+1)]) # increment: account for Index
end

function Base.getindex(ts::TS, i::AbstractVector{Int}, j::AbstractVector{T}) where {T<:Union{String, Symbol}}
    TS(ts.coredata[i, Cols(:Index, j)])
end

function Base.getindex(ts::TS, dt::AbstractVector{T}, j::AbstractVector{Int}) where {T<:TimeType}
    idx = map(d -> findfirst(x -> x == d, index(ts)), dt)
    if length(idx) == 0
        return nothing
    end
    ts[idx, j]
end

function Base.getindex(ts::TS, dt::AbstractVector{D}, j::AbstractVector{T}) where {D<:TimeType, T<:Union{String, Symbol}}
    idx = map(d -> findfirst(x -> x == d, index(ts)), dt)
    ts[idx, j]
end
###

### Inputs: row vector, column range; Output: TS
function Base.getindex(ts::TS, i::AbstractVector{Int}, j::UnitRange)
    ts[i, collect(j)]
end

function Base.getindex(ts::TS, dt::AbstractVector{T}, j::UnitRange) where {T<:TimeType}
    ts[dt, collect(j)]
end
###

### Inputs: row range, column scalar: return a vector
function Base.getindex(ts::TS, i::UnitRange, j::Int)
    ts[collect(i), j]
end

function Base.getindex(ts::TS, i::UnitRange, j::Union{String, Symbol})
    ts[collect(i), j]
end
###

### Inputs: row range, column vector: return TS
function Base.getindex(ts::TS, i::UnitRange, j::AbstractVector{Int})
    ts[collect(i), j]
end

function Base.getindex(ts::TS, i::UnitRange, j::AbstractVector{T}) where {T<:Union{String, Symbol}}
    ts[collect(i), j]
end
###

### Inputs: row range, column range: return TS
function Base.getindex(ts::TS, i::UnitRange, j::UnitRange)
    ts[collect(i), collect(j)]
end

###
## Row indexing interfaces
###

function Base.getindex(ts::TS, i::Int)
    ts[i, 1:TSx.ncol(ts)]
end

function Base.getindex(ts::TS, i::UnitRange)
    ts[i, 1:TSx.ncol(ts)]
end

function Base.getindex(ts::TS, i::AbstractVector{Int64})
    ts[i, 1:TSx.ncol(ts)]
end

function Base.getindex(ts::TS, dt::AbstractVector{T}) where {T<:TimeType}
    ts[dt, 1:TSx.ncol(ts)]
end

function Base.getindex(ts::TS, d::T) where {T<:TimeType}
    ts[[d], 1:TSx.ncol(ts)]
end

# By period
function Base.getindex(ts::TS, y::Year)
    sdf = filter(:Index => x -> Dates.Year(x) == y, ts.coredata)
    TS(sdf)
end

function Base.getindex(ts::TS, y::Year, q::Quarter)
    sdf = filter(:Index => x -> (Year(x), Quarter(x)) == (y, q), ts.coredata)
    TS(sdf)
end

# XXX: ideally, Dates.YearMonth class should exist
function Base.getindex(ts::TS, y::Year, m::Month)
    sdf = filter(:Index => x -> yearmonth(x) == (y, m), ts.coredata)
    TS(sdf)
end

function Base.getindex(ts::TS, y::Year, m::Month, d::Day)
    sdf = filter(:Index => x -> (Year(x), Month(x), Day(x)) == (y, m, d), ts.coredata)
    TS(sdf)
end

function Base.getindex(ts::TS, y::Year, m::Month, w::Week)
    sdf = filter(:Index => x -> (Year(x), Month(x), Week(x)) == (y, m, w), ts.coredata)
    TS(sdf)
end

# By string timestamp
function Base.getindex(ts::TS, i::String)
    ind = findall(x -> x == TSx._convert(eltype(ts.coredata[!, :Index]), i), ts.coredata[!, :Index]) # XXX: may return duplicate indices
    TS(ts.coredata[ind, :])     # XXX: check if data is being copied
end

# By {TimeType, Period} range
# function Base.getindex(ts::TS, r::StepRange{T, V}) where {T<:TimeType, V<:Period}
# end

###
## Column indexing with Colon
###

### Inputs: row colon, column scalar: return vector
function Base.getindex(ts::TS, ::Colon, j::Int)
    ts[1:TSx.nrow(ts), j]
end

function Base.getindex(ts::TS, ::Colon, j::Union{String, Symbol})
    ts[1:TSx.nrow(ts), j]
end
###

### Inputs: row colon, column vector: return TS
function Base.getindex(ts::TS, ::Colon, j::AbstractVector{Int})
    ts[1:TSx.nrow(ts), j]
end

function Base.getindex(ts::TS, ::Colon, j::AbstractVector{T}) where {T<:Union{String, Symbol}}
    ts[1:TSx.nrow(ts), j]
end
###

### Inputs: row colon, column range: return TS
function Base.getindex(ts::TS, i::Colon, j::UnitRange)
    ts[1:TSx.nrow(ts), collect(j)]
end
###


# ------------------------------------------------------------------------------
# Read converted NCA5 files and add 15 mm baseline correction to account for the
# 2000 - 2005 difference between both
# ------------------------------------------------------------------------------
module GriddedProjections
using NetCDF
using NCDatasets

function RunGriddedProjections(settings)
    println("\nGridded scenarios...")
    # Locations and time
    fn = settings["dir_NCA5"]*"NCA5_Low_grid.nc"
    lon = ncread(fn,"lon")
    lat = ncread(fn,"lat")
    years_NCA5 = ncread(fn,"years",start=[1],count=[14])

    # Create output NetCDF file
    fh = Dataset(settings["fn_proj_gri"],"c")
    fh.attrib["title"] = "Gridded projections"
    fh.attrib["description"] = "Gridded projections for the interagency report: Global and Regional Sea Level Rise Scenarios for the United States: Updated Mean Projections and Extreme Water Level Probabilities Along U.S. Coastlines"
    fh.attrib["processes"] = "AIS: Antarctic Ice Sheet, GIS: Greenland, glaciers: Glaciers and Ice Caps, landwaterstorage: Liquid water storage changes on land, oceandynamics: Ocean dynamics and global thermosteric expansion, verticallandmotion: Vertical land motion, total: All processes combined."

    defDim(fh,"years", length(years_NCA5))
    defDim(fh,"percentiles",3)
    defDim(fh,"lon", length(lon))
    defDim(fh,"lat", length(lat))

    defVar(fh,"lon",lon,("lon",),deflatelevel=5, attrib = Dict("description" => "Longitude", "units" => "degrees East"))
    defVar(fh,"lat",lat,("lat",),deflatelevel=5, attrib = Dict("description" => "Latitude", "units" => "degrees North"))
    defVar(fh,"years",years_NCA5,("years",),deflatelevel=5 ,attrib = Dict("units" => "years"))
    defVar(fh,"percentiles",convert.(Int32,[17,50,83]),("percentiles",),deflatelevel=5, attrib = Dict("description" => "The percentile of the projection. The 50th percentile shows the median projection, and the 17th and 83rd show the upper- and lower bound on the 1 sigma level.", "units" => "-"))
    # Write scenarios
    for scn in settings["NCA5_scenarios"]
        fn_in = settings["dir_NCA5"]*"NCA5_"*scn*"_grid.nc"
        for prc in settings["processes"]
            NCA5_prc = ncread(fn_in,prc,start=[1,1,1,1],count=[-1,-1,14,-1]);
            if (prc == "total") | (prc == "verticallandmotion")
                NCA5_prc .+= 15
            end
            defVar(fh,"rsl_"*prc*"_"*scn,NCA5_prc,("lon","lat","years","percentiles",),deflatelevel=5, attrib = Dict("description" => "Projected local sea level for scenario "*scn*" and process "*prc*", relative to year 2000.", "units" => "mm"))
        end
    end
    close(fh)
    println("Gridded scenarios done\n")
end

end


module GFlops
import Statistics
import BenchmarkTools
using BenchmarkTools:   @benchmark
using InteractiveUtils: peakflops
using Printf:           @printf
using PrettyTables:     pretty_table

export @gflops, @count_ops

include("overdub.jl")
include("Counter.jl")
include("count_ops.jl")

end # module


if VERSION > v"0.7-"
    push!(LOAD_PATH, Base.NamedEnv("SegmentTrees.jl"))
    using InteractiveUtils
    using Random
    using Profile
end
versioninfo()

using SegmentTrees
using CodecZlib

function readgff3(input::IO, chrom::String)
    segments = Segment{Int}[]
    for line in eachline(input)
        if startswith(line, "#")
            continue
        end
        values = split(line, '\t')
        if values[1] == chrom
            chromstart = parse(Int, values[4])
            chromend = parse(Int, values[5])
            push!(segments, Segment(chromstart, chromend))
        end
    end
    sort!(segments)
    return SegmentTree{Int,Segment{Int}}(segments)
end

function readgff3(filepath::String, chrom::String)
    stream = open(filepath)
    if endswith(filepath, ".gz")
        stream = GzipDecompressorStream(stream)
    end
    try
        return readgff3(stream, chrom)
    finally
        close(stream)
    end
end

isdir("data") || mkdir("data")
cd("data")
filename="Danio_rerio.GRCz11.92.gff3.gz"
isfile(filename)||
    download("ftp://ftp.ensembl.org/pub/release-92/gff3/danio_rerio/Danio_rerio.GRCz11.92.gff3.gz", filename)
cd("..")

tree = readgff3("data/Danio_rerio.GRCz11.92.gff3.gz", "1")

println("random query")
function random_query(tree, queries)
    n = 0
    for q in queries
        for _ in intersect(tree, q)
            n += 1
        end
    end
    return n
end
srand(1234)
segments = shuffle!(collect(tree))
for _ in 1:5
    @time random_query(tree, segments)
end
if "-p" in ARGS
    @profile random_query(tree, segments)
    Profile.print()
end


# This file is a part of InverseFunctions.jl, licensed under the MIT License (MIT).

using Test
using InverseFunctions

@testset "square" begin
    for x in (0.0, 0.73)
        @test InverseFunctions.square(x) ≈ x * x
    end

    @test_throws DomainError InverseFunctions.square(-1)
end


"""
This module provides:
- An extension of the `AbstractArray` interface in order to properly deal with mutable caches.
- A mechanism to generate lazy arrays resulting from operations between arrays.
- A collection of concrete implementations of `AbstractArray`.

The exported names in this module are:

$(EXPORTS)
"""
module Arrays

using InplaceArrays.Helpers
using InplaceArrays.Inference

using DocStringExtensions
using Test
using FillArrays
using Base: @propagate_inbounds

export array_cache
export getindex!
export getitems!
export testitem
export uses_hash
export test_array
export testitems
export array_caches

export CachedArray
export CachedMatrix
export CachedVector
export setsize!

export CompressedArray
export LocalToGlobalArray
export Table

export kernel_cache
export kernel_caches
export apply_kernels!
export apply_kernel!
export apply_kernel
export test_kernel
export bcast
export elem
export contract
export kernel_return_type
export kernel_return_types
export Kernel

export apply
export apply_all

import Base: size
import Base: getindex, setindex!
import Base: similar
import Base: IndexStyle

include("Interface.jl")

include("CachedArrays.jl")

include("Kernels.jl")

include("Apply.jl")

include("CompressedArrays.jl")

include("Tables.jl")

include("LocalToGlobalArrays.jl")

end # module


using .DataFrames

function meltdata(U::AbstractDataFrame, colgroups_::Vector{Col.GroupedColumn})
    um, un = size(U)

    colgroups = Set(colgroups_)

    # Figure out the size of the new melted matrix
    allcolumns = Set{Symbol}(names(U))

    vm = um
    grouped_columns = Set{Symbol}()
    for colgroup in colgroups
        if colgroup.columns===nothing # null => group all columns
            vm *= un
            grouped_columns = copy(allcolumns)
        else
            for j in colgroup.columns
                if !isa(j, Symbol)
                    error("DataFrame columns can only be grouped by (Symbol) names")
                end
                push!(grouped_columns, j)
            end
            vm *= length(colgroup.columns)
        end
    end

    ungrouped_columns = setdiff(allcolumns, grouped_columns)
    vn = length(colgroups) + length(ungrouped_columns)

    V = AbstractArray[]
    vnames = Symbol[]
    colmap = Dict{Any, Int}()

    # allocate vectors for grouped columns
    for (j, colgroup) in enumerate(colgroups)
        cols = colgroup.columns===nothing ? allcolumns : colgroup.columns

        # figure the grouped common column type
        firstcol = U[first(cols)]
        eltyp = eltype(firstcol)
        vectyp = isa(firstcol, Vector) ? Vector : DataVector
        for col in cols
            eltyp = promote_type(eltyp, typeof(U[col]))
            if !isa(U[col], Vector)
                vectyp = DataVector
            end
        end

        push!(V, eltyp == Vector ? Array{eltyp}(undef, vm) : Array{Union{Nothing,eltyp}}(undef, vm))
        name = gensym()
        push!(vnames, name)
        colmap[colgroup] = j
    end

    # allocate vectors for ungrouped columns
    for (j, col) in enumerate(ungrouped_columns)
        push!(V, similar(U[col], vm))
        colmap[col] = j + length(colgroups)
        push!(vnames, col)
    end

    # Indicator columns for each colgroup
    col_indicators = Array{Symbol}(undef, vm, length(colgroups))
    row_indicators = Array{Int}(undef, vm, length(colgroups))

    colidxs = [colgroup.columns===nothing ? collect(allcolumns) : colgroup.columns
               for colgroup in colgroups]

    vi = 1
    for ui in 1:um
        for colidx in Iterators.product(colidxs...)
            # copy grouped columns
            for (vj, uj) in enumerate(colidx)
                V[vj][vi] = U[ui, uj]
                col_indicators[vi, vj] = uj
                row_indicators[vi, vj] = ui
            end

            # copy ungrouped columns
            for (vj, uj) in enumerate(ungrouped_columns)
                V[vj + length(colgroups)][vi] = U[ui, uj]
            end

            vi += 1
        end
    end

    df = DataFrame(; collect(zip(vnames, V))...)
    return MeltedData(U, df, row_indicators, col_indicators, colmap)
end

evalmapping(source::MeltedData{T}, arg::Col.GroupedColumnValue) where T<:AbstractDataFrame =
    source.melted_data[source.colmap[Col.GroupedColumn(arg.columns)]]

evalmapping(source::AbstractDataFrame, arg::Symbol) = source[arg]
evalmapping(source::AbstractDataFrame, arg::AbstractString) = evalmapping(source, Symbol(arg))
evalmapping(source::AbstractDataFrame, arg::Integer) = source[arg]
evalmapping(source::AbstractDataFrame, arg::Expr) = with(source, arg)


using DataFrames
using Test
using Random: randstring

@testset "sort it" begin
M = 100_000
str_base = [randstring(8) for i in 1:1_000]
df = DataFrame(int = rand(Int32, M), float=rand(M), str = rand(str_base, M))

@time df1 = sort(df, :int);
@time df2 = fsort(df, :int);
@test df1 == df2
@test df != df2

@time df1 = sort(df, :str);
@time df2 = fsort(df, :str);
@test df1 == df2
@test df != df2

@time df1 = sort(df, [:str, :float]);
@time df2 = fsort(df, [:str, :float]);
@test df1 == df2
@test df != df2
end


using ArgParse
include("iPredict.jl")
using .iPredict

function parse_commandline()
    s = ArgParseSettings()

    @add_arg_table s begin
        "--mode"
            help = "Set to 'global' to do global simulations, set to 'gene' to simulate a single gene"
            required = true
            arg_type = String
        "--gene"
            help = "The gene to simulate in gene-level simulations"
            required = false
            arg_type = String
        "--countsMatrix"
            help = "Path to isoform-level counts matrix"
            required = true
            arg_type = String
        "--GeneIsoformRelationships"
            help = "Path to file containing gene-isoform relationships"
            required = true
            arg_type = String
        "--NumSimulations"
            help = "Number of simulations"
            required = true
            arg_type = Int64
        "--output"
            help="Path to save output at"
            required = true
            arg_type = String
        "--NumIsoformsToSimulate"
            help = "Max number of isoforms to simulate"
            required = false
            arg_type = Int64
        "--filteringThreshold"
            help = "All genes must express at least this number of isoforms"
            required = false
            arg_type = Int64
        "--alpha"
            help = "Beta distribution parameter alpha"
            required = false
            arg_type = Float64
        "--beta"
            help = "Beta distribution parameter beta"
            required = false
            arg_type = Float64
        "--pFP"
            required = false
            default = 0.01
            arg_type = Float64
        "--pFN"
            required = false
            default = 0.04
            arg_type = Float64

    end

    return parse_args(s)
end

function main()

    #Get command line args. Very basic atm, should add some argument validity
    #checking.
    parsed_args = parse_commandline()
    mode = parsed_args["mode"]
    countsMatrix = parsed_args["countsMatrix"]
    GeneIsoformRelationships = parsed_args["GeneIsoformRelationships"]
    NumSimulations = parsed_args["NumSimulations"]
    output = parsed_args["output"]
    NumIsoformsToSimulate = parsed_args["NumIsoformsToSimulate"]
    filteringThreshold = parsed_args["filteringThreshold"]
    alpha::Union{Nothing,Float64} = parsed_args["alpha"]
    beta::Union{Nothing,Float64} = parsed_args["beta"]
    pFP::Float64 = parsed_args["pFP"]
    pFN::Float64 = parsed_args["pFN"]

    if mode == "global"
        globalPredict(countsMatrix, GeneIsoformRelationships,
        NumSimulations, output, NumIsoformsToSimulate, filteringThreshold,
        false, alpha, beta, pFN, pFP)

    elseif mode == "gene"
        gene = parsed_args["gene"]
        oneGenePrediction(countsMatrix, GeneIsoformRelationships, gene,
        NumSimulations, output, false)

    else
        throw(ArgumentError("Unsupported mode - valid modes are gene or global.
        See help manual for details."))
    end
end

main()


# This file constains testset for Printer

@testset "PrinterTestSet" begin
    @info "Running PrinterTestSet ..."

    # Printer construction
    printer = Printer(input=Inport(2), buflen=100)
    @test typeof(printer.trigger) == Inpin{Float64}
    @test typeof(printer.handshake) == Outpin{Bool}
    @test size(printer.timebuf) == (100,)
    @test size(printer.databuf) == (2, 100)
    @test isa(printer.input, Inport)
    @test printer.plugin === nothing
    @test typeof(printer.callbacks) <: Nothing
    @test typeof(printer.sinkcallback) <: Callback

    # Driving Printer
    oport, iport, trg, hnd, tsk, tsk2 = prepare(printer)
    for t in 1 : 200
        put!(trg, t)
        put!(oport, ones(2) * t)
        take!(hnd)
        @test read(printer.timebuf) == t
        @test [read(pin.links[1].buffer) for pin in oport] == ones(2) * t
    end
    put!(trg, NaN)
    sleep(0.1)
    @test istaskdone(tsk)

    @info "Done PrinterTestSet ..."
end  # testset


# EXPERIMENTAL - Threaded version 3. Max number of tasks = 5 * Threads.nthreads()

function corkendallthreads_v3(X::Union{RealMatrix,RealOrMissingMatrix}, Y::Union{RealMatrix,RealOrMissingMatrix})
    nr = size(X, 2)
    nc = size(Y, 2)
    C = zeros(float(eltype(X)), nr, nc)

    numtasks = min(nc, 10 * Threads.nthreads())
    chunksize = Int(max(1, round(nc / numtasks)))
    numtasks = Int(round(nc / chunksize, RoundUp))

    tasks = Array{Task,1}(undef, numtasks)
    for j = 1:numtasks
        fromcol = (j - 1) * chunksize + 1
        tocol = min(fromcol + chunksize - 1, nc)
        tasks[j] = @spawn corkendall(X, Y[:,fromcol:tocol])
    end

    for j = 1:numtasks
        fromcol = (j - 1) * chunksize + 1
        tocol = min(fromcol + chunksize - 1, nc)
        C[:,fromcol:tocol] = fetch(tasks[j])
    end

    return C
end


"""Convert a `Colorant` to an `(r,g,b,a)` tuple ∈ [0,1]⁴, as accepted by Matplotlib."""
toRGBAtuple(c) = toRGBAtuple(RGBA(c))
toRGBAtuple(c::RGBA) = (c.r, c.g, c.b, c.alpha)

mplcolors = C0, C1, C2, C3, C4, C5, C6, C7, C9, C10 = parse.(RGB,
    ["#1f77b4", "#ff7f0e", "#2ca02c", "#d62728", "#9467bd",
     "#8c564b", "#e377c2", "#7f7f7f", "#bcbd22", "#17becf" ]
)

black = Gray(0)
white = Gray(1)
lightgrey = lightgray = Gray(0.77)

"""
Mix a color with white. `original` specifies how much is left of the original color.
`0`: output is pure white. `1`: output is the original color.
Visually equivalent to setting the color's alpha to `original`, on a white background.
"""
lighten(c::C, original = 0.8) where {C<:Color} = C(mix(RGB(c), RGB(white), 1 - original))
#   White goes last, because colors can't have negative channels (`b - a` in `mix`), even
#   with ColorVectorSpace.

"""
Mix a color with black. `original` specifies how much is left of the original color.
`0`: output is pure black. `1`: output is the original color.
"""
darken(c::C, original = 0.8) where {C<:Color} = C(mix(RGB(black), RGB(c), original))

"""Linearly interpolate ("lerp") between `a` (`t = 0`) and `b` (`t = 1`)."""
mix(a, b, t=0.5) = a + t * (b - a)

"""
De-emphasise part of an Axes by colouring it light grey.
`part` is one of {:xlabel, :ylabel, :xaxis, :yaxis}.
"""
function deemph(part::Symbol, ax; color = lightgrey)
    color = toRGBAtuple(color)
    if part == :xlabel
        ax.xaxis.get_label().set_color(color)
    elseif part == :ylabel
        ax.yaxis.get_label().set_color(color)
        hasproperty(ax, :hylabel) && ax.hylabel.set_color(color)
    elseif part == :xaxis
        foreach(loc -> ax.spines[loc].set_color(color), ["top", "bottom"])
        ax.tick_params(; axis = "x", which = "both", color, labelcolor = color)
    elseif part == :yaxis
        foreach(loc -> ax.spines[loc].set_color(color), ["left", "right"])
        ax.tick_params(; axis = "y", which = "both", color, labelcolor = color)
    end
end


include("conjugategradient.jl")
export HagerZhang


"""
Analysis data of Monte Carlo simulation
"""
@with_kw mutable struct MCAnalysis
    acc_rate::Float64 = 0.
    prop_local::Int = 0
    acc_local::Int = 0
    acc_rate_global::Float64 = 0.
    prop_global::Int = 0
    acc_global::Int = 0
end

"""
Parameters of Monte Carlo
"""
@with_kw struct MCParameters
    global_moves::Bool = false
    global_rate::Int = 5
    thermalization::Int = 0 # number of thermalization sweeps
    sweeps::Int = 1000 # number of sweeps (after thermalization)
    measure_rate::Int = 1 # measure at every nth sweep (after thermalization)
    print_rate::Int = 1000

    beta::Float64
end

"""
Monte Carlo simulation
"""
mutable struct MC{M<:Model, C} <: MonteCarloFlavor
    model::M
    conf::C

    thermalization_measurements::Dict{Symbol, AbstractMeasurement}
    measurements::Dict{Symbol, AbstractMeasurement}
    p::MCParameters
    a::MCAnalysis

    MC{M,C}() where {M,C} = new()
end

"""
    MC(m::M; kwargs...) where M<:Model

Create a Monte Carlo simulation for model `m` with keyword parameters `kwargs`.
"""
function MC(m::M;
        seed::Int=-1,
        thermalization_measurements = Dict{Symbol, AbstractMeasurement}(),
        measurements = :default,
        kwargs...
    ) where M<:Model

    conf = rand(MC, m)
    mc = MC{M, typeof(conf)}()
    mc.model = m
    kwdict = Dict(kwargs)
    if :T in keys(kwargs)
        kwdict[:beta] = 1/kwargs[:T]
        delete!(kwdict, :T)
    end
    mc.p = MCParameters(; kwdict...)

    init!(
        mc, seed = seed, conf = conf,
        thermalization_measurements = thermalization_measurements,
        measurements = measurements
    )
    return mc
end

"""
    MC(m::M, params::Dict)
    MC(m::M, params::NamedTuple)

Create a Monte Carlo simulation for model `m` with (keyword) parameters
as specified in the dictionary/named tuple `params`.
"""
MC(m::Model, params::Dict{Symbol, T}) where T = MC(m; params...)
MC(m::Model, params::NamedTuple) = MC(m; params...)

# convenience
@inline beta(mc::MC) = mc.p.beta
@inline model(mc::MC) = mc.model
@inline conf(mc::MC) = mc.conf

# cosmetics
Base.summary(mc::MC) = "MC simulation of $(summary(mc.model))"
function Base.show(io::IO, mc::MC)
    print(io, "Monte Carlo simulation\n")
    print(io, "Model: ", mc.model, "\n")
    print(io, "Beta: ", round(beta(mc), sigdigits=3), " (T ≈ $(round(1/beta(mc), sigdigits=3)))\n")
    N_th_meas = length(mc.thermalization_measurements)
    N_me_meas = length(mc.measurements)
    print(io, "Measurements: ", N_th_meas + N_me_meas, " ($N_th_meas + $N_me_meas)")
end
Base.show(io::IO, m::MIME"text/plain", mc::MC) = print(io, mc)

# implement MonteCarloFlavor interface
"""
    init!(mc::MC[; seed::Real=-1])

Initialize the Monte Carlo simulation `mc`.
If `seed !=- 1` the random generator will be initialized with `Random.seed!(seed)`.
"""
function init!(mc::MC;
        seed::Real=-1,
        conf=rand(MC, mc.model),
        thermalization_measurements = Dict{Symbol, AbstractMeasurement}(),
        measurements = :default
    )
    seed == -1 || Random.seed!(seed)
    mc.conf = conf
    mc.a = MCAnalysis()

    mc.thermalization_measurements = thermalization_measurements
    if measurements isa Dict{Symbol, AbstractMeasurement}
        mc.measurements = measurements
    elseif measurements == :default
        mc.measurements = default_measurements(mc, mc.model)
    else
        @warn(
            "`measurements` should be of type Dict{Symbol, AbstractMeasurement}, but are " *
            "$(typeof(measurements)). No measurements have been set."
        )
        mc.measurements = Dict{Symbol, AbstractMeasurement}()
    end

    nothing
end

"""
    run!(mc::MC[; verbose::Bool=true, sweeps::Int, thermalization::Int])

Runs the given Monte Carlo simulation `mc`.
Progress will be printed to `stdout` if `verbose=true` (default).
"""
function run!(mc::MC; verbose::Bool=true, sweeps::Int=mc.p.sweeps,
        thermalization=mc.p.thermalization)

    do_th_measurements = !isempty(mc.thermalization_measurements)
    do_me_measurements = !isempty(mc.measurements)
    !do_me_measurements && @warn(
        "There are no measurements set up for this simulation!"
    )
    total_sweeps = sweeps + thermalization

    start_time = now()
    verbose && println("Started: ", Dates.format(start_time, "d.u yyyy HH:MM"))

    _time = time()
    do_th_measurements && prepare!(mc.thermalization_measurements, mc, mc.model)
    for i in 1:total_sweeps
        sweep(mc)

        if mc.p.global_moves && mod(i, mc.p.global_rate) == 0
            mc.a.prop_global += 1
            mc.a.acc_global += global_move(mc, mc.model, conf(mc))
        end

        # For optimal performance whatever is most likely to fail should be
        # checked first.
        if i <= thermalization && iszero(mod(i, mc.p.measure_rate)) && do_th_measurements
            measure!(mc.thermalization_measurements, mc, mc.model, i)
        end
        if (i == thermalization+1)
            do_th_measurements && finish!(mc.thermalization_measurements, mc, mc.model)
            do_me_measurements && prepare!(mc.measurements, mc, mc.model)
        end
        if i > thermalization && iszero(mod(i, mc.p.measure_rate)) && do_me_measurements
            measure!(mc.measurements, mc, mc.model, i)
        end

        print_rate = mc.p.print_rate
        if print_rate != 0 && iszero(mod(i, print_rate))
            mc.a.acc_rate /= print_rate
            mc.a.acc_rate_global /= print_rate / mc.p.global_rate
            sweep_dur = (time() - _time)/print_rate
            if verbose
                println("\t", i)
                @printf("\t\tsweep dur: %.3fs\n", sweep_dur)
                @printf("\t\tacc rate (local) : %.1f%%\n", mc.a.acc_rate*100)
                if mc.p.global_moves
                    @printf("\t\tacc rate (global): %.1f%%\n", mc.a.acc_rate_global*100)
                    @printf("\t\tacc rate (global, overall): %.1f%%\n", mc.a.acc_global/mc.a.prop_global*100)
                end
            end

            mc.a.acc_rate = 0.0
            mc.a.acc_rate_global = 0.0
            flush(stdout)
            _time = time()
        end
    end
    do_me_measurements && finish!(mc.measurements, mc, mc.model)

    mc.a.acc_rate = mc.a.acc_local / mc.a.prop_local
    mc.a.acc_rate_global = mc.a.acc_global / mc.a.prop_global

    end_time = now()
    verbose && println("Ended: ", Dates.format(end_time, "d.u yyyy HH:MM"))
    verbose && @printf("Duration: %.2f minutes", (end_time - start_time).value/1000. /60.)

    nothing
end

"""
    sweep(mc::MC)

Performs a sweep of local moves.
"""
@inline function sweep(mc::MC)
    c = conf(mc)
    m = model(mc)
    β = beta(mc)
    @inbounds for i in eachindex(c)
        ΔE, Δsite = propose_local(mc, m, i, c)
        mc.a.prop_local += 1
        # Metropolis
        if ΔE <= 0 || rand() < exp(- β*ΔE)
            accept_local!(mc, m, i, c, Δsite, ΔE)
            mc.a.acc_rate += 1/nsites(m)
            mc.a.acc_local += 1
        end
    end

    nothing
end

include("MC_mandatory.jl")
include("MC_optional.jl")


 using Base

mutable struct LimitOrder
    price::UInt128
    size::UInt128
    trader_id::UInt128
    side::Bool
    order_id::UInt128
end

function Base.size(o::LimitOrder)::UInt128
    return o.size
end

function trader(o::LimitOrder)::UInt128
    return o.trader_id
end

function side(o::LimitOrder)::Bool
    return o.side
end

function id(o::LimitOrder)::UInt128
    return o.order_id
end


using Printf
using Oceananigans.Grids: size_summary, scalar_summary
using Oceananigans.Utils: prettysummary
using Oceananigans.BoundaryConditions: bc_str

location_str(::Type{Face})    = "Face"
location_str(::Type{Center})  = "Center"
location_str(::Type{Nothing}) = "⋅"
show_location(LX, LY, LZ) = "($(location_str(LX)), $(location_str(LY)), $(location_str(LZ)))"
show_location(field::AbstractField) = show_location(location(field)...)

function Base.summary(field::Field)
    LX, LY, LZ = location(field)
    prefix = string(size_summary(size(field)), " Field{$LX, $LY, $LZ}")

    grid_name = typeof(field.grid).name.wrapper
    reduced_dims = reduced_dimensions(field)

    suffix = reduced_dims === () ?
        string(" on ", grid_name, " on ", summary(architecture(field))) :
        string(" reduced over dims = ", reduced_dims,
               " on ", grid_name, " on ", summary(architecture(field)))

    return string(prefix, suffix)
end

data_summary(field) = string("max=", prettysummary(maximum(field)), ", ",
                             "min=", prettysummary(minimum(field)), ", ",
                             "mean=", prettysummary(mean(field)))

function Base.show(io::IO, field::Field)

    bcs = field.boundary_conditions

    prefix =
        string("$(summary(field))\n",
               "├── grid: ", summary(field.grid), '\n',
               "├── boundary conditions: ", summary(bcs), '\n',
               "│   └── west: ", bc_str(bcs.west), ", east: ", bc_str(bcs.east),
                     ", south: ", bc_str(bcs.south), ", north: ", bc_str(bcs.north),
                     ", bottom: ", bc_str(bcs.bottom), ", top: ", bc_str(bcs.top),
                     ", immersed: ", bc_str(bcs.immersed), '\n')

    middle = isnothing(field.operand) ? "" :
        string("├── operand: ", summary(field.operand), '\n',
               "├── status: ", summary(field.status), '\n')

    suffix = string("└── data: ", summary(field.data), '\n',
                    "    └── ", data_summary(field))

    print(io, prefix, middle, suffix)
end

Base.summary(status::FieldStatus) = "time=$(status.time)"

Base.summary(::ZeroField{N}) where N = "ZeroField{$N}"
Base.show(io::IO, z::ZeroField) = print(io, summary(z))

Base.show(io::IO, ::MIME"text/plain", f::AbstractField) = show(io, f)

const FieldTuple = Tuple{Field, Vararg{Field}}
const NamedFieldTuple = NamedTuple{S, <:FieldTuple} where S

function Base.show(io::IO, ft::NamedFieldTuple)
    names = keys(ft)
    N = length(ft)

    grid = first(ft).grid
    all_same_grid = true
    for field in ft
        if field.grid !== grid
            all_same_grid = false
        end
    end

    print(io, "NamedTuple with ", N, " Fields ")

    if all_same_grid
        print(io, "on ", summary(grid), ":\n")
    else
        print(io, "on different grids:", '\n')
    end

    for name in names[1:end-1]
        field = ft[name]
        print(io, "├── $name: ", summary(field), '\n')

        if !all_same_grid
            print(io, "│   └── grid: ", summary(field.grid), '\n')
        end
    end

    name = names[end]
    field = ft[name]
    print(io, "└── $name: ", summary(field))

    if !all_same_grid
        print(io, '\n')
        print(io, "    └── grid: ", summary(field.grid))
    end
end


export execute_chromosome, concatenate_outputs, create_count_function_list, increment_count_function_list, print_function_list
export evaluate_node, create_count_function_hash, increment_count_function_hash, print_count_function_hash, print_count_function_summary
export execute_chromosome_ft, evaluate_node_ft
using Printf

# Limits Funcs to 4 inputs
# TODO:  Write this function generally
function apply( f::Function,args)
  if length(args) == 0
    f()
  elseif length(args) == 1
    f(args[1])
  elseif length(args) == 2
    f(args[1],args[2])
  elseif length(args) == 3
    f(args[1],args[2],args[3])
  elseif length(args) == 4
    f(args[1],args[2],args[3],args[4])
  end
end

function evaluate_node(c::Chromosome, node::InputNode, context::Vector)
    #println("eval input node: ",node)
    #print("context[node.index]:")
    #Printf.@printf("0x0%4x\n",context[node.index])
    #=
    if node.active
      return node.cache
    end
    node.active = true
    node.cache = context[node.index]
    =#
    return context[node.index]
end

# Restored caching on 10/7/21
function evaluate_node(c::Chromosome, node::InteriorNode, context::Vector)
    #println("en eval interior node: ",node,"  contxt: ",context)
    prev_cache = node.cache
    func = node.func
    if node.active
      return node.cache
    end
    #println("node.inputs: ",node.inputs)
    args = map(node.inputs[1:func.arity]) do index
        #println("index: ",index)
        evaluate_node(c, c[index], context)
    end
    #args = map(x->context[x],node.inputs)
    #println("args: ",args)
    result = CGP.apply(func.func, args)
    #print("func: ",func,"  args: ",args,"  result: ")
    #Printf.@printf("0x%2x\n",result)
    if node.active
      @assert result == node.cache
    end
    node.active = true
    node.cache = result
    return result
end

# Evaluate an interior node in isolation using context as the input
# context should be the context for numinputs.
function eval_interior( node::InteriorNode, context::Vector )
    println("ei eval interior node: ",node)
    func = node.func
    args = [context[i] for i = 1:func.arity]
    #println("args: ",args)
    Main.CGP.apply(func.func, args)
end

function evaluate_node(c::Chromosome, node::OutputNode, context::Vector)
    index = node.input
    result = evaluate_node(c, c[index], context)
    #println("eval output node: ",node,"  index: ",index,"  result: ",result)
    # result = evaluate_node(c, c[index], context)
    return result
end

# Execute chromosme using context for the inputs and return outputs
function execute_chromosome(c::Chromosome, context::Vector; permute_context::Bool=false )
    #println("Executing chromosome")
    #print_chromosome(c)
    #println("context: ",context)
    return [evaluate_node(c, node, context) for node = c.outputs]
end

function evaluate_node_ft(c::Chromosome, node::OutputNode, context::Vector, perturb_index::Int64 )
    index = node.input
    #println("ft eval output node: ",node,"  index: ",index)
    if index == perturb_index
        #println("ft out index == perturb_index")
        result = NOT.func(evaluate_node_ft(c, c[index], context,perturb_index))
    else
       result = evaluate_node_ft(c, c[index], context,perturb_index)
    end
    #print("ft out index: ",index)
    #Printf.@printf("ft out result: 0x%2x\n",result)
    return result
end

function evaluate_node_ft(c::Chromosome, node::InputNode, context::Vector, perturb_index::Int64 )
    #println("ft eval input node: ",node)
    #Printf.@printf("ft result:  0x0%4x\n",context[node.index])
    return context[node.index]
end

function evaluate_node_ft(c::Chromosome, node::InteriorNode, context::Vector, perturb_index::Int64 )
    #println("ft eval interior node: ",node)
    func = node.func
    args = map(node.inputs[1:func.arity]) do index
        #println("index: ",index)
        if index == perturb_index
          #println("  ft int index == perturb_index")
          NOT.func(evaluate_node_ft(c, c[index], context,perturb_index))
        else
          evaluate_node_ft(c, c[index], context,perturb_index)
        end
    end
    result = apply(func.func, args)
    #print("ft int func: ",func,"  args: ",args )
    #Printf.@printf("  ft int result: 0x%2x\n",result)
    return result
end

# Fault tolerant version where the output of node c[perturb_index] is negated
# Note: ignores cache values for nodes
function execute_chromosome_ft(c::Chromosome, context::Vector, perturb_index::Int64 )
    #println("FT Executing chromosome")
    #print_chromosome(c)
    return [evaluate_node_ft(c, node, context,perturb_index) for node = c.outputs]
end

function print_function_list( function_list::Vector{MyFunc} )
  for i = convert(MyFunc,1):length(function_list)
    if function_list[i] > 0
      # The correct way to print when MyInt=UInt16, but wont work when MyInt is something else
      #Printf.@printf("0x%04x  %d\n",i-1,function_list[i])
    end
  end
end

# The purpose of the remaining functions in this file is to
#      generate random circuits and count the logic functions that are computed.
# Similar to results of Raman and Wagner 2011 to generate their Figure 2b.
function create_count_function_list( numinputs::Integer, numoutputs::Integer )
  return fill( convert(MyFunc,0), 2^(numoutputs*2^numinputs ))
end

function increment_count_function_list( funct::MyFunc, function_list::Vector{MyFunc} )
  function_list[funct+1] += 1
end

function create_count_function_hash( numinputs::Integer, numoutputs::Integer )
  Dict{MyFunc,Int64}()
end

function increment_count_function_hash( funct::MyFunc, counts::Dict{MyFunc,Int64} )
  counts[funct] = get!( counts, funct, 0 ) + 1
end

function concatenate_outputs( numinputs::Integer, outputs::Vector{Main.CGP.MyInt}  )
  shift = 2^numinputs
  result = convert(MyFunc,outputs[1])
  for i = 2:length(outputs)
    result = convert(MyFunc,outputs[i]) | result << 2^numinputs
  end
  result
end

function print_count_function_hash( f::IOStream, counts::Dict{MyFunc,Int64} )
  length_counts_list_max_for_printing = 50
  println(f,"function count")
  counts_list = [(k,counts[k]) for k in keys(counts) ]
  sort!( counts_list, by=x->x[2], rev=true )
  len = length(counts_list)
  sum = 0
  for c in counts_list[1:len]
    #=
    if len <= length_counts_list_max_for_printing
      Printf.@printf(f,"0x%04x   %d\n",c[1],c[2])
    end
    =#
    sum += c[2]
  end
  println(f,"length counts list: ",len,"  sum: ",sum)
end

function print_count_function_hash( counts::Dict{MyFunc,Int64} )
  print_count_function_hash( Base.stdout, counts )
end

function print_count_function_summary( f::IOStream,  counts::Dict{MyFunc,Int64} )
  length_print_count_max_for_printing = 150
  println(f,"count function hash: ")
  counts_list = [(k,counts[k]) for k in keys(counts) ]
  sort!( counts_list, by=x->x[2] )
  current_count = counts_list[1][2]
  accum_count = 1
  accum_sum = 0
  print_count = 0
  println(f,"funct freq  count")
  for i = 2:length(counts_list)
    if counts_list[i][2] == current_count
      accum_count += 1
    else
      if print_count <= length_print_count_max_for_printing
        println(f,"     ",current_count,":      ",accum_count)
      end
      print_count += 1
      accum_sum += current_count*accum_count
      current_count = counts_list[i][2]
      accum_count = 1
    end
  end
  accum_sum += current_count*accum_count
  if print_count <= length_print_count_max_for_printing
    println(f,"     ",current_count,":      ",accum_count)
  end
  println(f,"print_count: ",print_count,"  accum_sum: ",accum_sum)
end

function print_count_function_summary( counts::Dict{MyFunc,Int64} )
  print_count_function_summary( f, counts )
end


"""
    ThreadedScans.linear!(op, xs::AbstractVector) -> xs

Compute inclusive scan. Spawn, wait, and propagation of intermediate reductions
are all done sequentially.

# Keyword arguments
* `ntasks = Threads.nthreads()`: number of tasks
"""
function ThreadedScans.linear!(
    op,
    xs;
    ntasks::Integer = Threads.nthreads() + 1,
    spin::Union{Nothing,Integer} = nothing,
)
    ntasks = min(length(xs), ntasks)
    if ntasks <= 1
        scan!(op, xs)
        return xs
    end

    it = partitionto(xs, ntasks)
    chunk0, state = iterate(it)
    y = iterate(it, state)
    if y === nothing
        scan!(op, xs)
        return xs
    end

    tasks = Task[]
    fromleft = OneWayPromise{eltype(xs)}()
    @assert !isempty(chunk0)
    t = Threads.@spawn try
        put!($fromleft, scan!(op, $chunk0))
    catch
        close($fromleft)
        rethrow()
    end
    push!(tasks, t)
    chunk, state = y
    while true
        y = iterate(it, state)
        y === nothing && break
        @assert !isempty(chunk)
        toright = OneWayPromise{eltype(xs)}()
        t = Threads.@spawn try
            local accr = foldl(op, $chunk)
            local accl = take!($fromleft, spin)
            put!($toright, op(accl, accr))
            scan!(op, accl, $chunk)
        catch
            close($fromleft)
            close($toright)
            rethrow()
        end
        push!(tasks, t)
        fromleft = toright
        chunk, state = y
    end
    try
        scan!(op, take!(fromleft, spin), chunk)
    finally
        close(fromleft)
        foreach(wait, tasks)
    end
    return xs
end


for f in ["point", "bounds", "transform", "distance"]
    include("$f.jl")
end


# R2_IRI_assRCS_kfold_include.jl

include("../tcm_code/tcm_estimateRanking_CS.jl")
include("../tcm_code/tcm_predictRanking.jl")

include("../tcm_code/tcm_estimateRanking_kcv.jl")

include("../tcm_code/tcm_evaluateKL.jl")

include("../tcm_code/tcm_rankingStats.jl")

include("R2_IRI_assRCS_kfold_helperfn.jl")