@testset "intrinsics" begin

############################################################################################

@testset "math" begin
    buf = CuArray{Float32}(1)

    @eval function kernel_math_log10(a, i)
        a[1] = CUDAnative.log10(i)
        return nothing
    end

    @cuda (1, 1) kernel_math_log10(buf, Float32(100))
    val = Array(buf)
    @test val[1] ≈ 2.0
end

############################################################################################

@testset "I/O" begin

@testset "printing" begin
    _, out = @grab_output @on_device @cuprintf("")
    @test out == ""

    _, out = @grab_output @on_device @cuprintf("Testing...\n")
    @test out == "Testing...\n"

    _, out = @grab_output @on_device @cuprintf("Testing %ld...\n", Int64(42))
    @test out == "Testing 42...\n"

    _, out = @grab_output @on_device @cuprintf("Testing %u %u...\n", blockIdx().x, threadIdx().x)
    @test out == "Testing 1 1...\n"

    _, out = @grab_output @on_device begin
        @cuprintf("foo")
        @cuprintf("bar\n")
    end
    @test out == "foobar\n"
end

end

############################################################################################

# a composite type to test for more complex element types
@eval immutable RGB{T}
    r::T
    g::T
    b::T
end

@testset "shared memory" begin

n = 1024
types = [Int32, Int64, Float32, Float64]

@testset "constructors" begin
    # static
    @on_device @cuStaticSharedMem(Float32, 1)
    @on_device @cuStaticSharedMem(Float32, (1,2))
    @on_device @cuStaticSharedMem(Tuple{Float32, Float32}, 1)
    @on_device @cuStaticSharedMem(Tuple{Float32, Float32}, (1,2))
    @on_device @cuStaticSharedMem(Tuple{RGB{Float32}, UInt32}, 1)
    @on_device @cuStaticSharedMem(Tuple{RGB{Float32}, UInt32}, (1,2))

    # dynamic
    @on_device @cuDynamicSharedMem(Float32, 1)
    @on_device @cuDynamicSharedMem(Float32, (1, 2))
    @on_device @cuDynamicSharedMem(Tuple{Float32, Float32}, 1)
    @on_device @cuDynamicSharedMem(Tuple{Float32, Float32}, (1,2))
    @on_device @cuDynamicSharedMem(Tuple{RGB{Float32}, UInt32}, 1)
    @on_device @cuDynamicSharedMem(Tuple{RGB{Float32}, UInt32}, (1,2))

    # dynamic with offset
    @on_device @cuDynamicSharedMem(Float32, 1, 8)
    @on_device @cuDynamicSharedMem(Float32, (1,2), 8)
    @on_device @cuDynamicSharedMem(Tuple{Float32, Float32}, 1, 8)
    @on_device @cuDynamicSharedMem(Tuple{Float32, Float32}, (1,2), 8)
    @on_device @cuDynamicSharedMem(Tuple{RGB{Float32}, UInt32}, 1, 8)
    @on_device @cuDynamicSharedMem(Tuple{RGB{Float32}, UInt32}, (1,2), 8)
end

@testset "dynamic shmem" begin

@testset "statically typed" begin
    @eval function kernel_shmem_dynamic_typed(d, n)
        t = threadIdx().x
        tr = n-t+1

        s = @cuDynamicSharedMem(Float32, n)
        s[t] = d[t]
        sync_threads()
        d[t] = s[tr]

        return nothing
    end

    a = rand(Float32, n)
    d_a = CuArray(a)

    @cuda (1, n, n*sizeof(Float32)) kernel_shmem_dynamic_typed(d_a, n)
    @test reverse(a) == Array(d_a)
end

@testset "parametrically typed" begin
    @eval function kernel_shmem_dynamic_typevar{T}(d::CuDeviceArray{T}, n)
        t = threadIdx().x
        tr = n-t+1

        s = @cuDynamicSharedMem(T, n)
        s[t] = d[t]
        sync_threads()
        d[t] = s[tr]

        return nothing
    end

    for T in types
        a = rand(T, n)
        d_a = CuArray(a)

        @cuda (1, n, n*sizeof(T)) kernel_shmem_dynamic_typevar(d_a, n)
        @test reverse(a) == Array(d_a)
    end
end

end

@testset "static shmem" begin

@testset "statically typed" begin
    @eval function kernel_shmem_static_typed(d, n)
        t = threadIdx().x
        tr = n-t+1

        s = @cuStaticSharedMem(Float32, 1024)
        s2 = @cuStaticSharedMem(Float32, 1024)  # catch aliasing

        s[t] = d[t]
        s2[t] = 2*d[t]
        sync_threads()
        d[t] = s[tr]

        return nothing
    end

    a = rand(Float32, n)
    d_a = CuArray(a)

    @cuda (1, n) kernel_shmem_static_typed(d_a, n)
    @test reverse(a) == Array(d_a)
end

@testset "parametrically typed" begin
    @eval function kernel_shmem_static_typevar{T}(d::CuDeviceArray{T}, n)
        t = threadIdx().x
        tr = n-t+1

        s = @cuStaticSharedMem(T, 1024)
        s2 = @cuStaticSharedMem(T, 1024)  # catch aliasing

        s[t] = d[t]
        s2[t] = d[t]
        sync_threads()
        d[t] = s[tr]

        return nothing
    end

    for T in types
        a = rand(T, n)
        d_a = CuArray(a)

        @cuda (1, n) kernel_shmem_static_typevar(d_a, n)
        @test reverse(a) == Array(d_a)
    end
end

end

@testset "dynamic shmem consisting of multiple arrays" begin

# common use case 1: dynamic shmem consists of multiple homogeneous arrays
#                    -> split using `view`
@testset "homogeneous" begin
    @eval function kernel_shmem_dynamic_multi_homogeneous(a, b, n)
        t = threadIdx().x
        tr = n-t+1

        s = @cuDynamicSharedMem(eltype(a), 2*n)

        sa = view(s, 1:n)
        sa[t] = a[t]
        sync_threads()
        a[t] = sa[tr]

        sb = view(s, n+1:2*n)
        sb[t] = b[t]
        sync_threads()
        b[t] = sb[tr]

        return nothing
    end

    a = rand(Float32, n)
    d_a = CuArray(a)

    b = rand(Float32, n)
    d_b = CuArray(b)

    @cuda (1, n, 2*n*sizeof(Float32)) kernel_shmem_dynamic_multi_homogeneous(d_a, d_b, n)
    @test reverse(a) == Array(d_a)
    @test reverse(b) == Array(d_b)
end

# common use case 2: dynamic shmem consists of multiple heterogeneous arrays
#                    -> construct using pointer offset
@testset "heterogeneous" begin
    @eval function kernel_shmem_dynamic_multi_heterogeneous(a, b, n)
        t = threadIdx().x
        tr = n-t+1

        sa = @cuDynamicSharedMem(eltype(a), n)
        sa[t] = a[t]
        sync_threads()
        a[t] = sa[tr]

        sb = @cuDynamicSharedMem(eltype(b), n, n*sizeof(eltype(a)))
        sb[t] = b[t]
        sync_threads()
        b[t] = sb[tr]

        return nothing
    end

    a = rand(Float32, n)
    d_a = CuArray(a)

    b = rand(Int64, n)
    d_b = CuArray(b)

    @cuda (1, n, n*sizeof(Float32) + n*sizeof(Int64)) kernel_shmem_dynamic_multi_heterogeneous(d_a, d_b, n)
    @test reverse(a) == Array(d_a)
    @test reverse(b) == Array(d_b)
end

end

end

############################################################################################

@testset "data movement and conversion" begin

if capability(dev) >= v"3.0"
@testset "shuffle" begin

n = 14
types = [Int32, Int64, Float32, Float64]

@testset "down" begin
    @eval function kernel_shuffle_down{T}(d::CuDeviceArray{T}, n)
        t = threadIdx().x
        if t <= n
            d[t] += shfl_down(d[t], n÷2)
        end
        return nothing
    end

    for T in types
        a = T[i for i in 1:n]
        d_a = CuArray(a)

        threads = nearest_warpsize(dev, n)
        @cuda (1, threads) kernel_shuffle_down(d_a, n)

        a[1:n÷2] += a[n÷2+1:end]
        @test a == Array(d_a)
    end
end

end
end

end

############################################################################################

@testset "parallel synchronization and communication" begin

@testset "voting" begin

@testset "ballot" begin
    d_a = CuArray{UInt32}(1)

    @eval function kernel_vote_ballot(a, i)
        vote = vote_ballot(threadIdx().x == i)
        if threadIdx().x == 1
            a[1] = vote
        end

        return nothing
    end

    len = 4
    for i in 1:len
        @cuda (1,len) kernel_vote_ballot(d_a, i)
        @test Array(d_a) == [2^(i-1)]
    end
end

@testset "any" begin
    d_a = CuArray{UInt32}(1)

    @eval function kernel_vote_any(a, i)
        vote = vote_any(threadIdx().x >= i)
        if threadIdx().x == 1
            a[1] = vote
        end

        return nothing
    end

    @cuda (1,2) kernel_vote_any(d_a, 1)
    @test Array(d_a) == [1]

    @cuda (1,2) kernel_vote_any(d_a, 2)
    @test Array(d_a) == [1]

    @cuda (1,2) kernel_vote_any(d_a, 3)
    @test Array(d_a) == [0]
end

@testset "all" begin
    d_a = CuArray{UInt32}(1)

    @eval function kernel_vote_all(a, i)
        vote = vote_all(threadIdx().x >= i)
        if threadIdx().x == 1
            a[1] = vote
        end

        return nothing
    end

    @cuda (1,2) kernel_vote_all(d_a, 1)
    @test Array(d_a) == [1]

    @cuda (1,2) kernel_vote_all(d_a, 2)
    @test Array(d_a) == [0]

    @cuda (1,2) kernel_vote_all(d_a, 3)
    @test Array(d_a) == [0]
end

end

end

############################################################################################

end


Base.@kwdef struct TableMetaData
    name::Union{Nothing,String} = nothing
    id::Union{Nothing,String} = nothing
    provider::Union{Nothing,String} = nothing
    reference::Union{Nothing,String} = nothing
    content_type::Union{Nothing,String} = nothing
    description::Union{Nothing,String} = nothing
    comments::Union{Nothing,String} = nothing
    source_path::Union{Nothing,String} = nothing
end

mort_max_dur = 121
mort_max_issue_age = 120


#  MIPLearn: Extensible Framework for Learning-Enhanced Mixed-Integer Optimization
#  Copyright (C) 2020, UChicago Argonne, LLC. All rights reserved.
#  Released under the modified BSD license. See COPYING.md for more details.

"""
    FirstInfeasibleBranching()

Branching rule that always selects the first fractional variable.
"""
struct FirstInfeasibleBranching <: VariableBranchingRule end

function find_branching_var(
    rule::FirstInfeasibleBranching,
    node::Node,
    pool::NodePool,
)::Variable
    return node.fractional_variables[1]
end

"""
    LeastInfeasibleBranching()

Branching strategy that select the fractional variable whose value is the closest
to an integral value.
"""
struct LeastInfeasibleBranching <: VariableBranchingRule end

function find_branching_var(
    rule::LeastInfeasibleBranching,
    node::Node,
    pool::NodePool,
)::Variable
    scores = [max(v - floor(v), ceil(v) - v) for v in node.fractional_values]
    _, max_offset = findmax(scores)
    return node.fractional_variables[max_offset]
end

"""
    MostInfeasibleBranching()

Branching strategy that selects the fractional variable whose value is closest
to 1/2.
"""
struct MostInfeasibleBranching <: VariableBranchingRule end

function find_branching_var(
    rule::MostInfeasibleBranching,
    node::Node,
    pool::NodePool,
)::Variable
    scores = [min(v - floor(v), ceil(v) - v) for v in node.fractional_values]
    _, max_offset = findmax(scores)
    return node.fractional_variables[max_offset]
end


fname = joinpath(dirname(@__FILE__), "..", "data", "test-4d.dat")
sname = joinpath(dirname(@__FILE__), "..", "data", "tmp", "same.dat")

#x, y, z, s, t = read_dat(fname)

#@test size(x) == (30,)
#@test size(y) == (36,)
#@test size(z) == (28,)
#@test size(s) == (30,36,28,2)
#@test size(t) == (2,)

#@test maximum(x) == 72.5
#@test maximum(y) == 71.220001
#@test maximum(z) == 76.809998
#@test maximum(s) == 0.067409396
#@test maximum(t) == 0.24
#
#@test minimum(x) == -72.5
#@test minimum(y) == -103.779999
#@test minimum(z) == -58.189999
#@test minimum(s) == 0.0
#@test minimum(t) == 0.12

# write_dat(sname, x, y, z, s[:,:,:,:], t)

# x2, y2, z2, s2, t2 = read_dat(sname)

# @test x == x2
# @test y == y2
# @test z == z2
# @test s == s2
# @test t == t2

# fname = joinpath(dirname(@__FILE__), "..", "data", "test-3d.dat")
# x, y, z, s, t = read_dat(fname)

#@test size(x) == (30,)
#@test size(y) == (36,)
#@test size(z) == (28,)
#@test size(s) == (30,36,28,1)
#@test size(t) == (1,)

#@test maximum(x) == 72.5
#@test maximum(y) == 71.220001
#@test maximum(z) == 76.809998
#@test maximum(s) == 33.2692985535
#@test maximum(t) == 0

#@test minimum(x) == -72.5
#@test minimum(y) == -103.779999
#@test minimum(z) == -58.189999
#@test minimum(s) == -7.5189352036
#@test minimum(t) == 0


##########################################################################
# Copyright 2017 Samuel Ridler.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
##########################################################################

using LightGraphs
using SparseArrays

function readNetworkFiles(nodesFilename::String, arcsFilename::String)
	net = Network()
	fGraph = net.fGraph # shorthand
	fGraph.nodes = readNodesFile(nodesFilename)
	(fGraph.arcs, arcTravelTimes) = readArcsFile(arcsFilename)

	# create map
	mp = Map()
	mp.xMin = minimum(node -> node.location.x, fGraph.nodes) - 1
	mp.xMax = maximum(node -> node.location.x, fGraph.nodes) + 1
	mp.yMin = minimum(node -> node.location.y, fGraph.nodes) - 1
	mp.yMax = maximum(node -> node.location.y, fGraph.nodes) + 1

	JEMSS.initGraph!(fGraph)
	JEMSS.setArcDistances!(fGraph, mp)
	JEMSS.checkGraph(fGraph, mp)
	JEMSS.initFNetTravels!(net, arcTravelTimes)
	JEMSS.createRGraphFromFGraph!(net)
	JEMSS.checkGraph(net.rGraph, mp)
	JEMSS.createRNetTravelsFromFNetTravels!(net)
	JEMSS.setCommonFNodes!(net, Int[])

	return net
end

function fNetTravelShortestPathData(net::Network, travelModeIndex::Int)
	# Return spTimes and spSuccs for fNetTravel:
	# spTimes[i,j] gives the travel time from fNode i to fNode j along shortest path
	# spSuccs[i,j] gives index of successor node on shortest path from fNode i to fNode j

	# Most of this code is copied from the function JEMSS.calcRNetTravelShortestPaths!,
	# but changed to be for fNetTravel instead of rNetTravel.

	# shorthand
	fGraph = net.fGraph
	numNodes = length(fGraph.nodes)

	# calculate all-pairs shortest-paths data for the network with Dijkstra's algorithm
	revGraph = LightGraphs.reverse(fGraph.light) # reverse graph
	revArcTimes = spzeros(FloatSpTime, numNodes, numNodes)
	for arc in fGraph.arcs
		@assert(revArcTimes[arc.toNodeIndex, arc.fromNodeIndex] == 0)
		revArcTimes[arc.toNodeIndex, arc.fromNodeIndex] = net.fNetTravels[travelModeIndex].arcTimes[arc.index]
	end
	spTimes = Array{FloatSpTime,2}(undef, numNodes, numNodes) # spTimes[i,j] is the shortest-path travel time from node i to node j, to be calculated
	spSuccs = Array{IntRNode,2}(undef, numNodes, numNodes) # spSuccs[i,j] gives the successor node of i on shortest path from node i to node j, to be calculated
	for i = 1:numNodes
		# calculate shortest paths for origin node i
		spData = LightGraphs.dijkstra_shortest_paths(revGraph, i, revArcTimes) # type: LightGraphs.DijkstraState{Float}
		spTimes[:,i] = spData.dists
		spSuccs[:,i] = spData.parents
	end

	# change spSuccs where needed
	# where a path i -> l has spSuccs[i,l] = k, but spSuccs[i,k] = j,
	# need to change all cases where spSuccs[i,-] = k to instead have spSuccs[i,-] = j
	# this case can happen when an arc (i,k) has the same travel time as a path (i,j,k)
	for fArc in fGraph.arcs
		i = fArc.fromNodeIndex
		j = k = fArc.toNodeIndex
		while j != spSuccs[i,j]
			j = spSuccs[i,j]
		end
		if j != k
			# for any paths from i that have successor node k, change successor to be j
			spSuccs[i, spSuccs[i,:] .== k] .= j
		end
	end

	return spTimes, spSuccs
end

@testset "shortest path travel time" begin
	# Check shortestPathTravelTime function.
	# Compare against a simpler (but slower) function that calculates travel times for the whole full network,
	# unlike shortestPathTravelTime which takes advantage of the reduced network.

	testNetworksFolder = "data/network/small"
	for testFolderName in readdir(testNetworksFolder)
		folder = joinpath(testNetworksFolder, testFolderName)
		net = readNetworkFiles(joinpath(folder, "nodes.csv"), joinpath(folder, "arcs.csv"))
		for travelModeIndex = 1:length(net.fNetTravels)
			rtol = eps(FloatSpTime)
			atol = 2 * maximum(net.rNetTravels[travelModeIndex].arcTimes) * eps(FloatSpTime) # in NetTravel type, precision of fNodeToRNodeTime and fNodeFromRNodeTime (used in sp time calculation) are affected by rArc travel time. Multiplied by 2 since this can affect the travel time at both the start and end of the path.

			# compare travel time from shortestPathTravelTime function with spTimes for full network
			spTimes, spSuccs = fNetTravelShortestPathData(net, travelModeIndex)
			numNodes = length(net.fGraph.nodes) # shorthand
			allPass = true
			for i = 1:numNodes, j = 1:numNodes
				t1 = shortestPathTravelTime(net, travelModeIndex, i, j)
				t2 = spTimes[i,j]
				allPass &= abs(t1-t2) <= atol + rtol*max(t1,t2) # equivalent to isapprox(t1, t2, atol = atol, rtol = rtol) for julia v0.6
			end
			@test allPass
		end
	end
end

@testset "shortest path distance" begin
	# Check shortestPathDistance function.
	# Compare against a simpler (but slower) function that calculates distances based on arc traversed in full network,
	# unlike shortestPathDistance which takes advantage of the reduced network.

	# Calculate distance along shortest path between two nodes, using fGraph.
	# spSuccs[i,j] gives successor of node i in shortest path from node i to node j, see fNetTravelShortestPathData.
	function shortestPathDistance2(net::Network, spSuccs::Array{IntRNode,2}, startFNode::Int, endFNode::Int)::Float
		dist = 0.0
		fNode = startFNode
		while fNode != endFNode
			nextFNode = spSuccs[fNode, endFNode]
			arcIndex = net.fGraph.nodePairArcIndex[fNode, nextFNode]
			dist += net.fGraph.arcs[arcIndex].distance
			fNode = nextFNode
		end
		return dist
	end

	testNetworksFolder = "data/network/small"
	for testFolderName in readdir(testNetworksFolder)
		folder = joinpath(testNetworksFolder, testFolderName)
		net = readNetworkFiles(joinpath(folder, "nodes.csv"), joinpath(folder, "arcs.csv"))
		for travelModeIndex = 1:length(net.fNetTravels)
			# compare shortestPathDistance and shortestPathDistance2 for each pair of nodes in fGraph
			spTimes, spSuccs = fNetTravelShortestPathData(net, travelModeIndex)
			numNodes = length(net.fGraph.nodes) # shorthand
			allPass = true
			for i = 1:numNodes, j = 1:numNodes
				d1 = shortestPathDistance(net, travelModeIndex, i, j) # from JEMSS
				d2 = shortestPathDistance2(net, spSuccs, i, j)
				allPass &= isapprox(d1, d2, rtol = eps())
			end
			@test allPass
		end
	end
end


## Testing hand-coded trait definitions
#######################################

# All type belong to the empty trait, as it makes no restriction on
# the types:
@test istrait( Tuple{} )

immutable Tr1{X1} <: Traits.Trait{Tuple{}}
    methods::Traits.FDict
    constraints::Vector{Bool}
    assoctyps::Vector{Any}
    Tr1() = new(Traits.FDict(), Bool[], [])
end
immutable Tr2{X1,X2} <: Traits.Trait{Tuple{}}
    methods::Traits.FDict
    constraints::Vector{Bool}
    assoctyps::Vector{Any}
    Tr2() = new(Traits.FDict(), Bool[], [])
end
immutable Tr3{X1,X2} <: Traits.Trait{Tuple{Tr1{X1}, Tr2{X1,X2}}}
    methods::Traits.FDict
    constraints::Vector{Bool}
    assoctyps::Vector{Any}
    Tr3() = new(Traits.FDict(), Bool[], [])
end

@test istraittype(Tr1)
@test istraittype(Tr1{A1})
@test istraittype( Tuple{Tr1{A1},Tr2{A1,A2}} )

@test traitgetpara(Tr1{A1})==Tuple{A1}
@test traitgetsuper(Tr1{A1})==Tuple{}
@test traitgetsuper(Tr2{A1,A2})==Tuple{}
@test traitgetsuper(Tr3{A1,A2})==Tuple{Tr1{A1},Tr2{A1,A2}}

# any type is part of a unconstrained trait:
@test istrait(Tr1{Int}, verbose=verbose)
@test istrait(Tr2{DataType,Int})
@test istrait(Tr3{AbstractString,DataType})
@test_throws TraitException istrait(Tr3{:a,7})  # maybe this should error?

immutable D1{X1} <: Traits.Trait{Tuple{}}
    methods::Traits.FDict
    constraints::Vector{Bool}
    assoctyps::Vector{Any}
    function D1()
        new(Traits.FDict(
             sin => (_sin( ::Float64, ::X1) = nothing),
             cos => (_cos( ::Float64, ::X1) = nothing)
             ),
            Bool[],
            []
            )
    end
end

@test istrait(D1{Int}, verbose=verbose)
@test !istrait(D1{AbstractString})

immutable D2{X1,X2} <: Traits.Trait{Tuple{D1{X1}, D1{X2}}}
    methods::Traits.FDict
    constraints::Vector{Bool}
    assoctyps::Vector{Any}
    function D2()
        new(Traits.FDict(
             (+) => (_plus( ::Any, ::X1, ::X2) = nothing),
             (-) => (_minus( ::Any, ::X1, ::X2) = nothing)
             ),
            Bool[],
            []
            )
    end
end

@test istrait(D2{Int, Int})
@test !istrait(D2{Int, AbstractString})

immutable D3{X1} <: Traits.Trait{Tuple{}}
    methods::Traits.FDict
    constraints::Vector{Bool}
    assoctyps::Vector{Any}
    function D3()
        new(Traits.FDict(
             getkey => (_getkey( ::Any, ::X1,::Any, ::Any) = nothing),
             get!   => (_get!( ::Any, ::X1, ::Any, ::Any) = nothing)
             ),
            Bool[],
            []
            )
    end
end

immutable D4{X1,X2} <: Traits.Trait{Tuple{}} # like D2 but without supertraits
    methods::Traits.FDict
    constraints::Vector{Bool}
    assoctyps::Vector{Any}
    function D4()
        new(Traits.FDict(
             (+) => (_plus( ::Any, ::X1, ::X2) = nothing),
             (-) => (_minus( ::Any, ::X1, ::X2) = nothing)
             ),
            Bool[],
            []
            )
    end
end

@test istrait(D3{Dict{Int,Int}})
@test !istrait(D3{Int})

@test istrait(Tuple{D1{Int}, D2{Int, Int}} )

@test istrait(D4{Int,AbstractFloat})

### adding other constraints

immutable CTr1{X1,X2} <: Traits.Trait{Tuple{}}
    methods::Traits.FDict
    constraints::Vector{Bool}
    assoctyps::Vector{Any}
    function CTr1()
        new(Traits.FDict(
             (+) => (_plus( ::Any, ::X1, ::X2) = nothing),
             ),
            Bool[
                 X1==X2
                 ],
            []
            )
    end
end

@test !istrait(CTr1{Int32, Int})
@test istrait(CTr1{Int, Int})

### adding other associated types

immutable CTrAs{X1,X2} <: Traits.Trait{Tuple{}}
    methods::Traits.FDict
    constraints::Vector{Bool}
    assoctyps::Vector{Any}
    function CTrAs()
        R = promote_type(X1, X2)
        D = Tuple{X1,X2}<:Tuple{Integer,Integer} ? Float64 : promote_type(X1, X2)
        assoctyps = Any[TypeVar(:R, R), TypeVar(:D, D)]
        new(Traits.FDict(
                  (+) => (_plus( ::Any, ::X1, ::X2) = nothing),
                  (-) => (_minus( ::Any, ::X1, ::X2) = nothing)
             ),
            Bool[],
            assoctyps
            )
    end
end

@test istrait(CTrAs{Int32, Int})
# @test istrait(CTrAs{Integer, Integer}) # doesn't work because return type of /(Integer, Integer)==Any
@test istrait(CTrAs{Int, Int})
@test !istrait(CTrAs{Int, AbstractString})

# parametric methods
####################

# @traitdef Tr01{X} begin
#     g01{T<:X}(T, T) -> T
# end
immutable Tr01{X} <: Traits.Trait{Tuple{}}
    methods::Traits.FDict
    constraints::Vector{Bool}
    assoctyps::Vector{Any}
    function Tr01()
        new(Traits.FDict(
                         g01 => (_g01{T<:X}( ::Type{T}, ::T, ::T) = nothing)
                         ),
            Bool[],
            []
            )
    end
end

g01(::Int, ::Int) = Int
@test istrait(Tr01{Int}) # == true as constraints Int isleaftype
@test !istrait(Tr01{Integer})
g01{I<:Integer}(::I, ::I) = I
@test istrait(Tr01{Integer}) # == true

# @traitdef Tr02{X} begin
#     g02{T<:X}(T, T) -> T
# end
immutable Tr02{X} <: Traits.Trait{Tuple{}}
    methods::Traits.FDict
    constraints::Vector{Bool}
    assoctyps::Vector{Any}
    function Tr02()
        new(Traits.FDict(
                         g02 => (_g02{T<:X}( ::Type{T}, ::T, ::T) = nothing)
                         ),
            Bool[],
            []
            )
    end
end

g02{I<:Integer}(::I, ::I) = Integer
# By using Base.return_types it is not possible to figure out whether
# the returned value is constrained or not by I:
if function_types_bug1
    @test istrait(Tr02{Integer})
    # or throw an error/warning here saying parametric return types
    # are only supported for leaftypes
else
    @test !istrait(Tr02{Integer}) # if function types get implemented this should be possible to catch
end
@test !istrait(Tr02{Int})
g02{I<:Integer}(::I, ::I) = I
@test istrait(Tr02{Int})

# @traitdef Tr03{X} begin
#     g03{T<:X}(T, Vector{T})
# end
immutable Tr03{X} <: Traits.Trait{Tuple{}}
    methods::Traits.FDict
    constraints::Vector{Bool}
    assoctyps::Vector{Any}
    function Tr03()
        new(Traits.FDict(
                         g03 => (_g03{T<:X}( ::T, ::T, ::Vector{T}) = nothing)
                         ),
            Bool[],
            []
            )
    end
end

g03{I<:Integer}(::I, ::Vector{I}) = 1
@test istrait(Tr03{Integer})
@test istrait(Tr03{Int})

# ## If I ever need to get to the trait parameters, something like this should work:
# # @traitdef Tr04{X} begin
# #     g04{T<:X}(T, Vector{T})
# # end
# immutable Tr04{X} <: Traits.Trait{Tuple{}}
#     methods::Traits.FDict
#     constraints::Vector{Bool}
#     assoctyps::Vector{Any}
#     function Tr04()
#         A1 = getassoc(X)
#         new(Traits.FDict(
#                          g04 => (_g04{T<:X}( ::T, ::T, ::Vector{T}, ::A1) = nothing)
#                          ),
#             Bool[],
#             [A1]
#             )
#     end
#     function Tr04(::Type{Traits._TestTraitPara})
#         A1 = Traits._TestAssoc{:A1}
#         new(Traits.FDict(
#                          g04 => (_g04{T<:X}( ::T, ::T, ::Vector{T}, ::A1) = nothing)
#                          ),
#             Bool[],
#             [A1]
#             )
#     end
# end
# getassoc{T<:Integer}(::Type{T}) = UInt
# g04{I<:Integer}(::I, ::Vector{I}, ::Integer) = 1


# generation of some containers filled with random values

if VERSION < v"1.2.0-DEV.257"
    Base.:(!=)(x) = Base.Fix2(!=, x)
end

function make_argument(param)
    if param isa Symbol
        param
    else
        param::Expr
        @assert param.head == :(::)
        if param.args[1] isa Symbol
            param.args[1]
        else
            @assert param.args[1].head == :curly
            @assert param.args[1].args[1] == :Type
            @assert param.args[1].args[2] isa Symbol
            param.args[1].args[2]
        end
    end
end

# arg names must not be "rng" nor "X", already in use
# (this is for nicer output of methods, to avoid cryptic names with gensym)
macro make_container(margs...)
    definitions = []
    argss = []
    for a in margs
        push!(argss,
              a isa Expr && a.head == :vect ? # optional
                [nothing, a.args[1]] :
              a isa Expr && a.head == :call && a.args[1] == :(=>) ? # curly
                [a.args[2] => a.args[3]] :
              [a])

    end
    pushfirst!(argss, [(), :X, :(::Type{X}) => :X]) # for Sampler

    for as0 in Iterators.product(argss...)
        as1 = filter(!=(nothing), collect(Any, as0))
        curlys = []
        replace!(as1) do a
            if a isa Pair
                push!(curlys, a[2])
                a.first
            else
                a
            end
        end
        for def in (:(rand(rng::AbstractRNG) where {} = rand(rng, make())),
                    :(rand() where {} = rand(GLOBAL_RNG, make())))
            append!(def.args[1].args[1].args, filter(!=(()), as1))
            as2 = copy(as1)
            as2[1], as2[2] = as2[2], as2[1]
            filter!(!=(()), as2)
            append!(def.args[2].args[2].args[3].args, map(make_argument, as2))
            append!(def.args[1].args, curlys)
            push!(definitions, def)
        end
    end
    esc(Expr(:block, definitions...))
end

@make_container(::Type{String}, [n::Integer])

# we disable smthg like rand(String, 2, 3) because it looks to similar to, but is too different
# than, rand(String, 2) (Array of String vs String)

string_error() = throw(ArgumentError(
 "rand([rng], String, d1::Integer, d2::Integer, ...) is unsupported,
 use rand(rng, make(String), d1, d2, ...) or rand(rng, String, (d1, d2, ...)) instead"))
rand(rng::AbstractRNG, ::Type{String}, n::Integer...) = string_error()
rand(                  ::Type{String}, n::Integer...) = string_error()

# sparse vectors & matrices
@make_container(::Type{SparseVector},    p::AbstractFloat, m::Integer)
@make_container(::Type{SparseVector},    p::AbstractFloat, d::Dims{1})
@make_container(::Type{SparseMatrixCSC}, p::AbstractFloat, m::Integer, n::Integer)
@make_container(::Type{SparseMatrixCSC}, p::AbstractFloat, d::Dims{2})

# Tuple as a container
@make_container(T::Type{<:Tuple})
@make_container(::Type{Tuple}, n::Integer)
@make_container(T::Type{NTuple{N,TT} where N} => TT, n::Integer)
@make_container(T::Type{<:NamedTuple{K}} => K)

if VERSION < v"1.1.0"
    # disambiguate
    rand(rng::AbstractRNG, ::Type{NamedTuple}) = rand(rng, make(NamedTuple))
end

## arrays (same as in Random, but with explicit type specification, e.g. rand(Int, Array, 4)

macro make_array_container(Cont)
    definitions =
        [ :(rand(rng::AbstractRNG,            $Cont, dims::Dims) =                 rand(rng,        _make_cont(t, dims))),
          :(rand(                             $Cont, dims::Dims) =                 rand(GLOBAL_RNG, _make_cont(t, dims))),
          :(rand(rng::AbstractRNG,            $Cont, dims::Integer...) =           rand(rng,        _make_cont(t, Dims(dims)))),
          :(rand(                             $Cont, dims::Integer...) =           rand(GLOBAL_RNG, _make_cont(t, Dims(dims)))),

          :(rand(rng::AbstractRNG, X,         $Cont, dims::Dims) =                 rand(rng,        _make_cont(t, X, dims))),
          :(rand(                  X,         $Cont, dims::Dims) =                 rand(GLOBAL_RNG, _make_cont(t, X, dims))),
          :(rand(rng::AbstractRNG, X,         $Cont, dims::Integer...) =           rand(rng,        _make_cont(t, X, Dims(dims)))),
          :(rand(                  X,         $Cont, dims::Integer...) =           rand(GLOBAL_RNG, _make_cont(t, X, Dims(dims)))),

          :(rand(rng::AbstractRNG, ::Type{X}, $Cont, dims::Dims)       where {X} = rand(rng,        _make_cont(t, X, dims))),
          :(rand(                  ::Type{X}, $Cont, dims::Dims)       where {X} = rand(GLOBAL_RNG, _make_cont(t, X, dims))),
          :(rand(rng::AbstractRNG, ::Type{X}, $Cont, dims::Integer...) where {X} = rand(rng,        _make_cont(t, X, Dims(dims)))),
          :(rand(                  ::Type{X}, $Cont, dims::Integer...) where {X} = rand(GLOBAL_RNG, _make_cont(t, X, Dims(dims)))),
        ]
    esc(Expr(:block, definitions...))
end

_make_cont(args...) = make(args...)

@make_array_container(t::Type{<:Array})
@make_array_container(t::Type{<:BitArray})
@make_array_container(t::AbstractFloat)
_make_cont(t::AbstractFloat, x, dims::Dims{1}) = make(SparseVector,    x, t, dims)
_make_cont(t::AbstractFloat,    dims::Dims{1}) = make(SparseVector,       t, dims)
_make_cont(t::AbstractFloat, x, dims::Dims{2}) = make(SparseMatrixCSC, x, t, dims)
_make_cont(t::AbstractFloat,    dims::Dims{2}) = make(SparseMatrixCSC,    t, dims)

## sets/dicts

function _rand!(rng::AbstractRNG, A::SetDict, n::Integer, sp::Sampler)
    empty!(A)
    while length(A) < n
        push!(A, rand(rng, sp))
    end
    A
end

rand!(                  A::SetDict, X=default_sampling(A)) = rand!(GLOBAL_RNG, A, X)
rand!(rng::AbstractRNG, A::SetDict, X=default_sampling(A)) = _rand!(rng, A, length(A), sampler(rng, X))
rand!(                  A::SetDict, ::Type{X}) where {X}  = rand!(GLOBAL_RNG, A, X)
rand!(rng::AbstractRNG, A::SetDict, ::Type{X}) where {X}  = rand!(rng, A, Sampler(rng, X))

@make_container(T::Type{<:SetDict}, n::Integer)


using Base.Sort, Base.Order

refs(v::PooledArray) = v.refs
refs(v::AbstractArray) = v
refs(v::StructArray) = StructArray(map(refs, fieldarrays(v)))

function Base.permute!!(c::StructArray, p::AbstractVector{<:Integer})
    Base.invoke(Base.permute!!, Tuple{AbstractArray, AbstractVector{<:Integer}}, refs(c), p)
    return c
end

struct GroupPerm{V<:AbstractVector, P<:AbstractVector{<:Integer}, U<:AbstractUnitRange}
    vec::V
    perm::P
    within::U
end

GroupPerm(vec, perm=sortperm(vec)) = GroupPerm(vec, perm, axes(vec, 1))

Base.sortperm(g::GroupPerm) = g.perm
Base.parent(g::GroupPerm) = g.vec

function Base.iterate(g::GroupPerm, i = first(g.within))
    vec, perm = g.vec, g.perm
    l = last(g.within)
    i > l && return nothing
    @inbounds pi = perm[i]
    i1 = i+1
    @inbounds while i1 <= l && roweq(vec, pi, perm[i1])
        i1 += 1
    end
    return (i:(i1-1), i1)
end

Base.IteratorSize(::Type{<:GroupPerm}) = Base.SizeUnknown()

Base.eltype(::Type{<:GroupPerm}) = UnitRange{Int}

@inline function roweq(x::AbstractVector, i, j)
    r = refs(x)
    @inbounds eq = isequal(r[i], r[j])
    return eq
end

@generated function roweq(c::StructVector{D,C}, i, j) where {D,C}
    N = fieldcount(C)
    ex = :(roweq(getfield(fieldarrays(c),1), i, j))
    for n in 2:N
        ex = :(($ex) && (roweq(getfield(fieldarrays(c),$n), i, j)))
    end
    ex
end

function uniquesorted(keys, perm=sortperm(keys))
    (keys[perm[idxs[1]]] for idxs in GroupPerm(keys, perm))
end

function finduniquesorted(keys, perm=sortperm(keys))
    func = function (idxs)
        p_idxs = perm[idxs]
        return keys[p_idxs[1]] => p_idxs
    end
    (func(idxs) for idxs in GroupPerm(keys, perm))
end

function Base.sortperm(c::StructVector{T}) where {T<:Union{Tuple, NamedTuple}}
    cols = fieldarrays(c)
    x = cols[1]
    p = sortperm(x)
    if length(cols) > 1
        y = cols[2]
        refine_perm!(p, cols, 1, x, y, 1, length(x))
    end
    return p
end

Base.sort!(c::StructArray{<:Union{Tuple, NamedTuple}}) = (Base.permute!!(c, sortperm(c)); c)
Base.sort(c::StructArray{<:Union{Tuple, NamedTuple}}) = c[sortperm(c)]

# Given an ordering `p`, return a vector `v` such that `Perm(Forward, v)` is
# equivalent to `p`. Return `nothing` if such vector is not found.
forward_vec(p::Perm{ForwardOrdering}) = p.data
forward_vec(::Ordering) = nothing

# Methods from IndexedTables to refine sorting:
# # assuming x[p] is sorted, sort by remaining columns where x[p] is constant
function refine_perm!(p, cols, c, x, y′, lo, hi)
    temp = similar(p, 0)
    order = Perm(Forward, y′)
    y = something(forward_vec(order), y′)
    nc = length(cols)
    for idxs in GroupPerm(x, p, lo:hi)
        i, i1 = extrema(idxs)
        if i1 > i
            sort_sub_by!(p, i, i1, y, order, temp)
            if c < nc-1
                z = cols[c+2]
                refine_perm!(p, cols, c+1, y, z, i, i1)
            end
        end
    end
end

# sort the values in v[i0:i1] in place, by array `by`
Base.@noinline function sort_sub_by!(v, i0, i1, by, order, temp)
    empty!(temp)
    sort!(v, i0, i1, MergeSort, order, temp)
end

Base.@noinline function sort_sub_by!(v, i0, i1, by::AbstractVector{T}, order, temp) where T<:Integer
    min = max = by[v[i0]]
    @inbounds for i = i0+1:i1
        val = by[v[i]]
        if val < min
            min = val
        elseif val > max
            max = val
        end
    end
    rangelen = max-min+1
    n = i1-i0+1
    if rangelen <= n
        sort_int_range_sub_by!(v, i0-1, n, by, rangelen, min, temp)
    else
        empty!(temp)
        sort!(v, i0, i1, MergeSort, order, temp)
    end
    v
end

# in-place counting sort of x[ioffs+1:ioffs+n] by values in `by`
function sort_int_range_sub_by!(x, ioffs, n, by, rangelen, minval, temp)
    offs = 1 - minval

    where = fill(0, rangelen+1)
    where[1] = 1
    @inbounds for i = 1:n
        where[by[x[i+ioffs]] + offs + 1] += 1
    end
    cumsum!(where, where)

    length(temp) < n && resize!(temp, n)
    @inbounds for i = 1:n
        xi = x[i+ioffs]
        label = by[xi] + offs
        wl = where[label]
        temp[wl] = xi
        where[label] = wl+1
    end

    @inbounds for i = 1:n
        x[i+ioffs] = temp[i]
    end
    x
end


module Projectors
using ..Coords: AbstractCoords

abstract type AbstractProjector end

"""
Coordinate transformation module
"""
struct Projector{Domain, Codomain} where {Domain, Codomain <: AbstractCoords}
    pipeline::Union{Projector, Nothing}
    domain::Type{Domain}
    codomain::Type{Codomain}
end

prj(x) = 2x
inv(x) = x/2

struct Mask <: AbstractProjector

end

end


# Implementation of the Next Subvolume Method

############################ NSM ###################################
#NOTE state vector u is a matrix. u[i,j] is species i, site j
#NOTE hopping_constants is a matrix. hopping_constants[i,j] is species i, site j
mutable struct NSMJumpAggregation{J,T,RX,HOP,RNG,DEPGR,VJMAP,JVMAP,PQ,SS} <: AbstractSSAJumpAggregator
    next_jump::SpatialJump{J} #some structure to identify the next event: reaction or hop
    prev_jump::SpatialJump{J} #some structure to identify the previous event: reaction or hop
    next_jump_time::T
    end_time::T
    rx_rates::RX
    hop_rates::HOP
    # rates::F1 #rates for constant-rate jumps
    # affects!::F2 #affects! function determines the effect of constant-rate jumps
    save_positions::Tuple{Bool,Bool}
    rng::RNG
    dep_gr::DEPGR #dep graph is same for each site
    vartojumps_map::VJMAP #vartojumps_map is same for each site
    jumptovars_map::JVMAP #jumptovars_map is same for each site
    pq::PQ
    spatial_system::SS
    numspecies::Int #number of species
end

function NSMJumpAggregation(nj::SpatialJump{J}, njt::T, et::T, rx_rates::RX, hop_rates::HOP, sps::Tuple{Bool,Bool},
                                      rng::RNG, spatial_system::SS; num_specs, vartojumps_map=nothing, jumptovars_map=nothing, dep_graph=nothing, kwargs...) where {J,T,RX,HOP,RNG,SS}

    # a dependency graph is needed
    if dep_graph === nothing
        dg = make_dependency_graph(num_specs, rx_rates.ma_jumps)
    else
        dg = dep_graph
        # make sure each jump depends on itself
        add_self_dependencies!(dg)
    end

    # a species-to-reactions graph is needed
    if vartojumps_map === nothing
        vtoj_map = var_to_jumps_map(num_specs, rx_rates.ma_jumps)
    else
        vtoj_map = vartojumps_map
    end

    if jumptovars_map === nothing
        jtov_map = jump_to_vars_map(rx_rates.ma_jumps)
    else
        jtov_map = jumptovars_map
    end

    pq = MutableBinaryMinHeap{T}()

    NSMJumpAggregation{J,T,RX,HOP,RNG,typeof(dg),typeof(vtoj_map),typeof(jtov_map),typeof(pq),SS}(nj, nj, njt, et, rx_rates, hop_rates, sps, rng, dg, vtoj_map, jtov_map, pq, spatial_system, num_specs)
end

############################# Required Functions ##############################
# creating the JumpAggregation structure (function wrapper-based constant jumps)
function aggregate(aggregator::NSM, starting_state, p, t, end_time, constant_jumps, ma_jumps, save_positions, rng; hopping_constants, spatial_system, kwargs...)
    num_species = size(starting_state,1)
    majumps = ma_jumps
    if majumps === nothing
        majumps = MassActionJump(Vector{typeof(end_time)}(), Vector{Vector{Pair{Int,Int}}}(), Vector{Vector{Pair{Int,Int}}}())
    end

    next_jump = SpatialJump{Int}(typemax(Int),typemax(Int),typemax(Int)) #a placeholder
    next_jump_time = typemax(typeof(end_time))
    rx_rates = RxRates(num_sites(spatial_system), majumps)
    hop_rates = HopRates(hopping_constants, spatial_system)

    NSMJumpAggregation(next_jump, next_jump_time, end_time, rx_rates, hop_rates, save_positions, rng, spatial_system; num_specs = num_species, kwargs...)
end

# set up a new simulation and calculate the first jump / jump time
function initialize!(p::NSMJumpAggregation, integrator, u, params, t)
    p.end_time = integrator.sol.prob.tspan[2]
    fill_rates_and_get_times!(p, u, t)
    generate_jumps!(p, integrator, params, u, t)
    nothing
end

# calculate the next jump / jump time
function generate_jumps!(p::NSMJumpAggregation, integrator, params, u, t)
    p.next_jump_time, site = top_with_handle(p.pq)
    p.next_jump_time >= p.end_time && return nothing
    p.next_jump = sample_jump_direct(p, site)
    nothing
end

# execute one jump, changing the system state
function execute_jumps!(p::NSMJumpAggregation, integrator, u, params, t)
    # execute jump
    update_state!(p, integrator)

    # update current jump rates and times
    update_dependent_rates_and_firing_times!(p, integrator.u, t)
    nothing
end

######################## SSA specific helper routines ########################
"""
    fill_rates_and_get_times!(aggregation::NSMJumpAggregation, u, t)

reset all stucts, reevaluate all rates, recalculate tentative site firing times, and reinit the priority queue
"""
function fill_rates_and_get_times!(aggregation::NSMJumpAggregation, u, t)
    @unpack spatial_system, rx_rates, hop_rates, rng = aggregation

    reset!(rx_rates)
    reset!(hop_rates)

    rxs = 1:num_rxs(rx_rates)
    species = 1:aggregation.numspecies
    num_nodes = num_sites(spatial_system)

    pqdata = Vector{typeof(t)}(undef, num_nodes)
    @inbounds for site in 1:num_nodes
        update_rx_rates!(rx_rates, rxs, u, site)
        update_hop_rates!(hop_rates, species, u, site, spatial_system)
        pqdata[site] = t + randexp(rng) / (total_site_rate(rx_rates, hop_rates, site))
    end

    aggregation.pq = MutableBinaryMinHeap(pqdata)
    nothing
end

"""
    update_dependent_rates_and_firing_times!(p, u, t)

recalculate jump rates for jumps that depend on the just executed jump (p.prev_jump)
"""
function update_dependent_rates_and_firing_times!(p::NSMJumpAggregation, u, t)
    jump = p.prev_jump
    if is_hop(p, jump)
        source_site = jump.src
        target_site = jump.dst
        update_rates_after_hop!(p, u, source_site, target_site, jump.jidx)
        update_site_time!(p, source_site, t)
        update_site_time!(p, target_site, t)
    else
        site = jump.src
        update_rates_after_reaction!(p, u, site, reaction_id_from_jump(p,jump))
        update_site_time!(p, site, t)
    end
    nothing
end

"""
    update_site_time!(p::NSMJumpAggregation, site, t)

update the time of site in the priority queue
"""
function update_site_time!(p::NSMJumpAggregation, site, t)
    site_rate = (total_site_rate(p.rx_rates, p.hop_rates, site))
    if site_rate > zero(typeof(site_rate))
        update!(p.pq, site, t + randexp(p.rng) / site_rate)
    else
        update!(p.pq, site, typemax(t))
    end
    nothing
end

"""
    num_constant_rate_jumps(aggregator::NSMJumpAggregation)

number of constant rate jumps
"""
num_constant_rate_jumps(aggregator::NSMJumpAggregation) = 0


import Chipmunk
const Cp = Chipmunk

abstract type Object{T} <: AbstractVector{T} end
Base.IndexStyle(::Type{<:Object}) = IndexLinear()
Base.size(p::Object) = (2,)
Base.length(p::Object) = 2
function Base.getindex(p::Object, i::Int)
    if i == 1
        return Cp.get_position(p.body).x # TODO: needs too long # TODO: tooks too many memory
    elseif i == 2
        return Cp.get_position(p.body).y # TODO: needs too long # TODO: tooks too many memory
    end
    return 0.0
end
# function Base.setindex!(p::Object, v, i::Int)
#     if i == 1
#         Cp.set_position(p.body, Cp.Vect(v, p[2]))
#     elseif i == 2
#         Cp.set_position(p.body, Cp.Vect(p[1], v))
#     end
# end
get_position(p::Object) = [Cp.get_position(p.body).x, Cp.get_position(p.body).y]
set_position!(p::Object, new_position::AbstractVector{<:Number}) = Cp.set_position(p.body, Cp.Vect(new_position...))
get_velocity(p::Object) = [Cp.get_velocity(p.body).x, Cp.get_velocity(p.body).y]
set_velocity!(p::Object, new_velocity::AbstractVector{<:Number}) = Cp.set_velocity(p.body, Cp.Vect(new_velocity...)) # TODO: tooks too many memory
set_velocity!(p::Object, new_velocity_x::Number, new_velocity_y::Number) = Cp.set_velocity(p.body, Cp.Vect(new_velocity_x, new_velocity_y)) # TODO: tooks too many memory
get_angular_velocity(p::Object) = Cp.get_angular_velocity(p.body)
get_size(p::Object) = Cp.get_radius(p.shape)
get_mass(p::Object) = Cp.get_mass(p.shape)
get_friction(p::Object) = Cp.get_friction(p.shape)

"""
    Robot

The current state of an robot.

# Arguments

position::AbstractVector - 2d Vector with the position in the plane
body_orientation::T      - angle (in radiens) relative to x axis with [-π, π], determined from `atan2`
"""
mutable struct Robot{T<:Number} <: Object{T}
    # position::AbstractVector{T}
    body_orientation::T
    head_orientation::T
    # velocity::T
    # angularvelocity::T
    # size::T
    # mass::T
    hfov::T
    max_velocity::T
    max_angularvelocity::T
    body::Cp.Body
    shape::Cp.Shape
    order::Order
end
function Robot(position::AbstractVector{T}, body_orientation::T, head_orientation::T, velocity::AbstractVector{T}, angularvelocity::T, size::T, mass::T, hfov::T, max_velocity::T, max_angularvelocity::T, order::Order) where T
    body = Cp.Body(0, 0) # TODO: KinematicBody
    shape = Cp.CircleShape(body, size, Cp.Vect(0, 0))
    Cp.set_position(body, Cp.Vect(position...))
    Cp.set_friction(shape, 0.8) #TODO: add command line argument
    Cp.set_elasticity(shape, 0) #TODO: add command line argument
    Cp.set_mass(shape, mass)
    Cp.set_collision_type(shape, 0)
    Cp.set_velocity(body, Cp.Vect(velocity...))
    Cp.set_angular_velocity(body, angularvelocity)
    Robot(body_orientation, head_orientation, hfov, max_velocity, max_angularvelocity, body, shape, order)
end
# http://doc.aldebaran.com/2-1/family/robots/index_robots.html
Robot(;position::AbstractVector{T}=[0.0, 0.0],
       body_orientation::T=0.0,
       head_orientation::T=0.0,
       velocity::AbstractVector{T}=[0.0, 0.0],
       angularvelocity::T=0.0,
       size::T=0.15,
       mass::T=5.305350006,
       hfov::T=deg2rad(60.97),
       max_velocity::T=1.0,
       max_angularvelocity::T=Float64(π/2),
       order::Order=NoOrder()) where T<:Number =
    Robot(position, body_orientation, head_orientation, velocity, angularvelocity, size, mass, hfov, max_velocity, max_angularvelocity, order)
Robot(r::Robot) = Robot(get_position(r), r.body_orientation, r.head_orientation, get_velocity(r), get_angular_velocity(r), get_size(r), get_mass(r), r.hfov, r.max_velocity, r.max_angularvelocity, r.order)

Robot(position::AbstractVector{T}, body_orientation::T) where T<:Number =
    Robot(position=position, body_orientation=body_orientation)
NaoV5(position::AbstractVector{T}, body_orientation::T) where T<:Number = Robot(position, body_orientation)
Nao(position::AbstractVector{T}, body_orientation::T) where T<:Number = NaoV5(position, body_orientation)

get_angularvelocity(r::Robot) = Cp.get_angular_velocity(r.body)

apply_order!(robot::Robot) = apply_order!(robot, robot.order)

"""
    Obstacle

The current state of an Obstacle.
"""
mutable struct Obstacle{T<:Number} <: Object{T}
    # position::AbstractVector{T}
    # velocity::T
    # size::T
    # mass::T
    max_velocity::T
    deceleration::T
    friction::T
    fixed::Bool
    body::Cp.Body
    shape::Cp.Shape
    color::AbstractVector{<:Real} # TODO: switch to Color Object
end
function Obstacle(position::AbstractVector{T}, velocity::AbstractVector{T}, size::T, mass::T, max_velocity::T, friction::T, gravity::T, fixed::Bool, color::AbstractVector{<:Real}) where T
    if fixed
        body = Cp.StaticBody()
    else
        body = Cp.Body(0, 0)
        Cp.set_velocity(body, Cp.Vect(velocity...))
    end
    Cp.set_position(body, Cp.Vect(position...))
    Cp.set_velocity(body, Cp.Vect(velocity...))
    shape = Cp.CircleShape(body, size, Cp.Vect(0, 0))
    Cp.set_friction(shape, friction) #TODO: add command line argument
    Cp.set_elasticity(shape, 1.0) #TODO: add command line argument
    Cp.set_collision_type(shape, 1)
    Cp.set_mass(shape, mass)
    Obstacle(max_velocity, friction*gravity, friction, fixed, body, shape, color)
end
Obstacle(;position::AbstractVector{T}=[0.0, 0.0],
          velocity::AbstractVector{T}=[0.0, 0.0],
          size::T=0.05,
          mass::T=0.048,
          max_velocity::T=0.25,
          friction::T=0.1,
          gravity::T=9.81,
          fixed::Bool=false,
          color::AbstractVector{<:Real}=[1.0, 0.1, 0.1, 1.0]) where T<:Number =
    Obstacle(position, velocity, size, mass, max_velocity, friction, gravity, fixed, color)
Obstacle(o::Obstacle) = Obstacle(get_position(o), get_velocity(o), get_size(o), get_mass(o), o.max_velocity, o.friction, o.deceleration/o.friction, o.fixed, o.color)
apply_order!(obstacle::Obstacle) = nothing

Ball(position::AbstractVector{T}) where T<:Number = Obstacle(position=position)

"""
    WorldState

Data clump holding all information about the game.
"""
struct WorldState{R<:AbstractVector{<:Robot{<:Number}}, O<:AbstractVector{<:Obstacle{<:Number}}}
    myPlayers::R # indices of myPlayers
    opponents::R # indices of opponents
    obstacles::O
    # time::Number
    # max_time::Number
    # v_off::Number
    # player::AbstractVector{<:Robot{<:T}}
    # all::AbstractVector{<:Object{<:T}}
end
# WorldState(myPlayers, opponents, obstacles) = WorldState(myPlayers, opponents, obstacles, vcat(myPlayers, opponents), vcat(myPlayers, opponents, obstacles))
function WorldState(ws::WorldState)
    myPlayers = Robot.(ws.myPlayers)
    opponents = Robot.(ws.opponents)
    obstacles = Obstacle.(ws.obstacles)
    return WorldState(myPlayers, opponents, obstacles)
end

# struct WorldStateIter
#     ws::WorldState
#     start::Integer

#     iterate_obstacles::Bool
#     exclude::Object
# end


# Batched matrix multiplication
# 1st argument is produced by NNlib.storage_type(A)
NNlib._batched_gemm!(::Type{<:CuArray}, transA::Char, transB::Char, α::Number, A, B, β::Number, C) =
     CUBLAS.gemm_strided_batched!(transA, transB, α, A, B, β, C)

Base.unsafe_convert(::Type{CuPtr{T}}, A::NNlib.BatchedAdjOrTrans{T}) where {T} =
    Base.unsafe_convert(CuPtr{T}, parent(A))


"""
Surprise Actor-critic
---------------------

Actor-Critic with a surprise-modulated learning rate for the actor.

See actor-critic.jl for some more implementational info.
See learner/actorcriticlearner.jl and learner/actorcriticmodulatedlearner.jl
for the core implementation.

Note 1
------
With particle filtering we need to run multiple times with different seeds.
As final LL we report the MEAN across seeds.
For signals/results we keep track of the ones of all seeds
and in the end report the one with the MAXIMUM LL.

Note 2
------
get_all_puzzles(), get_all_episodes_in_puzzle(), get_data_by_puzzle_episode()
are in rl_data_toolsforalgos.jl

"""
function getLL_ActorCritic_TDLambda_modulatedActor(data::Array{Int,2},
                                            alpha::Float64,
                                            surpriseprobability::Float64,
                                            stochasticity::Float64,
                                            eta::Float64,
                                            gamma::Float64,
                                            lambda_actor::Float64,
                                            lambda_critic::Float64,
                                            temp::Float64,
                                            nparticles::Union{Int, Float64};
                                            particlefilterseeds = [1,2,3],
                                            init_policy=nothing,
                                            init_Vs = nothing,
                                            use_replacing_trace=true,
                                            do_debug_print = false)
    nparticles = converttoInt(nparticles)
    data_stats, nr_states, nr_actions, nr_samples = get_data_stats_and_dimensions(data)
    if data_stats.nr_subjects != 1
        error("Invalid argument: getLL_ActorCritic_TDLambda_modulatedActor supports only one subject")
    end

    action_selection_probs_seeds = Array{Array{Float64, 1}, 1}(undef, length(particlefilterseeds))
    [action_selection_probs_seeds[i_seed] = zeros(Float64, nr_samples) for i_seed in 1:length(particlefilterseeds)]
    LL_seeds = zeros(Float64, length(particlefilterseeds))

    Tsas_seeds = Array{Any, 1}(undef, length(particlefilterseeds))

    # signals:
    TD_seeds =Array{Array{Float64,1}, 1}(undef, length(particlefilterseeds))
    [TD_seeds[i_seed] = zeros(Float64, nr_samples) for i_seed in 1:length(particlefilterseeds)]
    Surprise_seeds =Array{Array{Float64,1}, 1}(undef, length(particlefilterseeds))
    [Surprise_seeds[i_seed] = zeros(Float64, nr_samples) for i_seed in 1:length(particlefilterseeds)]
    V_state_seeds =Array{Array{Float64,1}, 1}(undef, length(particlefilterseeds))
    [V_state_seeds[i_seed] = zeros(Float64, nr_samples + data_stats.nr_rewards) for i_seed in 1:length(particlefilterseeds)]

    γ_surprise_seeds =Array{Array{Float64,1}, 1}(undef, length(particlefilterseeds))
    [γ_surprise_seeds[i_seed] = zeros(Float64, nr_samples) for i_seed in 1:length(particlefilterseeds)]
    SPE_seeds =Array{Array{Float64,1}, 1}(undef, length(particlefilterseeds))
    [SPE_seeds[i_seed] = zeros(Float64, nr_samples) for i_seed in 1:length(particlefilterseeds)]

    finalV_record_seed = Array{Dict{Int,Any}, 1}(undef, length(particlefilterseeds))
    [finalV_record_seed[i_seed] = Dict{Int,Any}() for i_seed in 1:length(particlefilterseeds)]
    final_policyparameters_record_seed = Array{Dict{Int,Any}, 1}(undef, length(particlefilterseeds))
    [final_policyparameters_record_seed[i_seed] = Dict{Int,Any}() for i_seed in 1:length(particlefilterseeds)]

    PolicyParametersImbalance_record_seed  = Array{Array{Float64,1}, 1}(undef, length(particlefilterseeds))
    [PolicyParametersImbalance_record_seed[i_seed] = zeros(Float64, nr_samples) for i_seed in 1:length(particlefilterseeds)]
    PolicyImbalance_record_seed = Array{Array{Float64,1}, 1}(undef, length(particlefilterseeds))
    [PolicyImbalance_record_seed[i_seed] = zeros(Float64, nr_samples) for i_seed in 1:length(particlefilterseeds)]

    PolicyParametersDiffChosenVsUnchosen_record_seed =Array{Array{Float64,1}, 1}(undef, length(particlefilterseeds))
    [PolicyParametersDiffChosenVsUnchosen_record_seed[i_seed] = zeros(Float64, nr_samples) for i_seed in 1:length(particlefilterseeds)]

    PolicyDiffChosenVsUnchosen_record_seed = Array{Array{Float64,1}, 1}(undef, length(particlefilterseeds))
    [PolicyDiffChosenVsUnchosen_record_seed[i_seed] = zeros(Float64, nr_samples) for i_seed in 1:length(particlefilterseeds)]

    PolicyParameters_record_seed = Array{Array{Float64,1}, 1}(undef, length(particlefilterseeds))
    [PolicyParameters_record_seed[i_seed] = zeros(Float64, nr_samples) for i_seed in 1:length(particlefilterseeds)]

    for i_seed in 1:length(particlefilterseeds)
        current_puzzle_id = -1
        Testimate = []
        learner = []
        j = 1
        for i = 1:nr_samples
            if current_puzzle_id != data[i,Int(c_puzzle)]
                # Puzzle = independent block of the experiment.
                # The code was made for general purposes, in this experiment we
                # have only ONE puzzle.
                current_puzzle_id = data[i,Int(c_puzzle)]
                if do_debug_print
                    println("new puzzle: $current_puzzle_id")
                end

                # Particle Filter
                Testimate = TParticleFilterJump(nr_states = nr_states,
                                        nr_actions = nr_actions,
                                        nparticles = nparticles,
                                        surpriseprobability = surpriseprobability,
                                        stochasticity = stochasticity,
                                        seed = particlefilterseeds[i_seed])

                if init_policy == nothing # providing init_policy w/o init_Vs is considered a bug
                    learner = ActorCriticModulated(nr_states = nr_states,
                                        nr_actions = nr_actions,
                                        α_critic = eta,
                                        α_actor = alpha,
                                        γ = gamma,
                                        λ_critic = lambda_critic,
                                        λ_actor = lambda_actor,
                                        use_replacing_trace = use_replacing_trace,
                                        do_debug_print = do_debug_print)
                else
                    learner = ActorCriticModulated(nr_states = nr_states,
                                        nr_actions = nr_actions,
                                        α_critic = eta,
                                        α_actor = alpha,
                                        γ = gamma,
                                        λ_critic = lambda_critic,
                                        λ_actor = lambda_actor,
                                        policyparameters = copy(init_policy[current_puzzle_id]),
                                        V = copy(init_Vs[current_puzzle_id]),
                                        use_replacing_trace = use_replacing_trace,
                                        do_debug_print = do_debug_print)
                end
            end
            state = data[i, Int(c_state)]
            action = data[i, Int(c_action)]
            reward::Float64= data[i, Int(c_reward)]
            next_state = data[i, Int(c_nextState)]
            done = reward>0 ? true : false

            p_actions = getSoftmaxProbs(learner.policyparameters[state,:], temp, [0., 0.])

            unbiased_p_sa = p_actions[action]

            action_selection_probs_seeds[i_seed][i] = copy(unbiased_p_sa)
            LL_seeds[i_seed] += log(unbiased_p_sa)

            # Particle Filtering model learner
            updatet!(Testimate, state, action, next_state, done)
            γ_surprise_seeds[i_seed][i] = copy(Testimate.γ_surprise[1])
            Surprise_seeds[i_seed][i] = copy(Testimate.Sgm[1])
            Tsas_seeds[i_seed]= deepcopy(Testimate.Ps1a0s0)
            SPE_seeds[i_seed][i] = copy(Testimate.SPE[1])

            # Track V values (used as fmri regressor)
            V_state_seeds[i_seed][j] = learner.V[state]
            if done
                j+=1
                if j < (nr_samples + data_stats.nr_rewards)
                    V_state_seeds[i_seed][j] = learner.V[next_state] # Track V value of landing state in case it is the Goal
                end
            end
            j+=1

            PolicyParametersImbalance_record_seed[i_seed][i] = abs(learner.policyparameters[state, 1] - learner.policyparameters[state, 2])
            PolicyImbalance_record_seed[i_seed][i] = abs(p_actions[1] - p_actions[2])
            PolicyParametersDiffChosenVsUnchosen_record_seed[i_seed][i] = learner.policyparameters[state, action] - learner.policyparameters[state, mod(action,2)+1]
            PolicyDiffChosenVsUnchosen_record_seed[i_seed][i] = p_actions[action] - p_actions[mod(action,2)+1]
            PolicyParameters_record_seed[i_seed][i] = learner.policyparameters[state, action]

            # Actor-Critic
            update!(learner, state, action, next_state, reward, unbiased_p_sa, (1. - Testimate.γ_surprise[1]))
            TD_seeds[i_seed][i] = copy(learner.td_error[1])

            finalV_record_seed[i_seed][current_puzzle_id] = copy(learner.V)
            final_policyparameters_record_seed[i_seed][current_puzzle_id] = copy(learner.policyparameters)

        end
    end

    LL = get_meanLL_acrossseeds(LL_seeds)

    action_selection_probs_cat = hcat(action_selection_probs_seeds...)
    action_selection_probs = mean(action_selection_probs_cat, dims=2)[:]
    trial_by_trial_p_sa = (action_selection_probs)

    # For signals: return the ones corresponding to the Max LL across seeds
    LL_max, seed_idx = findmax(LL_seeds)

    TD_record = copy(TD_seeds[seed_idx])
    SPE_record = copy(SPE_seeds[seed_idx])
    trial_by_trial_p_sa_max = (action_selection_probs_seeds[seed_idx])

    trace_type = use_replacing_trace ? "ReplTrc" : "EligTrc"
    algo = "Actor-Critic (TD-Lambda, $trace_type) modulated Actor"
    param_v = [alpha, surpriseprobability, stochasticity, eta, gamma, lambda_actor, lambda_critic, temp, nparticles]
    param_str = "alpha(actor)=$(round(alpha,digits=4)),
                surpriseprobability=$(round(surpriseprobability,digits=4)),
                stochasticity=$(round(stochasticity,digits=4)),
                eta(critic)=$(round(eta,digits=4)),
                gamma=$(round(gamma,digits=4)),
                lambda_actor=$(round(lambda_actor,digits=4)),
                lambda_critic=$(round(lambda_critic,digits=4)),
                temp=$(round(temp,digits=4)),
                nparticles=$nparticles,
                init_policy=$(repr(init_policy)),
                init_Vs=$(repr(init_Vs)),
                use_replacing_trace=$use_replacing_trace"

    results_dict = Dict{String,Any}(
        "trialByTrial_Vs" => V_state_seeds[seed_idx],
        "final_V" => finalV_record_seed[seed_idx],
        "final_PolicyParameters" => final_policyparameters_record_seed[seed_idx],
        "final_Tsas" => Tsas_seeds[seed_idx],
        "Surprise" => Surprise_seeds[seed_idx],
        "gamma_surprise" => γ_surprise_seeds[seed_idx],
        "nparticles" => nparticles,
        "particlefilterseeds" => particlefilterseeds,
        "trial_by_trial_p_sa_max" => trial_by_trial_p_sa_max,
        "PolicyParametersImbalance" => PolicyParametersImbalance_record_seed[seed_idx],
        "PolicyImbalance" => PolicyImbalance_record_seed[seed_idx],
        "PolicyParametersDiffChosenVsUnchosen" => PolicyParametersDiffChosenVsUnchosen_record_seed[seed_idx],
        "PolicyDiffChosenVsUnchosen" => PolicyDiffChosenVsUnchosen_record_seed[seed_idx],
        "PolicyParameters" => PolicyParameters_record_seed[seed_idx],
        )

    signals = Step_by_step_signals(
    data_stats.all_subject_ids[1],
    TD_record,  # RPE
    SPE_record,  # no SPE
    trial_by_trial_p_sa,
    LL,
    data_stats,
    algo,
    param_v,
    param_str,
    99, # dummy value
    results_dict)
    return LL, signals
end


import Random

# TODO: remove Random.CloseOpen01{BigFloat} parameters
# TODO: remove setprecision
# possibly fixed by https://github.com/JuliaLang/julia/pull/38169

function Random.Sampler(
    RNG::Type{<:Random.AbstractRNG},
    st::Random.SamplerType{Acb},
    n::Random.Repetition,
)
    return Random.SamplerSimple(
        Random.SamplerType{Acb}(),
        Random.SamplerBigFloat{Random.CloseOpen01{BigFloat}}(precision(Acb)),
    )
end

function Random.Sampler(
    ::Type{<:Random.AbstractRNG},
    ::Random.CloseOpen01{T},
    ::Random.Repetition,
) where {T<:Union{Arf,Arb}}
    return Random.SamplerSimple(
        Random.SamplerType{T}(),
        Random.SamplerBigFloat{Random.CloseOpen01{BigFloat}}(precision(T)),
    )
end

function Random.Sampler(
    RNG::Type{<:Random.AbstractRNG},
    x::TOrRef,
    n::Random.Repetition,
) where {TOrRef<:Union{Arf,ArfRef,Arb,ArbRef,Acb,AcbRef}}
    T = _nonreftype(TOrRef)
    return Random.SamplerSimple(
        Random.SamplerType{T}(),
        Random.SamplerBigFloat{Random.CloseOpen01{BigFloat}}(precision(x)),
    )
end

Random.rand(rng::Random.AbstractRNG, sp::Random.SamplerSimple{Random.SamplerType{Arf}}) =
    setprecision(BigFloat, sp.data.prec) do
        Arf(rand(rng, sp.data), prec = sp.data.prec)
    end

Random.rand(rng::Random.AbstractRNG, sp::Random.SamplerSimple{Random.SamplerType{Arb}}) =
    setprecision(BigFloat, sp.data.prec) do
        Arb(rand(rng, sp.data), prec = sp.data.prec)
    end

Random.rand(rng::Random.AbstractRNG, sp::Random.SamplerSimple{Random.SamplerType{Acb}}) =
    setprecision(BigFloat, sp.data.prec) do
        Acb(rand(rng, sp.data), rand(rng, sp.data), prec = sp.data.prec)
    end


function ao_2e3c(BS::BasisSet, auxBS::BasisSet, T::DataType = Float64)

    ATM_SLOTS = 6
    BAS_SLOTS = 8

    natm = BS.natoms
    nbas = BS.nbas + auxBS.nbas
    nshells = BS.nshells + auxBS.nshells

    lc_atm = zeros(Cint, natm*ATM_SLOTS)
    lc_bas = zeros(Cint, nshells*BAS_SLOTS)
    env = zeros(Cdouble, length(BS.lc_env) + length(auxBS.lc_env) - 3*natm)

    # Prepare the lc_atom input
    off = 0
    ib = 0
    for i = eachindex(BS.molecule.atoms)
        A = BS.molecule.atoms[i]
        # lc_atom has ATM_SLOTS (6) "spaces" for each atom
        # The first one (Z_INDEX) is the atomic number
        lc_atm[1 + ATM_SLOTS*(i-1)] = Cint(A.Z)
        # The second one is the env index address for xyz
        lc_atm[2 + ATM_SLOTS*(i-1)] = off
        env[off+1:off+3] .= A.xyz ./ Fermi.PhysicalConstants.bohr_to_angstrom
        off += 3
        # The remaining 4 slots are zero.
    end

    for i = eachindex(BS.molecule.atoms)
        A = BS.molecule.atoms[i]
        # Prepare the lc_bas input
        for j = eachindex(BS.basis[A])
            B = BS.basis[A][j]
            Ne = length(B.exp)
            Nc = length(B.coef)
            # lc_bas has BAS_SLOTS for each basis set
            # The first one is the index of the atom starting from 0
            lc_bas[1 + BAS_SLOTS*ib] = i-1
            # The second one is the angular momentum
            lc_bas[2 + BAS_SLOTS*ib] = B.l
            # The third is the number of primitive functions
            lc_bas[3 + BAS_SLOTS*ib] = Nc
            # The fourth is the number of contracted functions
            lc_bas[4 + BAS_SLOTS*ib] = 1
            # The fifth is a κ parameter
            lc_bas[5 + BAS_SLOTS*ib] = 0
            # Sixth is the env index address for exponents
            lc_bas[6 + BAS_SLOTS*ib] = off
            env[off+1:off+Ne] .= B.exp
            off += Ne
            # Seventh is the env index address for contraction coeff
            lc_bas[7 + BAS_SLOTS*ib] = off
            env[off+1:off+Nc] .= B.coef
            off += Nc
            # Eigth, nothing
            ib += 1
        end
    end

    for i = eachindex(auxBS.molecule.atoms)
        A = auxBS.molecule.atoms[i]
        # Prepare the lc_bas input
        for j = eachindex(auxBS.basis[A])
            B = auxBS.basis[A][j]
            Ne = length(B.exp)
            Nc = length(B.coef)
            # lc_bas has BAS_SLOTS for each basis set
            # The first one is the index of the atom starting from 0
            lc_bas[1 + BAS_SLOTS*ib] = i-1
            # The second one is the angular momentum
            lc_bas[2 + BAS_SLOTS*ib] = B.l
            # The third is the number of primitive functions
            lc_bas[3 + BAS_SLOTS*ib] = Nc
            # The fourth is the number of contracted functions
            lc_bas[4 + BAS_SLOTS*ib] = 1
            # The fifth is a κ parameter
            lc_bas[5 + BAS_SLOTS*ib] = 0
            # Sixth is the env index address for exponents
            lc_bas[6 + BAS_SLOTS*ib] = off
            env[off+1:off+Ne] .= B.exp
            off += Ne
            # Seventh is the env index address for contraction coeff
            lc_bas[7 + BAS_SLOTS*ib] = off
            env[off+1:off+Nc] .= B.coef
            off += Nc
            # Eigth, nothing
            ib += 1
        end
    end

    # Allocate output array
    #N = (BS.nbas^2 + BS.nbas) >> 1
    #out = zeros(T, N, auxBS.nbas)
    out = zeros(T, BS.nbas, BS.nbas, auxBS.nbas)

    # Save a list containing the angular momentum number for each shell
    lvals = [Libcint.CINTcgtos_spheric(i-1, BS.lc_bas) for i = 1:BS.nshells]
    Plvals = [Libcint.CINTcgtos_spheric(P-1, auxBS.lc_bas) for P = 1:auxBS.nshells]
    ao_offset = [sum(lvals[1:(i-1)]) for i = 1:BS.nshells]

    buf_arrays = [zeros(Cdouble, maximum(lvals)^2*maximum(Plvals)) for _ = 1:Threads.nthreads()]

    @sync for p in 1:auxBS.nshells
        Threads.@spawn begin
            @inbounds begin
                buf = buf_arrays[Threads.threadid()]
                Lp = Plvals[p]
                poff = sum(Plvals[1:(p-1)])
                Bp = p + BS.nshells - 1
                for i in 1:BS.nshells
                    Li = lvals[i]
                    ioff = ao_offset[i]
                    for j in i:BS.nshells
                        Lj = lvals[j]
                        joff = ao_offset[j]

                        # Call libcint
                        cint3c2e_sph!(buf, Cint.([i-1, j-1, Bp]), lc_atm, natm, lc_bas, nbas, env)

                        # Loop through shell block and save unique elements
                        for ps = 1:Lp
                            P = poff + ps
                            for js = 1:Lj
                                J = joff + js
                                for is = 1:Li
                                    I = ioff + is
                                    J < I ? break : nothing
                                    out[I,J,P] = buf[is + Li*(js-1) + Li*Lj*(ps-1)]
                                    out[J,I,P] = out[I,J,P]
                                end
                            end
                        end
                    end
                end
            end #inbounds
        end #spwan
    end #sync
    return out
end


# this doesn't work yet, but we can load images using other tools anyway
## "Read image from file"
## function read_image(path::AbstractString, typ::Int)
##     # @assert rows*cols == length(data) "rows * cols != length of data!"
##     T = Float64
##     # typ = CCV_IO_RGB_COLOR | CCV_IO_ANY_FILE
##     typ = CCV_IO_GRAY | CCV_IO_ANY_FILE
##     matout = Ref{Ptr{CCVDenseMatrixType{T}}}()
##     path = "minions.jpg"
##     # scanline = sizeof(T)
##     # typ = ccv_type(T) | CCV_IO_ANY_FILE
##     code = ccall((:ccv_read_impl, libccv), Cint,
##                  (Ptr{Cchar}, Ptr{Ptr{CCVDenseMatrixType{T}}},
##                   Cint, Cint, Cint, Cint),
##                  path, matout, typ, 0, 0, 0)
##     if code != CCV_IO_FINAL
##         throw(ErrorException("ccv_read_impl() exited with error: $code"))
##     else
##         ptr = matout[]
##         return CCVDenseMatrix{T}(ptr)
##     end
## end

if Pkg.installed("Images") != nothing

    using Images, Colors

    Base.convert{T}(::Type{Matrix{Float64}}, img::Image{Gray{T}}) =
        convert(Array{Float64, 2}, convert(Array, img))

    Base.convert(::Type{Matrix{Float64}}, img::Image) =
        convert(Array{Float64, 2}, convert(Image{Gray}, img))

    Base.convert(::Type{Array{Float64, 3}}, img::Image) =
        convert(Array{Float64}, separate(img).data)

    function CCVDenseMatrix(img::Image)
        arr = convert(Matrix{Float64}, img)
        return CCVDenseMatrix(arr)
    end

end


using Test
using CausalDiscovery
using Random

aexpr = au"""(program
(= GRID_SIZE 16)

(object Particle (Cell 0 0 "blue"))

(: particles (List Particle))
(= particles
   (initnext (list)
             (updateObj (prev particles) (--> obj (Particle (uniformChoice (adjPositions (.. obj origin))))))))

(on clicked (= particles (addObj (prev particles) (Particle (Position (.. click x) (.. click y))))))
)"""
#
compiledMod = eval(compiletojulia(aexpr))

# time 0
state = compiledMod.init(nothing, nothing, nothing, nothing, nothing, MersenneTwister(0))
@test compiledMod.render(state.scene) == []

# time 1
state = compiledMod.next(state, nothing, nothing, nothing, nothing, nothing)
@test compiledMod.render(state.scene) == []

# time 2
state = compiledMod.next(state, compiledMod.Click(5,5), nothing, nothing, nothing, nothing)
@test compiledMod.render(state.scene) == [compiledMod.Cell(compiledMod.Position(5, 5), "blue", 0.8)]

# time 3
state = compiledMod.next(state, nothing, nothing, nothing, nothing, nothing)
@test compiledMod.render(state.scene) == [compiledMod.Cell(compiledMod.Position(4, 5), "blue", 0.8)]

# time 4
state = compiledMod.next(state, nothing, nothing, nothing, nothing, nothing)
@test compiledMod.render(state.scene) == [compiledMod.Cell(compiledMod.Position(3, 5), "blue", 0.8)]


export fit_tree!, fit_tree

using ...JLBoostTrees: is_left_child
using Tables

function viewtable(table, rows, cols)
    t = Tables.columns(table)
    v = map(cols) do c
      col = Tables.getcolumn(t, c)
      c => view(col, rows)
    end
    (; v...)
  end

function tree_diag_print(jlt)
    if jlt.parent===nothing
        parent_feature=nothing
        parent_split=nothing
        split_sign = ""
    else
        parent_feature=jlt.parent.splitfeature
        parent_split=jlt.parent.split
        split_sign = is_left_child(jlt) ? "<=" : ">"
    end
    jlt.splitfeature, jlt.split, parent_feature, split_sign, parent_split
end

"""
	_fit_tree(loss, df, target, features, warm_start, jlt, node_colsample_strategy)

Fit a tree by following a algorithm

Parameters:
* loss
    A SupervisedLoss from LossFunctions where `deriv` and `deriv2` are defined
* tbl
    A Tables.jl compatible table
* target
    The column name. Likely to be a `Symbol` or a `String`
* features
    The iterable of features name. The type of the elements should be Symbol or String
* warm_start
    A vector weight to serve as starting weights
* jlt
    The JLBoost tree to update
* stopping_criterion: Function
* tree_growth: Function
    A function to control where to grow the tree
* lambda = 0
    The L1 Norm regulization constnat
* gamma = 0
    The L2 Norm regulization constnat
* colsample_bynode = 1 (NOT IMPLEMENTED YET)
    What proportion of features to sample for each node
* colsample_bylevel = 1 (NOT IMPLEMENTED YET)
    What proportion of features to sample for each level
"""
function _fit_tree!(loss, tbl, target, features, warm_start,
    jlt::AbstractJLBoostTree = JLBoostTree(0.0),
    tree_growth = depth_wise,
    stopping_criterion = max_depth_stopping_criterion(6);
    lambda = 0, gamma = 0,
    verbose = false, #colsample_bynode = 1, colsample_bylevel = 1,
	kwargs...)

    @assert Tables.istable(tbl)

    tblc = Tables.columns(tbl)
    dfo = DataFrame(tbl)

    @assert nrow(tblc) >= 2 # seriously? you have so few records

	# make absolutely sure that target is not part of features
    features = setdiff(features, [target])

    if verbose
        println("`_fit_tree!`: Current state of tree $jlt")
    end

    no_more_gains_to_found = false

    while !no_more_gains_to_found && !stopping_criterion(jlt)
        if verbose
            println("Tree Depth: $(treedepth(jlt))")
        end
        # at the beginning there is only one leaf node which is the parent for all nodes
        # amongst the end nodes compute the best split and choose the best split based on the leaf
        # nodes keep only those where a split has not been decided
        # println("the nodes considered for expansion are $(get_leaf_nodes(jlt) |> length)")

        # println.(get_leaf_nodes(jlt) .|> tree_diag_print)

        leaf_nodes = filter(x->ismissing(x.splitfeature) && ismissing(x.gain), get_leaf_nodes(jlt))

        # println("the nodes considered for expansion are $(length(leaf_nodes))")
        # println.(leaf_nodes .|> tree_diag_print)
        # keep track of the best gains at each node as we do not want to store the gain in the tree

        best_split_dict = Dict()

        # for all nodes eligible for splitting
        # compute the best split feature and the best split point
        # set the split point
        # leaf_node = leaf_nodes[1]
        for leaf_node in leaf_nodes
            if leaf_node.parent === nothing
                # if the node is the parent
                tblc_filtered = tblc
                warm_start_filtered = warm_start
            else
                keeprow = keeprow_vec(tbl, leaf_node)
                #tblc_filtered = view(tblc, keeprow, :)
                tblc_filtered = Tables.columns(DataFrame(view(dfo, keeprow, :)))
                #println("TYPE")
                #println(typeof(tblc_filtered))
                #print(keeprow)
                #println(size(keeprow))
                #println(typeof(tblc))
                #tblc_filtered = tblc[keeprow, :]
                warm_start_filtered = view(warm_start, keeprow)
                #warm_start_filtered = warm_start[keeprow]
                if sum(keeprow) <= 2
                    # no rows are kept, so move to next node
                    leaf_node.split = typemin(Float64)
                    leaf_node.splitfeature = Symbol("too few records for split")
                    leaf_node.gain = typemin(Float64)
                    #println("this branch has too few records $(leaf_node)")
                    continue
                else
                    # println("detective")
                    # println(extrema(tblc_filtered.SepalLength))
                end
            end

            # compute the gain for all splits for all features
            split_with_best_gain = find_best_split(loss, tblc_filtered, features[1], target, warm_start_filtered,
                                lambda, gamma; verbose=verbose, kwargs...)

            for feature in Iterators.drop(features, 1)

                feature_split =find_best_split(loss, tblc_filtered, feature, target, warm_start_filtered,
                                    lambda, gamma; verbose=verbose, kwargs...)

                if feature_split.gain > split_with_best_gain.gain
                    split_with_best_gain = feature_split
                end
            end

            # remember the split but do not set children
            if verbose
                println("found a best split at $(split_with_best_gain.feature) $(split_with_best_gain.split_at) $(split_with_best_gain.gain) $(split_with_best_gain.further_split) for $(leaf_node)")
            end
            best_split_dict[leaf_node] = split_with_best_gain
            # set the parent tree node
            leaf_node.split = split_with_best_gain.split_at
            leaf_node.splitfeature = split_with_best_gain.feature
            leaf_node.gain = split_with_best_gain.gain

            key = collect(keys(best_split_dict))[1]
            # println("Type is $(typeof(key))")
            # println(key)
            #println(typeof(key))
            # reset the best split
            split_with_best_gain = ()
        end

        # tree_growth phase
        # select the node to grow based on growth function
        # the tree_growth function will return the list of
        # nodes_to_split = tree_growth(jlt)
        nodes_to_split::Vector{<:AbstractJLBoostTree} = tree_growth(jlt)
        # println("Nodes to split")
        # println.(nodes_to_split)
        # println("Dict nodes")
        # println.(keys(best_split_dict))

        #@assert haskey(best_split_dict, nodes_to_split[1])
        # println(best_split_dict)
        no_more_gains_to_found = true
        for node_to_split in nodes_to_split

            # there needs to be positive gain then apply split to the tree
            # print(best_split_dict)
            # println(best_split_dict)
            # println("node to split is next line")
            # # println(typeof(node_to_split))

            # println("mehmehmeh")

            if !haskey(best_split_dict, node_to_split)
                if verbose
                    println("Won't split on $(node_to_split)")
                end
                continue
            else
                if verbose
                    println("Will split on $(node_to_split)")
                end
            end
            split_with_best_gain = best_split_dict[node_to_split]

            if split_with_best_gain.further_split && (split_with_best_gain.gain > 0)
                no_more_gains_to_found = false
                left_treenode = JLBoostTree(split_with_best_gain.lweight; parent = node_to_split)
                right_treenode = JLBoostTree(split_with_best_gain.rweight; parent = node_to_split)
                node_to_split.children = [left_treenode, right_treenode]
            end
        end
    end # end !stopping_criterion(jlt)
    jlt
end


function combine(obj::MelbourneWubbenaCombination, arg0::ObservationDataSet)
    return jcall(obj, "combine", CombinedObservationDataSet, (ObservationDataSet,), arg0)
end

function get_name(obj::MelbourneWubbenaCombination)
    return jcall(obj, "getName", JString, ())
end


using GRIB

function grib_area(file::String)
    GribFile(file) do reader
        m = Message(reader)
        lons, lats = data(m)
        min_lon = minimum(lons)
        max_lon = maximum(lons)
        if min_lon > 180 || max_lon > 180
            min_lon -= 360
            max_lon -= 360
        end
        if min_lon < -180 || max_lon < -180
            min_lon += 360
            max_lon += 360
        end
        return [maximum(lats), min_lon, minimum(lats), max_lon]
    end
end

function get_key_values(file::String, key::String)
    key_values = Vector()
    GribFile(file) do reader
        for msg in reader
            push!(key_values, string(msg[key]))
        end
    end
    return unique(key_values)
end

function get_keys(file::String)::Vector{String}
    keylist = Vector{String}()
    GribFile(file) do reader
        for msg in reader
            for key in keys(msg)
                push!(keylist, key)
            end
        end
    end
    keylist |> unique
end


struct Factor{T<:Unsigned} <: AbstractFactor{T}
    name::String
    levels::AbstractVector{<:AbstractString}
    data::AbstractVector{T}
end

Base.length(factor::Factor{T}) where {T<:Unsigned} = length(factor.data)

function Factor{T}(name::String, length::Integer, levels::Vector{String}, datpath::String) where {T<:Unsigned}
    data = Vector{T}(length)
    open(datpath) do f
        read!(f, data)
    end
    Factor{T}(name, levels, data)
end

function Factor(name::String, data::AbstractVector{<:AbstractString})
    maxlevelcount = length(unique(data))
    if maxlevelcount <= typemax(UInt8) + 1
        levelmap = Dict{String, UInt8}()
        levels = Vector{String}()
        resdata = Vector{UInt8}()
        for v in data
            if v == "" || v == MISSINGLEVEL
                push!(resdata, 0)
            else
                levelcount = length(levelmap)
                levelindex = get(levelmap, v, levelcount + 1)
                if levelindex > levelcount
                    push!(levels, v)
                end
                push!(resdata, levelindex)
            end
        end
        Factor{UInt8}(name, levels, resdata)
    elseif maxlevelcount <= typemax(UInt16) + 1
        levelmap = Dict{String, UInt16}()
        levels = Vector{String}()
        resdata = Vector{UInt16}()
        for v in data
            if v == "" || v == MISSINGLEVEL
                push!(resdata, 0)
            else
                levelcount = length(levelmap)
                levelindex = get(levelmap, v, levelcount + 1)
                if levelindex > levelcount
                    push!(levels, v)
                end
                push!(resdata, levelindex)
            end
        end
        Factor{UInt16}(name, levels, resdata)
    else
        levelmap = Dict{String, UInt32}()
        levels = Vector{String}()
        resdata = Vector{UInt32}()
        for v in data
            if v == "" || v == MISSINGLEVEL
                push!(resdata, 0)
            else
                levelcount = length(levelmap)
                levelindex = get(levelmap, v, levelcount + 1)
                if levelindex > levelcount
                    push!(levels, v)
                end
                push!(resdata, levelindex)
            end
        end
        Factor{UInt32}(name, levels, resdata)
    end
end

function slice(factor::Factor{T}, fromobs::Integer, toobs::Integer, slicelength::Integer) where {T<:Unsigned}
    slice(factor.data, fromobs, toobs, slicelength)
end

function Base.map(factor::Factor, dataframe::AbstractDataFrame)
    dataframe[getname(factor)]
    #PermuteFactor(factor, dataframe[getname(factor)])
end


using HDF5
using FileIO
using Formatting

# interprete argument
srcDir = ARGS[1]
dstFile = ARGS[2]

# parameters

# read images
aff = zeros(Float32, (1600,1600,107,3))

lst_fname = readdir(srcDir)

for c in 1:3
    for z in 1:107
	info("section $z")
	#fname = joinpath(srcDir, "c$(format(c-1, width=5, zeropadding=true))_z$(format(z-1, width=5, zeropadding=true))_y20000_x17000.tiff")
        fname = joinpath(srcDir, "c$(format(c-1, width=1, zeropadding=true))_z$(format(z-1, width=5, zeropadding=true)).tiff")
        aff2d = Array{Float32,2}( load(fname).data )
        # transpose the image for tif loading
        #aff2d = permutedims(aff2d, [2,1])
        # also swap the x and y channel
        aff[:,:,z,c] = aff2d
    end
end

# correct the affinity definition
sx,sy,sz,sc = size(aff)
aff[2:sx, :, :, 1] = aff[1:sx-1, :, :, 1]
aff[:, 2:sy, :, 2] = aff[:, 1:sy-1, :, 2]
aff[:, :, 2:sz, 3] = aff[:, :, 1:sz-1, 3]

# write result
h5write(dstFile, "main", aff)


#!/usr/bin/env julia

using GZip

exomf = ARGS[1] # Exome file e.g: Exome_region.bed
vcf  = ARGS[2] # Variant Calling file e.g: chr3.vcf.gz

function read(file)
    if ismatch(r".gz$", file)   # Gzipped .gz files
		f = GZip.open(file)
		lines = readlines(f)
		close(f)
    else
		f = open(file)
		lines = readlines(f)
		close(f)
	end
	return lines
end

function expand(locci)
    list = []
    pos = split(locci, ':')
        chr = pos[1]
        region = split(pos[2], '-')
        n = parse(Int64, region[1])
        while n <= parse(Int64, region[end])
            #out = "$chr" * ":" * "$n"
            push!(list, n)
            n +=  1
        end
    return list
end

function getchr(records=read(vcf))
    chr = split(records[5], '\t')[1]
	return chr
end

function readBED(bed_file, chr::AbstractString=getchr())
    pos = []
    bed = read(bed_file)
    for line in bed
        bed = split(line, '\t')
        chrm = replace(bed[1], "chr", "")
        if chrm == chr
            start, last = bed[2], bed[3]
            chroms = "$chrm" * ":" * "$start" * "-" * "$last"
            region = expand(chroms)
            append!(pos, region)
        end
    end
    pos = unique(pos)
    return pos
end

function checkExome(vcf_line, exomf=exomf)
    exompos = readBED(exomf)
    snp = split(vcf_line, '\t')[2]
    out = in(snp, exompos)
    return out
end

function filterExonic(vcf_path)
	lines = read(vcf_path)
	println(chomp(lines[1]))
	for line = lines[2:end]
		exomic = checkExome(line)
		if exomic == true
            continue
        else
			print(line)
		end
	end
end

filterExonic(vcf)


function plotmodel(modelnames, project)
    data_to_plot = []
    key_labels =[]

	rpoB_eq_table = project["rpoB_equiv_table"]
	reads_table = project["read_counts_table"]
	hmm_database_info = project["hmm_database_info"]
	sample_names = map(string,names(project["rpoB_equiv_table"])[2:end])

    #Getting data out of the big table and putting it into a small data frame using the melt command to pivot that dataframe
    for (i, modelname) in enumerate(modelnames)
        single_model = melt(rpoB_eq_table[rpoB_eq_table[:PFAM_Model] .== modelname, :], :PFAM_Model)
        single_model_reads = melt(reads_table[reads_table[:PFAM_Model] .== modelname, :], :PFAM_Model)

        single_reads = []
        for readnum in single_model_reads[:value]
            push!(single_reads,string(readnum))
        end
        single_model[:reads] = single_reads
        single_model[:sample] = sample_names
		single_model[:PFAM_full] = hmm_database_info[hmm_database_info[:NAME] .== modelname, :DESC][1]
    relative_plot_position = 1:length(sample_names)
    relative_plot_position = relative_plot_position + 0.5 - i*(1/length(modelnames))
    single_model[:read_plot_pos] = relative_plot_position
        push!(data_to_plot, single_model)
    	push!(key_labels, hmm_database_info[hmm_database_info[:NAME] .== modelname, :DESC][1])
	end

    all_data_to_plot = data_to_plot[1]
    if length(data_to_plot) > 1
        for i in 2:length(data_to_plot)
            all_data_to_plot = vcat(all_data_to_plot, data_to_plot[i])
        end
    end

    #plotting
    modelplot = plot(all_data_to_plot, x=:value,y=:sample, colour=:PFAM_full, Geom.bar(position=:dodge,orientation=:horizontal),
    Guide.ylabel(""),
    Guide.xlabel("rpoB equivalents"),
    	Theme(bar_highlight=color(colorant"black"),
    	key_position=:bottom,
    	default_color=color(colorant"black"),
    	panel_stroke=color(colorant"black"),
    	grid_color=color(colorant"gray"),
    	major_label_font="Helvetica",
    	major_label_color=color(colorant"black"),
    	key_title_color=color(colorant"white"),
    	minor_label_font="Helvetica",
    	key_label_font="Helvetica",
    	minor_label_color=color(colorant"black"),
		point_label_font_size=6pt),
	Guide.annotation(compose(context(),
    text(all_data_to_plot[:value]+0.01,all_data_to_plot[:read_plot_pos],all_data_to_plot[:reads],[hleft])))
    )

	return modelplot
end

function perform_pca(project)

	rpoB_eq_table = project["rpoB_equiv_table"]
	sample_names = map(string,names(project["rpoB_equiv_table"])[2:end])

	rpoB_eq_matrix = convert(Array, rpoB_eq_table)
	rpoB_eq_matrix_converted = convert(Array{Float64,2},rpoB_eq_matrix[:,2:end])
	PCA_model = fit(PCA, rpoB_eq_matrix_converted)

	PC1 = transform(PCA_model, rpoB_eq_matrix_converted)[1,:]
	PC2 = transform(PCA_model, rpoB_eq_matrix_converted)[2,:]

	pca_plot = plot(x=PC1,y=PC2, Guide.annotation(compose(context(),text(PC1,PC2,map(string,sample_names),[hleft]))))

	return pca_plot
end

function plotcorrelation(modelnames,parameter,project)

	rpoB_eq_table = project["rpoB_equiv_table"]
	metadata_table = project["project_table"]
	hmm_database_info = project["hmm_database_info"]

	data_to_plot = []
    key_labels =[]

    #Getting data out of the big table and putting it into a small data frame using the melt command to pivot that dataframe
    for modelname in modelnames
        single_model = melt(rpoB_eq_table[rpoB_eq_table[:PFAM_Model] .== modelname, :], :PFAM_Model)
        single_model[:param] = reverse(metadata_table[parameter])
        single_model[:PFAM_full] = hmm_database_info[hmm_database_info[:NAME] .== modelname, :DESC][1]
        push!(data_to_plot, single_model)
	end

    #Putting all the data frames for each PFAM model together
    all_data_to_plot = data_to_plot[1]
    if length(data_to_plot) > 1
        for i in 2:length(data_to_plot)
            all_data_to_plot = vcat(all_data_to_plot, data_to_plot[i])
        end
    end

    #Plotting all of that data together
    plot(all_data_to_plot, x=:param,y=:value, color=:PFAM_full,
    Theme(default_color=color(colorant"black"),
    panel_stroke=color(colorant"black"),
    grid_color=color(colorant"gray"),
    major_label_font="Helvetica",
    major_label_color=color(colorant"black"),
    key_title_color=color(colorant"white"),
    minor_label_font="Helvetica",
    key_label_font="Helvetica",
    minor_label_color=color(colorant"black"),
    key_position=:bottom),
    Guide.xlabel(string(parameter)),
    Guide.ylabel("rpoB equivalents")
    #Guide.annotation(compose(context(), text(rpoB_eq,metadata_subset,sample_names)))
    )
end


# loaders.jl
# utilities for loading in data / images

using FileIO
using Colors

function ir_load_brainweb_t1_256()
	file = dirname(pathof(MIRT)) * "/../data/mri/brainweb_t1_256.gif"
	return Float32.(Gray.(load(file)))' # transpose!
end


function generate_html(integrators, figures, figures_detail, prefix, suffix, fig_dir, output_dir, output_file;
                        title="", params=nothing, max_height="200px")
    header = """
    <!DOCTYPE html>

            <title>GeometricExamples.jl $title</title>
            <meta charset="utf-8" />
            <meta name="authors" content="Michael Kraus" />

    """

    footer = """

    """

    # 1. Header
    # 2. Simulation information (runid, Δt, nt, ...)
    # 3. Figures with caption: integrator, tableau
    # 4. Libray settings (solver, atol, rtol, stol, ...)
    # 5. Footer

    # generate empty output string
    html = ""

    # add header
    html = html * header

    # add title
    if title != ""
        html = html * "
" * title * "
\n"
    end

    # write simulation parameters
    if params != nothing
        html = html * "
\n"
        html = html * "
\n"
        html = html * "
\n"
        html = html * "
\n"
        html = html * "Simulation Parameters	\n"
        html = html * "	\n"
        html = html * "
\n"
        html = html * "
\n"
        html = html * "
\n"

        for (key, value) in params
            html = html * "
\n"
            html = html * "" * string(key) * "	\n"
            html = html * "" * string(value) * "	\n"
            html = html * "
\n"
        end

        html = html * "
\n"
        html = html * "
\n"
        html = html * "

\n"
    end

    # start table
    html = html * "
\n"

    # table header
    html = html * "
\n"
    html = html * "
\n"
    html = html * "Run ID	\n"
    html = html * "Integrator	\n"
    html = html * "Tableau	\n"

    for fig in figures
        html = html * "	\n"
    end

    html = html * "
\n"
    html = html * "
\n"

    # plots for each integrator/tableau combination
    html = html * "
\n"
    for run in integrators

        int  = string(run[1])[(lastindex(findlast("Integrator", string(run[1]))) + 1):end]
        tab  = string(run[2].name)
        rid  = prefix * run[3]

        html = html * "
\n"
        html = html * "" * "" * rid * "" * "	\n"
        html = html * "" * int * "	\n"
        html = html * "" * tab * "	\n"

        for fig in figures
            fig_file = fig_dir * "/" * rid * fig * suffix

            html = html * ""
            if isfile(output_dir * "/" * fig_file)
                html = html * ""
                html = html * ""
                html = html * ""
            end
            html = html * "	\n"
        end

        html = html * "
\n"

        # generate empty output string for run file
        html_run = ""

        # add header
        html_run = html_run * header

        # add title
        if title != ""
            html_run = html_run * "
" * title * "
\n"
        end

        for fig in figures_detail
            fig_file = rid * fig * suffix

            if isfile(output_dir * "/" * fig_dir * "/" * fig_file)
                html_run = html_run * "\n"
                html_run = html_run * "
\n"
                html_run = html_run * fig_file * "

\n\n"
            end
        end

        # add footer
        html_run = html_run * footer

        # write file
        open(output_dir * "/" * rid * ".html", "w") do f
            write(f, html_run)
        end

    end
    html = html * "
\n"

    # finish table
    html = html * "
\n"

    # write library settings
    settings = get_config_dictionary()

    html = html * "
\n"
    html = html * "
\n"
    html = html * "
\n"
    html = html * "
\n"
    html = html * "Library Settings	\n"
    html = html * "	\n"
    html = html * "
\n"
    html = html * "
\n"
    html = html * "
\n"

    for key in sort(collect(keys(settings)))
        html = html * "
\n"
        html = html * "" * string(key) * "	\n"
        html = html * "" * string(settings[key]) * "	\n"
        html = html * "
\n"
    end

    html = html * "
\n"
    html = html * "
\n"
    html = html * "

\n"

    # add footer
    html = html * footer

    # write file
    open(output_dir * "/" * output_file, "w") do f
        write(f, html)
    end

end


using Test
using SCIP

@testset "direct library calls" begin
    include("direct_library_calls.jl")
end

@testset "managed memory" begin
    include("scip_data.jl")
end

# new type definitions in module (needs top level)
include("conshdlr_support.jl")

@testset "constraint handlers" begin
    include("conshdlr.jl")
end

if SCIP.SCIP_versionnumber() >= v"7"
    include("sepa_support.jl")
    @testset "separators" begin
        include("sepa.jl")
    end
    @testset "cut callbacks" begin
        include("cutcallback.jl")
    end
else
    @info "Separation and callbacks not tested for SCIP versions below 7"
end

const MOI_BASE_EXCLUDED = [
    "Semicontinuous",
    "ScalarAffineFunction_Semiinteger",
    "ScalarQuadraticFunction_Semiinteger",
    "VectorAffineFunction_GeometricMeanCone",
    "Indicator_GreaterThan",
    "Indicator_LessThan",
    "Indicator_ACTIVATE_ON_ZERO", # odd MOI bug?
    "test_constraint_get_ConstraintIndex", # accessing constraint from string name
    "test_model_ListOfConstraintAttributesSet",
    "BoundAlreadySet", # see TODO,
    "ListOfConstraintIndices", # MathOptInterface.ListOfModelAttributesSet
    "ListOfConstraintTypesPresent",
    "ScalarAffineFunction_ConstraintName", # get(::SCIP.Optimizer, ::Type{MathOptInterface.ConstraintIndex}, ::String)
    "UnsupportedAttribute",    # test_model_copy_to_UnsupportedAttribute: MOI.copy_to(model, BadVariableAttributeModel())
    "UnsupportedConstraint",
    "test_model_delete", # MOI.ListOfConstraintTypesPresent
    "duplicate_VariableName", # two identical variable names should error
    "test_modification_delete_variables_in_a_batch",
    "test_modification_set_function_single_variable",
    "test_modification_set_scalaraffine_",
    "test_modification_set_singlevariable_",
    "test_modification_transform_",
    "test_nonlinear_", # None of tests provide expression graphs in the evaluator.
    "FEASIBILITY_SENSE", # TODO
    "ObjectiveFunction_ScalarAffineFunction",
    "test_objective_set_via_modify", # ListOfModelAttributesSet
]

# second order cone
# SCIP needs non-negative variable on right-hand side of second-order cone constraint
append!(MOI_BASE_EXCLUDED,[
    "test_conic_RotatedSecondOrderCone_INFEASIBLE",
    "test_conic_RotatedSecondOrderCone_VectorAffineFunction",
    "test_conic_RotatedSecondOrderCone_VectorOfVariables",
    "test_conic_RotatedSecondOrderCone_out_of_order",
    "test_conic_SecondOrderCone_INFEASIBLE",
    "test_conic_SecondOrderCone_Non",
    "test_conic_SecondOrderCone_VectorAffineFunction",
    "test_conic_SecondOrderCone_VectorOfVariables",
    "test_conic_SecondOrderCone_negative_initial_bound",
    "test_conic_SecondOrderCone_negative_post_bound",
    "test_conic_SecondOrderCone_no_initial_bound",
    "test_conic_SecondOrderCone_out_of_order",
    "test_constraint_PrimalStart_DualStart_SecondOrderCone",
    "test_quadratic_Integer_SecondOrderCone",
])

@testset "MathOptInterface tests (bridged)" begin
    include("MOI_wrapper_bridged.jl")
end

@testset "MathOptInterface tests (bridged & cached)" begin
    include("MOI_wrapper_cached.jl")
end

@testset "MathOptInterface tests (direct)" begin
    include("MOI_wrapper_direct.jl")
end

@testset "MathOptInterface additional tests" begin
    include("MOI_additional.jl")
end

@testset "MathOptInterface nonlinear expressions" begin
    include("MOI_nonlinear_exprs.jl")
end

include("MOI_conshdlr.jl")