module Structures

include("mdl.jl")

mutable struct GenericStructure
    structure_type::String
    structure_role::String
    n_edges_total::Int64
    function GenericStructure(structure_role::String, n_edges_total::Int64)
        return new("generic", structure_role, n_edges_total)
    end
end

mutable struct Clique
    structure_type::String
    n_nodes_total::Int64
    n_edges_total::Int64
    nodes::Array{Int64,1}
    function Clique(n_edges_total::Int64, nodes::Array{Int64,1})
        length_nodes = length(nodes)
        @assert n_edges_total <= (length_nodes * (length_nodes - 1)) / 2 "Impossible number of edges! nv=$(length_nodes), ne=$(n_edges_total)"
        @assert length_nodes == length(unique(nodes)) "Input nodes must be unique!"
        return new("clique", length_nodes, n_edges_total, nodes)
    end
end

mutable struct Star
    structure_type::String
    n_nodes_total::Int64
    n_edges_total::Int64
    n_nodes_in_spokes::Int64
    n_edges_in_spokes::Int64
    hub::Int64
    spokes::Array{Int64,1}
    function Star(n_edges_total::Int64, hub::Int64, spokes::Array{Int64,1})
        length_spokes = length(spokes)
        @assert n_edges_total >= length_spokes "Too few edges!"
        @assert n_edges_total <= length_spokes + length_spokes * (length_spokes - 1) / 2 "Too many edges!"
        @assert length_spokes + 1 == length(unique([spokes...,hub])) "Input nodes must be unique!"
        return new("star", length_spokes + 1, n_edges_total, length_spokes, n_edges_total - length_spokes, hub, spokes)
    end
end

mutable struct Biclique
    structure_type::String
    n_nodes_total::Int64
    n_edges_total::Int64
    n_nodes_in_left::Int64
    n_nodes_in_right::Int64
    n_edges_in_left::Int64
    n_edges_in_right::Int64
    n_edges_across::Int64
    left_nodes::Array{Int64,1}
    right_nodes::Array{Int64,1}
    function Biclique(n_edges_total::Int64, left_nodes::Array{Int64,1}, right_nodes::Array{Int64,1}, n_edges_in_left, n_edges_in_right)
        left_length = length(left_nodes)
        right_length = length(right_nodes)
        n_edges_across = n_edges_total - n_edges_in_left - n_edges_in_right
        max_edges_in_left = left_length * (left_length - 1) / 2
        max_edges_in_right = right_length * (right_length - 1) / 2
        max_edges_across = left_length * right_length
        @assert n_edges_in_left <= max_edges_in_left "Too many edges in left!"
        @assert n_edges_in_right <= max_edges_in_right "Too many edges in right!"
        @assert n_edges_across <= max_edges_across "Too many edges in across!"
        @assert left_length + right_length == length(unique([left_nodes..., right_nodes...])) "Input nodes must be unique!"
        return new("biclique", left_length + right_length, n_edges_total, left_length, right_length,
            n_edges_in_left, n_edges_in_right, n_edges_across, left_nodes, right_nodes)
    end
end

mutable struct Starclique
    structure_type::String
    n_nodes_total::Int64
    n_edges_total::Int64
    n_nodes_in_left::Int64
    n_nodes_in_right::Int64
    n_edges_in_left::Int64
    n_edges_in_right::Int64
    n_edges_across::Int64
    left_nodes::Array{Int64,1}
    right_nodes::Array{Int64,1}
    function Starclique(n_edges_total::Int64, left_nodes::Array{Int64,1}, right_nodes::Array{Int64,1}, n_edges_in_left, n_edges_in_right)
        left_length = length(left_nodes)
        right_length = length(right_nodes)
        n_edges_across = n_edges_total - n_edges_in_left - n_edges_in_right
        max_edges_in_left = left_length * (left_length - 1) / 2
        max_edges_in_right = right_length * (right_length - 1) / 2
        max_edges_across = left_length * right_length
        @assert n_edges_in_left <= max_edges_in_left "Too many edges in left!"
        @assert n_edges_in_right <= max_edges_in_right "Too many edges in right!"
        @assert n_edges_across <= max_edges_across "Too many edges in across!"
        @assert left_length + right_length == length(unique([left_nodes..., right_nodes...])) "Input nodes must be unique!"
        return new("starclique", left_length + right_length, n_edges_total, left_length, right_length,
            n_edges_in_left, n_edges_in_right, n_edges_across, left_nodes, right_nodes)
    end
end

Structure = Union{GenericStructure, Clique, Star, Biclique, Starclique}
GenericBipartite = Union{Biclique, Starclique}

Base.show(io::IO, GS::GenericStructure) = print(io, "Generic Structure: $(GS.structure_role) with $(GS.n_edges_total) edges")
Base.show(io::IO, C::Clique) = print(io, "Clique with $(C.n_nodes_total) nodes and $(C.n_edges_total) edges")
Base.show(io::IO, S::Star) = print(io, "Star with $(S.n_nodes_in_spokes) spokes and $(S.n_edges_in_spokes) edges between spokes")
Base.show(io::IO, B::Biclique) = print(io, "Biclique with $(B.n_nodes_in_left) left nodes, $(B.n_nodes_in_right) right nodes, $(B.n_edges_in_left) edges in left, $(B.n_edges_in_right) edges in right, and $(B.n_edges_across) edges across")
Base.show(io::IO, B::Starclique) = print(io, "Starclique with $(B.n_nodes_in_left) left nodes, $(B.n_nodes_in_right) right nodes, $(B.n_edges_in_left) edges in left, $(B.n_edges_in_right) edges in right, and $(B.n_edges_across) edges across")

"""
generic structure description length
"""
function compute_description_length(structure::GenericStructure, maximum_n_loopy::Int64)
    @assert structure.structure_role in ["Overall number of edges", "Loop-freeness"] "Unexpected generic structure with no proper description length computation!"
    if structure.structure_role == "Loop-freeness"
        return 1.
    else
        return MDL.log2_zero(maximum_n_loopy)
    end
end

function compute_rest(clique::Clique)
    max_n_edges = convert(Int64, MDL.choose(clique.n_nodes_total, 2))
    cost_sparse_or_dense = 1
    cost_n_edges = MDL.log2_zero(MDL.log2_zero(floor(max_n_edges/2))) + MDL.log2_zero(min(clique.n_edges_total, max_n_edges - clique.n_edges_total))
    #cost_n_edges = MDL.universal_integer(min(clique.n_edges_total, max_n_edges - clique.n_edges_total) + 1)
    return cost_sparse_or_dense + cost_n_edges
end

### CLIQUES #######################################################################################################################################

"""
clique description length without connector or with path connector
(note that total_n_remaining is computed differently depending on the connector)
"""
function compute_description_length(clique::Clique, total_n_remaining::Int64, path::Bool; with_node_ids_cost::Bool=true)
    cost_n_nodes = path ? MDL.universal_integer(clique.n_nodes_total - 1) : MDL.universal_integer(clique.n_nodes_total)
    if with_node_ids_cost
        cost_node_ids = path ? MDL.log2_choose(total_n_remaining, clique.n_nodes_total - 1) : MDL.log2_choose(total_n_remaining, clique.n_nodes_total)
    else
        cost_node_ids = 0
    end
    rest = compute_rest(clique)
    return cost_n_nodes + cost_node_ids + rest
end

"""
clique description length with overlap connector
(note that total_n_remaining is computed differently depending on the connector)
"""
function compute_description_length(clique::Clique, parent_size::Int64, overlap_size::Int64, total_n_remaining::Int64)
    cost_n_nodes = MDL.universal_integer(clique.n_nodes_total - overlap_size)
    cost_node_ids = MDL.log2_choose(parent_size, overlap_size) + MDL.log2_choose(total_n_remaining, clique.n_nodes_total - overlap_size)
    rest = compute_rest(clique)
    return cost_n_nodes + cost_node_ids + rest
end

### STARS #########################################################################################################################################

"""
helper to remove redundancy in star description length computation
"""
function compute_rest(star::Star)
    cost_of_hub = MDL.log2_zero(star.n_nodes_total)
    cost_n_edges = MDL.log2_zero(MDL.log2_zero((star.n_nodes_in_spokes*(star.n_nodes_in_spokes-1)/2))) + MDL.log2_zero(star.n_edges_in_spokes)
    #cost_n_edges = MDL.universal_integer(star.n_edges_in_spokes + 1)
    return cost_of_hub + cost_n_edges
end

"""
star description length without connector or with path connector
(note that total_n_remaining is computed differently depending on the connector)
"""
function compute_description_length(star::Star, total_n_remaining::Int64, path::Bool; with_node_ids_cost::Bool=true)
    cost_n_spokes = path ? MDL.universal_integer(star.n_nodes_in_spokes - 1) : MDL.universal_integer(star.n_nodes_in_spokes)
    if with_node_ids_cost
        cost_node_ids = path ? MDL.log2_choose(total_n_remaining, star.n_nodes_total - 1) : MDL.log2_choose(total_n_remaining, star.n_nodes_total)
    else
        cost_node_ids = 0
    end
    rest = compute_rest(star)
    return cost_n_spokes + cost_node_ids + rest
end

"""
star description length with overlap connector
(note that total_n_remaining is computed differently depending on the connector)
"""
function compute_description_length(star::Star, parent_size::Int64, overlap_size::Int64, total_n_remaining::Int64)
    cost_n_spokes = MDL.universal_integer(star.n_nodes_in_spokes)
    cost_node_ids = MDL.log2_choose(parent_size, overlap_size) + MDL.log2_choose(total_n_remaining, star.n_nodes_total - overlap_size)
    rest = compute_rest(star)
    return cost_n_spokes + cost_node_ids + rest
end

### BICLIQUES AND STARCLIQUES #####################################################################################################################

"""
helper to remove redundancy in biclique description length computation
"""
function compute_rest(biclique::GenericBipartite)
    cost_n_left = MDL.log2_zero(biclique.n_nodes_total)
    cost_left_ids = MDL.log2_choose(biclique.n_nodes_total, biclique.n_nodes_in_left)
    # across, we definitely want density
    max_across_edges = biclique.n_nodes_in_left * biclique.n_nodes_in_right
    cost_n_across = MDL.log2_zero(MDL.log2_zero(max_across_edges)) + MDL.log2_zero(biclique.n_edges_across)
    max_left_edges = convert(Int64, MDL.choose(biclique.n_nodes_in_left, 2))
    max_right_edges = convert(Int64, MDL.choose(biclique.n_nodes_in_right, 2))
    # left we can have sparsity or density
    cost_n_left_edges = MDL.log2_zero(MDL.log2_zero(floor(max_left_edges/2))) + MDL.log2_zero(min(biclique.n_edges_in_left, max_left_edges - biclique.n_edges_in_left))
    # right we want sparsity
    cost_n_right_edges = MDL.log2_zero(MDL.log2_zero(floor(max_right_edges/2))) + MDL.log2_zero(biclique.n_edges_in_right)
    return (cost_n_left + cost_left_ids
            + cost_n_across + cost_n_left_edges + cost_n_right_edges)
end

"""
biclique description length without connector or with path connector
(note that total_n_remaining is computed differently depending on the connector)
"""
function compute_description_length(biclique::GenericBipartite, total_n_remaining::Int64, path::Bool; with_node_ids_cost::Bool=true)
    cost_n_nodes = path ? MDL.universal_integer(biclique.n_nodes_total - 1) : MDL.universal_integer(biclique.n_nodes_total)
    if with_node_ids_cost
        cost_node_ids = path ? MDL.log2_choose(total_n_remaining, biclique.n_nodes_total - 1) : MDL.log2_choose(total_n_remaining, biclique.n_nodes_total)
    else
        cost_node_ids = 0
    end
    rest = compute_rest(biclique)
    return (cost_n_nodes + cost_node_ids + rest)
end

"""
biclique description length with overlap connector
(note that total_n_remaining is computed differently depending on the connector)
"""
function compute_description_length(biclique::GenericBipartite, parent_size::Int64, overlap_size::Int64, total_n_remaining::Int64)
    cost_n_nodes = MDL.universal_integer(biclique.n_nodes_total - overlap_size)
    cost_node_ids = MDL.log2_choose(parent_size, overlap_size) + MDL.log2_choose(total_n_remaining, biclique.n_nodes_total - overlap_size)
    rest = compute_rest(biclique)
    return (cost_n_nodes + cost_node_ids + rest)
end

"""
get all nodes from a clique
"""
function get_node_set(clique::Clique)
    return clique.nodes
end

"""
get all nodes from a star
"""
function get_node_set(star::Star)
    return [star.hub, star.spokes...]
end

"""
get all nodes from a biclique
"""
function get_node_set(biclique::GenericBipartite)
    return [biclique.left_nodes..., biclique.right_nodes...]
end

end


function estimation_step(obj::KalmanEstimator, arg0::ObservedMeasurement)
    return jcall(obj, "estimationStep", Vector{AbstractIntegratedPropagator}, (ObservedMeasurement,), arg0)
end

function get_current_date(obj::KalmanEstimator)
    return jcall(obj, "getCurrentDate", AbsoluteDate, ())
end

function get_current_measurement_number(obj::KalmanEstimator)
    return jcall(obj, "getCurrentMeasurementNumber", jint, ())
end

function get_estimated_measurements_parameters(obj::KalmanEstimator)
    return jcall(obj, "getEstimatedMeasurementsParameters", ParameterDriversList, ())
end

function get_orbital_parameters_drivers(obj::KalmanEstimator, arg0::jboolean)
    return jcall(obj, "getOrbitalParametersDrivers", ParameterDriversList, (jboolean,), arg0)
end

function get_physical_estimated_covariance_matrix(obj::KalmanEstimator)
    return jcall(obj, "getPhysicalEstimatedCovarianceMatrix", RealMatrix, ())
end

function get_physical_estimated_state(obj::KalmanEstimator)
    return jcall(obj, "getPhysicalEstimatedState", RealVector, ())
end

function get_propagation_parameters_drivers(obj::KalmanEstimator, arg0::jboolean)
    return jcall(obj, "getPropagationParametersDrivers", ParameterDriversList, (jboolean,), arg0)
end

function process_measurements(obj::KalmanEstimator, arg0::Iterable)
    return jcall(obj, "processMeasurements", Vector{AbstractIntegratedPropagator}, (Iterable,), arg0)
end

function set_observer(obj::KalmanEstimator, arg0::KalmanObserver)
    return jcall(obj, "setObserver", void, (KalmanObserver,), arg0)
end


using LibSodium
using Test

@testset "Public-key signatures - Ed25519 - Signing" begin

    sodium_init()

    string_to_sign = "test"

    seed_hex = "3ba564b9eb1a7e4549da027dc6dea3fb8ba0c2970f65cdcca13bd3a8dc8e7de7"
    seed = hex2bytes(seed_hex)

    # Generate public & secret key
    pk = Vector{Cuchar}(undef, crypto_sign_ed25519_PUBLICKEYBYTES)
    sk = Vector{Cuchar}(undef, crypto_sign_ed25519_SECRETKEYBYTES)
    crypto_sign_ed25519_seed_keypair(pk, sk, seed)

    @test bytes2hex(sk) == "3ba564b9eb1a7e4549da027dc6dea3fb8ba0c2970f65cdcca13bd3a8dc8e7de7e539bc6e0faa5df63176178f8809bfcbd4a6ac3178803a89e697ebe309384879"
    @test sizeof(sk) == 64

    signed_message = Vector{Cuchar}(undef, crypto_sign_ed25519_BYTES + sizeof(string_to_sign))
    signed_message_len = Ref{Culonglong}(0)

    signature = Vector{Cuchar}(undef, crypto_sign_ed25519_BYTES)
    signature_len = Ref{Culonglong}(0)

    crypto_sign_ed25519(signed_message, signed_message_len, string_to_sign, sizeof(string_to_sign), sk)
    crypto_sign_ed25519_detached(signature, signature_len, string_to_sign, sizeof(string_to_sign), sk)

    @test bytes2hex(signed_message) == "13247659998dd51b975a273d39f646bbbf1e48b61168605b6c727803cef30c6ef5370e9f4cb8faf2cb773fc82886bffe5d5603165293b62c83edc34a86af0d0774657374"
    @test signed_message_len[] == 68
    @test bytes2hex(signature) == "13247659998dd51b975a273d39f646bbbf1e48b61168605b6c727803cef30c6ef5370e9f4cb8faf2cb773fc82886bffe5d5603165293b62c83edc34a86af0d07"
    @test signature_len[] == 64

end


function compute_zenith_delay(obj::DiscreteTroposphericModel, arg0::RealFieldElement, arg1::Vector{RealFieldElement}, arg2::FieldAbsoluteDate)
    return jcall(obj, "computeZenithDelay", Vector{RealFieldElement}, (RealFieldElement, Vector{RealFieldElement}, FieldAbsoluteDate), arg0, arg1, arg2)
end

function compute_zenith_delay(obj::DiscreteTroposphericModel, arg0::jdouble, arg1::Vector{jdouble}, arg2::AbsoluteDate)
    return jcall(obj, "computeZenithDelay", Vector{jdouble}, (jdouble, Vector{jdouble}, AbsoluteDate), arg0, arg1, arg2)
end

function get_parameters(obj::MappingFunction)
    return jcall(obj, "getParameters", Vector{jdouble}, ())
end

function get_parameters(obj::MappingFunction, arg0::Field)
    return jcall(obj, "getParameters", Vector{RealFieldElement}, (Field,), arg0)
end

function get_parameters_drivers(obj::MappingFunction)
    return jcall(obj, "getParametersDrivers", List, ())
end

function mapping_factors(obj::MappingFunction, arg0::RealFieldElement, arg1::RealFieldElement, arg2::Vector{RealFieldElement}, arg3::FieldAbsoluteDate)
    return jcall(obj, "mappingFactors", Vector{RealFieldElement}, (RealFieldElement, RealFieldElement, Vector{RealFieldElement}, FieldAbsoluteDate), arg0, arg1, arg2, arg3)
end

function mapping_factors(obj::MappingFunction, arg0::jdouble, arg1::jdouble, arg2::Vector{jdouble}, arg3::AbsoluteDate)
    return jcall(obj, "mappingFactors", Vector{jdouble}, (jdouble, jdouble, Vector{jdouble}, AbsoluteDate), arg0, arg1, arg2, arg3)
end

function path_delay(obj::DiscreteTroposphericModel, arg0::RealFieldElement, arg1::RealFieldElement, arg2::Vector{RealFieldElement}, arg3::FieldAbsoluteDate)
    return jcall(obj, "pathDelay", RealFieldElement, (RealFieldElement, RealFieldElement, Vector{RealFieldElement}, FieldAbsoluteDate), arg0, arg1, arg2, arg3)
end

function path_delay(obj::DiscreteTroposphericModel, arg0::jdouble, arg1::jdouble, arg2::Vector{jdouble}, arg3::AbsoluteDate)
    return jcall(obj, "pathDelay", jdouble, (jdouble, jdouble, Vector{jdouble}, AbsoluteDate), arg0, arg1, arg2, arg3)
end


# ref: https://github.com/apache/parquet-format/blob/master/Encodings.md

const MSB = 0x80
const MASK7 = 0x7f
const MASK8 = 0xff
const MASK3 = 0x07
function MASKN(nbits)
    T = byt2uitype_small(bit2bytewidth(nbits))
    O = convert(T, 0x1)
    (O << nbits) - O
end

bitwidth(i) = ceil(Int, log(2, i+1))
bytewidth(i) = bit2bytewidth(bitwidth(i))
bit2bytewidth(i) = ceil(Int, i/8)
byt2itype(i) = (i <= 4) ? Int32 : (i <= 8) ? Int64 : Int128
byt2uitype(i) = (i <= 4) ? UInt32 : (i <= 8) ? UInt64 : UInt128
byt2uitype_small(i) = (i <= 1) ? UInt8 : (i <= 2) ? UInt16 : (i <= 4) ? UInt32 : (i <= 8) ? UInt64 : UInt128

read_fixed(io::IO, typ::Type{UInt32}) = _read_fixed(io, convert(UInt32,0), 4)
read_fixed(io::IO, typ::Type{UInt64}) = _read_fixed(io, convert(UInt64,0), 8)
read_fixed(io::IO, typ::Type{Int32}) = reinterpret(Int32, _read_fixed(io, convert(UInt32,0), 4))
read_fixed(io::IO, typ::Type{Int64}) = reinterpret(Int64, _read_fixed(io, convert(UInt64,0), 8))
read_fixed(io::IO, typ::Type{Int128}) = reinterpret(Int128, _read_fixed(io, convert(UInt128, 0), 12))   # INT96: 12 bytes little endian
read_fixed(io::IO, typ::Type{Float32}) = reinterpret(Float32, _read_fixed(io, convert(UInt32,0), 4))
read_fixed(io::IO, typ::Type{Float64}) = reinterpret(Float64, _read_fixed(io, convert(UInt64,0), 8))
function _read_fixed(io::IO, ret::T, N::Int) where {T <: Unsigned}
    for n in 0:(N-1)
        byte = convert(T, read(io, UInt8))
        ret |= (byte << *(8,n))
    end
    ret
end

function _read_varint(io::IO, ::Type{T}) where {T <: Integer}
    res = zero(T)
    n = 0
    byte = UInt8(MSB)
    while (byte & MSB) != 0
        byte = read(io, UInt8)
        res |= (convert(T, byte & MASK7) << (7*n))
        n += 1
    end
    # in case of overflow, consider it as missing field and return default value
    if (n-1) > sizeof(T)
        #@debug("overflow reading $T. returning 0")
        return zero(T)
    end
    res
end

# parquet types:           BOOLEAN,      INT32,    INT64,    INT96,   FLOAT,   DOUBLE, BYTE_ARRAY, FIXED_LEN_BYTE_ARRAY
# enum values:                   0,          1,        2,        3,       4,        5,          6,                   7
const PLAIN_PROTOBUF_TYPES = ("bool", "sint32", "sint64", "sint64", "float", "double",    "bytes",              "bytes")

# parquet types:           BOOLEAN, INT32, INT64, INT96,    FLOAT,   DOUBLE, BYTE_ARRAY, FIXED_LEN_BYTE_ARRAY
# enum values:                   0,     1,     2,     3,        4,        5,          6,                    7
const PLAIN_THRIFT_TYPES = ("bool", "i32", "i64", "i64", "double", "double",   "binary",            "binary")

# parquet types:   BOOLEAN, INT32, INT64,  INT96,   FLOAT,  DOUBLE, BYTE_ARRAY, FIXED_LEN_BYTE_ARRAY
# enum values:           0,     1,     2,      3,       4,       5,          6,                    7
const PLAIN_JTYPES = (Bool, Int32, Int64, Int128, Float32, Float64,      UInt8,               UInt8)

# read plain encoding (PLAIN = 0)
function read_plain(io::IO, typ::Int32, jtype::Type{T}=PLAIN_JTYPES[typ+1]) where {T}
    if typ == _Type.FIXED_LEN_BYTE_ARRAY
        #@debug("reading fixedlenbytearray length:$count")
        read!(io, Array{UInt8}(count))
    elseif typ == _Type.BYTE_ARRAY
        count = read_fixed(io, Int32)
        #@debug("reading bytearray length:$count")
        read!(io, Array{UInt8}(undef, count))
    elseif typ == _Type.BOOLEAN
        error("not implemented")
    else
        #@debug("reading type:$jtype, typenum:$typ")
        read_fixed(io, jtype)
    end
end

# read plain values or dictionary (PLAIN_DICTIONARY = 2)
function read_plain_values(io::IO, count::Integer, typ::Int32)
    @debug("reading plain values type:$typ, count:$count")
    arr = [read_plain(io, typ) for i in 1:count]
    @debug("read $(length(arr)) plain values")
    arr
end

# read rle dictionary (RLE_DICTIONARY = 8, or PLAIN_DICTIONARY = 2 in a data page)
function read_rle_dict(io::IO, count::Integer)
    bits = read(io, UInt8)
    @debug("reading rle dictionary bits:$bits")
    arr = read_hybrid(io, count, bits; read_len=false)
    @debug("read $(length(arr)) dictionary values")
    arr
end

# read RLE or bit backed format (RLE = 3)
function read_hybrid(io::IO, count::Integer, bits::Integer, byt::Int=bit2bytewidth(bits), typ::Type{T}=byt2itype(byt), arr::Vector{T}=Array{T}(undef, count); read_len::Bool=true) where {T <: Integer}
    len = read_len ? read_fixed(io, Int32) : Int32(0)
    @debug("reading hybrid data length:$len, count:$count, bits:$bits")
    arrpos = 1
    while arrpos <= count
        runhdr = _read_varint(io, Int)
        isbitpack = ((runhdr & 0x1) == 0x1)
        runhdr >>= 1
        nitems = isbitpack ? min(runhdr*8, count - arrpos + 1) : runhdr
        #@debug("nitems=$nitems, isbitpack:$isbitpack, runhdr:$runhdr, remaininglen: $(count - arrpos + 1)")
        #@debug("creating sub array for $nitems items at $arrpos, total length:$(length(arr))")
        #subarr = pointer_to_array(pointer(arr, arrpos), nitems)
        subarr = unsafe_wrap(Array, pointer(arr, arrpos), nitems, own=false)

        if isbitpack
            read_bitpacked_run(io, runhdr, bits, byt, typ, subarr)
        else # rle
            read_rle_run(io, runhdr, bits, byt, typ, subarr)
        end
        arrpos += nitems
    end
    arr
end

function read_rle_run(io::IO, count::Integer, bits::Integer, byt::Int=bit2bytewidth(bits), typ::Type{T}=byt2itype(byt), arr::Vector{T}=Array{T}(count)) where {T <: Integer}
    @debug("read_rle_run. count:$count, typ:$T, nbits:$bits, nbytes:$byt")
    arr[1:count] .= reinterpret(T, _read_fixed(io, zero(byt2uitype(byt)), byt))
    arr
end

function read_bitpacked_run(io::IO, grp_count::Integer, bits::Integer, byt::Int=bit2bytewidth(bits), typ::Type{T}=byt2itype(byt), arr::Vector{T}=Array{T}(undef, grp_count*8)) where {T <: Integer}
    count = min(grp_count * 8, length(arr))
    # multiple of 8 values at a time are bit packed together
    nbytes = bits * grp_count # same as: round(Int, (bits * grp_count * 8) / 8)
    #@debug("read_bitpacked_run. grp_count:$grp_count, count:$count, nbytes:$nbytes, nbits:$bits, available:$(bytesavailable(io))")
    data = Array{UInt8}(undef, min(nbytes, bytesavailable(io)))
    read!(io, data)

    mask = MASKN(bits)
    V = typeof(mask)
    bitbuff = zero(V)
    nbitsbuff = 0
    shift = 0

    arridx = 1
    dataidx = 1
    while arridx <= count
        #@debug("arridx:$arridx nbitsbuff:$nbitsbuff shift:$shift bits:$bits")
        if nbitsbuff > 0
            # we have leftover bits, which must be appended
            arr[arridx] = bitbuff & MASKN(nbitsbuff)
            shift = nbitsbuff
            nbitsbuff = 0
        end

        # fill buffer
        while (nbitsbuff + shift) < bits
             # shift 8 bits and read directly into bitbuff
            bitbuff <<= 8
            bitbuff |= data[dataidx]
            dataidx += 1
            nbitsbuff += 8
        end

        # set values
        while ((nbitsbuff + shift) >= bits) && (arridx <= count)
            if shift > 0
                remshift = bits - shift
                #@debug("setting part from bitbuff nbitsbuff:$nbitsbuff, shift:$shift, remshift:$remshift")
                arr[arridx] |= convert(T, (bitbuff << shift) & mask)
                bitbuff >>= remshift
                nbitsbuff -= remshift
                shift = 0
            else
                #@debug("setting all from bitbuff nbitsbuff:$nbitsbuff")
                arr[arridx] = convert(T, bitbuff & mask)
                bitbuff >>= bits
                nbitsbuff -= bits
            end
            arridx += 1
        end
    end
    arr
end

# read bit packed in deprecated format (BIT_PACKED = 4)
function read_bitpacked_run_old(io::IO, count::Integer, bits::Integer, byt::Int=bit2bytewidth(bits), typ::Type{T}=byt2itype(byt), arr::Vector{T}=Array{T}(undef, count)) where {T <: Integer}
    # multiple of 8 values at a time are bit packed together
    nbytes = round(Int, (bits * count) / 8)
    @debug("read_bitpacked_run. count:$count, nbytes:$nbytes, nbits:$bits")
    data = Array{UInt8}(undef, nbytes)
    read!(io, data)

    # the mask is of the smallest bounding type for bits
    # T is one of the types that map on to the appropriate Julia type in Parquet (which may be larger than the mask type)
    mask = MASKN(bits)
    V = typeof(mask)
    bitbuff = zero(V)
    nbitsbuff = 0

    arridx = 1
    dataidx = 1
    while arridx <= count
        diffnbits = bits - nbitsbuff
        while diffnbits > 8
            # shift 8 bits and read directly into bitbuff
            bitbuff <<= 8
            bitbuff |= data[dataidx]
            dataidx += 1
            nbitsbuff += 8
            diffnbits -= 8
        end

        if diffnbits > 0
            # read next byte from input
            nxtdata = data[dataidx]
            dataidx += 1
            # shift bitbuff by diffnbits, add diffnbits and set result
            nbitsbuff = 8 - diffnbits
            arr[arridx] = convert(T, ((bitbuff << diffnbits) | (nxtdata >> nbitsbuff)) & mask)
            arridx += 1
            # keep remaining bits in bitbuff
            bitbuff <<= 8
            bitbuff |= nxtdata
        else
            # set result
            arr[arridx] = convert(T, (bitbuff >> abs(diffnbits)) & mask)
            arridx += 1
            nbitsbuff -= bits
        end
    end
    arr
end


# An implementation of delimited continuations (the shift/reset operators) in
# Julia. Works by transforming all Julia code to continuation passing style.
# The `shift` operator then just has to return the continuation.

# https://en.wikipedia.org/wiki/Delimited_continuation
# https://en.wikipedia.org/wiki/Continuation-passing_style

using IRTools.All

struct Func
  f # Avoid over-specialising on the continuation object.
end

(f::Func)(args...) = f.f(args...)

function captures(ir, vs)
  us = Set()
  for v in vs
    isexpr(ir[v].expr) || continue
    foreach(x -> x isa Variable && push!(us, x), ir[v].expr.args)
  end
  return setdiff(us, vs)
end

rename(env, x) = x
rename(env, x::Variable) = env[x]
rename(env, x::Expr) = Expr(x.head, rename.((env,), x.args)...)
rename(env, x::Statement) = stmt(x, expr = rename(env, x.expr))

excluded = [GlobalRef(Base, :getindex)]

function continuation!(bl, ir, env, vs, ret)
  rename(x) = Main.rename(env, x)
  local v, st
  while true
    isempty(vs) && return return!(bl, rename(Expr(:call, ret, returnvalue(block(ir, 1)))))
    v = popfirst!(vs)
    st = ir[v]
    isexpr(st.expr, :call) && !(st.expr.args[1] ∈ excluded) && break
    isexpr(st.expr, :lambda) &&
      (st = stmt(st, expr = Expr(:lambda, cpslambda(st.expr.args[1]), st.expr.args[2:end]...)))
    env[v] = push!(bl, rename(st))
  end
  cs = [ret, setdiff(captures(ir, vs), [v])...]
  if isempty(vs)
    next = rename(ret)
  else
    next = push!(bl, Expr(:lambda, continuation(ir, vs, cs, v, ret), rename.(cs)...))
    next = xcall(Main, :Func, next)
  end
  ret = push!(bl, stmt(st, expr = xcall(Main, :cps, next, rename(st.expr).args...)))
  return!(bl, ret)
end

function continuation(ir, vs, cs, in, ret)
  bl = empty(ir)
  env = Dict()
  self = argument!(bl)
  env[in] = argument!(bl)
  for (i, c) in enumerate(cs)
    env[c] = pushfirst!(bl, xcall(:getindex, self, i))
  end
  continuation!(bl, ir, env, vs, ret)
end

cpslambda(ir) = cpstransform(ir, true)

function cpstransform(ir, lambda = false)
  lambda || (ir = functional(ir))
  k = argument!(ir, at = lambda ? 2 : 1)
  bl = empty(ir)
  env = Dict()
  for arg in arguments(ir)
    env[arg] = argument!(bl)
  end
  continuation!(bl, ir, env, keys(ir), k)
end

cps(k, f::Core.IntrinsicFunction, args...) = k(f(args...))
cps(k, f::IRTools.Lambda{<:Tuple{typeof(cps),Vararg{Any}}}, args...) = f(k, args...)
cps(k, ::typeof(cond), c, t, f) = c ? cps(k, t) : cps(k, f)
cps(k, ::typeof(cps), args...) = k(cps(args...))

# Speed up compilation
for f in [Broadcast.broadcasted, Broadcast.materialize]
  @eval cps(k, ::typeof($f), args...) = k($f(args...))
end

@dynamo function cps(k, args...)
  ir = IR(args...)
  ir == nothing && return :(args[1](args[2](args[3:end]...)))
  cpstransform(IR(args...))
end

# shift/reset

reset(f) = cps(identity, f)
shift(f) = error("`shift` must be called inside `reset`")
cps(k, ::typeof(shift), f) = f(k)

macro reset(ex)
  :(reset(() -> $(esc(ex))))
end

k = @reset begin
  shift(k -> k)^2
end

k(4) == 16


# Geometry which displays points at given (x, y) positions.

struct PointGeometry <: Gadfly.GeometryElement
    tag::Symbol
end
PointGeometry(; tag=empty_tag) = PointGeometry(tag)

const point = PointGeometry

element_aesthetics(::PointGeometry) = [:x, :y, :size, :color, :shape]

# Generate a form for a point geometry.
#
# Args:
#   geom: point geometry.
#   theme: the plot's theme.
#   aes: aesthetics.
#
# Returns:
#   A compose Form.
#
function render(geom::PointGeometry, theme::Gadfly.Theme, aes::Gadfly.Aesthetics)
    Gadfly.assert_aesthetics_defined("Geom.point", aes, :x, :y)
    Gadfly.assert_aesthetics_equal_length("Geom.point", aes, :x, :y)

    default_aes = Gadfly.Aesthetics()
    default_aes.shape = Function[Shape.circle]
    default_aes.color = discretize_make_ia(RGBA{Float32}[theme.default_color])
    default_aes.size = Measure[theme.point_size]
    aes = inherit(aes, default_aes)

    if eltype(aes.size) <: Int
      size_min, size_max = extrema(aes.size)
      size_range = size_max - size_min
      point_size_range = theme.point_size_max - theme.point_size_min
      interpolate_size(x) = theme.point_size_min + (x-size_min) / size_range * point_size_range
    end

    ctx = context()

    for (x, y, color, size, shape) in Compose.cyclezip(aes.x, aes.y, aes.color, aes.size, aes.shape)
        shapefun = typeof(shape) <: Function ? shape : theme.point_shapes[shape]
        sizeval = typeof(size) <: Int ? interpolate_size(size) : size
        strokecolor = aes.color_key_continuous != nothing && aes.color_key_continuous ?
                    theme.continuous_highlight_color(color) :
                    theme.discrete_highlight_color(color)
        class = svg_color_class_from_label(aes.color_label([color])[1])
        compose!(ctx, (context(),
              (context(), shapefun([x], [y], [sizeval]), svgclass("marker")),
              fill(color), stroke(strokecolor), svgclass(class)))
    end

    compose!(ctx, linewidth(theme.highlight_width))

    return compose!(context(order=4), svgclass("geometry"), ctx)
end


using Pkg
Pkg.add("http://github/AdversarialPrediction")
Pkg.add("PyCall")
using AdversarialPrediction
using PyCall


@testset "DTM" begin
    sample_file = joinpath(dirname(@__FILE__), "data", "poem.txt")

    fd = FileDocument(sample_file)
    sd = StringDocument(text(fd))

    crps = Corpus(Any[fd, sd])

    m = DocumentTermMatrix(crps)
    dtm(m)
    dtm(m, :dense)

    update_lexicon!(crps)

    m = DocumentTermMatrix(crps)
    dtm(m)
    dtm(m, :dense)

    tf_idf(dtm(m, :dense))

    dtv(crps[1], lexicon(crps))

    hash_dtv(crps[1], TextHashFunction())
    hash_dtv(crps[1])

    dtm1 = dtm(crps)
    dtm1sp = sparse(dtm(crps))
    hash_dtm(crps)

    tdm(crps)
    hash_tdm(crps)

    # construct a DocumentTermMatrix from a crps and a custom terms vector
    terms = ["And", "notincrps"]
    m = DocumentTermMatrix(crps,terms)
    @test size(dtm(m),1) == length(terms)
    @test terms == m.terms
    @test size(dtm(m),2) == length(crps)

    # construct a DocumentTermMatrix from a crps and a custom lexicon
    lex = Dict("And"=>1, "notincrps"=>4)
    m = DocumentTermMatrix(crps,lex)
    @test size(dtm(m),1) == length(keys(lex))
    @test size(dtm(m),1) == length(m.terms)
    @test size(dtm(m),2) == length(crps)

    # construct a DocumentTermMatrix from a dtm and terms vector
    terms = m.terms
    m2 = DocumentTermMatrix(dtm1,terms)
    @test m.column_indices == m2.column_indices
    m2 = DocumentTermMatrix(dtm1sp,terms)
    @test m.column_indices == m2.column_indices
end


# -----------------------------------------------------------------------
# Test set for critical functions
# -----------------------------------------------------------------------
using Test
include("GA.jl")

# Test set for stepper movements
function coordsteps_test()
    @testset "Coordinates and steps" begin
        @testset "One cylinder" begin
            p1 = OneCylType(0.0,0.0)
            p2 = OneCylType(10.0,0.0)
            p3 = OneCylType(20.0,0.0)
            p2_cor = steps2coord(StepperMotionType([41,16],[0]),xorig=[-8.8],d=2.4, Rh=80.0, Re=10.0)
            p3_cor = steps2coord(StepperMotionType([82,-10],[0]),xorig=[-8.8],d=2.4, Rh=80.0, Re=10.0)
            @test coord2steps(p1,p2,xorig=[-8.8],d=2.4, Rh=80.0, Re=10.0)[1].step ≈ [41,16] atol=1
            @test coord2steps(p1,p3,xorig=[-8.8],d=2.4, Rh=80.0, Re=10.0)[1].step ≈ [82,-10] atol=1
            @test coord2steps(p2,p3,xorig=[-8.8],d=2.4, Rh=80.0, Re=10.0)[1].step ≈ [41,-25] atol=1
            @test coord2steps(p1,p3,xorig=[-8.8],d=2.4, Rh=80.0, Re=10.0)[1].strategy == [1,2]
            @test coord2steps(p3,p2,xorig=[-8.8],d=2.4, Rh=80.0, Re=10.0)[1].strategy == [1,2]
            @test convert(Array, p2_cor) ≈ convert(Array, p2) atol=0.2
            @test convert(Array, p3_cor) ≈ convert(Array, p3) atol=0.2
        end
        @testset "Two cylinders" begin
            p1 = TwoCylType(0.0,0.0,2.4,0.0)
            p2 = TwoCylType(10.0,0.0,2.4,0.0)
            p3 = TwoCylType(20.0,0.0,2.4,0.0)
            p2_cor = steps2coord(StepperMotionType([41,16,41,47],[0]),xorig=[-8.8,-18.8],d=2.4, Rh=80.0, Re=10.0)
            p3_cor = steps2coord(StepperMotionType([82,-10,82,53],[0]),xorig=[-8.8,-18.8],d=2.4, Rh=80.0, Re=10.0)
            @test coord2steps(p1,p2,xorig=[-8.8,-18.8],d=2.4, Rh=80.0, Re=10.0)[1].step ≈ [41,16,41,47] atol=1
            @test coord2steps(p1,p3,xorig=[-8.8,-18.8],d=2.4, Rh=80.0, Re=10.0)[1].step ≈ [82,-10,82,53] atol=1
            @test coord2steps(p2,p3,xorig=[-8.8,-18.8],d=2.4, Rh=80.0, Re=10.0)[1].step ≈ [41,-25,41,6] atol=1
            @test coord2steps(p1,p3,xorig=[-8.8,-18.8],d=2.4, Rh=80.0, Re=10.0)[1].strategy == [3,4,1,2]
            @test coord2steps(p3,p2,xorig=[-8.8,-18.8],d=2.4, Rh=80.0, Re=10.0)[1].strategy == [1,2,3,4]
            @test convert(Array, p2_cor) ≈ convert(Array, p2) atol=0.2
            @test convert(Array, p3_cor) ≈ convert(Array, p3) atol=0.2
        end
        @testset "Three cyliders" begin
            p1 = ThreeCylType(0.0,0.0,2.4,0.0,2.4,0.0)
            p2 = ThreeCylType(10.0,0.0,2.4,0.0,2.4,0.0)
            p3 = ThreeCylType(20.0,0.0,2.4,0.0,2.4,0.0)
            p2_cor = steps2coord(StepperMotionType([41,16,41,47,42,80],[0]),xorig=[-8.8,-18.8,-28.8],d=2.4, Rh=80.0, Re=10.0)
            p3_cor = steps2coord(StepperMotionType([82,-10,82,53,83,117],[0]),xorig=[-8.8,-18.8,-28.8],d=2.4,Rh=80.0, Re=10.0)
            @test coord2steps(p1,p2,xorig=[-8.8,-18.8,-28.8],d=2.4, Rh=80.0, Re=10.0)[1].step ≈ [41,16,41,47,42,80] atol=1
            @test coord2steps(p1,p3,xorig=[-8.8,-18.8,-28.8],d=2.4, Rh=80.0, Re=10.0)[1].step ≈ [82,-10,82,53,83,117] atol=1
            @test coord2steps(p2,p3,xorig=[-8.8,-18.8,-28.8],d=2.4, Rh=80.0, Re=10.0)[1].step ≈ [41,-25,41,6,41,37] atol=1
            @test coord2steps(p1,p3,xorig=[-8.8,-18.8,-28.8],d=2.4, Rh=80.0, Re=10.0)[1].strategy ≈ [5,6,3,4,1,2]
            @test coord2steps(p3,p2,xorig=[-8.8,-18.8,-28.8],d=2.4, Rh=80.0, Re=10.0)[1].strategy ≈ [1,2,3,4,5,6]
            @test convert(Array, p2_cor) ≈ convert(Array, p2) atol=0.2
            @test convert(Array, p3_cor) ≈ convert(Array, p3) atol=0.2
        end
    end
end

function bitsconversion_tests()
    @testset "Bits and coordinates transformations" begin

        c11 = OneCylType(1.0, 2.0)
        c11_bit = cylpos2bits(c11, size=[6,5])
        @test bits2cylpos(c11_bit) == c11
        c12 = OneCylType(1.0, -2.0)
        c12_bit = cylpos2bits(c12, size=[6,5])
        @test bits2cylpos(c12_bit) == c12

        c21 = TwoCylType(2.0,1.0,3.0,4.0)
        c21_bit = cylpos2bits(c21, size=[6,5,4,4])
        @test bits2cylpos(c21_bit) == c21
        c22 = TwoCylType(2.0,-1.0,3.0,-4.0)
        c22_bit = cylpos2bits(c22, size=[6,5,4,4])
        @test bits2cylpos(c22_bit) == c22

        c31 = ThreeCylType(10.0,2.0,2.0,3.0,2.0,1.0)
        c31_bit = cylpos2bits(c31, size=[6,5,4,4,4,4])
        @test bits2cylpos(c31_bit) == c31
        c32 = ThreeCylType(10.0,-2.0,2.0,-3.0,2.0,-1.0)
        c32_bit = cylpos2bits(c32, size=[6,5,4,4,4,4])
        @test bits2cylpos(c32_bit) == c32
        c33 = ThreeCylType(0,0,0,0,0,0)
        c33_bit = cylpos2bits(c33, size=[6,5,4,4,4,4])
        @test bits2cylpos(c33_bit) == c33

    end
end

# TEST ALL
function testall()
    coordsteps_test()
    bitsconversion_tests()
end