using PastaQ
using PastaQ.ITensors
using Test
using LinearAlgebra
using Random
using Observers
using JLD2
using HDF5
using Optimisers

@testset "write and read samples" begin
  N = 3
  depth = 4
  nshots = 100
  circuit = randomcircuit(N; depth = depth)
  path = "test_data_writesamples.h5"

  X = runcircuit(circuit)
  data = getsamples(X, nshots)
  writesamples(data, path)
  datatest = readsamples(path)
  @test datatest ≈ data
  writesamples(data, X, path)
  datatest, Xtest = readsamples(path)
  @test X ≈ Xtest
  @test datatest ≈ data

  X = runcircuit(circuit; noise = ("DEP",(p=0.01,)))
  data = getsamples(X, nshots)
  writesamples(data, path)
  datatest = readsamples(path)
  @test datatest ≈ data
  writesamples(data, X, path)
  datatest, Xtest = readsamples(path)
  @test X ≈ Xtest
  @test datatest ≈ data

  X = randomstate(N; ξ = 2, χ = 3)
  writesamples(data, X, path)
  datatest, Xtest = readsamples(path)
  @test X.X ≈ Xtest.X
  @test datatest ≈ data

  X = runcircuit(circuit)
  bases = randombases(N, 10)
  data = getsamples(X, bases, nshots)
  writesamples(data, path)
  datatest = readsamples(path)
  @test all(data .== datatest)
  writesamples(data, X, path)
  datatest, Xtest = readsamples(path)
  @test X ≈ Xtest
  @test all(data .== datatest)

  X = runcircuit(circuit; noise = ("DEP",(p=0.01,)))
  bases = randombases(N, 10)
  data = getsamples(X, bases, nshots)
  writesamples(data, path)
  datatest = readsamples(path)
  @test all(data .== datatest)
  writesamples(data, X, path)
  datatest, Xtest = readsamples(path)
  @test X ≈ Xtest
  @test all(data .== datatest)

  X = randomstate(N; ξ = 2, χ = 3)
  writesamples(data, X, path)
  datatest, Xtest = readsamples(path)
  @test X.X ≈ Xtest.X
  @test all(data .== datatest)

  X = runcircuit(circuit; process = true)
  preps = randompreparations(N, 8)
  bases = randombases(N, 10)
  data = getsamples(X, preps, bases, nshots)
  writesamples(data, path)
  datatest = readsamples(path)
  @test all(data .== datatest)
  writesamples(data, X, path)
  datatest, Xtest = readsamples(path)
  @test X ≈ Xtest
  @test all(data .== datatest)

  X = runcircuit(circuit; noise = ("DEP",(p=0.01,)), process = true)
  preps = randompreparations(N, 8)
  bases = randombases(N, 10)
  data = getsamples(X, preps, bases, nshots)
  writesamples(data, path)
  datatest = readsamples(path)
  @test all(data .== datatest)
  writesamples(data, X, path)
  datatest, Xtest = readsamples(path)
  @test all(data .== datatest)

  X = randomstate(N; ξ = 2, χ = 3)
  writesamples(data, X, path)
  datatest, Xtest = readsamples(path)
  @test X.X ≈ Xtest.X
  @test all(data .== datatest)
end

@testset "circuit observer: MPS" begin
  N = 8
  depth = 9
  R = 5
  Random.seed!(1234)
  circuit = randomcircuit(N; depth = depth)
  layer = Tuple[]
  push!(circuit, [("CX",(1,N)),])

  sites = siteinds("Qubit", N)

  outputpath = "simulation"
  ϕ = randomstate(sites; χ = 10, normalize=true)
  ψ = runcircuit(sites, circuit)
  Ftest = fidelity(ψ,ϕ)
  f(ψ::MPS; kwargs...) = fidelity(ψ, ϕ)#; kwargs...) = fidelity(ψ, ϕ)
  obs = Observer(["f" => f])
  ψ = runcircuit(sites, circuit; (observer!)=obs, move_sites_back_before_measurements=true,
                 outputpath = outputpath, savestate = true, outputlevel = 0)
  @test Ftest ≈ results(obs, "f")[end]
  @test length(results(obs, "f")) == depth+1

  obs2 = load("simulation_observer.jld2")
  for (k,v) in obs2
    @test last(v) ≈ results(obs, k)
  end

  fin = h5open("simulation_state.h5","r")
  M = read(fin, "state", MPS)
  close(fin)
  @test M ≈ ψ
end

@testset "circuit observer: MPO" begin
  N = 4
  depth = 5
  #circuit = Vector{Vector{<:Any}}(undef, depth)
  sites = siteinds("Qubit", N)
  circuit = randomcircuit(N; depth = depth)

  @disable_warn_order begin
    L = randomstate(sites; χ = 10, ξ = 3, normalize=true)
    ϱ = MPO(L)
    ρ = runcircuit(sites, circuit; noise = ("DEP",(p=0.001,)))
    Ftest = fidelity(ϱ,ρ)
    g(ρ::MPO; kwargs...) = fidelity(ρ, ϱ)#; kwargs...) = fidelity(ψ, ϕ)
    obs = Observer(["g" => g])
    outputpath = "simulation"
    ρ = runcircuit(sites, circuit;
                   (observer!)=obs,
                   move_sites_back_before_measurements=true,
                   outputlevel = 0,
                   noise = ("DEP",(p=0.001,)),
                   outputpath = outputpath, savestate = true)
 end
  @test Ftest ≈ results(obs, "g")[end]
  @test length(results(obs, "g")) == depth
  obs2 = load("simulation_observer.jld2")
  for (k,v) in obs2
    @test last(v) ≈ results(obs, k)
  end

  fin = h5open("simulation_state.h5","r")
  M = read(fin, "state", MPO)
  close(fin)
  @test M ≈ ρ
end

@testset "state tomography observer output: MPS" begin
  Random.seed!(1234)
  N = 4
  depth = 4
  nshots = 100
  circuit = randomcircuit(N; depth = depth)
  Ψ = runcircuit(circuit)
  bases = randombases(N,2)
  data = getsamples(Ψ, bases, nshots)
  test_data = copy(data[1:10, :])

  N = length(Ψ)     # Number of productstate
  χ = maxlinkdim(Ψ) # Bond dimension of variational MPS
  ψ0 = randomstate(Ψ; χ=χ, σ=0.1)
  opt = Optimisers.Descent(0.01)

  F(ψ::MPS; kwargs...) = fidelity(ψ, Ψ)
  obs = Observer(["F" => F])
  epochs = 18

  batchsize = 10
  observe_step = 3

  outputpath = "simulation"
  ψ = tomography(
    data,
    ψ0;
    test_data=test_data,
    batchsize=10,
    epochs=epochs,
    (observer!)=obs,
    observe_step = observe_step,
    outputpath = outputpath,
    savestate = true,
    outputlevel = 0
   )
  @test length(results(obs, "F")) == epochs ÷ observe_step

  obs2 = load("simulation_observer.jld2")
  for (k,v) in obs2
    @test last(v) ≈ results(obs, k)
  end

  fin = h5open("simulation_state.h5","r")
  M = read(fin, "state", MPS)
  close(fin)
  @test M ≈ ψ
end

@testset "state tomography observer output: LPDO" begin
  Random.seed!(1234)
  N = 4
  depth = 4
  nshots = 100
  circuit = randomcircuit(N; depth = depth)
  ϱ = runcircuit(circuit; noise = ("DEP",(p=0.01,)))
  bases = randombases(N,2)
  data = getsamples(ϱ, bases, nshots)
  test_data = copy(data[1:10, :])

  N = length(ϱ)     # Number of productstate
  χ = maxlinkdim(ϱ) # Bond dimension of variational MPS
  ρ = randomstate(ϱ; χ=χ÷2, ξ=2)
  opt = Optimisers.Descent(0.01)

  F(ρ::LPDO; kwargs...) = fidelity(ρ, ϱ)
  obs = Observer(["F" => F])
  epochs = 9

  batchsize = 10
  observe_step = 3

  outputpath = "simulation"
  ρ = tomography(
    data,
    ρ;
    test_data=test_data,
    batchsize=10,
    epochs=epochs,
    (observer!)=obs,
    observe_step = observe_step,
    outputpath = outputpath,
    savestate = true,
    outputlevel = 0
  )
  @test length(results(obs, "F")) == epochs ÷ observe_step

  obs2 = load("simulation_observer.jld2")
  for (k,v) in obs2
    @test last(v) ≈ results(obs, k)
  end

  fin = h5open("simulation_state.h5","r")
  M = read(fin, "state", LPDO{MPO})
  close(fin)
  @test M.X ≈ ρ.X
end

@testset "process tomography observer output: MPO" begin
  Random.seed!(1234)
  N = 3
  depth = 4
  nshots = 100
  circuit = randomcircuit(N; depth =  depth)
  V = runcircuit(circuit; process = true)
  preps = randompreparations(N,2)
  bases = randombases(N,2)
  data = getsamples(V, preps, bases, nshots)
  test_data = copy(data[1:10, :])
  N = length(V)     # Number of productstate
  χ = maxlinkdim(V) # Bond dimension of variational MPS
  U0 = randomprocess(V; χ=χ, σ=0.1)
  opt = Optimisers.Descent(0.01)

  F(U::MPO; kwargs...) = fidelity(U, V; process=true)
  obs = Observer(["F" => F])
  epochs = 9

  batchsize = 10
  observe_step = 3

  outputpath = "simulation"
  U = tomography(
    data,
    U0;
    test_data=test_data,
    batchsize=10,
    epochs=epochs,
    (observer!)=obs,
    observe_step = observe_step,
    outputpath = outputpath,
    savestate = true,
    outputlevel = 0
   )

  @test length(results(obs, "F")) == epochs ÷ observe_step

  obs2 = load("simulation_observer.jld2")
  for (k,v) in obs2
    @test last(v) ≈ results(obs, k)
  end
  fin = h5open("simulation_state.h5","r")
  M = read(fin, "state", MPO)
  close(fin)
  @test M ≈ U
end

@testset "state tomography observer output: LPDO" begin
  Random.seed!(1234)
  N = 2
  depth = 4
  nshots = 100
  circuit = randomcircuit(N; depth = depth)
  Φ = runcircuit(circuit; process = true, noise = ("DEP",(p=0.01,)))
  preps = randompreparations(N,2)
  bases = randombases(N,2)
  data = getsamples(Φ, preps, bases, nshots)
  test_data = copy(data[1:10, :])

  N = length(Φ)     # Number of productstate
  χ = maxlinkdim(Φ) # Bond dimension of variational MPS
  Λ = randomprocess(Φ; χ=χ÷2, ξ=2)
  opt = Optimisers.Descent(0.01)

  F(Λ::LPDO; kwargs...) = fidelity(Λ, Φ)
  obs = Observer(["F" => F])
  epochs = 9

  batchsize = 10
  observe_step = 3

  outputpath = "simulation"
  Λ = tomography(
    data,
    Λ;
    test_data=test_data,
    batchsize=10,
    epochs=epochs,
    (observer!)=obs,
    observe_step = observe_step,
    outputpath = outputpath,
    savestate = true,
    outputlevel = 0
  )
  @test length(results(obs, "F")) == epochs ÷ observe_step

  obs2 = load("simulation_observer.jld2")
  for (k,v) in obs2
    @test last(v) ≈ results(obs, k)
  end

  fin = h5open("simulation_state.h5","r")
  M = read(fin, "state", LPDO{MPO})
  close(fin)
  @test M.X ≈ Λ.X
end


using Gtk, Graphics, Cairo, Colors
using svgPath

include("samples.jl")
path = parsePathData(Computer)
#path = ScalePath(path, 2)

c = @GtkCanvas()
win = GtkWindow(c, "Canvas")
@guarded draw(c) do widget
    ctx = getgc(c)
    h = height(c)
    w = width(c)
    rectangle(ctx, 50,50, 50, 50)
    set_source_rgb(ctx, 1, 1, 1)
    fill(ctx)

    set_line_width(ctx, 1)
    set_source_rgb(ctx, 0, 0, 0)
    # stroke(ctx)
    drawPath(ctx, path)
    #println(get_current_point(ctx))
    #println(copy_path_flat(ctx))

    stroke(ctx)
end
show(c)


function RBMs.zerosum!(rbm::ConvRBM)
    RBMs.zerosum_visible!(rbm)
    RBMs.zerosum_hidden!(rbm)
    return rbm
end

RBMs.zerosum_visible!(rbm::ConvRBM{<:AbstractLayer, <:AbstractLayer}) = rbm
RBMs.zerosum_hidden!(rbm::ConvRBM{<:AbstractLayer, <:AbstractLayer})  = rbm

function RBMs.zerosum_visible!(rbm::ConvRBM{<:RBMs.Potts, <:AbstractLayer})
    RBMs.zerosum!(visible(rbm))
    RBMs.zerosum!(weights(rbm); dims = 1)
    return nothing
end

function RBMs.zerosum_hidden!(rbm::ConvRBM{<:AbstractLayer, <:RBMs.Potts})
    RBMs.zerosum!(hidden(rbm))
    RBMs.zerosum!(weights(rbm); dims = ndims(weights(rbm)) - ndims(hidden(rbm)) + 1)
    return nothing
end

function RBMs.zerosum!(rbm::ConvRBM{<:RBMs.Potts, <:RBMs.Potts})
    RBMs.zerosum!(visible(rbm))
    RBMs.zerosum!(hidden(rbm))
    RBMs.zerosum!(weights(rbm); dims = 1)
    RBMs.zerosum!(weights(rbm); dims = ndims(weights(rbm)) - ndims(hidden(rbm)) + 1)
    return nothing
end


# Julia wrapper for header: /Users/dls/jl/openssl-1.0.1e/include/openssl/sha.h
# Automatically generated using Clang.jl wrap_c, version 0.0.0

@c Int32 SHA_Init (Ptr{SHA_CTX},) libcrypto
@c Int32 SHA_Update (Ptr{SHA_CTX}, Ptr{None}, Csize_t) libcrypto
@c Int32 SHA_Final (Ptr{Uint8}, Ptr{SHA_CTX}) libcrypto
@c Ptr{Uint8} SHA (Ptr{Uint8}, Csize_t, Ptr{Uint8}) libcrypto
@c None SHA_Transform (Ptr{SHA_CTX}, Ptr{Uint8}) libcrypto
@c Int32 SHA1_Init (Ptr{SHA_CTX},) libcrypto
@c Int32 SHA1_Update (Ptr{SHA_CTX}, Ptr{None}, Csize_t) libcrypto
@c Int32 SHA1_Final (Ptr{Uint8}, Ptr{SHA_CTX}) libcrypto
@c Ptr{Uint8} SHA1 (Ptr{Uint8}, Csize_t, Ptr{Uint8}) libcrypto
@c None SHA1_Transform (Ptr{SHA_CTX}, Ptr{Uint8}) libcrypto
@c Int32 SHA224_Init (Ptr{SHA256_CTX},) libcrypto
@c Int32 SHA224_Update (Ptr{SHA256_CTX}, Ptr{None}, Csize_t) libcrypto
@c Int32 SHA224_Final (Ptr{Uint8}, Ptr{SHA256_CTX}) libcrypto
@c Ptr{Uint8} SHA224 (Ptr{Uint8}, Csize_t, Ptr{Uint8}) libcrypto
@c Int32 SHA256_Init (Ptr{SHA256_CTX},) libcrypto
@c Int32 SHA256_Update (Ptr{SHA256_CTX}, Ptr{None}, Csize_t) libcrypto
@c Int32 SHA256_Final (Ptr{Uint8}, Ptr{SHA256_CTX}) libcrypto
@c Ptr{Uint8} SHA256 (Ptr{Uint8}, Csize_t, Ptr{Uint8}) libcrypto
@c None SHA256_Transform (Ptr{SHA256_CTX}, Ptr{Uint8}) libcrypto
@c Int32 SHA384_Init (Ptr{SHA512_CTX},) libcrypto
@c Int32 SHA384_Update (Ptr{SHA512_CTX}, Ptr{None}, Csize_t) libcrypto
@c Int32 SHA384_Final (Ptr{Uint8}, Ptr{SHA512_CTX}) libcrypto
@c Ptr{Uint8} SHA384 (Ptr{Uint8}, Csize_t, Ptr{Uint8}) libcrypto
@c Int32 SHA512_Init (Ptr{SHA512_CTX},) libcrypto
@c Int32 SHA512_Update (Ptr{SHA512_CTX}, Ptr{None}, Csize_t) libcrypto
@c Int32 SHA512_Final (Ptr{Uint8}, Ptr{SHA512_CTX}) libcrypto
@c Ptr{Uint8} SHA512 (Ptr{Uint8}, Csize_t, Ptr{Uint8}) libcrypto
@c None SHA512_Transform (Ptr{SHA512_CTX}, Ptr{Uint8}) libcrypto


#adapted from https://stackoverflow.com/questions/51038294/resize-image-using-nearest-neighborhood-with-cuda

function _CUDA_resize(pIn,pOut)
    _CUDA_resize(pIn,pOut,size(pIn,1),size(pIn,2),size(pOut,1),size(pOut,2))
end

function _CUDA_resize(pIn,pOut,w_in,h_in,w_out,h_out)

    index_i = blockDim().y * (blockIdx().y-1) + threadIdx().y
    index_j = blockDim().x * (blockIdx().x-1) + threadIdx().x

    stride_i = blockDim().y * gridDim().y
    stride_j = blockDim().x * gridDim().x

    for j=index_j:stride_j:h_out
        for i=index_i:stride_i:w_out
            jIn = div(j*h_in, h_out)
            iIn = div(i*w_in, w_out)
            @inbounds pOut[i,j] = pIn[iIn,jIn]
        end
    end
    return
end

function CUDA_resize(pIn,pOut)
    CUDA.@sync @cuda threads=(16,16) _CUDA_resize(pIn,pOut)
end

function _CUDA_resize4(pIn,pOut,w_in,h_in,w_out,h_out,n)

    c = blockIdx().y
    b = blockIdx().x
    a = threadIdx().x

    if ((a <= h_out)&&(b <= w_out))&&(c <=n)
        jIn = div(b*h_in, h_out)
        iIn = div(a*w_in, w_out)
        @inbounds pOut[a,b,1,c] = pIn[iIn,jIn,1,c]
    end

    return
end

function _CUDA_resize4(pIn,pOut)
    _CUDA_resize4(pIn,pOut,size(pIn,1),size(pIn,2),size(pOut,1),size(pOut,2),size(pIn,4))
end

function CUDA_resize4(pIn,pOut)
    numblocks_x = size(pOut,2)
    numblocks_y = size(pOut,4)

    CUDA.@sync @cuda threads=256 blocks=(numblocks_x,numblocks_y) _CUDA_resize4(pIn,pOut)
end

function _CUDA_normalize_images(pIn,meanImg,h_out,w_out,n)
    index_i = blockDim().y * (blockIdx().y-1) + threadIdx().y
    index_j = blockDim().x * (blockIdx().x-1) + threadIdx().x

    stride_i = blockDim().y * gridDim().y
    stride_j = blockDim().x * gridDim().x

    for k=1:n
        for j=index_j:stride_j:h_out
            for i=index_i:stride_i:w_out
                @inbounds pIn[i,j,1,k] = pIn[i,j,1,k] / 255
                @inbounds pIn[i,j,1,k] = pIn[i,j,1,k] - meanImg[i,j]
            end
        end
    end
    return
end

function _CUDA_normalize_images(pIn,meanImg)
    _CUDA_normalize_images(pIn,meanImg,size(pIn,1),size(pIn,2),size(pIn,4))
end

function CUDA_normalize_images(pIn,meanImg)
    CUDA.@sync @cuda threads=256 _CUDA_normalize_images(pIn,meanImg)
end

function CUDA_preprocess(pIn,pOut)
    n = size(pIn,4)

    w_in=size(pIn,1)
    h_in=size(pIn,2)
    w_out=size(pOut,1)
    h_out=size(pOut,2)

    CUDA.@sync @cuda threads=(16,16) _CUDA_preprocess(pIn,pOut,w_in,h_in,w_out,h_out,n)
end

function _CUDA_preprocess(pIn,pOut,w_in,h_in,w_out,h_out,n)
    index_i = blockDim().y * (blockIdx().y-1) + threadIdx().y
    index_j = blockDim().x * (blockIdx().x-1) + threadIdx().x

    stride_i = blockDim().y * gridDim().y
    stride_j = blockDim().x * gridDim().x

    for k=1:n
        for j=index_j:stride_j:h_out
            for i=index_i:stride_i:w_out
                jIn = div(j*h_in, h_out)
                iIn = div(i*w_in, w_out)
                @inbounds pOut[j,i,1,k] = Float32(pIn[iIn,jIn,1,k])
            end
        end
    end
    return
end

function _CUDA_blur_x(pIn,pOut,w_in,h_in,gauss,n)

    c = blockIdx().z
    b = blockIdx().y
    a = threadIdx().x

    while (a<=w_in)
        if ((a <= w_in)&&(b <= h_in))&&(c <=n)
            temp = 0.0
            for yy = 1:9

                new_y = yy-5 + a
                if new_y<1
                    new_y=abs(new_y)+1
                end
                if new_y>w_in
                    new_y = w_in - (new_y - w_in)
                end

                @inbounds temp = temp + Float32(pIn[new_y,b,1,c]) * gauss[yy]
            end
            @inbounds pOut[a,b,1,c] = temp
        end
        a += blockDim().x*blockIdx().x
    end
    return nothing
end

function _CUDA_blur_x(pIn,pOut,gauss)
    _CUDA_blur_x(pIn,pOut,size(pIn,1),size(pIn,2),gauss,size(pIn,4))
end

function CUDA_blur_x(pIn,pOut,gauss)
    numblocks_x = ceil(Int,size(pOut,1)/ 256)
    numblocks_y = size(pOut,2)
    numblocks_z = size(pOut,4)
    CUDA.@sync @cuda threads=256 blocks=(numblocks_x,numblocks_y,numblocks_z) _CUDA_blur_x(pIn,pOut,gauss)
end

function _CUDA_blur_y(pIn,pOut,gauss)
    _CUDA_blur_y(pIn,pOut,size(pIn,1),size(pIn,2),gauss,size(pIn,4))
end

function CUDA_blur_y(pIn,pOut,gauss)
    numblocks_x = ceil(Int,size(pOut,2)/ 256)
    numblocks_y = size(pOut,1)
    numblocks_z = size(pOut,4)
    CUDA.@sync @cuda threads=256 blocks=(numblocks_x,numblocks_y,numblocks_z) _CUDA_blur_y(pIn,pOut,gauss)
end

function _CUDA_blur_y(pIn,pOut,w_in,h_in,gauss,n)

    c = blockIdx().z
    b = blockIdx().y
    a = threadIdx().x

    while (a<=h_in)
        if ((a <= h_in)&&(b <= w_in))&&(c <=n)
            temp = 0.0
            for yy = 1:9

                new_y = yy-5 + a
                if new_y<1
                    new_y=abs(new_y)+1
                end
                if new_y>h_in
                    new_y = h_in - (new_y - h_in)
                end

                @inbounds temp = temp + Float32(pIn[b,new_y,1,c]) * gauss[yy]
            end
            @inbounds pOut[b,a,1,c] = temp
        end
        a += blockDim().x*blockIdx().x
    end
    return nothing
end

function _CUDA_flip_xy(pIn,pOut,w_in,h_in,n)

    c = blockIdx().y
    b = blockIdx().x
    a = threadIdx().x

    if ((a <= h_in)&&(b <= w_in))&&(c <=n)
        @inbounds pOut[a,b,1,c] = pIn[b,a,1,c]
    end

    return
end

function _CUDA_flip_xy(pIn,pOut)
    _CUDA_flip_xy(pIn,pOut,size(pIn,1),size(pIn,2),size(pIn,4))
end

function CUDA_flip_xy(pIn,pOut)
    numblocks_x = size(pOut,2)
    numblocks_y = size(pOut,4)

    CUDA.@sync @cuda threads=256 blocks=(numblocks_x,numblocks_y) _CUDA_flip_xy(pIn,pOut)
end


mutable struct BasicContUnvParameterNState{N<:Real} <: ParameterNState{Continuous, Univariate}
  value::Vector{N}
  loglikelihood::Vector{N}
  logprior::Vector{N}
  logtarget::Vector{N}
  gradloglikelihood::Vector{N}
  gradlogprior::Vector{N}
  gradlogtarget::Vector{N}
  tensorloglikelihood::Vector{N}
  tensorlogprior::Vector{N}
  tensorlogtarget::Vector{N}
  dtensorloglikelihood::Vector{N}
  dtensorlogprior::Vector{N}
  dtensorlogtarget::Vector{N}
  diagnosticvalues::Matrix
  monitor::Vector{Bool}
  n::Integer
  diagnostickeys::Vector{Symbol}

  function BasicContUnvParameterNState{N}(
    n::Integer,
    monitor::Vector{Bool}=[true; fill(false, 12)],
    diagnostickeys::Vector{Symbol}=Symbol[],
    ::Type{N}=Float64,
    diagnosticvalues::Matrix=Array{Any}(length(diagnostickeys), isempty(diagnostickeys) ? 0 : n)
  ) where N<:Real
    instance = new()

    l = Array{Integer}(13)
    for i in 1:13
      l[i] = (monitor[i] == false ? zero(Integer) : n)
    end

    fnames = fieldnames(BasicContUnvParameterNState)
    for i in 1:13
      setfield!(instance, fnames[i], Array{N}(l[i]))
    end

    instance.diagnosticvalues = diagnosticvalues
    instance.monitor = monitor
    instance.n = n
    instance.diagnostickeys = diagnostickeys

    instance
  end
end

BasicContUnvParameterNState(
  n::Integer,
  monitor::Vector{Bool}=[true; fill(false, 12)],
  diagnostickeys::Vector{Symbol}=Symbol[],
  ::Type{N}=Float64,
  diagnosticvalues::Matrix=Array{Any}(length(diagnostickeys), isempty(diagnostickeys) ? 0 : n)
) where N<:Real =
  BasicContUnvParameterNState{N}(n, monitor, diagnostickeys, N, diagnosticvalues)

function BasicContUnvParameterNState(
  n::Integer,
  monitor::Vector{Symbol},
  diagnostickeys::Vector{Symbol}=Symbol[],
  ::Type{N}=Float64,
  diagnosticvalues::Matrix=Array{Any}(length(diagnostickeys), isempty(diagnostickeys) ? 0 : n)
) where N<:Real
  fnames = fieldnames(BasicContUnvParameterNState)
  BasicContUnvParameterNState(n, [fnames[i] in monitor ? true : false for i in 1:13], diagnostickeys, N, diagnosticvalues)
end

const ContUnvMarkovChain = BasicContUnvParameterNState

function copy!(nstate::BasicContUnvParameterNState, state::BasicContUnvParameterState, i::Integer)
  fnames = fieldnames(BasicContUnvParameterNState)

  for j in 1:13
    if nstate.monitor[j]
      getfield(nstate, fnames[j])[i] = getfield(state, fnames[j])
    end
  end

  if !isempty(nstate.diagnosticvalues)
    nstate.diagnosticvalues[:, i] = state.diagnosticvalues
  end
end

eltype{N<:Real}(::Type{BasicContUnvParameterNState{N}}) = N
eltype{N<:Real}(::BasicContUnvParameterNState{N}) = N

==(z::S, w::S) where {S<:BasicContUnvParameterNState} = reduce(&, [getfield(z, n) == getfield(w, n) for n in fieldnames(S)[1:16]])

isequal(z::S, w::S) where {S<:BasicContUnvParameterNState} =
  reduce(&, [isequal(getfield(z, n), getfield(w, n)) for n in fieldnames(S)[1:16]])

function show(io::IO, nstate::BasicContUnvParameterNState{N}) where N<:Real
  fnames = fieldnames(BasicContUnvParameterNState)
  fbool = map(n -> !isempty(getfield(nstate, n)), fnames[1:13])
  indentation = "  "

  println(io, "BasicContUnvParameterNState:")

  println(io, indentation*"eltype: $(eltype(nstate))")
  println(io, indentation*"number of states = $(nstate.n)")

  print(io, indentation*"monitored components:")
  if !any(fbool)
    println(io, " none")
  else
    print(io, "\n")
    for i in 1:13
      if fbool[i]
        println(io, string(indentation^2, fnames[i]))
      end
    end
  end

  print(io, indentation*"diagnostics:")
  if isempty(nstate.diagnostickeys)
    print(io, " none")
  else
    for k in nstate.diagnostickeys
      print(io, "\n")
      print(io, string(indentation^2, k))
    end
  end
end


using Colors, FileIO

import Base: getindex, setindex!, size, show

export Image, width, height
export pixels

struct Image{T<:Colorant}
   pixels::Matrix{T}
end

"""
    Image(filename)
Creates an image from an image file stored on disk. If loading
a `gif` image, this will produce multiple images
"""
function Image(filename::AbstractString)
    pixels = load(filename)
    dims = ndims(pixels)
    if dims == 2
        Image(pixels)
    elseif dims == 3
        map(1:size(pixels, 3)) do i
           Image(pixels[:, :, i])
        end
    else
        error("Don't support loading imagse with $dims dimensions")
    end
end

"""
    Image(w::Int, h::Int)
Creates a blank image of width `w` and height `h`.
"""
function Image(w::Int, h::Int)
    Image(fill(colorant"black", h, w))
end

"""
    Image(w::Int, h::Int, c::Colorant)
Creates image of width `w` and height `h` filled with color `c`.
"""
function Image(w::Int, h::Int, c::Colorant)
    Image(fill(c, h, w))
end

getindex(img::Image, x::Int, y::Int) = img.pixels[y+1, x+1]

function setindex!(img::Image, color::Colorant, x::Int, y::Int)
    img.pixels[y+1, x+1] = color
end

function getindex(img::Image, X::UnitRange{Int}, Y::UnitRange{Int})
    I = first(Y)+1:last(Y)
    J = first(X)+1:last(X)
    Image(img.pixels[I, J])
end

function getindex(img::Image, x::Int, Y::UnitRange{Int})
    I = first(Y)+1:last(Y)
    j = x+1
    img.pixels[I, j]
end

function getindex(img::Image, X::UnitRange{Int}, y::Int)
    i = y+1
    J = first(X)+1:last(X)
    img.pixels[i, J]
end

width(img::Image) = size(img.pixels, 2)
height(img::Image) = size(img.pixels, 1)

"""
    size(img::Image) -> Size
Gets the width and the height of image `img`.
"""
size(img::Image) = Size(width(img), height(img))

show(io::IO, mime::MIME"text/plain", img::Image)    = show(io, mime, img.pixels)
show(io::IO, mime::MIME"image/svg+xml", img::Image) = show(io, mime, img.pixels)

function pixels(img::Image)
   img.pixels
end


function init(start_time, m::JuniperProblem; incumbent = nothing)
    hash_val = string(hash(hcat(m.l_var,m.u_var)))
    node = BnBNode(1, 1, m.l_var, m.u_var, m.relaxation_solution, 0, :Branch, m.relaxation_status, m.relaxation_objval,[],hash_val)
    obj_gain_m = zeros(m.num_disc_var)
    obj_gain_p = zeros(m.num_disc_var)
    obj_gain_mc = zeros(Int64, m.num_disc_var)
    obj_gain_pc = zeros(Int64, m.num_disc_var)
    obj_gain_inf = zeros(Int64, m.num_disc_var)
    obj_gain = GainObj(obj_gain_m, obj_gain_p, obj_gain_mc, obj_gain_pc, obj_gain_inf)
    disc2var_idx = zeros(m.num_disc_var)
    var2disc_idx = zeros(m.num_var)
    disc_i = 1
    for i=1:m.num_var
        if m.var_type[i] != :Cont
            disc2var_idx[disc_i] = i
            var2disc_idx[i] = disc_i
            disc_i += 1
        end
    end
    factor = 1
    if m.obj_sense == :Min
        factor = -1
    end
    bnbTree = BnBTreeObj()
    bnbTree.m           = m
    bnbTree.obj_gain    = obj_gain
    bnbTree.disc2var_idx = disc2var_idx
    bnbTree.var2disc_idx = var2disc_idx
    bnbTree.options     = m.options
    bnbTree.obj_fac     = factor
    bnbTree.start_time  = start_time
    bnbTree.nsolutions  = 0
    bnbTree.branch_nodes = [node]
    bnbTree.best_bound  = m.relaxation_objval
    bnbTree.limit = :None
    bnbTree.global_solver = is_global_status(m.status)

    if incumbent !== nothing
        bnbTree.incumbent = incumbent
        bnbTree.nsolutions += 1
        if m.options.incumbent_constr
            add_incumbent_constr(m,bnbTree.incumbent)
            m.ncuts += 1
        end
    end

    return bnbTree
end

function new_default_node(idx, level, l_var, u_var, solution;
                            var_idx=0, state=:Solve, relaxation_state=MOI.OPTIMIZE_NOT_CALLED, best_bound=NaN, path=[])

    l_var = copy(l_var)
    u_var = copy(u_var)
    solution = copy(solution)
    hash_val = string(hash(hcat(l_var,u_var)))
    return BnBNode(idx, level, l_var, u_var, solution, var_idx, state, relaxation_state, best_bound, path, hash_val)
end

function new_left_node(node, u_var;
                       var_idx=0, state=:Solve, relaxation_state=MOI.OPTIMIZE_NOT_CALLED, best_bound=NaN, path=[])
    l_var = copy(node.l_var)
    u_var = copy(u_var)
    solution = NaN*ones(length(node.solution))
    hash_val = string(hash(hcat(l_var,u_var)))
    return BnBNode(node.idx*2, node.level+1, l_var, u_var, solution, var_idx, state, relaxation_state, best_bound, path, hash_val)
end

function new_right_node(node, l_var;
                       var_idx=0, state=:Solve, relaxation_state=MOI.OPTIMIZE_NOT_CALLED, best_bound=NaN, path=[])
    l_var = copy(l_var)
    u_var = copy(node.u_var)
    solution = NaN*ones(length(node.solution))
    hash_val = string(hash(hcat(l_var,u_var)))
    return BnBNode(node.idx*2+1, node.level+1, l_var, u_var, solution, var_idx, state, relaxation_state, best_bound, path, hash_val)
end

function init_time_obj()
    return TimeObj(0.0,0.0,0.0,0.0,0.0)
end

function new_default_step_obj(m,node)
    gains_m = zeros(m.num_disc_var)
    gains_mc = zeros(Int64, m.num_disc_var)
    gains_p = zeros(m.num_disc_var)
    gains_pc = zeros(Int64, m.num_disc_var)
    gains_inf = zeros(Int64, m.num_disc_var)
    gains = GainObj(gains_m, gains_p, gains_mc, gains_pc, gains_inf)
    idx_time = 0.0
    node_idx_time = 0.0
    upd_gains_time = 0.0
    node_branch_time = 0.0
    branch_time = 0.0
    step_obj = StepObj()
    step_obj.node             = node
    step_obj.var_idx          = 0
    step_obj.state            = :None
    step_obj.nrestarts        = 0
    step_obj.gain_gap         = 0.0
    step_obj.obj_gain         = gains
    step_obj.idx_time         = idx_time
    step_obj.node_idx_time    = node_idx_time
    step_obj.upd_gains_time   = upd_gains_time
    step_obj.node_branch_time = node_branch_time
    step_obj.branch_time      = branch_time
    step_obj.integral         = []
    step_obj.branch           = []
    step_obj.counter          = 0
    step_obj.upd_gains        = :None
    step_obj.strong_disc_vars = Int64[]
    step_obj.branch_strategy  = :None
    step_obj.strong_branching = nothing
    return step_obj
end

_ci(vi::MOI.VariableIndex, S::Type) = MOI.ConstraintIndex{MOI.VariableIndex, S}(vi.value)
function init_juniper_problem!(jp::JuniperProblem, model::MOI.AbstractOptimizer)
    jp.num_var = MOI.get(model, MOI.NumberOfVariables())

    jp.status = MOI.OPTIMIZE_NOT_CALLED
    jp.relaxation_status = MOI.OPTIMIZE_NOT_CALLED
    jp.nlp_data = nothing
    jp.objective = nothing
    for attr in MOI.get(model, MOI.ListOfModelAttributesSet())
        if attr isa MOI.NLPBlock
            jp.nlp_data = MOI.get(model, MOI.NLPBlock())
        elseif attr isa MOI.ObjectiveFunction
            jp.objective = MOI.get(model, attr)
        end
    end

    jp.objval = NaN
    jp.best_bound = NaN
    jp.solution = fill(NaN, jp.num_var)
    jp.nsolutions = 0
    jp.solutions = []
    jp.num_disc_var = 0
    jp.nintvars = 0
    jp.nbinvars = 0
    jp.nnodes = 1 # is set to one for the root node
    jp.ncuts = 0
    jp.nbranches = 0
    jp.nlevels = 1
    jp.relaxation_time = 0.0

    jp.start_time = time()

    jp.nl_solver = model.options.nl_solver

    if model.options.mip_solver !== nothing
        jp.mip_solver = model.options.mip_solver
    end
    jp.options = model.options
    if MOI.get(model, MOI.ObjectiveSense()) == MOI.MIN_SENSE
        jp.obj_sense = :Min
    else
        jp.obj_sense = :Max
    end
    vis = MOI.get(model, MOI.ListOfVariableIndices())
    jp.l_var = map(vis) do vi
        lb = MOIU.get_bounds(model, Float64, vi)[1]
        MOI.is_valid(model, _ci(vi, MOI.ZeroOne)) ? max(lb, 0.0) : lb
    end
    jp.u_var = map(vis) do vi
        ub = MOIU.get_bounds(model, Float64, vi)[2]
        MOI.is_valid(model, _ci(vi, MOI.ZeroOne)) ? min(ub, 1.0) : ub
    end
    integer_bool_arr = map(vis) do vi
        MOI.is_valid(model, _ci(vi, MOI.Integer))
    end
    binary_bool_arr = map(vis) do vi
        MOI.is_valid(model, _ci(vi, MOI.ZeroOne))
    end
    primal_start_arr = map(vis) do vi
        start = MOI.get(model, MOI.VariablePrimalStart(), vi)
        start === nothing ? 0.0 : start
    end
    jp.primal_start = primal_start_arr
    jp.nintvars = sum(integer_bool_arr)
    jp.nbinvars = sum(binary_bool_arr)
    jp.num_disc_var = sum(integer_bool_arr)+sum(binary_bool_arr)
    jp.var_type = [:Cont for i in 1:jp.num_var]
    jp.var_type[integer_bool_arr .== true] .= :Int
    jp.var_type[binary_bool_arr .== true] .= :Bin
    jp.disc2var_idx = zeros(jp.num_disc_var)
    jp.var2disc_idx = zeros(jp.num_var)
    int_i = 1
    for i=1:jp.num_var
        if jp.var_type[i] != :Cont
            jp.disc2var_idx[int_i] = i
            jp.var2disc_idx[i] = int_i
            int_i += 1
        end
    end
end

function new_default_strong_branch_step_obj(var_idx)
    strong_step_obj = StrongBranchStep()
    strong_step_obj.var_idx                 = var_idx
    strong_step_obj.l_relaxation_state      = MOI.OPTIMIZE_NOT_CALLED
    strong_step_obj.r_relaxation_state      = MOI.OPTIMIZE_NOT_CALLED
    strong_step_obj.init_restart            = false
    return strong_step_obj
end


#==============================================================================#
# Options to give in parameter file
#==============================================================================#

#------------------------------------------------------------------------------#
# Files
#------------------------------------------------------------------------------#

# set genotype, map, and pedigree file names
#genotype_file = "ALGP2_Cycle_1_5_Geno_BLUPF90.txt_clean"
genotype_file = "genotypes.txt"
map_file      = "ChrInfoBLUPF90.txt"
ped_file      = "dam_1_5.ped"

#------------------------------------------------------------------------------#
# General Options
#------------------------------------------------------------------------------#

#------------------------------------------------------------------------------#
# Map File Options
#------------------------------------------------------------------------------#

# Map file options
map_file_delimiter = ' '      # [char] needs to be a character with ' not "
map_file_header    = false    # [logical] is there a header on the map file
map_file_start_row = 1        # [Int] Line to start reading from

#------------------------------------------------------------------------------#
# Genotype File Options
#------------------------------------------------------------------------------#

# SNP format:
# "dense" =      ID1     0501102020102052
# "space" =      ID1     0 5 0 1 1 0 2 0 2 0 1 0 2 0 5 2
SNP_format = "dense"

# headers on map file or genotype file
genotype_file_delimiter   = ' '        # [char] set what separates in the genotype file
genotype_file_start_row   = 1          # [Int] row to start reading
genotype_missing_value    = 5          # [Int] set missing value for genotypes (will take anything not 0,1,2)

# NOT USED
n_skip_genotype_file      = 0          # [Int] number of lines to skip in the genotype file
genotype_header           = false      # [logical] ask if there is a header
genotype_start_col        = 2          # [Int]

# number of SNPs to sample
#n_sample_M = 0              # Select all SNPs
n_sample_M = 50000                  # [Int] number to sample from genotype file

#------------------------------------------------------------------------------#
# QC Options
#------------------------------------------------------------------------------#

# Call Rates (minimum proportion of non-missing genotypes for each)
call_rate_animals  = 0.90
call_rate_SNPs = 0.90

# MAF threshold
MAF_threshold = 0.05

# Exclude IDs
exclude_animals = []
exclude_SNPs    = []
exclude_Chr     = []
exclude_SNP     = []

#----------------------------------------#
# PCA Options
#----------------------------------------#

# set number of Principal components to use and plot bivariate scatterplots
n_PCs = 3

#----------------------------------------#
# Save Options
#----------------------------------------#

# save clean SNPs
save_clean = "yes"

# Save Raw Relationship Matrices
saveG = "no"
saveA = "no"
saveH = "no"

# Save Inverses of relationship matrices
saveGinv = "no"
saveAinv = "no"
saveHinv = "no"


using DataStructures: OrderedDict

@testset "Datasets" begin
  data = [rand(4,5,12), rand(4,5,12), rand(4,5)]
  axlist1 = [RangeAxis("XVals",1.0:4.0), CategoricalAxis("YVals",[1,2,3,4,5]), RangeAxis("Time",Date(2001,1,15):Month(1):Date(2001,12,15))]
  axlist2 = [RangeAxis("XVals",1.0:4.0), CategoricalAxis("YVals",[1,2,3,4,5])]
  props = [Dict("att$i"=>i) for i=1:3]
  c1,c2,c3 = (YAXArray(axlist1, data[1], props[1]),
  YAXArray(axlist1, data[2], props[2]),
  YAXArray(axlist2, data[3], props[3])
  )
  ds = Dataset(avar = c1, something = c2, smaller = c3)
  @testset "Basic functions" begin
    b = IOBuffer()
    show(b,ds)
    s = split(String(take!(b)),"\n")
    s2 = """
    YAXArray Dataset
    Dimensions:
    XVals               Axis with 4 Elements from 1.0 to 4.0
    YVals               Axis with 5 elements: 1 2 3 4 5
    Time                Axis with 12 Elements from 2001-01-15 to 2001-12-15
    Variables: avar something smaller """
    s2 = split(s2,"\n")
#     @test s[[1,2,6]] == s2[[1,2,6]]
#     @test all(i->in(i,s2), s[3:5])
    for n in [:avar, :something, :smaller, :XVals, :Time, :YVals]
      @test n in propertynames(ds)
      @test n in propertynames(ds, true)
    end
    @test :axes ∉ propertynames(ds)
    @test :cubes ∉ propertynames(ds)
    @test :axes ∈ propertynames(ds, true)
    #Test getproperty
    @test all(i->in(i,values(ds.axes)),axlist1)
    @test collect(keys(ds.cubes)) == [:avar, :something, :smaller]
    @test collect(values(ds.cubes)) == [c1,c2,c3]
    @test ds.avar === c1
    @test ds.something === c2
    @test ds.smaller === c3
    @test ds[:avar] === c1
    ds2 = ds[[:avar, :smaller]]
    @test collect(keys(ds2.cubes)) == [:avar, :smaller]
    @test collect(values(ds2.cubes)) == [c1,c3]
    @test YAXArrays.Datasets.fuzzyfind("hal", ["hallo","bla","something"]) == 1
    ds3 = ds[["av", "some"]]
    @test collect(keys(ds3.cubes)) == [:av, :some]
    @test collect(values(ds3.cubes)) == [c1,c2]
    @test ds["avar"] === c1
  end
  @testset "Dataset interface" begin
    struct MockDataset
      vars
      dims
      attrs
      path
    end
    Base.getindex(d::MockDataset,i) = d.vars[i]
    Base.haskey(d::MockDataset,i) = haskey(d.vars,i)
    YAXArrayBase.get_varnames(d::MockDataset) = (keys(d.vars)...,)
    YAXArrayBase.get_var_dims(d::MockDataset,name) = d.dims[name]
    YAXArrayBase.get_var_attrs(d::MockDataset, name) = d.attrs[name]
    YAXArrayBase.allow_missings(d::MockDataset) = !occursin("nomissings",d.path)
    function YAXArrayBase.create_empty(::Type{MockDataset},path)
      mkpath(dirname(path))
      open(_->nothing, path,"w")
      MockDataset(Dict(),Dict(),Dict(),path)
    end
    function YAXArrayBase.add_var(ds::MockDataset, T, name, s, dimlist, atts;kwargs...)
      data = Array{T}(undef,s...)
      ds.vars[name] = data
      ds.dims[name] = dimlist
      ds.attrs[name] = atts
      data
    end
    YAXArrayBase.backendlist[:mock] = MockDataset
    push!(YAXArrayBase.backendregex,r".mock$"=>MockDataset)
    data1,data2,data3,d1,d2,d3 = (rand(12,5,2),rand(12,5),rand(12,5,2),1:12, 0.1:0.1:0.5, ["One","Two"])
    att1 =  Dict("att1"=>5,"_ARRAY_OFFSET"=>(2,0,0))
    att2 =  Dict("att2"=>6,"_ARRAY_OFFSET"=>(2,0))
    attd1 = Dict("_ARRAY_OFFSET"=>2, "units"=>"days since 2001-01-01", "calendar"=>"gregorian")
    attd2 = Dict("attd"=>"d")
    attd3 = Dict("attd"=>"d")
    function MockDataset(p)
        MockDataset(
        OrderedDict("Var1"=>data1, "Var2"=>data2, "Var3"=>data3, "time"=>d1,"d2"=>d2, "d3"=>d3),
        Dict("Var1"=>("time","d2","d3"),"Var2"=>("time","d2"),"Var3"=>("time","d2","d3"),"time"=>("time",),"d2"=>["d2"],"d3"=>["d3"]),
        Dict("Var1"=>att1,"Var2"=>att2,"Var3"=>att1,"time"=>attd1,"d2"=>attd2,"d3"=>attd3),
        p
        )
    end
    m = MockDataset("testpath.mock")
    @testset "collectdims" begin
      dcollect = YAXArrays.Datasets.collectdims(m)
      @test dcollect["time"].ax isa RangeAxis
      @test YAXArrays.Cubes.Axes.axname(dcollect["time"].ax) == "time"
      @test dcollect["time"].ax.values == DateTime(2001,1,4):Day(1):DateTime(2001,1,13)
      @test dcollect["time"].offs == 2
      @test dcollect["d2"].ax isa RangeAxis
      @test YAXArrays.Cubes.Axes.axname(dcollect["d2"].ax) == "d2"
      @test dcollect["d2"].ax.values == 0.1:0.1:0.5
      @test dcollect["d2"].offs == 0
      @test dcollect["d3"].ax isa CategoricalAxis
      @test YAXArrays.Cubes.Axes.axname(dcollect["d3"].ax) == "d3"
      @test dcollect["d3"].ax.values == ["One","Two"]
      @test dcollect["d3"].offs == 0
      a1 = [0.1,0.2,0.3,0.4]
      a2 = [0.1,0.21,0.3,0.4]
      @test YAXArrays.Datasets.testrange(a1)== 0.1:0.1:0.4
      @test YAXArrays.Datasets.testrange(a2) isa Array
      @test YAXArrays.Datasets.testrange(a2) == [0.1,0.21,0.3,0.4]
    end
    @testset "open_dataset" begin
      ds = open_dataset("test.mock")
      @test size(ds.Var1) == (10,5,2)
      @test size(ds.Var2) == (10,5)
      @test all(in(keys(ds.axes)),(:time,:d2,:d3))
      ar = Cube(ds)
      @test ar isa YAXArray
      @test size(ar) == (10,5,2,2)
      @test YAXArrays.Cubes.Axes.axname.(ar.axes) == ["time","d2","d3","Variable"]
      @test ar.axes[4].values == ["Var1","Var3"]
    end
    @testset "Dataset creation" begin
      al = [RangeAxis("Time",Date(2001):Month(1):Date(2001,12,31)), CategoricalAxis("Variable",["A","B"]), RangeAxis("Xvals",1:10)]
      #Basic
      newds, newds2 = YAXArrays.Datasets.createdataset(MockDataset,al)
      @test YAXArrays.Cubes.axsym.(newds2.axes) == [:Time, :Xvals, :Variable]
      @test newds2.axes[1].values == Date(2001):Month(1):Date(2001,12,31)
      @test newds2.axes[3].values == ["A","B"]
      @test newds2.axes[2].values == 1:10
      @test newds2.data isa YAXArrays.Cubes.DiskArrayTools.DiskArrayStack
      # A bit more advanced
      fn = string(tempname(),".mock")
      newds, newds2 = YAXArrays.Datasets.createdataset(MockDataset,al,path = fn, persist = false,
      chunksize = (4,2,4), chunkoffset = (2,0,3), properties = Dict("att1"=>5), datasetaxis="A")
      @test size(newds.data) == (12,2,10)
      @test size(newds.data.parent) == (14,2,13)
      @test eltype(newds.data) <: Union{Float32,Missing}
      @test newds.properties["att1"] == 5
      @test isfile(fn)
      newds = nothing
      newds2 = nothing
      # Without missings
      fn = string(tempname(),"nomissings.mock")
      newds = YAXArrays.Datasets.createdataset(MockDataset,al,path = fn,datasetaxis="A")
    end
  end
end


# This file is a part of Julia. License is MIT: http://julialang.org/license

## IP ADDRESS HANDLING ##
abstract type IPAddr end

Base.isless{T<:IPAddr}(a::T, b::T) = isless(a.host, b.host)
Base.convert(dt::Type{<:Integer}, ip::IPAddr) = dt(ip.host)

struct IPv4 <: IPAddr
    host::UInt32
    IPv4(host::UInt32) = new(host)
    IPv4(a::UInt8,b::UInt8,c::UInt8,d::UInt8) = new(UInt32(a)<<24|
                                                    UInt32(b)<<16|
                                                    UInt32(c)<<8|
                                                    d)
    function IPv4(a::Integer,b::Integer,c::Integer,d::Integer)
        if !(0<=a<=255 && 0<=b<=255 && 0<=c<=255 && 0<=d<=255)
            throw(ArgumentError("IPv4 field out of range (must be 0-255)"))
        end
        IPv4(UInt8(a),UInt8(b),UInt8(c),UInt8(d))
    end
end

"""
    IPv4(host::Integer) -> IPv4

Returns an IPv4 object from ip address `host` formatted as an `Integer`.

```jldoctest
julia> IPv4(3223256218)
ip"192.30.252.154"
```
"""
function IPv4(host::Integer)
    if host < 0
        throw(ArgumentError("IPv4 address must be positive"))
    elseif typemax(typeof(host)) > typemax(UInt32) && host > typemax(UInt32)
        throw(ArgumentError("IPv4 address must fit within 32 bits"))
    else
        return IPv4(UInt32(host))
    end
end

# constructor: ("1.2.3.4")
IPv4(str::AbstractString) = parse(IPv4, str)

show(io::IO,ip::IPv4) = print(io,"ip\"",ip,"\"")
print(io::IO,ip::IPv4) = print(io,dec((ip.host&(0xFF000000))>>24),".",
                                  dec((ip.host&(0xFF0000))>>16),".",
                                  dec((ip.host&(0xFF00))>>8),".",
                                  dec(ip.host&0xFF))

struct IPv6 <: IPAddr
    host::UInt128
    IPv6(host::UInt128) = new(host)
    IPv6(a::UInt16,b::UInt16,c::UInt16,d::UInt16,
     e::UInt16,f::UInt16,g::UInt16,h::UInt16) = new(UInt128(a)<<(7*16)|
                            UInt128(b)<<(6*16)|
                            UInt128(c)<<(5*16)|
                            UInt128(d)<<(4*16)|
                            UInt128(e)<<(3*16)|
                            UInt128(f)<<(2*16)|
                            UInt128(g)<<(1*16)|
                            h)
    function IPv6(a::Integer,b::Integer,c::Integer,d::Integer,
          e::Integer,f::Integer,g::Integer,h::Integer)
        if !(0<=a<=0xFFFF && 0<=b<=0xFFFF && 0<=c<=0xFFFF && 0<=d<=0xFFFF &&
             0<=e<=0xFFFF && 0<=f<=0xFFFF && 0<=g<=0xFFFF && 0<=h<=0xFFFF)
            throw(ArgumentError("IPv6 field out of range (must be 0-65535)"))
        end
        IPv6(UInt16(a),UInt16(b),UInt16(c),UInt16(d),
             UInt16(e),UInt16(f),UInt16(g),UInt16(h))
    end
end

"""
    IPv6(host::Integer) -> IPv6

Returns an IPv6 object from ip address `host` formatted as an `Integer`.

```jldoctest
julia> IPv6(3223256218)
ip"::c01e:fc9a"
```
"""
function IPv6(host::Integer)
    if host < 0
        throw(ArgumentError("IPv6 address must be positive"))
        # We allow passing bigger integer types, but need to be careful to avoid overflow
        # Let's hope promotion rules are sensible
    elseif typemax(typeof(host)) > typemax(UInt128) && host > typemax(UInt128)
        throw(ArgumentError("IPv6 address must fit within 128 bits"))
    else
        return IPv6(UInt128(host))
    end
end

IPv6(str::AbstractString) = parse(IPv6, str)

# Suppress leading '0's and "0x"
print_ipv6_field(io,field::UInt16) = print(io,hex(field))

print_ipv6_field(io,ip,i) = print_ipv6_field(io,ipv6_field(ip,i))
function ipv6_field(ip::IPv6,i)
    if i < 0 || i > 7
        throw(BoundsError())
    end
    UInt16(ip.host&(UInt128(0xFFFF)<<(i*16))>>(i*16))
end

show(io::IO, ip::IPv6) = print(io,"ip\"",ip,"\"")
# RFC 5952 compliant show function
# http://tools.ietf.org/html/rfc5952
function print(io::IO,ip::IPv6)
    i = 8
    m = 0
    longest_sub_i = -1
    while i!=0
        i-=1
        field = ipv6_field(ip,i)
        if field == 0 && longest_sub_i == -1
            # Find longest subsequence of 0
            longest_sub_i,j,m,c = i,i,1,1
            while j != 0
                j-=1
                if ipv6_field(ip,j) == 0
                    c += 1
                else
                    c = 0
                end
                if c > m
                    if j+c != longest_sub_i+1
                        longest_sub_i = j+c-1
                    end
                    m = c
                end
            end
            # Prevent single 0 from contracting to :: as required
            if m == 1
                longest_sub_i = 9
            end
        end
        if i == longest_sub_i
            print(io,":")
            i -= m-1
            if i == 0
                print(io,":")
                break
            end
        else
            if i != 7
                print(io,":")
            end
            print_ipv6_field(io,field)
        end
    end
end

# Parsing

const ipv4_leading_zero_error = """
Leading zeros in IPv4 addresses are disallowed due to ambiguity.
If the address is in octal or hexadecimal, convert it to decimal, otherwise remove the leading zero.
"""

function parse(::Type{IPv4}, str::AbstractString)
    fields = split(str,'.')
    i = 1
    ret = 0
    for f in fields
        if isempty(f)
            throw(ArgumentError("empty field in IPv4 address"))
        end
        if length(f) > 1 && f[1] == '0'
            throw(ArgumentError(ipv4_leading_zero_error))
        else
            r = parse(Int,f,10)
        end
        if i != length(fields)
            if r < 0 || r > 255
                throw(ArgumentError("IPv4 field out of range (must be 0-255)"))
            end
            ret |= UInt32(r) << ((4-i)*8)
        else
            if r > ((UInt64(1)<<((5-length(fields))*8))-1)
                throw(ArgumentError("IPv4 field too large"))
            end
            ret |= r
        end
        i+=1
    end
    IPv4(ret)
end

function parseipv6fields(fields,num_fields)
    if length(fields) > num_fields
        throw(ArgumentError("too many fields in IPv6 address"))
    end
    cf = 7
    ret = UInt128(0)
    for f in fields
        if f == ""
            # ::abc:... and ..:abc::
            if cf != 7 && cf != 0
                cf -= num_fields-length(fields)
            end
            cf -= 1
            continue
        end
        ret |= UInt128(parse(Int,f,16))<<(cf*16)
        cf -= 1
    end
    ret
end
parseipv6fields(fields) = parseipv6fields(fields,8)

function parse(::Type{IPv6}, str::AbstractString)
    fields = split(str,':')
    if length(fields) > 8
        throw(ArgumentError("too many fields in IPv6 address"))
    elseif length(fields) == 8
        return IPv6(parseipv6fields(fields))
    elseif in('.',fields[end])
        return IPv6((parseipv6fields(fields[1:(end-1)],6))
            | parse(IPv4, fields[end]).host )
    else
        return IPv6(parseipv6fields(fields))
    end
end

#
# This support IPv4 addresses in the common dot (IPv4) or colon (IPv6)
# separated formats. Most other common formats use a standard integer encoding
# of the appropriate size and should use the appropriate constructor
#

function parse(::Type{IPAddr}, str::AbstractString)
    if ':' in str
        return parse(IPv6, str)
    else
        return parse(IPv4, str)
    end
end

macro ip_str(str)
    return parse(IPAddr, str)
end

struct InetAddr{T<:IPAddr}
    host::T
    port::UInt16
end

InetAddr(ip::IPAddr, port) = InetAddr{typeof(ip)}(ip, port)

## SOCKETS ##

mutable struct TCPSocket <: LibuvStream
    handle::Ptr{Void}
    status::Int
    buffer::IOBuffer
    readnotify::Condition
    connectnotify::Condition
    closenotify::Condition
    sendbuf::Nullable{IOBuffer}
    lock::ReentrantLock
    throttle::Int

    function TCPSocket(handle::Ptr{Void}, status)
        tcp = new(
                handle,
                status,
                PipeBuffer(),
                Condition(),
                Condition(),
                Condition(),
                nothing,
                ReentrantLock(),
                DEFAULT_READ_BUFFER_SZ)
        associate_julia_struct(tcp.handle, tcp)
        finalizer(tcp, uvfinalize)
        return tcp
    end
end
function TCPSocket()
    tcp = TCPSocket(Libc.malloc(_sizeof_uv_tcp), StatusUninit)
    err = ccall(:uv_tcp_init, Cint, (Ptr{Void}, Ptr{Void}),
                eventloop(), tcp.handle)
    uv_error("failed to create tcp socket", err)
    tcp.status = StatusInit
    return tcp
end

mutable struct TCPServer <: LibuvServer
    handle::Ptr{Void}
    status::Int
    connectnotify::Condition
    closenotify::Condition

    function TCPServer(handle::Ptr{Void}, status)
        tcp = new(
            handle,
            status,
            Condition(),
            Condition())
        associate_julia_struct(tcp.handle, tcp)
        finalizer(tcp, uvfinalize)
        return tcp
    end
end
function TCPServer()
    tcp = TCPServer(Libc.malloc(_sizeof_uv_tcp), StatusUninit)
    err = ccall(:uv_tcp_init, Cint, (Ptr{Void}, Ptr{Void}),
                eventloop(), tcp.handle)
    uv_error("failed to create tcp server", err)
    tcp.status = StatusInit
    return tcp
end

isreadable(io::TCPSocket) = isopen(io) || nb_available(io) > 0
iswritable(io::TCPSocket) = isopen(io) && io.status != StatusClosing

## VARIOUS METHODS TO BE MOVED TO BETTER LOCATION

_jl_connect_raw(sock::TCPSocket, sockaddr::Ptr{Void}) =
    ccall(:jl_connect_raw, Int32, (Ptr{Void}, Ptr{Void}, Ptr{Void}), sock.handle, sockaddr, uv_jl_connectcb::Ptr{Void})
_jl_sockaddr_from_addrinfo(addrinfo::Ptr{Void}) =
    ccall(:jl_sockaddr_from_addrinfo, Ptr{Void}, (Ptr{Void}, ), addrinfo)
_jl_sockaddr_set_port(ptr::Ptr{Void}, port::UInt16) =
    ccall(:jl_sockaddr_set_port, Void, (Ptr{Void}, UInt16), ptr, port)

accept(server::TCPServer) = accept(server, TCPSocket())

# Libuv will internally reset the readable and writable flags on
# this pipe after it has successfully accepted the connection, to
# remember that before that this is an invalid pipe
accept(server::PipeServer) = accept(server, init_pipe!(PipeEndpoint();
    readable=false, writable=false, julia_only=true))

# UDP

mutable struct UDPSocket <: LibuvStream
    handle::Ptr{Void}
    status::Int
    recvnotify::Condition
    sendnotify::Condition
    closenotify::Condition

    function UDPSocket(handle::Ptr{Void}, status)
        udp = new(
            handle,
            status,
            Condition(),
            Condition(),
            Condition())
        associate_julia_struct(udp.handle, udp)
        finalizer(udp, uvfinalize)
        return udp
    end
end
function UDPSocket()
    this = UDPSocket(Libc.malloc(_sizeof_uv_udp), StatusUninit)
    err = ccall(:uv_udp_init, Cint, (Ptr{Void}, Ptr{Void}),
                eventloop(), this.handle)
    uv_error("failed to create udp socket", err)
    this.status = StatusInit
    return this
end

show(io::IO, stream::UDPSocket) = print(io, typeof(stream), "(", uv_status_string(stream), ")")

function _uv_hook_close(sock::UDPSocket)
    sock.handle = C_NULL
    sock.status = StatusClosed
    notify(sock.closenotify)
    notify(sock.sendnotify)
    notify_error(sock.recvnotify,EOFError())
end

# Disables dual stack mode.
const UV_TCP_IPV6ONLY = 1

# Disables dual stack mode. Only available when using ipv6 binf
const UV_UDP_IPV6ONLY = 1

# Indicates message was truncated because read buffer was too small. The
# remainder was discarded by the OS.
const UV_UDP_PARTIAL = 2

# Indicates if SO_REUSEADDR will be set when binding the handle in uv_udp_bind. This sets
# the SO_REUSEPORT socket flag on the BSDs and OS X. On other Unix platforms, it sets the
# SO_REUSEADDR flag. What that means is that multiple threads or processes can bind to the
# same address without error (provided they all set the flag) but only the last one to bind
# will receive any traffic, in effect "stealing" the port from the previous listener.
const UV_UDP_REUSEADDR = 4

##

_bind(sock::TCPServer, host::IPv4, port::UInt16, flags::UInt32 = UInt32(0)) = ccall(:jl_tcp_bind, Int32, (Ptr{Void}, UInt16, UInt32, Cuint),
            sock.handle, hton(port), hton(host.host), flags)

_bind(sock::TCPServer, host::IPv6, port::UInt16, flags::UInt32 = UInt32(0)) = ccall(:jl_tcp_bind6, Int32, (Ptr{Void}, UInt16, Ptr{UInt128}, Cuint),
            sock.handle, hton(port), Ref(hton(host.host)), flags)

_bind(sock::UDPSocket, host::IPv4, port::UInt16, flags::UInt32 = UInt32(0)) = ccall(:jl_udp_bind, Int32, (Ptr{Void}, UInt16, UInt32, UInt32),
            sock.handle, hton(port), hton(host.host), flags)

_bind(sock::UDPSocket, host::IPv6, port::UInt16, flags::UInt32 = UInt32(0)) = ccall(:jl_udp_bind6, Int32, (Ptr{Void}, UInt16, Ptr{UInt128}, UInt32),
            sock.handle, hton(port), Ref(hton(host.host)), flags)

"""
    bind(socket::Union{UDPSocket, TCPSocket}, host::IPAddr, port::Integer; ipv6only=false, reuseaddr=false, kws...)

Bind `socket` to the given `host:port`. Note that `0.0.0.0` will listen on all devices.

* The `ipv6only` parameter disables dual stack mode. If `ipv6only=true`, only an IPv6 stack is created.
* If `reuseaddr=true`, multiple threads or processes can bind to the same address without error
  if they all set `reuseaddr=true`, but only the last to bind will receive any traffic.
"""
function bind(sock::Union{TCPServer, UDPSocket}, host::IPAddr, port::Integer; ipv6only = false, reuseaddr = false, kws...)
    if sock.status != StatusInit
        error("$(typeof(sock)) is not in initialization state")
    end
    flags = 0
    if isa(host,IPv6) && ipv6only
        flags |= isa(sock, UDPSocket) ? UV_UDP_IPV6ONLY : UV_TCP_IPV6ONLY
    end
    if isa(sock, UDPSocket) && reuseaddr
        flags |= UV_UDP_REUSEADDR
    end
    err = _bind(sock, host, UInt16(port), UInt32(flags))
    if err < 0
        if err != UV_EADDRINUSE && err != UV_EACCES && err != UV_EADDRNOTAVAIL
            #TODO: this codepath is not currently tested
            throw(UVError("bind", err))
        else
            return false
        end
    end
    sock.status = StatusOpen
    isa(sock, UDPSocket) && setopt(sock; kws...)
    return true
end

bind(sock::TCPServer, addr::InetAddr) = bind(sock, addr.host, addr.port)

"""
    setopt(sock::UDPSocket; multicast_loop = nothing, multicast_ttl=nothing, enable_broadcast=nothing, ttl=nothing)

Set UDP socket options.

* `multicast_loop`: loopback for multicast packets (default: `true`).
* `multicast_ttl`: TTL for multicast packets (default: `nothing`).
* `enable_broadcast`: flag must be set to `true` if socket will be used for broadcast messages,
  or else the UDP system will return an access error (default: `false`).
* `ttl`: Time-to-live of packets sent on the socket (default: `nothing`).
"""
function setopt(sock::UDPSocket; multicast_loop = nothing, multicast_ttl=nothing, enable_broadcast=nothing, ttl=nothing)
    if sock.status == StatusUninit
        error("Cannot set options on uninitialized socket")
    end
    if multicast_loop !== nothing
        uv_error("multicast_loop",ccall(:uv_udp_set_multicast_loop,Cint,(Ptr{Void},Cint),sock.handle,multicast_loop) < 0)
    end
    if multicast_ttl !== nothing
        uv_error("multicast_ttl",ccall(:uv_udp_set_multicast_ttl,Cint,(Ptr{Void},Cint),sock.handle,multicast_ttl))
    end
    if enable_broadcast !== nothing
        uv_error("enable_broadcast",ccall(:uv_udp_set_broadcast,Cint,(Ptr{Void},Cint),sock.handle,enable_broadcast))
    end
    if ttl !== nothing
        uv_error("ttl",ccall(:uv_udp_set_ttl,Cint,(Ptr{Void},Cint),sock.handle,ttl))
    end
end

"""
    recv(socket::UDPSocket)

Read a UDP packet from the specified socket, and return the bytes received. This call blocks.
"""
function recv(sock::UDPSocket)
    addr, data = recvfrom(sock)
    return data
end

"""
    recvfrom(socket::UDPSocket) -> (address, data)

Read a UDP packet from the specified socket, returning a tuple of `(address, data)`, where
`address` will be either IPv4 or IPv6 as appropriate.
"""
function recvfrom(sock::UDPSocket)
    # If the socket has not been bound, it will be bound implicitly to ::0 and a random port
    if sock.status != StatusInit && sock.status != StatusOpen && sock.status != StatusActive
        error("UDPSocket is not initialized and open")
    end
    if ccall(:uv_is_active, Cint, (Ptr{Void},), sock.handle) == 0
        uv_error("recv_start", ccall(:uv_udp_recv_start, Cint, (Ptr{Void}, Ptr{Void}, Ptr{Void}),
                                    sock.handle, uv_jl_alloc_buf::Ptr{Void}, uv_jl_recvcb::Ptr{Void}))
    end
    sock.status = StatusActive
    return stream_wait(sock, sock.recvnotify)::Tuple{Union{IPv4, IPv6}, Vector{UInt8}}
end

alloc_buf_hook(sock::UDPSocket, size::UInt) = (Libc.malloc(size), size)

function uv_recvcb(handle::Ptr{Void}, nread::Cssize_t, buf::Ptr{Void}, addr::Ptr{Void}, flags::Cuint)
    # C signature documented as (*uv_udp_recv_cb)(...)
    sock = @handle_as handle UDPSocket
    if nread < 0
        Libc.free(buf_addr)
        notify_error(sock.recvnotify, UVError("recv", nread))
    elseif flags & UV_UDP_PARTIAL > 0
        Libc.free(buf_addr)
        notify_error(sock.recvnotify, "Partial message received")
    else
        buf_addr = ccall(:jl_uv_buf_base, Ptr{Void}, (Ptr{Void},), buf)
        buf_size = ccall(:jl_uv_buf_len, Csize_t, (Ptr{Void},), buf)
        # need to check the address type in order to convert to a Julia IPAddr
        addrout = if addr == C_NULL
                      IPv4(0)
                  elseif ccall(:jl_sockaddr_in_is_ip4, Cint, (Ptr{Void},), addr) == 1
                      IPv4(ntoh(ccall(:jl_sockaddr_host4, UInt32, (Ptr{Void},), addr)))
                  else
                      tmp = [UInt128(0)]
                      ccall(:jl_sockaddr_host6, UInt32, (Ptr{Void}, Ptr{UInt8}), addr, pointer(tmp))
                      IPv6(ntoh(tmp[1]))
                  end
        buf = unsafe_wrap(Array, convert(Ptr{UInt8}, buf_addr), Int(nread), true)
        notify(sock.recvnotify, (addrout, buf))
    end
    ccall(:uv_udp_recv_stop, Cint, (Ptr{Void},), sock.handle)
    sock.status = StatusOpen
    nothing
end

function _send(sock::UDPSocket, ipaddr::IPv4, port::UInt16, buf)
    ccall(:jl_udp_send, Cint, (Ptr{Void}, UInt16, UInt32, Ptr{UInt8}, Csize_t, Ptr{Void}),
          sock.handle, hton(port), hton(ipaddr.host), buf, sizeof(buf), uv_jl_sendcb::Ptr{Void})
end

function _send(sock::UDPSocket, ipaddr::IPv6, port::UInt16, buf)
    ccall(:jl_udp_send6, Cint, (Ptr{Void}, UInt16, Ptr{UInt128}, Ptr{UInt8}, Csize_t, Ptr{Void}),
          sock.handle, hton(port), &hton(ipaddr.host), buf, sizeof(buf), uv_jl_sendcb::Ptr{Void})
end

"""
    send(socket::UDPSocket, host, port::Integer, msg)

Send `msg` over `socket` to `host:port`.
"""
function send(sock::UDPSocket,ipaddr,port,msg)
    # If the socket has not been bound, it will be bound implicitly to ::0 and a random port
    if sock.status != StatusInit && sock.status != StatusOpen && sock.status != StatusActive
        error("UDPSocket is not initialized and open")
    end
    uv_error("send", _send(sock, ipaddr, UInt16(port), msg))
    stream_wait(sock, sock.sendnotify)
    nothing
end

function uv_sendcb(handle::Ptr{Void}, status::Cint)
    sock = @handle_as handle UDPSocket
    if status < 0
        notify_error(sock.sendnotify, UVError("UDP send failed", status))
    end
    notify(sock.sendnotify)
    Libc.free(handle)
    nothing
end

##

mutable struct DNSError <: Exception
    host::AbstractString
    code::Int32
end

function show(io::IO, err::DNSError)
    print(io, "DNSError: ", err.host, ", ", struverror(err.code),
                                      " (", uverrorname(err.code), ")")
end

callback_dict = ObjectIdDict()

function uv_getaddrinfocb(req::Ptr{Void}, status::Cint, addrinfo::Ptr{Void})
    data = ccall(:jl_uv_getaddrinfo_data, Ptr{Void}, (Ptr{Void},), req)
    data == C_NULL && return
    cb = unsafe_pointer_to_objref(data)::Function
    pop!(callback_dict,cb) # using pop forces an error if cb not in callback_dict
    if status != 0 || addrinfo == C_NULL
        cb(UVError("uv_getaddrinfocb received an unexpected status code", status))
    else
        freeaddrinfo = addrinfo
        while addrinfo != C_NULL
            sockaddr = ccall(:jl_sockaddr_from_addrinfo, Ptr{Void}, (Ptr{Void},), addrinfo)
            if ccall(:jl_sockaddr_is_ip4, Int32, (Ptr{Void},), sockaddr) == 1
                cb(IPv4(ntoh(ccall(:jl_sockaddr_host4, UInt32, (Ptr{Void},), sockaddr))))
                break
            #elseif ccall(:jl_sockaddr_is_ip6, Int32, (Ptr{Void},), sockaddr) == 1
            #    host = Array{UInt128}(1)
            #    scope_id = ccall(:jl_sockaddr_host6, UInt32, (Ptr{Void}, Ptr{UInt128}), sockaddr, host)
            #    cb(IPv6(ntoh(host[1])))
            #    break
            end
            addrinfo = ccall(:jl_next_from_addrinfo, Ptr{Void}, (Ptr{Void},), addrinfo)
        end
        ccall(:uv_freeaddrinfo, Void, (Ptr{Void},), freeaddrinfo)
    end
    Libc.free(req)
    nothing
end

function getaddrinfo(cb::Function, host::String)
    isascii(host) || error("non-ASCII hostname: $host")
    callback_dict[cb] = cb
    status = ccall(:jl_getaddrinfo, Int32, (Ptr{Void}, Cstring, Ptr{UInt8}, Any, Ptr{Void}),
                   eventloop(), host, C_NULL, cb, uv_jl_getaddrinfocb::Ptr{Void})
    if status == UV_EINVAL
        throw(ArgumentError("Invalid uv_getaddrinfo() agument"))
    elseif status in [UV_ENOMEM, UV_ENOBUFS]
        throw(OutOfMemoryError())
    elseif status < 0
        throw(UVError("uv_getaddrinfo returned an unexpected error code", status))
    end
    return nothing
end
getaddrinfo(cb::Function, host::AbstractString) = getaddrinfo(cb, String(host))

"""
    getaddrinfo(host::AbstractString) -> IPAddr

Gets the IP address of the `host` (may have to do a DNS lookup)
"""
function getaddrinfo(host::String)
    c = Condition()
    getaddrinfo(host) do IP
        notify(c,IP)
    end
    r = wait(c)
    if isa(r, UVError)
        r = r::UVError
        code = r.code
        if code in (UV_EAI_ADDRFAMILY, UV_EAI_AGAIN, UV_EAI_BADFLAGS,
                    UV_EAI_BADHINTS, UV_EAI_CANCELED, UV_EAI_FAIL,
                    UV_EAI_FAMILY, UV_EAI_NODATA, UV_EAI_NONAME,
                    UV_EAI_OVERFLOW, UV_EAI_PROTOCOL, UV_EAI_SERVICE,
                    UV_EAI_SOCKTYPE)
            throw(DNSError(host, code))
        elseif code == UV_EAI_MEMORY
            throw(OutOfMemoryError())
        else
            throw(SystemError("uv_getaddrinfocb", -code))
        end
    end
    return r::IPAddr
end
getaddrinfo(host::AbstractString) = getaddrinfo(String(host))

const _sizeof_uv_interface_address = ccall(:jl_uv_sizeof_interface_address,Int32,())

"""
    getipaddr() -> IPAddr

Get the IP address of the local machine.
"""
function getipaddr()
    addr_ref = Ref{Ptr{UInt8}}(C_NULL)
    count_ref = Ref{Int32}(1)
    lo_present = false
    err = ccall(:jl_uv_interface_addresses, Int32, (Ref{Ptr{UInt8}}, Ref{Int32}), addr_ref, count_ref)
    uv_error("getlocalip", err)
    addr, count = addr_ref[], count_ref[]
    for i = 0:(count-1)
        current_addr = addr + i*_sizeof_uv_interface_address
        if 1 == ccall(:jl_uv_interface_address_is_internal, Int32, (Ptr{UInt8},), current_addr)
            lo_present = true
            continue
        end
        sockaddr = ccall(:jl_uv_interface_address_sockaddr, Ptr{Void}, (Ptr{UInt8},), current_addr)
        if ccall(:jl_sockaddr_in_is_ip4, Int32, (Ptr{Void},), sockaddr) == 1
            rv = IPv4(ntoh(ccall(:jl_sockaddr_host4, UInt32, (Ptr{Void},), sockaddr)))
            ccall(:uv_free_interface_addresses, Void, (Ptr{UInt8}, Int32), addr, count)
            return rv
        # Uncomment to enbable IPv6
        #elseif ccall(:jl_sockaddr_in_is_ip6, Int32, (Ptr{Void},), sockaddr) == 1
        #   host = Array{UInt128}(1)
        #   ccall(:jl_sockaddr_host6, UInt32, (Ptr{Void}, Ptr{UInt128}), sockaddrr, host)
        #   return IPv6(ntoh(host[1]))
        end
    end
    ccall(:uv_free_interface_addresses, Void, (Ptr{UInt8}, Int32), addr, count)
    return lo_present ? ip"127.0.0.1" : error("No networking interface available")
end

##

function connect!(sock::TCPSocket, host::IPv4, port::Integer)
    if sock.status != StatusInit
        error("TCPSocket is not in initialization state")
    end
    if !(0 <= port <= typemax(UInt16))
        throw(ArgumentError("port out of range, must be 0 ≤ port ≤ 65535, got $port"))
    end
    uv_error("connect", ccall(:jl_tcp4_connect, Int32, (Ptr{Void}, UInt32, UInt16, Ptr{Void}),
                             sock.handle, hton(host.host), hton(UInt16(port)), uv_jl_connectcb::Ptr{Void}))
    sock.status = StatusConnecting
    nothing
end

function connect!(sock::TCPSocket, host::IPv6, port::Integer)
    if sock.status != StatusInit
        error("TCPSocket is not in initialization state")
    end
    if !(0 <= port <= typemax(UInt16))
        throw(ArgumentError("port out of range, must be 0 ≤ port ≤ 65535, got $port"))
    end
    uv_error("connect", ccall(:jl_tcp6_connect, Int32, (Ptr{Void}, Ptr{UInt128}, UInt16, Ptr{Void}),
                             sock.handle, &hton(host.host), hton(UInt16(port)), uv_jl_connectcb::Ptr{Void}))
    sock.status = StatusConnecting
    nothing
end

connect!(sock::TCPSocket, addr::InetAddr) = connect!(sock, addr.host, addr.port)

# Default Host to localhost

"""
    connect([host], port::Integer) -> TCPSocket

Connect to the host `host` on port `port`.
"""
connect(sock::TCPSocket, port::Integer) = connect(sock,IPv4(127,0,0,1), port)
connect(port::Integer) = connect(IPv4(127,0,0,1), port)

# Valid connect signatures for TCP
connect(host::AbstractString, port::Integer) = connect(TCPSocket(), host, port)
connect(addr::IPAddr, port::Integer) = connect(TCPSocket(), addr, port)
connect(addr::InetAddr) = connect(TCPSocket(), addr)

default_connectcb(sock,status) = nothing

function connect!(sock::TCPSocket, host::AbstractString, port::Integer)
    if sock.status != StatusInit
        error("TCPSocket is not in initialization state")
    end
    ipaddr = getaddrinfo(host)
    sock.status = StatusInit
    connect!(sock,ipaddr,port)
    sock.status = StatusConnecting
    return sock
end

##

"""
    listen([addr, ]port::Integer; backlog::Integer=BACKLOG_DEFAULT) -> TCPServer

Listen on port on the address specified by `addr`.
By default this listens on `localhost` only.
To listen on all interfaces pass `IPv4(0)` or `IPv6(0)` as appropriate.
`backlog` determines how many connections can be pending (not having
called [`accept`](@ref)) before the server will begin to
reject them. The default value of `backlog` is 511.
"""
function listen(addr; backlog::Integer=BACKLOG_DEFAULT)
    sock = TCPServer()
    !bind(sock, addr) && error("cannot bind to port; may already be in use or access denied")
    listen(sock; backlog=backlog)
    return sock
end
listen(port::Integer; backlog::Integer=BACKLOG_DEFAULT) = listen(IPv4(UInt32(0)), port; backlog=backlog)
listen(host::IPAddr, port::Integer; backlog::Integer=BACKLOG_DEFAULT) = listen(InetAddr(host, port); backlog=backlog)

function listen(callback, server::Union{TCPSocket, UDPSocket})
    @async begin
        local client = TCPSocket()
        while isopen(server)
            err = accept_nonblock(server, client)
            if err == 0
                callback(client)
                client = TCPSocket()
            elseif err != UV_EAGAIN
                uv_error("accept", err)
            else
                stream_wait(server, server.connectnotify)
            end
        end
    end
    return sock
end

##

function accept_nonblock(server::TCPServer, client::TCPSocket)
    if client.status != StatusInit
        error("client TCPSocket is not in initialization state")
    end
    err = ccall(:uv_accept, Int32, (Ptr{Void}, Ptr{Void}), server.handle, client.handle)
    if err == 0
        client.status = StatusOpen
    end
    return err
end

function accept_nonblock(server::TCPServer)
    client = TCPSocket()
    uv_error("accept", accept_nonblock(server, client))
    return client
end

## Utility functions

"""
    listenany([host::IPAddr,] port_hint) -> (UInt16, TCPServer)

Create a `TCPServer` on any port, using hint as a starting point. Returns a tuple of the
actual port that the server was created on and the server itself.
"""
function listenany(host::IPAddr, default_port)
    addr = InetAddr(host, default_port)
    while true
        sock = TCPServer()
        if bind(sock, addr) && trylisten(sock) == 0
            return (addr.port, sock)
        end
        close(sock)
        addr = InetAddr(addr.host, addr.port + 1)
        if addr.port == default_port
            error("no ports available")
        end
    end
end

listenany(default_port) = listenany(IPv4(UInt32(0)), default_port)

"""
    getsockname(sock::Union{TCPServer, TCPSocket}) -> (IPAddr, UInt16)

Get the IP address and the port that the given `TCPSocket` is connected to
(or bound to, in the case of `TCPServer`).
"""
function getsockname(sock::Union{TCPServer,TCPSocket})
    rport = Ref{Cushort}(0)
    raddress = zeros(UInt8, 16)
    rfamily = Ref{Cuint}(0)
    r = if isa(sock, TCPServer)
        ccall(:jl_tcp_getsockname, Int32,
                (Ptr{Void}, Ref{Cushort}, Ptr{Void}, Ref{Cuint}),
                sock.handle, rport, raddress, rfamily)
    else
        ccall(:jl_tcp_getpeername, Int32,
                (Ptr{Void}, Ref{Cushort}, Ptr{Void}, Ref{Cuint}),
                sock.handle, rport, raddress, rfamily)
    end
    uv_error("cannot obtain socket name", r)
    if r == 0
        port = ntoh(rport[])
        if rfamily[] == 2 # AF_INET
            addrv4 = raddress[1:4]
            naddr = ntoh(unsafe_load(Ptr{Cuint}(pointer(addrv4)), 1))
            addr = IPv4(naddr)
        elseif rfamily[] == @static is_windows() ? 23 : (@static is_apple() ? 30 : 10) # AF_INET6
            naddr = ntoh(unsafe_load(Ptr{UInt128}(pointer(raddress)), 1))
            addr = IPv6(naddr)
        else
            error("unsupported address family: $(getindex(rfamily))")
        end
    else
        error("cannot obtain socket name")
    end
    return addr, port
end


# Types used in Arena.jl

const lmpoint = HyperSphere(Point(0.,0,0), 0.05)
const greenMat = MeshPhongMaterial(color=RGBA(0, 1, 0, 0.5))
const redMat = MeshPhongMaterial(color=RGBA(0, 1, 0, 0.5))

"""
	$(TYPEDEF)
Type for 2d visualization
"""
struct BotVis2
    vis::Visualizer
    # poses::Dict{Symbol, NTuple{3,Float64}}
    # landmarks::Dict{Symbol, NTuple{3,Float64}}
    cachevars::Dict{Symbol, Tuple{Symbol, Vector{Bool}, Vector{Float64}}}
end

mutable struct SubmapColorCheat
  colors
  SubmapColorCheat(;colors::Vector=
    [ RGB(0.651, 0.808, 0.890),
      RGB(0.122, 0.471, 0.706),
      RGB(0.698, 0.875, 0.541),
      RGB(0.200, 0.627, 0.172),
      RGB(0.984, 0.604, 0.600),
      RGB(0.890, 0.102, 0.110),
      RGB(0.992, 0.749, 0.043)]
  ) = new(colors)
end

mutable struct ArcPointsRangeSolve <: Function
  x1::Vector{Float64}
  x2::Vector{Float64}
  x3::Vector{Float64}
  r::Float64
  center::Vector{Float64}
  angle::Float64
  axis::Vector{Float64}
  ArcPointsRangeSolve(x1::Vector{Float64}, x2::Vector{Float64}, r::Float64) = new(x1,x2,zeros(0),r, zeros(2), 0.0, zeros(3))
  ArcPointsRangeSolve(x1::Vector{Float64}, x2::Vector{Float64}, x3::Vector{Float64}, r::Float64) = new(x1,x2,x3,r, zeros(3), 0.0, zeros(3))
end


export iauUtctai
"""
Time scale transformation:  Coordinated Universal Time, UTC, to
International Atomic Time, TAI.

This function is part of the International Astronomical Union's
SOFA (Standards of Fundamental Astronomy) software collection.

Status:  canonical.

Given:
   utc1,utc2  double   UTC as a 2-part quasi Julian Date (Notes 1-4)

Returned:
   tai1,tai2  double   TAI as a 2-part Julian Date (Note 5)

Returned (function value):
               int      status: +1 = dubious year (Note 3)
                                 0 = OK
                              -1 = unacceptable date

Notes:

   1. utc1+utc2 is quasi Julian Date (see Note 2), apportioned in any
   convenient way between the two arguments, for example where utc1
   is the Julian Day Number and utc2 is the fraction of a day.

   2. JD cannot unambiguously represent UTC during a leap second unless
   special measures are taken.  The convention in the present
   function is that the JD day represents UTC days whether the
   length is 86399, 86400 or 86401 SI seconds.  In the 1960-1972 era
   there were smaller jumps (in either direction) each time the
   linear UTC(TAI) expression was changed, and these "mini-leaps"
   are also included in the SOFA convention.

   3. The warning status "dubious year" flags UTCs that predate the
   introduction of the time scale or that are too far in the future
   to be trusted.  See iauDat for further details.

   4. The function iauDtf2d converts from calendar date and time of day
   into 2-part Julian Date, and in the case of UTC implements the
   leap-second-ambiguity convention described above.

   5. The returned TAI1,TAI2 are such that their sum is the TAI Julian
   Date.

Called:
   iauJd2cal    JD to Gregorian calendar
   iauDat       delta(AT) = TAI-UTC
   iauCal2jd    Gregorian calendar to JD

References:

   McCarthy, D. D., Petit, G. (eds.), IERS Conventions (2003),
   IERS Technical Note No. 32, BKG (2004)

   Explanatory Supplement to the Astronomical Almanac,
   P. Kenneth Seidelmann (ed), University Science Books (1992)

This revision:  2013 July 26

SOFA release 2018-01-30

Copyright (C) 2018 IAU SOFA Board.  See notes at end.
"""

# int iauUtctai(double utc1, double utc2, double *tai1, double *tai2)

function iauUtctai(utc1::Real, utc2::Real)
   ref_tai1 = Ref{Float64}(0.0)
   ref_tai2 = Ref{Float64}(0.0)

   status = ccall((:iauUtctai, libsofa_c), Cint,
                   (Cdouble, Cdouble, Ref{Cdouble}, Ref{Cdouble}),
                   convert(Float64, utc1), convert(Float64, utc2),
                   ref_tai1, ref_tai2)

   return status, ref_tai1[], ref_tai2[]
end


module WhiskerTracking

#Standard Library
using Statistics,Random,Distributed,SharedArrays,DelimitedFiles,LinearAlgebra, Libdl, Dates

#Deep Learning Libraries
#using CuArrays, CuArrays.CUFFT, CUDAnative, Knet
using CUDA, Knet

using Gtk.ShortNames, Cairo, Images, ImageFiltering, MAT, JLD2, Interpolations, DSP,
FFTW, IterTools, FFMPEG, StackedHourglass, Polynomials, StatsBase

if VERSION > v"0.7-"
    const Void = Nothing
    const setproperty! = set_gtk_property!
    const getproperty = get_gtk_property
    const unshift! = pushfirst!
    const is_windows() = Sys.iswindows()
    const is_unix() = Sys.isunix()
    const find = findall
    const indmax = argmax
end

include("config.jl")

function __init__()
    ccall((:Load_Params_File,libwhisk_path),Int32,(Cstring,),jt_parameters)
end

include("types.jl")

include("deep_learning/helper.jl")
include("deep_learning/save_load.jl")
include("deep_learning/prediction.jl")
include("deep_learning/config.jl")
include("deep_learning/training.jl")

include("gui.jl")
include("janelia_tracker.jl")
include("processing.jl")
include("image_processing.jl")
include("analysis.jl")
include("save_load.jl")
include("discrete.jl")

include("gui/manual.jl")
include("gui/analog.jl")
include("gui/whisker_pad.jl")
include("gui/discrete.jl")
include("gui/mask.jl")
include("gui/pole.jl")
include("gui/view.jl")
include("gui/tracing.jl")
include("gui/image.jl")
include("gui/janelia.jl")
include("gui/export.jl")
include("gui/contact.jl")
include("gui/deeplearning.jl")

include("drawing_tools/draw.jl")
include("drawing_tools/erase.jl")
include("drawing_tools/shapes.jl")

include("analysis/follicle.jl")
include("analysis/kinematics.jl")
include("analysis/forces.jl")
include("analysis/touch.jl")
include("analysis/math.jl")

include("precompile.jl")
_precompile()

end


@cdef :PyFloat_FromDouble PyPtr (Cdouble,)
@cdef :PyFloat_AsDouble Cdouble (PyPtr,)

const PyFloat_Type__ref = Ref(PyNULL)
PyFloat_Type() = pyglobal(PyFloat_Type__ref, :PyFloat_Type)

PyFloat_Check(o) = Py_TypeCheck(o, PyFloat_Type())

PyFloat_CheckExact(o) = Py_TypeCheckExact(o, PyFloat_Type())

PyFloat_From(o::Union{Float16,Float32,Float64}) = PyFloat_FromDouble(o)

# "Floatable" means a 'float' or anything with a '__float__' method
PyFloatable_TryConvertRule_convert(o, ::Type{S}) where {S} = begin
    x = PyFloat_AsDouble(o)
    ism1(x) && PyErr_IsSet() && return -1
    putresult(convert(S, x))
end

PyFloatable_TryConvertRule_tryconvert(o, ::Type{S}) where {S} = begin
    x = PyFloat_AsDouble(o)
    ism1(x) && PyErr_IsSet() && return -1
    putresult(tryconvert(S, x))
end


using DataFrames  # readtable function
# using Requests
type Bus
   nodeID::Int
   root::Int
   Pd::Float64
   Qd::Float64
   Vmax::Float64
   Vmin::Float64
   B::Float64
   R::Float64
   X::Float64
   children::Vector{Int}
   ancestor::Vector{Int}
   genids::Int
   function Bus(nodeID, root, Pd, Qd, B, R, X, Vmax, Vmin)
      b = new(nodeID, root, Pd, Qd)
      b.Vmax = Vmax
      b.Vmin = Vmin
      b.R = R
      b.X = X
      b.B = B
      b.children = Int[]
      b.ancestor = Int[]
      # b.genids = 0
      return b
   end
end
#################################################################
type Generator
   genID::Int
   busidx::Int
   Pgmax::Float64
   Pgmin::Float64
   Qgmax::Float64
   Qgmin::Float64
   cost::Float64
   function Generator( busidx, Pgmax, Pgmin, Qgmax, Qgmin, cost)
      g = new(busidx)
      g.busidx = busidx
      g.cost = cost
      g.Pgmax = Pgmax
      g.Pgmin = Pgmin
      g.Qgmax = Qgmax
      g.Qgmin = Qgmin
      return g
   end
end

##################################################################
type Line
   arcID::Int
   tail::Int # the "to" node
   head::Int # the "from" node
   r::Float64 # the resistance value
   x::Float64 # the reactance value
   u::Float64 # the capacity of the line
   function Line(arcID, tail, head, r, x, u)
      line = new(arcID, tail, head, r, x)
      line.u = u
      return line
   end
end
#########################################
## storage data
type Storage
   ID::Int
   busidx::Int
   Pkmax::Float64
   Ekmax::Float64
   Aleph::Float64
   Co::Float64
   Cp::Float64
   Ce::Float64
   function Storage(ID, busidx, Pkmax, Ekmax, Aleph, Co, Cp, Ce)
      s = new(ID)
      s.ID = ID
      s.busidx = busidx
      s.Pkmax = Pkmax
      s.Ekmax = Ekmax
      s.Aleph = Aleph
      s.Co = Co
      s.Cp = Cp
      s.Ce = Ce
      return s
   end
end
#########################################
## station data
type Station
   ID::Int
   busidx::Int
   Cf::Float64
   function Station(ID, busidx, Cf)
      st = new(ID)
      st.ID = ID
      st.busidx = busidx
      st.Cf = Cf
      return st
   end
end
######################################################################################################

function DataImport(filename_Node, filename_Generator, filename_Line, filename_Storage, filename_Station, filename_SMP)

   ######################################################################################################
   # Bus/Node Data
   # busmat = readtable(filename_Node)
   busmat = readcsv(filename_Node, header=true)[1]

   buses = Bus[]
   busIDmap = Dict()
   for i in 1:size(busmat,1)

      nodeID = i
      busIDmap[busmat[i,1]] = i

      if i==1
         root = busIDmap[busmat[i,1]]
      else
         root = 0
      end

      Pd = busmat[i,2]
      Qd = busmat[i,3]
      R = busmat[i,6]
      X = busmat[i,7]

      Vmax = busmat[i,4]
      Vmin = busmat[i,5]
      B = busmat[i,8]

      b = Bus(nodeID, root, Pd, Qd, B, R, X, Vmax, Vmin)
      push!(buses, b)
   end

   #######################################################################################################
   ## generator data
   generatorlist = Int[]
   generators = Generator[]
   # genmat = readtable(filename_Generator)
   genmat = readcsv(filename_Generator, header=true)[1]

   for i in 1:size(genmat,1)
      busidx = genmat[i,1]
      Pgmax = genmat[i,2]
      Pgmin = genmat[i,3]
      Qgmax = genmat[i,4]
      Qgmin = genmat[i,5]
      cost = genmat[i,6]

      g = Generator(busidx, Pgmax, Pgmin, Qgmax, Qgmin, cost)
      push!(generators, g)
      # setg(buses[busidx], i)
   end
   #for g in 1:length(generators)
   #   generators[g].cost = genmat[g,4]
   #end
   #################################################################
   ## branch data
   # branchmat = readtable(filename_Line)
   branchmat = readcsv(filename_Line, header=true)[1]

   lines = Line[]
   for i in 1:size(branchmat,1)
      fbus = busIDmap[branchmat[i,2]]
      tbus = busIDmap[branchmat[i,1]]
      abus = busIDmap[branchmat[i,1]]
      x = branchmat[i,4]
      r = branchmat[i,3]
      u = branchmat[i,6]  # flow limit
      push!(buses[tbus].children, fbus)#children
      push!(buses[fbus].ancestor, abus)#ancestor
      l = Line(i, tbus, fbus, r, x, u)
      push!(lines,l)
   end
   #########################################
   ## storage data
   # storagemat = readtable(filename_Storage)
   storagemat = readcsv(filename_Storage, header=true)[1]

   storages = Storage[]
   for i in 1:size(storagemat,1)
      storageID = i
      busidx = 1
      Pkmax = storagemat[i,2]
      Ekmax = storagemat[i,3]
      Aleph = storagemat[i,4]
      Co = storagemat[i,5]
      Cp = storagemat[i,6]
      Ce = storagemat[i,7]

      temp = Storage(storageID, busidx, Pkmax, Ekmax, Aleph, Co, Cp, Ce)
      push!(storages,temp)
   end

   #########################################
   ## charging/discharing station data
   # stationmat = readtable(filename_Station)
   stationmat = readcsv(filename_Station, header=true)[1]
   stations = Station[]
   for i in 1:size(stationmat,1)
      stationID = i
      busidx = stationmat[i,2]
      Cf = stationmat[i,3]

      temp = Station(stationID, busidx, Cf)
      push!(stations,temp)
   end

   #########################################
   SMP_raw = readcsv(filename_SMP, header=true)[1]
   SMP = SMP_raw[3:end]

   return buses, generators,lines, storages, stations, SMP
end


@testset "`Samples` API" begin
    root = mktempdir()
    signals = Signal[]
    possible_recordings = (uuid4(), uuid4(), uuid4())
    expected_sample_types = (UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64)
    expected_parameters = [(recording=rand(possible_recordings),
                            file_path=joinpath(root, "x_$(i)_file"),
                            file_format=rand(("lpcm", "lpcm.zst")),
                            kind="x_$i",
                            channels=["a_$i", "b_$i", "c_$i"],
                            sample_unit="unit_$i",
                            sample_resolution_in_unit=rand((0.25, 1)),
                            sample_offset_in_unit=rand((-0.25, 0.25)),
                            sample_type=expected_sample_types[i],
                            sample_rate=rand((128, 50.5)),
                            start=Second(rand(0:30))) for i in 1:length(expected_sample_types)]
    for params in expected_parameters
        info = Onda.extract_samples_info(params)
        data = rand(params.sample_type, 3, sample_count(info, Second(rand(1:15))))
        samples = Samples(data, info, true)
        start = Second(rand(0:30))
        signal = store(params.file_path, params.file_format, samples, params.recording, params.start)
        push!(signals, signal)
    end
    for (expected, signal) in zip(expected_parameters, signals)
        expected_info = Onda.extract_samples_info(expected)
        for encoded in (false, true)
            s = @compat load(signal; encoded)
            s1 = @compat load(expected.file_path, expected.file_format, expected_info; encoded)
            s2 = @compat load(expected.file_path, Onda.format(expected.file_format, expected_info), expected_info; encoded)
            @test s == s1 == s2
            @test s.info == expected_info
            @test TimeSpans.istimespan(s)
            @test TimeSpans.duration(s) == TimeSpans.time_from_index(s.info.sample_rate, size(s.data, 2) + 1)
            @test channel_count(s) == channel_count(s.info) == length(s.info.channels)
            @test sample_count(s) == sample_count(s.info, TimeSpans.duration(s)) == size(s.data, 2)
            encoded || continue # everything after this assumes `encoded` is `true`
            if signal.file_format == "lpcm"
                s1_mmap = Onda.mmap(signal)
                s2_mmap = Onda.mmap(signal.file_path, s.info)
                s3_mmap = open(io -> Onda.mmap(io, s.info), signal.file_path)
                @test s == s1_mmap == s2_mmap == s3_mmap
            else
                @test_throws ArgumentError Onda.mmap(signal)
            end
            @test sizeof(Matrix(s.data)) == sizeof_samples(s.info, TimeSpans.duration(s))
            @test encode(s) === s
            tmp = similar(s.data)
            tmp_dither_storage = zeros(size(s.data))
            encode!(tmp, s.info.sample_resolution_in_unit,
                    s.info.sample_offset_in_unit, decode(s).data,
                    tmp_dither_storage)
            @test tmp == encode(sample_type(s.info), s.info.sample_resolution_in_unit,
                                s.info.sample_offset_in_unit, decode(s).data + tmp_dither_storage,
                                nothing)
            tmp = similar(s.data)
            encode!(tmp, s)
            @test tmp == s.data
            d = decode(s)
            @test decode(d) === d
            tmp = similar(d.data)
            decode!(tmp, d)
            @test tmp == d.data
            tmp = similar(d.data)
            decode!(tmp, s)
            @test tmp == d.data
            tmp = similar(d.data)
            decode!(tmp, s.info.sample_resolution_in_unit, s.info.sample_offset_in_unit, s.data)
            @test tmp == d.data
            @test d.data == (s.data .* s.info.sample_resolution_in_unit .+ s.info.sample_offset_in_unit)
            if sizeof(sample_type(s.info)) >= 8
                # decoding from 64-bit to floating point is fairly lossy
                tmp = similar(s.data)
                @test isapprox(encode(d, nothing).data, s.data, rtol = 10)
                encode!(tmp, d, nothing)
                @test isapprox(tmp, s.data, rtol = 10)
                @test isapprox(encode(d, missing).data, s.data, rtol = 10)
                encode!(tmp, d, missing)
                @test isapprox(tmp, s.data, rtol = 10)
            else
                tmp = similar(s.data)
                encode!(tmp, d, nothing)
                @test tmp == s.data
                @test encode(d, nothing).data == s.data
                encode!(tmp, d, missing)
                @test isapprox(tmp, s.data, rtol = 1)
                @test isapprox(encode(d, missing).data, s.data, rtol = 1)
            end
            chs = s.info.channels
            i = 27
            @test s[:, i].data == s.data[:, i:i]
            t = TimeSpans.time_from_index(s.info.sample_rate, i)
            t2 = TimeSpans.time_from_index(s.info.sample_rate, i + 15)
            j = TimeSpans.index_from_time(s.info.sample_rate, t2) - 1
            for (ch_inds, reg) in ((:, r""), (1:2, r"[ab]"), (2:3, r"[bc]"), (1:3, r"[abc]"), ([3,1], :skip), ([2,3,1], :skip), ([1,2,3], r"[abc]"))
                @test s[chs[ch_inds], t].data == s[ch_inds, i].data
                @test s[chs[ch_inds], TimeSpan(t, t2)].data == s.data[ch_inds, i:j]
                @test s[chs[ch_inds], i:j].data == s.data[ch_inds, i:j]
                @test s[ch_inds, t].data == s[ch_inds, i].data
                @test s[ch_inds, TimeSpan(t, t2)].data == s.data[ch_inds, i:j]
                @test s[ch_inds, i:j].data == s.data[ch_inds, i:j]
                reg === :skip && continue # can't represent the index with regex
                @test s[reg, t].data == s[ch_inds, i].data
                @test s[reg, TimeSpan(t, t2)].data == s.data[ch_inds, i:j]
                @test s[reg, i:j].data == s.data[ch_inds, i:j]
            end
            @test size(s[:, TimeSpan(0, Second(1))].data, 2) == floor(s.info.sample_rate)
            for i in 1:length(chs)
                @test channel(s, chs[i]) == i
                @test channel(s, i) == chs[i]
                @test channel(s.info, chs[i]) == i
                @test channel(s.info, i) == chs[i]
            end
            @test s[:, TimeSpan(0, TimeSpans.duration(s))].data == s.data
        end
    end
end

@testset "`Samples` pretty printing" begin
    info = SamplesInfo(kind="eeg",
                       channels=["a", "b", "c-d"],
                       sample_unit="unit",
                       sample_resolution_in_unit=0.25,
                       sample_offset_in_unit=-0.5,
                       sample_type=Int16,
                       sample_rate=50.2)
    samples = Samples(rand(Random.MersenneTwister(0), sample_type(info), 3, 5), info, true)
    M = VERSION >= v"1.6" ? "Matrix{Int16}" : "Array{Int16,2}"
    @test sprint(show, samples, context=(:compact => true)) == "Samples(3×5 $M)"
    @test sprint(show, samples) == """
                                   Samples (00:00:00.099601594):
                                     info.kind: "eeg"
                                     info.channels: ["a", "b", "c-d"]
                                     info.sample_unit: "unit"
                                     info.sample_resolution_in_unit: 0.25
                                     info.sample_offset_in_unit: -0.5
                                     sample_type(info): Int16
                                     info.sample_rate: 50.2 Hz
                                     encoded: true
                                     data:
                                   3×5 $M:
                                    20032  4760  27427  -20758   24287
                                    14240  5037   5598   -5888   21784
                                    16885   600  20880  -32493  -19305"""
end

@testset "Onda.VALIDATE_SAMPLES_DEFAULT" begin
    info = SamplesInfo(kind="kind",
                       channels=["a", "b", "c"],
                       sample_unit="microvolt",
                       sample_resolution_in_unit=1.0,
                       sample_offset_in_unit=0.0,
                       sample_type=Int16,
                       sample_rate=100.0)
    @test Onda.VALIDATE_SAMPLES_DEFAULT[]
    @test_throws ArgumentError Samples(rand(4, 10), info, false)
    @test_throws ArgumentError Samples(rand(Int32, 3, 10), info, true)
    Onda.VALIDATE_SAMPLES_DEFAULT[] = false
    @test Samples(rand(4, 10), info, false) isa Samples
    @test Samples(rand(Int32, 3, 10), info, true) isa Samples
    @test_throws ArgumentError Onda.validate(Samples(rand(4, 10), info, false))
    @test_throws ArgumentError Onda.validate(Samples(rand(Int32, 3, 10), info, true))
    Onda.VALIDATE_SAMPLES_DEFAULT[] = true
    @test_throws ArgumentError Samples(rand(4, 10), info, false)
    @test_throws ArgumentError Samples(rand(Int32, 3, 10), info, true)
end


"""

```
mddf_frame!(iframe::Int, framedata::FrameData, options::Options, RNG, R::Result)
```

Computes the MDDF for a single frame, modifies the data in the `R` (type `Result`) structure.

"""
function mddf_frame!(iframe::Int, framedata::FrameData, options::Options, RNG, R::Result)

  # Simplify code by assigning some shortened names
  trajectory = framedata.trajectory
  volume_frame = framedata.volume_frame
  rdf_count_random_frame = framedata.rdf_count_random_frame
  md_count_random_frame = framedata.md_count_random_frame
  dc = framedata.dc
  dmin_mol = framedata.dmin_mol
  dref_mol = framedata.dref_mol
  x_solvent_random = framedata.x_solvent_random
  lc_solvent = framedata.lc_solvent
  solute = trajectory.solute
  solvent = trajectory.solvent
  x_solute = trajectory.x_solute
  x_solvent = trajectory.x_solvent

  # Reset counters for this frame
  reset!(volume_frame)
  @. rdf_count_random_frame = 0.
  @. md_count_random_frame = 0.

  # get pbc sides in this frame
  sides = getsides(trajectory,iframe)

  volume_frame.total = sides[1]*sides[2]*sides[3]
  R.volume.total = R.volume.total + volume_frame.total

  R.density.solute = R.density.solute + (solute.nmols / volume_frame.total)
  R.density.solvent = R.density.solvent + (solvent.nmols / volume_frame.total)

  # Add the box side information to the box structure, in this frame
  box = Box(options.lcell,sides,R.cutoff)

  # Will wrap everthing relative to the reference atom of the first molecule
  # and move everything such that that center is in the origin. This is important
  # to simplify the computation of cell indexes, as the minimum coordinates are
  # automatically -side/2 at each direction. The reason for wrapping to the solute
  # center and not to the origin is that if the solute is a protein, for example,
  # the protein gets centered, and hardly ever it will be necessary to wrap the
  # coordinates afterwards, accelerating the computation.
  solute_center = x_solute[R.irefatom]
  wrap!(x_solute,sides,solute_center)
  center_to_origin!(x_solute,solute_center)
  wrap!(x_solvent,sides,solute_center)
  center_to_origin!(x_solvent,solute_center)

  # Initialize linked cells
  initcells!(x_solvent,box,lc_solvent)

  # Check if the cutoff is not too large considering the periodic cell size
  if R.cutoff > sides[1]/2. || R.cutoff > sides[2]/2. || R.cutoff > sides[3]/2.
    error("""
    in MDDF: cutoff or dbulk > periodic_dimension/2 in frame: $iframe
             max(cutoff,dbulk) = $(R.cutoff)
             sides read from file = $(sides)
             If sides are zero it is likely that the PBC information is not available
             in the trajectory file.
    """)
  end

  local n_dmin_in_bulk
  n_solvent_in_bulk = 0.
  for isolute in 1:solute.nmols

    # We need to do this one solute molecule at a time to avoid exploding the memory
    # requirements
    x_this_solute = viewmol(isolute,x_solute,solute)

    # Compute all distances between solute and solvent atoms which are smaller than the
    # cutoff (this is the most computationally expensive part), the distances are returned
    # in the dc structure
    cutoffdistances!(R.cutoff,x_this_solute,x_solvent,lc_solvent,box,dc)

    # For each solute molecule, update the counters (this is highly suboptimal, because
    # within updatecounters there are loops over solvent molecules, in such a way that
    # this will loop with cost nsolute*nsolvent. However, I cannot see an easy solution
    # at this point with acceptable memory requirements
    n_dmin_in_bulk, n_dref_in_bulk = updatecounters!(R,solute,solvent,dc,dmin_mol,dref_mol)
    n_solvent_in_bulk += n_dref_in_bulk
  end
  n_solvent_in_bulk = n_solvent_in_bulk / solute.nmols

  #
  # Computing the random-solvent distribution to compute the random minimum-distance count
  #
  bulk_range = (solvent.nmols-n_dmin_in_bulk+1):solvent.nmols
  for i in 1:options.n_random_samples

    # generate random solvent box, and store it in x_solvent_random
    for j in 1:solvent.nmols
      # Choose randomly one molecule from the bulk, if there are actually bulk molecules
      if n_dmin_in_bulk != 0
        jmol = dmin_mol[random(RNG,bulk_range)].jmol
      else
        jmol = random(RNG,1:solvent.nmols)
      end
      # Indexes of this molecule in the x_solvent array
      x_ref = viewmol(jmol,x_solvent,solvent)
      # Indexes of the random molecule in random array
      x_rnd = viewmol(j,x_solvent_random,solvent)
      # Generate new random coordinates (translation and rotation) for this molecule
      random_move!(x_ref,R.irefatom,sides,x_rnd,RNG)
    end

    # wrap random solvent coordinates to box, with the center at the origin
    wrap!(x_solvent_random,sides)

    # Initialize linked cells
    initcells!(x_solvent_random,box,lc_solvent)

    # Choose randomly one solute molecule to be the solute in this sample
    i_rand_mol = random(RNG,1:solute.nmols)
    x_this_solute = viewmol(i_rand_mol,x_solute,solute)

    # Compute all distances between solute and solvent atoms which are smaller than the
    # cutoff (this is the most computationally expensive part), the distances are returned
    # in the dc structure
    cutoffdistances!(R.cutoff,x_this_solute,x_solvent_random,lc_solvent,box,dc)

    # Update the counters of the random distribution
    updatecounters!(R,rdf_count_random_frame,md_count_random_frame,solvent,dc,dmin_mol,dref_mol)

  end # random solvent sampling

  # Update counters with the data of this frame
  update_counters_frame!(R, rdf_count_random_frame, md_count_random_frame, volume_frame,
                         solute, solvent, n_solvent_in_bulk)

  nothing
end


# SPD-License-Identifier: MIT

using ArgParse
using BayesianModels
using CUDA
using HDF5
using JLD2
using LinearAlgebra
using Random

CUDA.allowscalar(false)

include("utils.jl")

function parse_commandline()
    s = ArgParseSettings()
    @add_arg_table s begin
        "--compression-level", "-c"
            arg_type = Int
            default = 0
            help = "compression level of the HDF5 archive"
        "model"
            required = true
            help = "input GMM"
        "features"
            required = true
            help = "HDF5 archive containing the features"
        "out"
            required = true
            help = "output posteriors in HDF5 format"
    end
    s.description = """
    Compute the GMM's components posterior. The list of utterances
    to use is read from the standard input.
    """
    parse_args(s)
end

function main(args)
    model, _ = loadmodel(args["model"])

    h5open(args["out"], "w") do fout
        h5open(args["features"], "r") do ffea
            for utt in eachline(stdin)
                X = read(ffea[utt])
                fout[utt, compress = args["compression-level"]] = posterior(model, X)
            end
        end
    end
end

args = parse_commandline()
main(args)