#=
Copyright (c) 2015, Intel Corporation
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
- Redistributions of source code must retain the above copyright notice,
  this list of conditions and the following disclaimer.
- Redistributions in binary form must reproduce the above copyright notice,
  this list of conditions and the following disclaimer in the documentation
  and/or other materials provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
THE POSSIBILITY OF SUCH DAMAGE.
=#

module ParForTest

using ParallelAccelerator
#ParallelAccelerator.DomainIR.set_debug_level(3)
#ParallelAccelerator.ParallelIR.set_debug_level(3)
#ParallelAccelerator.CGen.set_debug_level(3)
@acc function parfor1(n)
 A = Array(Int, n, n)
 @par for i in 1:n, j in 1:n
    A[i,j] = i * j
 end
 return sum(A)
end

@acc function parfor2(n)
 A = Array(Int, n, n)
 s::Int = 0
 m::Int = 0
 @par s(+) m(+) for i in 1:n, j = 1:n
    A[i,j] = i * j
    s = s + A[i,j]
    m = m + 1
 end
 return s * m
end

@acc function parfor3(n)
 A = Array(Int, n, n)
 s::Array{Int, 1} = zeros(n)
 m::Int = 0
 @par s(.+) m(+) for i in 1:n
    for j = 1:n
      A[j,i] = i * j
    end
    s = s .+ A[:,i]
    m = m + 1
 end
 return s .* m
end

function test1()
  parfor1(10) == @noacc parfor1(10)
end

function test2()
  parfor2(10) == @noacc parfor2(10)
end

function test3()
  parfor3(10) == @noacc parfor3(10)
end

end

using Base.Test
println("Testing parfor support via @par macro...")
@test ParForTest.test1()
@test ParForTest.test2()
@test ParForTest.test3()
println("Done testing parfor.")


function Sum()
    return Sum(())
end

function Sum(arg0::Sum)
    return Sum((Sum,), arg0)
end

function aggregate(obj::Sum, arg0::Sum)
    return jcall(obj, "aggregate", void, (Sum,), arg0)
end

function clear(obj::Sum)
    return jcall(obj, "clear", void, ())
end

function copy(obj::Sum)
    return jcall(obj, "copy", Sum, ())
end

function evaluate(obj::Sum, arg0::Vector{jdouble}, arg1::Vector{jdouble}, arg2::jint, arg3::jint)
    return jcall(obj, "evaluate", jdouble, (Vector{jdouble}, Vector{jdouble}, jint, jint), arg0, arg1, arg2, arg3)
end

function evaluate(obj::Sum, arg0::Vector{jdouble}, arg1::jint, arg2::jint)
    return jcall(obj, "evaluate", jdouble, (Vector{jdouble}, jint, jint), arg0, arg1, arg2)
end

function get_n(obj::Sum)
    return jcall(obj, "getN", jlong, ())
end

function get_result(obj::Sum)
    return jcall(obj, "getResult", jdouble, ())
end

function increment(obj::Sum, arg0::jdouble)
    return jcall(obj, "increment", void, (jdouble,), arg0)
end


using SEML, Test

@time begin include("test1.jl") end


using OptionalUnits
using Test
using Unitful

@testset "OptionalUnits.jl" begin
    @testset "BasicFunctions" begin
        @optionalunits function f1(a::Unitful.Length → u"m", b::Unitful.Length → u"mm")
            a + b
        end
        @test f1(1.0, 2.0) == 1.002u"m"
        @test f1(1, 2u"m") == 3u"m"
        @test f1(1, 2.0u"m") == 3.0u"m"
        @test f1(1u"mm", 2) == 3u"mm"
        @test f1(1u"km", 2u"km") == 3u"km"
        @test_throws MethodError f1(1u"m", 2u"m/s")
    end
    @testset "BasicFunctionsVectors" begin
        @optionalunits function f2(a::Vector{Unitful.Length → u"m"}, b::Unitful.Length → u"mm")
            a[1] += b
            return a
        end
        @test f2([1.0,2.0], 2.0) == [1.002u"m",2.0u"m"]
        @test f2([1,2], 2u"m") == [3u"m",2u"m"]
        @test f2([1,2], 2.0u"m") == [3.0u"m",2.0u"m"]
        @test f2([1u"mm",2u"mm"], 2) == [3u"mm",2u"mm"]
        @test f2([1,2]u"km", 2u"km") == [3u"km",2u"km"]
        @test_throws MethodError f2([1u"m",2], 3)
    end
    @testset "FunctionsWithDefaults" begin
        @optionalunits function f3(a::Unitful.Length → u"m", b::Unitful.Length → u"mm"=1u"mm")
            a + b
        end
        @test f3(1.0) == 1.001u"m"
        @test f3(1, 2u"m") == 3u"m"
        @test f3(1u"mm") == 2u"mm"
        @test f3(1u"km", 2u"km") == 3u"km"
        @test_throws MethodError f3(1u"m", 2u"m/s")
    end
    @testset "FunctionsWithDefaultsVector" begin
        @optionalunits function f4(a::Unitful.Length → u"m", b::Vector{Unitful.Length → u"mm"}=[1u"mm",2u"mm"])
            b[1] += a
            return b
        end
        @test f4(1) == [1001u"mm",2u"mm"]
        @test f4(1, [2u"m",3u"m"]) == [3u"m", 3u"m"]
        @test f4(1u"mm") == [2u"mm",2u"mm"]
        @test f4(1u"km", [2u"km", 4u"mm"]) == [3u"km",4u"mm"]
        @test_throws MethodError f4(1u"m", [2u"m",2u"m/s"])
    end
    @testset "Struct" begin
        @optionalunits struct Point
            x::typeof(1u"m")
            y::typeof(1.0u"mm")
            size::Int
        end

        @test Point(1,1,1)==Point(1u"m",1.0u"mm",1)
        @test Point(2u"m",5u"m",2)==Point(2u"m",5000.0u"mm",2)

        @test_throws InexactError Point(1u"mm",1,1)
        @test_throws InexactError Point(1.5,1,1)
    end
    @testset "StructVector" begin
        @optionalunits struct MyVector
            values::Vector{typeof(1.0u"mm")}
        end

        @test MyVector([1.0,1.0,1.0]).values==MyVector([1.0u"mm",1.0u"mm",1.0u"mm"]).values
        @test MyVector([2u"m",5u"m",2u"mm"]).values==MyVector([2000.0u"mm",5000.0u"mm",2.0u"mm"]).values

        @test_throws AssertionError MyVector([1u"mm",1,1])
    end
end


mutable struct Matrix{T} <: ElementalMatrix{T}
    obj::Ptr{Cvoid}
end

for (elty, ext) in ((:ElInt, :i),
                    (:Float32, :s),
                    (:Float64, :d),
                    (:ComplexF32, :c),
                    (:ComplexF64, :z))
    @eval begin

        # destructor to be used in finalizer. Don't call explicitly
        function destroy(A::Matrix{$elty})
            ElError(ccall(($(string("ElMatrixDestroy_", ext)), libEl), Cuint,
                (Ptr{Cvoid},), A.obj))
            return nothing
        end

        function Matrix(::Type{$elty})
            obj = Ref{Ptr{Cvoid}}(0)
            ElError(ccall(($(string("ElMatrixCreate_", ext)), libEl), Cuint,
                (Ref{Ptr{Cvoid}},), obj))
            A = Matrix{$elty}(obj[])
            finalizer(destroy, A)
            return A
        end

        function getindex(A::Matrix{$elty}, i::Integer, j::Integer)
            x = Ref{$elty}(0)
            ElError(ccall(($(string("ElMatrixGet_", ext)), libEl), Cuint,
                (Ptr{Cvoid}, ElInt, ElInt, Ref{$elty}),
                A.obj, i - 1, j - 1, x))
            return x[]
        end

        function resize!(A::Matrix{$elty}, i::Integer, j::Integer = 1) # to mimic vector behavior
            ElError(ccall(($(string("ElMatrixResize_", ext)), libEl), Cuint,
                (Ptr{Cvoid}, ElInt, ElInt),
                A.obj, i, j))
            return A
        end

        function height(A::Matrix{$elty})
            rs = Ref{ElInt}(0)
            ElError(ccall(($(string("ElMatrixHeight_", ext)), libEl), Cuint,
                (Ptr{Cvoid}, Ref{ElInt}),
            A.obj, rs))
            return rs[]
        end

        function width(A::Matrix{$elty})
            rs = Ref{ElInt}(0)
            ElError(ccall(($(string("ElMatrixWidth_", ext)), libEl), Cuint,
                (Ptr{Cvoid}, Ref{ElInt}),
            A.obj, rs))
            return rs[]
        end

        function lockedBuffer(A::Matrix{$elty})
            rp = Ref{Ptr{$elty}}(0)
            ElError(ccall(($(string("ElMatrixLockedBuffer_", ext)), libEl), Cuint,
                (Ptr{Cvoid}, Ref{Ptr{$elty}}),
                A.obj, rp))
            return rp[]
        end

        function setindex!(A::Matrix{$elty}, x::Number, i::Integer, j::Integer)
            ElError(ccall(($(string("ElMatrixSet_", ext)), libEl), Cuint,
                (Ptr{Cvoid}, ElInt, ElInt, $elty),
                A.obj, i - 1, j - 1, x))
            return A
        end
    end
end

Matrix() = Matrix(Float64)

# Julia convenience

pointer(A::Matrix) = lockedBuffer(A)

function similar(::Matrix, ::Type{T}, sz::Dims) where {T}
    A = Matrix(T)
    resize!(A, sz...)
    return A
end


include("plot_vb_results.jl")

# Read CB data
y, markernames = let
  function separatesamples(Y)
    sample_ids = sort(unique(Y.sample_id))
    return Matrix.(Y[Y.sample_id .== i, Not(:sample_id)]
                   for i in sample_ids)
  end

  path_to_data = "../data/cb_transformed_reduced.csv"
  Y = coalesce.(CSV.read(path_to_data), NaN)
  (separatesamples(Y), names(Y))
end

paths = [joinpath(root, file)
         for (root, dirs, files) in walkdir("results")
         for file in files
         if file == "output.bson"]

for path in paths
  println("Processing: $(path)")
  makeplots(y=y, path_to_output=path,
            markernames=markernames, nlam=30)
end


"""
    area(x, y)

Creates an `EChart` where region below plotted line filled with color.

## Methods
```julia
area(x::AbstractVector, y::AbstractVector{<:Union{Missing, Real}})
area(x::AbstractVector, y::AbstractArray{<:Union{Missing, Real},2})
area(df::AbstractDataFrame, x::Symbol, y::Symbol)
area(df::AbstractDataFrame, x::Symbol, y::Symbol, group::Symbol)
area(k::KernelDensity.UnivariateKDE)
```

## Arguments
* `mark::Union{String, AbstractVector} = "line"` : how to display plotted points
* `fill::Union{Bool, AbstractVector} = true` : fill area below marks with color?
* `stack::Union{Bool, AbstractVector, Void} = nothing` : stack (add together) when multple series present?
* `step::Union{String, Void} = nothing` : one of {"start", "end", "middle", nothing}
* `legend::Bool` : display legend?
* `scale::Bool = false` : show full Y-axis or truncated
* `kwargs` : varargs to set any field of resulting `EChart` struct

## Notes

Reasonable defaults set for different methods of `area`, such as displaying a legend when two or more series present.

## Examples
```julia
x = ["Monday","Tuesday","Wednesday","Thursday","Friday","Saturday","Sunday"]
y = [11, 11, 15, 13, 12, 13, 10]
ar = area(x, y)
```
"""
function area(x::AbstractVector, y::AbstractVector{<:Union{Missing, Real}};
			mark::Union{String, AbstractVector} = "line",
			fill::Union{Bool, AbstractVector} = true,
			stack::Union{Bool, AbstractVector, Nothing} = nothing,
			step::Union{String, Nothing} = nothing,
			legend::Bool = false,
			scale::Bool = false,
			kwargs...)

	ec = xy_plot(x, y; stack = stack, mark = mark, step = step, legend = legend, scale = scale, kwargs...)
	ec.xAxis[1].boundaryGap = false

	# Fill area if requested
	fill!(ec, 1, fill)

	return ec

end

#Basically same function as above, except automatically puts a legend when multiple series present
function area(x::AbstractVector, y::AbstractArray{<:Union{Missing, Real},2};
			mark::Union{String, AbstractVector} = "line",
			fill::Union{Bool, AbstractVector} = true,
			stack::Union{Bool, AbstractVector, Nothing} = true,
			step::Union{String, Nothing} = nothing,
			legend::Bool = true,
			scale::Bool = false,
			kwargs...)

	ec = xy_plot(x, y; stack = stack, mark = mark, step = step, legend = legend, scale = scale, kwargs...)
	ec.xAxis[1].boundaryGap = false

	# Fill area if requested
	fill!(ec, size(y)[2], fill)

	return ec

end

#dataframe, single series
function area(df::AbstractDataFrame, x::Symbol, y::Symbol;
			mark::Union{String, AbstractVector} = "line",
			fill::Union{Bool, AbstractVector} = true,
			stack::Union{Bool, AbstractVector, Nothing} = nothing,
			step::Union{String, Nothing} = nothing,
			legend::Bool = false,
			scale::Bool = false,
			kwargs...)

			ec = xy_plot(df, x, y, mark = mark, stack = stack, step = step, legend = legend, scale = scale, kwargs...)

			ec.xAxis[1].boundaryGap = false

			fill!(ec, 1, fill)

			return ec

end

#dataframe, group argument
function area(df::AbstractDataFrame, x::Symbol, y::Symbol, group::Symbol;
			mark::Union{String, AbstractVector} = "line",
			fill::Union{Bool, AbstractVector} = true,
			stack::Union{Bool, AbstractVector, Nothing} = true,
			step::Union{String, Nothing} = nothing,
			legend::Bool = true,
			scale::Bool = false,
			kwargs...)

			ec = xy_plot(df, x, y, group, mark = mark, stack = stack, step = step, legend = legend, scale = scale, kwargs...)

			ec.xAxis[1].boundaryGap = false

			fill!(ec, length(ec.series), fill)

			return ec

end

function area(k::KernelDensity.UnivariateKDE;
				mark::Union{String, AbstractVector} = "line",
				fill::Union{Bool, AbstractVector} = true,
				stack::Union{Bool, AbstractVector, Nothing} = nothing,
				step::Union{String, Nothing} = nothing,
				legend::Bool = false,
				scale::Bool = false,
				kwargs...)

	ec = area(collect(k.x), k.density, mark = mark, fill = fill, stack = stack, step = step, legend = legend, scale = scale, kwargs...)

	return ec

end


using Test, ParticleDA, MPI, Random

MPI.Init()
my_rank = MPI.Comm_rank(MPI.COMM_WORLD)
my_size = MPI.Comm_size(MPI.COMM_WORLD)

nprt_per_rank = 3
n = nprt_per_rank * my_size

@testset for size in ((1,1,1,nprt_per_rank), (1,1,1,1,nprt_per_rank))

    local_particles = float(collect(my_rank * nprt_per_rank + 1 : (my_rank + 1) * nprt_per_rank))

    for i = 1:my_size
        if i == my_rank + 1
            println("rank ",my_rank,": particle states: ", local_particles)
        end
        MPI.Barrier(MPI.COMM_WORLD)
    end

    buffer = zeros(size)

    Random.seed!(1234)
    indices = rand(1:n, n)

    if my_rank == 0
        println()
        println("Resampling particles to indices ", indices)
        println()
    end

    ParticleDA.copy_states!(reshape(local_particles, size), buffer, indices, my_rank, nprt_per_rank)

    for i = 1:my_size
        if i == my_rank + 1
            test = local_particles == float(indices[my_rank * nprt_per_rank + 1 : (my_rank + 1) * nprt_per_rank])
            println("rank ",my_rank,": particle states: ", local_particles, " -- ",test)
        end
        MPI.Barrier(MPI.COMM_WORLD)
    end

    @test local_particles == float(indices[my_rank * nprt_per_rank + 1 : (my_rank + 1) * nprt_per_rank])

end


@testset "meta-pkg" begin
    rr = RidgeRegressor()
    d = MLJBase.info_dict(rr)
    @test d[:package_name] == "MLJLinearModels"
    @test d[:package_url] == "https://github.com/alan-turing-institute/MLJLinearModels.jl"
    @test d[:package_license] == "MIT"
    @test d[:is_pure_julia] == true
    @test d[:is_wrapper] == false
end

@testset "meta-reg" begin
    lr = LinearRegressor()
    d = MLJBase.info_dict(lr)
    @test d[:input_scitype] == MLJBase.Table(MLJBase.Continuous)
    @test d[:target_scitype] == AbstractVector{MLJBase.Continuous}
    @test d[:supports_weights] == false
    @test !isempty(d[:docstring])
    @test d[:load_path] == "MLJLinearModels.LinearRegressor"
end

@testset "meta-clf" begin
    lr = LogisticClassifier()
    d = MLJBase.info_dict(lr)
    @test d[:input_scitype] == MLJBase.Table(MLJBase.Continuous)
    @test d[:target_scitype] == AbstractVector{<:MLJBase.Finite}
    @test d[:supports_weights] == false
    @test !isempty(d[:docstring])
    @test d[:load_path] == "MLJLinearModels.LogisticClassifier"
end


struct GeoTiff
    raster::AbstractRaster
end
GeoTiff(path::AbstractString) = GeoTiff(_to_rgb(path))
Base.parent(geotiff::GeoTiff) = geotiff.raster

function _to_rgb(fn)
    rast = Raster(fn)
    rast = reverse(rast, dims = 2)
    red = view(rast, Rasters.Band(1)) |> read
    green = view(rast, Rasters.Band(2)) |> read
    blue = view(rast, Rasters.Band(3)) |> read

    replace!.(x -> isnan(x) ? 0.0 : x, [red, green, blue])

    rgb = fill(RGBf(0,0,0), dims(rast, :X), dims(rast, :Y));

    for i in eachindex(red)
        rgb[i] = RGBf(red[i], green[i], blue[i])
    end
    eltype(rast) == UInt8 ? rgb./255 : rgb
end


maze = zeros(Int8, 8, 6)
#maze = [i+j*width  for j = 0:height for i = 0:width]

function generate_maze(width,height,arg...)
    maze = zeros(Int8,width,height)
    maze = [(i+j*width)%10 for i = 0:width-1,j = 0:height-1]
    return (m=maze,w=width,h=height,a=arg)
end

function print_maze(maze)
    for r = 1:size(maze,2)
        txt = ""
        for c in maze[:,r]
            txt *= "$c"
        end
        println(txt)
    end
end

function save_maze(maze,file)
    file = open(file,"w")
    for r = 1:size(maze,2)
        txt = ""
        for c in maze[:,r]
            txt *= "$c"
        end
        txt *= "\n"
        write(file,txt)
    end
    close(file)
end

res = generate_maze(8,6,"dungeon maze")
println(res)
print_maze(res.m)
save_maze(res.m,"maze.txt")

using Sockets
ip = "www.baidu.com"
connect(ip,80)
getaddrinfo(ip)
#print(maze[2:3,5:7])
cmd = `echo hello`
run(cmd)


module ConstantCellValues

using Gridap

export ConstantCellValue
export ConstantCellNumber
export ConstantCellArray
export ConstantCellVector
export ConstantCellMatrix
export ConstantCellMap

import Gridap: apply
import Gridap: reindex
import Gridap: evaluate
import Base: size
import Base: getindex
import Base: broadcast
import Base: ==, ≈

struct ConstantCellValue{T} <: IndexCellValue{T,1}
  value::T
  length::Int
end

size(self::ConstantCellValue) = (self.length,)

function getindex(c::ConstantCellValue,i::Integer)
  c.value
end

const ConstantCellNumber = ConstantCellValue{<:NumberLike}

const ConstantCellArray{T,N} = ConstantCellValue{Array{T,N}}

const ConstantCellVector{T} = ConstantCellArray{T,1}

const ConstantCellMatrix{T} = ConstantCellArray{T,2}

const ConstantCellMap{S,M,T,N} = ConstantCellValue{<:Map{S,M,T,N}}

function ConstantCellArray(v::AbstractArray{T,N},l::Integer) where {T,N}
  ConstantCellArray{T,N}(v,l)
end

function ConstantCellVector(v::AbstractVector{T},l::Integer) where T
  ConstantCellVector{T}(v,l)
end

function ConstantCellMatrix(v::AbstractMatrix{T},l::Integer) where T
  ConstantCellMatrix{T}(v,l)
end

function ConstantCellMap(v::Map,l::Integer)
  ConstantCellValue(v,l)
end

function ConstantCellNumber(v::NumberLike,l::Integer)
  ConstantCellValue(v,l)
end

function (==)(a::ConstantCellNumber,b::ConstantCellNumber)
  _eq_kernel(==,a,b)
end

function (≈)(a::ConstantCellNumber,b::ConstantCellNumber)
  _eq_kernel(≈,a,b)
end

function (==)(a::ConstantCellArray,b::ConstantCellArray)
   _eq_kernel(==,a,b)
end

function (≈)(a::ConstantCellArray,b::ConstantCellArray)
  _eq_kernel(≈,a,b)
end

function _eq_kernel(op,a,b)
  !(op(a.value,b.value)) && return false
  length(a) != length(b) && return false
  return true
end

function apply(k::NumberKernel,v::Vararg{<:ConstantCellValue})
  vals = [vi.value for vi in v]
  l = _compute_l(v...)
  val = compute_value(k,vals...)
  ConstantCellNumber(val,l)
end

function apply(k::ArrayKernel,v::Vararg{<:ConstantCellValue})
  vals = [vi.value for vi in v]
  l = _compute_l(v...)
  val = compute_value(k,vals...)
  ConstantCellArray(val,l)
end

function apply(k::ArrayKernel,m::ConstantCellMap)
  w = apply(k,m.value)
  ConstantCellMap(w,m.length)
end

function apply(k::ArrayKernel,m::ConstantCellMap,v::Vararg{<:ConstantCellValue})
  vals = [vi.value for vi in v]
  l = _compute_l(m,v...)
  w = apply(k,m.value,vals...)
  ConstantCellMap(w,l)
end

function evaluate(cm::ConstantCellMap{S,M},ca::ConstantCellArray{<:S,M}) where {S,M}
  @assert length(cm) == length(ca)
  m = cm.value
  a = ca.value
  r = evaluate(m,a)
  ConstantCellArray(r,length(cm))
end

function _compute_l(v...)
  @assert length(v) > 0
  v1, = v
  l1 = length(v1)
  @assert all([ length(vi) == l1 for vi in v ])
  l1
end

function reindex(values::ConstantCellValue, indices::CellValue{<:IndexLike})
  ConstantCellValue(values.value,length(indices))
end

function reindex(values::ConstantCellValue, indices::IndexCellValue{<:IndexLike})
  ConstantCellValue(values.value,length(indices))
end

broadcast(f,cvs::ConstantCellValue) = ConstantCellValue(f(cvs.value),cvs.length)

end # module ConstantCellValues


import Gaugefields:Traceless_antihermitian_add!

struct DomainwallFermiAction{Dim,Dirac,fermion,gauge} <: FermiAction{Dim,Dirac,fermion,gauge}
    hascovnet::Bool
    covneuralnet::Union{Nothing,CovNeuralnet{Dim}}
    diracoperator::Dirac
    _temporary_fermionfields::Vector{fermion}
    _temporary_gaugefields::Vector{gauge}

    function DomainwallFermiAction(D::Dirac_operator{Dim},hascovnet,covneuralnet) where Dim
        num = 8
        temps = get_temporaryvectors(D)
        x = temps[1]
        xtype = typeof(x)
        _temporary_fermionfields = Array{xtype,1}(undef,num)
        for i=1:num
            _temporary_fermionfields[i] = similar(x)
        end

        Utemp = D.U[1]
        Utype = typeof(Utemp)
        numU = 2
        _temporary_gaugefields = Array{Utype,1}(undef,numU)
        for i=1:numU
            _temporary_gaugefields[i] = similar(Utemp)
        end

        return new{Dim,typeof(D),xtype,Utype }(hascovnet,covneuralnet,D,
                        _temporary_fermionfields,
                        _temporary_gaugefields)

    end
end

function evaluate_FermiAction(fermi_action::DomainwallFermiAction{Dim,Dirac,fermion,gauge },U,ϕ::AbstractFermionfields) where {Dim,Dirac,fermion,gauge }
    W = fermi_action.diracoperator(U)
    η = fermi_action._temporary_fermionfields[1]
    solve_DinvX!(η,W',ϕ)
    Sf = dot(η,η)
    return real(Sf)
end

function calc_UdSfdU!(UdSfdU::Vector{<: AbstractGaugefields},fermi_action::DomainwallFermiAction{Dim,Dirac,fermion,gauge } ,U::Vector{<: AbstractGaugefields},ϕ::AbstractFermionfields) where  {Dim,Dirac,fermion,gauge }
    #println("------dd")
    QD5DW = fermi_action.diracoperator.D5DW(U)
    Q = D5DWdagD5DW_Wilson_operator(QD5DW)
    D5_PV = fermi_action.diracoperator.D5DW_PV(U)

    temps_dw =fermi_action._temporary_fermionfields[1]

    X = fermi_action._temporary_fermionfields[6]
    Y = fermi_action._temporary_fermionfields[5]

    mul!(temps_dw,D5_PV',ϕ)

    solve_DinvX!(X,Q,temps_dw)
    #set_wing_fermion!(X)
    set_wing_fermion!(ϕ)

    clear_U!(UdSfdU)

    calc_UdSfdU_fromX!(UdSfdU,Y,ϕ,fermi_action,U,X)

    #println("----aa--")
    set_wing_U!(UdSfdU)
end

function calc_UdSfdU_fromX!(UdSfdU::Vector{<: AbstractGaugefields},Y,ϕ,
        fermi_action::DomainwallFermiAction{Dim,Dirac,fermion,gauge } ,U,X;coeff = 1) where {Dim,Dirac,fermion,gauge }
    W = fermi_action.diracoperator.D5DW(U)
    temps_dw = fermi_action._temporary_fermionfields[2]
    mul!(temps_dw,W,X)
    clear_fermion!(Y)
    add_fermion!(Y,-1,ϕ,1,temps_dw)
    set_wing_fermion!(Y)

    temp0_g = fermi_action._temporary_gaugefields[1]

    κ = 1/2
    Dwilson = W.wilsonoperator

    L5 = fermi_action.diracoperator.D5DW.L5

    if L5 != X.L5
        @assert L5 % 2 == 0
        irange = Int64[]
        irange_out = Int64[]
        #irange = 1:L5
        #irange_out = (L5+1):X.L5

        for i5=1:X.L5
            if i5 <= div(L5,2) || i5 >= X.L5-div(L5,2)+1
                push!(irange,i5)
            else
                push!(irange_out,i5)
            end

        end

    else
        irange = 1:L5
    end

#    for i5=1:X.L5
    for i5 in irange

        temp0_f = fermi_action._temporary_fermionfields[1].w[i5] #F_field
        temp1_f = fermi_action._temporary_fermionfields[2].w[i5] #F_field

        for μ=1:Dim
            #!  Construct U(x,mu)*P1
            Xs = X.w[i5]
            Ys = Y.w[i5]

            # U_{k,μ} X_{k+μ}
            Xsplus = shift_fermion(Xs,μ)

            #@time mul!(temp0_f,U[μ],X)
            mul!(temp0_f,U[μ],Xsplus)

            # (r-γ_μ) U_{k,μ} X_{k+μ}
            mul!(temp1_f,view(Dwilson.rminusγ,:,:,μ),temp0_f)

            # κ (r-γ_μ) U_{k,μ} X_{k+μ}
            mul!(temp0_f,κ,temp1_f)

            # κ ((r-γ_μ) U_{k,μ} X_{k+μ}) ⊗ Y_k
            mul!(temp0_g,temp0_f,Ys')

            add_U!(UdSfdU[μ],-coeff,temp0_g)

            #!  Construct P2*U_adj(x,mu)
            # Y_{k+μ}^dag U_{k,μ}^dag
            Ysplus = shift_fermion(Ys,μ)
            mul!(temp0_f,Ysplus',U[μ]')

            # Y_{k+μ}^dag U_{k,μ}^dag*(r+γ_μ)
            mul!(temp1_f,temp0_f,view(Dwilson.rplusγ,:,:,μ))

            # κ Y_{k+μ}^dag U_{k,μ}^dag*(r+γ_μ)
            mul!(temp0_f,κ,temp1_f)

            # X_k ⊗ κ Y_{k+μ}^dag U_{k,μ}^dag*(r+γ_μ)
            #println(getvalue(temp0_g,1,1,1,1,1,1))
            #println(temp0_g[1,1,1,1,1,1])
            mul!(temp0_g,Xs,temp0_f)
            #println("temp0_g , ",getvalue(temp0_g,1,1,1,1,1,1))
            #println("temp0_g , ",temp0_g[1,1,1,1,1,1])
            #println("coeff ", coeff)
            #println("before ",getvalue(UdSfdU[μ],1,1,1,1,1,1))
            #println("before ",UdSfdU[μ][1,1,1,1,1,1])

            add_U!(UdSfdU[μ],coeff,temp0_g)
            #error("h")
            #println("after ",getvalue(UdSfdU[μ],1,1,1,1,1,1))
            #println("after ",UdSfdU[μ][1,1,1,1,1,1])

        end

    end

end

function calc_p_UdSfdU!(p,
    fermi_action::DomainwallFermiAction{Dim,Dirac,fermion,gauge } ,U::Vector{<: AbstractGaugefields},ϕ::AbstractFermionfields,coeff = 1) where {Dim,Dirac,fermion,gauge }
    #println("------dd")
    QD5DW = fermi_action.diracoperator.D5DW(U)
    Q = D5DWdagD5DW_Wilson_operator(QD5DW)
    D5_PV = fermi_action.diracoperator.D5DW_PV(U)

    temps_dw =fermi_action._temporary_fermionfields[1]

    X = fermi_action._temporary_fermionfields[end]
    Y = fermi_action._temporary_fermionfields[end-1]

    mul!(temps_dw,D5_PV',ϕ)

    solve_DinvX!(X,Q,temps_dw)

    #set_wing_fermion!(ϕ)

    calc_p_UdSfdU_fromX!(p,Y,ϕ,fermi_action,U,X,coeff= coeff)
    #println("----aa--")
    #set_wing_U!(UdSfdU)
end

function calc_p_UdSfdU_fromX!(p,Y,ϕ,
    fermi_action::DomainwallFermiAction{Dim,Dirac,fermion,gauge }  ,U,X;coeff = 1) where {Dim,Dirac,fermion,gauge }
    W = fermi_action.diracoperator.D5DW(U)
    temps_dw = fermi_action._temporary_fermionfields[1]
    mul!(temps_dw,W,X)
    clear_fermion!(Y)
    add_fermion!(Y,-1,ϕ,1,temps_dw)
    set_wing_fermion!(Y)

    temp0_g = fermi_action._temporary_gaugefields[1]

    κ = 1/2
    Dwilson = W.wilsonoperator
    for i5=1:X.L5

        temp0_f = fermi_action._temporary_fermionfields[1].w[i5] #F_field
        temp1_f = fermi_action._temporary_fermionfields[2].w[i5] #F_field

        for μ=1:Dim
            #!  Construct U(x,mu)*P1
            Xs = X.w[i5]
            Ys = Y.w[i5]

            # U_{k,μ} X_{k+μ}
            Xsplus = shift_fermion(Xs,μ)

            #@time mul!(temp0_f,U[μ],X)
            mul!(temp0_f,U[μ],Xsplus)

            # (r-γ_μ) U_{k,μ} X_{k+μ}
            mul!(temp1_f,view(Dwilson.rminusγ,:,:,μ),temp0_f)

            # κ (r-γ_μ) U_{k,μ} X_{k+μ}

            mul!(temp0_f,κ,temp1_f)

            # κ ((r-γ_μ) U_{k,μ} X_{k+μ}) ⊗ Y_k
            mul!(temp0_g,temp0_f,Ys')

            Traceless_antihermitian_add!(p[μ],-coeff,temp0_g)
            #println(p[μ][1,1,1,1,1])

            #!  Construct P2*U_adj(x,mu)
            # Y_{k+μ}^dag U_{k,μ}^dag
            Ysplus = shift_fermion(Ys,μ)
            mul!(temp0_f,Ysplus',U[μ]')

            # Y_{k+μ}^dag U_{k,μ}^dag*(r+γ_μ)
            mul!(temp1_f,temp0_f,view(Dwilson.rplusγ,:,:,μ))

            # κ Y_{k+μ}^dag U_{k,μ}^dag*(r+γ_μ)
            mul!(temp0_f,κ,temp1_f)

            # X_k ⊗ κ Y_{k+μ}^dag U_{k,μ}^dag*(r+γ_μ)
            mul!(temp0_g,Xs,temp0_f)

            Traceless_antihermitian_add!(p[μ],coeff,temp0_g)
        end
    end

end

function gauss_sampling_in_action!(η::AbstractFermionfields,U,fermi_action::DomainwallFermiAction{Dim,Dirac,fermion,gauge } ) where {Dim,Dirac,fermion,gauge }
    #gauss_distribution_fermion!(η)
    gauss_distribution_fermion!(η,rand)
end

using InteractiveUtils

function sample_pseudofermions!(ϕ::AbstractFermionfields,U,fermi_action::DomainwallFermiAction{Dim,Dirac,fermion,gauge } ,ξ::AbstractFermionfields) where {Dim,Dirac,fermion,gauge }
    W = fermi_action.diracoperator(U)
    mul!(ϕ,W',ξ)
    set_wing_fermion!(ϕ)
end


ontol = parseontology("c:\\home\\resource\\geneontology\\go-basic.obo")
annot = load_annot_long("c:\\home\\resource\\Xt\\9.1\\go_b2g_lf.map", ontol)
gographs = ontologygraphs(ontol)
exannot = extendannotation(annot, gographs)

### enrichment tests
function loadclusters(file="c:\\home\\projects\\pot\\gc\\pot_xen_tpm_gc_corrected_gp_smoothed.tsv")
    CSV.read(file)
end
data = loadclusters()
#
background = data.Gene
genesel = data.ClusterBothBIC .== 6
enrichtest(data.ClusterBothBIC .== 6, background, exannot, ontol) |> display

rexannot = reverseannotation(exannot)
rannot = reverseannotation(annot)

Z = enrichtest(data.ClusterBothBIC .== 6, background, exannot, ontol)

for row in eachrow(Z)
    id = goid_to_num(row.GT)
    for g in row.Genes
        @assert id ∈ exannot["P"][g]
    end
end


using QCMaterial
using PyCall
using LinearAlgebra
using HDF5
import Random
import PyCall: pyimport

#make taus list file.h5
#length=8
function taus_list(file_name)
    x = Float64[]
    for i in 1:10
        tau = 0.005*(i-1) + exp((0.1*(i-1))) -1
        push!(x,tau)
    end
    fid = h5open(file_name,"w")
    fid["/test/data"] = x
    close(fid)
end

function taus_list(file_name2)
    x = Float64[]
    for i in 1:10
        tau = 0.005*(i-1) + exp((0.1*(i-1))) -1
        push!(x,tau)
    end
    fid = h5open(file_name,"w")
    fid["/test/data2"] = x
    close(fid)
end

#read taus list file.h5
function read_taus_list(file_name)
    fid = h5open(file_name,"r")
    x = fid["/test/data"][:]
    y = fid["/test/data2"][:]
    close(fid)
    return x
end

#read taus list file.txt
function read_and_parse_float(file_name)
    x = zeros(Float64, 0)
    open(file_name, "r") do fp
        num_elm = parse(Int64,readline(fp))
        for i in 1:num_elm
            push!(x, parse(Float64, readline(fp)))
        end
    end
    return x
end

function hev(n_qubit, depth)
    circuit = QulacsParametricQuantumCircuit(n_qubit)
    for d in 1:depth
        for i in 1:n_qubit
            add_parametric_RY_gate!(circuit, i, 0.0)
            add_parametric_RZ_gate!(circuit, i, 0.0)
        end
        for i in 1:n_qubit÷2
            add_CNOT_gate!(circuit, 2*i-1, 2*i)
        end
        for i in 1:n_qubit÷2-1
            add_CNOT_gate!(circuit, 2*i, 2*i+1)
        end
    end
    for i in 1:n_qubit
        add_parametric_RY_gate!(circuit, i, 0.0)
        add_parametric_RZ_gate!(circuit, i, 0.0)
    end
    circuit = QulacsVariationalQuantumCircuit(circuit)
    circuit
end

nsite = 2
n_qubit = 2 * nsite
U = 1.0
V = 1.0
μ = U/2
ε = 1.0
d_theta = 1e-5
verbose = QCMaterial.MPI_rank == 0
Random.seed!(100)

#Hamiltonian
ham_op = generate_impurity_ham_with_1imp_multibath(U, V, μ, ε, nsite)

n_electron_gs = 2
@assert mod(n_electron_gs, 2) == 0
sparse_mat = get_number_preserving_sparse_operator(ham_op, n_qubit, n_electron_gs);
enes_ed = eigvals(sparse_mat.toarray());
println("Ground_enegry_ED=", minimum(enes_ed))
#N+1系のground energy
#n_electron_ex = 3
#sparse_mat_ex = get_number_preserving_sparse_operator(ham_op, n_qubit, n_electron_ex);
#enes_ex_ed = eigvals(sparse_mat_ex.toarray());

#debug
#N = particle number
#println("Ground energy_2_ED=",minimum(enes_ed))
#println("Ground_enegry_3_ED=", minimum(enes_ex_ed))

#1 system
#n_electron_1 = 1
#sparse_mat_enum_1 = get_number_preserving_sparse_operator(ham_op, n_qubit, n_electron_1);
#enes_ed_enum_1 = eigvals(sparse_mat_enum_1.toarray());
#debug
#println("Ground energy_ED=",minimum(enes_ed_enum_1))
#println("Ground energy_1_ED=",minimum(enes_ed_enum_1))

#1 system
#n_electron_4 = 4
#sparse_mat_enum_4 = get_number_preserving_sparse_operator(ham_op, n_qubit, n_electron_4);
#enes_ed_enum_4 = eigvals(sparse_mat_enum_4.toarray());
#debug
#println("Ground energy_ED=",minimum(enes_ed_enum_1))
#println("Ground energy_4_ED=",minimum(enes_ed_enum_4))

#ansatz
state0 = create_hf_state(n_qubit, n_electron_gs)
vc = uccgsd(n_qubit, nocc = 1, orbital_rot=true, uccgsd = true, p_uccgsd = false)
#depth = n_qubit
#vc = hev(n_qubit, depth )
theta_init = rand(num_theta(vc))

#Perform VQE
cost_history, thetas_opt =
QCMaterial.solve_gs(jordan_wigner(ham_op), vc, state0, theta_init=theta_init, verbose=true,
    comm=QCMaterial.MPI_COMM_WORLD
)

#debug
println("Ground energy_VQE=",cost_history[end])

#c^{dag},c
up1 = up_index(1)
down1 = down_index(1)
up2 = up_index(2)
down2 = down_index(2)

if ARGS[1] == "plus_true"
    right_op = FermionOperator("$(up1)^ ", 1.0)
    right_op = jordan_wigner(right_op)
    left_op = FermionOperator("$(up1) ", 1.0)
    left_op = jordan_wigner(left_op)

    n_electron_ex = 3

    sign = 1
end

if ARGS[2] == "minus_true"
    right_op = FermionOperator("$(up1) ", 1.0)
    right_op = jordan_wigner(right_op)
    left_op = FermionOperator("$(up1)^ ", 1.0)
    left_op = jordan_wigner(left_op)

    n_electron_ex = 1

    sign = -1
end

vc_ex = uccgsd(n_qubit, nocc = 1, orbital_rot = true, uccgsd = true, p_uccgsd = false)
#vc_ex = hev(n_qubit, depth )
state_gs = create_hf_state(n_qubit, n_electron_gs)
update_circuit_param!(vc, thetas_opt)
update_quantum_state!(vc, state_gs)

state0_ex = create_hf_state(n_qubit, n_electron_ex)

#taus = collect(range(0.0, 0.02, length = 2))

#make taus list file
#taus_list("dimer_plus.h5")
#generate taus list file
#taus = read_taus_list("dimer_plus.h5")
#println("taus=",taus)

taus = read_and_parse_float(ARGS[3])
println("taus=",taus)

Gfunc_ij_list, norm =  compute_gtau(
    jordan_wigner(ham_op),
    left_op,
    right_op,
    vc_ex,
    state_gs,
    state0_ex,
    taus,
    d_theta,
    verbose = verbose,
    algorithm = "vqs",
    recursive = true
)

Gfunc_ij_list *= sign

println("Gfunc_ij_list_plus=", Gfunc_ij_list)
println("norm=", norm)

function write_to_txt(file_name, x, y)
    open(file_name, "w") do fp
        for i = 1:length(x)
            println(fp, x[i], " ", real(y[i]))
        end
    end
end

function write_to_txt_(file_name, x, y)
    open(file_name, "w") do fp
        for i = 1:length(x)
            println(fp, 1000-x[i], " ", -real(y[i]))
        end
    end
end

write_to_txt("gf_dimer_plus_shotnoise1000_proc1_relative_thetadot.txt", taus, Gfunc_ij_list)
write_to_txt("dimer_plus_vqs_norm__relativeerr_1000_thetadot.txt", taus, norm)
#write_to_txt_("gf_dimer_recurisive_direct_minus3_freq.txt", taus, Gfunc_ij_list)
println("done!!")


mutable struct Tensor{T, N} <: AbstractArray{T, N}
    type::Type
    ndims::Int64
    pointer::Ptr
    data::Union{Nothing,Array{T, N}}

    function Tensor{T, N}(p::Ptr, data) where {T, N}
        if !haskey(TYPE_MAP, T)
            error("Type $T is not supported.")
        end
        ret = new(T, N, p, data)
        finalizer(ret) do t
            ccall((:tensor_destroy, libtorch_capi),
                  Cvoid, (Ptr{Cvoid},),
                  t.pointer)
        end
        return ret
    end

end

Base.setproperty!(::Tensor, ::Symbol, x) = error("Can't change field of Tensor.")

function Tensor{T}(array::Array{U, N};
                   detach=false, requires_grad=false) where {T, U, N}
    if !haskey(TYPE_MAP, T)
        error("Type $T is not supported.")
    end

    dims = collect(size(array))
    stri = collect(strides(array))
    if T != U
        array = convert.(T, array)
    end
    grad = requires_grad ? 1 : 0
    copy_data = detach ? 1 : 0

    ptr = ccall((:tensor_from_data, libtorch_capi),
                Ptr{Cvoid},
                (Ptr{Cvoid}, Csize_t, Cchar,
                 Ptr{Clonglong}, Ptr{Clonglong}, Csize_t, Cint, Cint),
                array, sizeof(array), TYPE_MAP[T], dims, stri, N, copy_data, grad)
    Tensor{T, N}(ptr, detach ? nothing : array)
end

function Tensor(array::Array{T, N}; detach=false, requires_grad=false) where {T, N}
    Tensor{T}(array, detach=detach, requires_grad=requires_grad)
end

# 0-dim Tensor
function Tensor(s::Int64; requires_grad=false)
    grad = requires_grad ? 1 : 0
    ptr = ccall((:tensor_int64_0dim, libtorch_capi),
                Ptr{Cvoid},
                (Clonglong, Cint), s, grad)
    Tensor{Int64, 0}(ptr, nothing)
end

function Tensor(s::T; requires_grad=false) where {T <: TorchNumber}
    data = T[s]
    grad = requires_grad ? 1 : 0
    ptr = ccall((:tensor_from_data, libtorch_capi),
                Ptr{Cvoid},
                (Ptr{Cvoid}, Csize_t, Cchar,
                 Ptr{Clonglong}, Ptr{Clonglong}, Csize_t, Cint, Cint),
                data, sizeof(T), TYPE_MAP[T], C_NULL, C_NULL, 0, 1, grad)
    Tensor{T, 0}(ptr, nothing)
end

function Tensor(a0::Array{T, 0}; requires_grad=false) where {T <: TorchNumber}
    Tensor(a0[], requires_grad=requires_grad)
end

function tensor_from_ptr(p::Ptr)
    n_dims = ccall((:tensor_method_ndimension, libtorch_capi),
                   Clonglong, (Ptr{Cvoid},),
                   p)
    # sizes = zeros(Int64, n_dims)
    # ccall((:tensor_method_sizes, libtorch_capi),
    #       Cvoid, (Ptr{Cvoid}, Ptr{Cvoid}),
    #       p, sizes)
    dtype = ccall((:tensor_method_dtype, libtorch_capi),
                  Cchar, (Ptr{Cvoid},),
                  p)
    Tensor{REVERSE_TYPE_MAP[dtype], n_dims}(p, nothing)
end

function Base.convert(::Type{Array}, t::Tensor{T, N}) where {T, N}
    if t.data != nothing
        return t.data
    end
    dims = size(t)
    ret = Array{T, N}(undef, reverse(dims))
    ccall((:tensor_method_data_copy, libtorch_capi),
          Cvoid, (Ptr{Cvoid}, Ptr{Cvoid}, Csize_t),
          t.pointer, ret, sizeof(T) * prod(dims))
    if strides(t)[1] != 1
        return permutedims(ret, collect(N:-1:1))
    end
    return reshape(ret, dims)
end
Base.convert(::Type{T}, x::Tensor{T, 0}) where T = x[]

function Base.string(t::Tensor)
    str = ccall((:tensor_to_string, libtorch_capi),
                Ptr{UInt8}, (Ptr{Cvoid},),
                t.pointer)
    ret = unsafe_string(str)
    ccall(:free, Cvoid, (Ptr{Cvoid},), str)
    return ret
end

function Base.show(io::IO, t::Tensor{T, N}) where {T, N}
    write(io, "PyTorch.Tensor{$T, $N}:\n")
    write(io, string(t))
    write(io, "\n")
end

function Base.display(t::Tensor)
    show(stdout, t)
end

# array interface
Base.eltype(::Type{Tensor{T}}) where {T} = Tensor{T, 0}
Base.ndims(t::Tensor{T, N}) where {T, N} = N

eltype_id(::Tensor{T}) where {T} = Int(TYPE_MAP[T])
eltype_id(::Type{T}) where {T <: TorchNumber} = Int(TYPE_MAP[T])

function Base.strides(t::Tensor)
    n_dims = ccall((:tensor_method_ndimension, libtorch_capi),
                   Clonglong, (Ptr{Cvoid},),
                   t.pointer)
    strides = zeros(Int64, n_dims)
    ccall((:tensor_method_strides, libtorch_capi),
          Cvoid, (Ptr{Cvoid}, Ptr{Cvoid}),
          t.pointer, strides)
    strides
end

function Base.size(t::Tensor{T, N}) where {T, N}
    n_dims = ccall((:tensor_method_ndimension, libtorch_capi),
                   Clonglong, (Ptr{Cvoid},),
                   t.pointer)
    @assert N == n_dims "Dimension mismatch!"
    sizes = zeros(Int64, n_dims)
    ccall((:tensor_method_sizes, libtorch_capi),
          Cvoid, (Ptr{Cvoid}, Ptr{Cvoid}),
          t.pointer, sizes)
    tuple(sizes...)
end

function _tensor_indices(t::Tensor, I)
    indices = collect.(to_indices(t, I))
    shape = collect(Iterators.flatten(size.(indices)))
    indices = map(x -> x .- 1, indices)
    collect(indices), shape
end

_to_dim_0(t::Tensor) = ThC.opt_reshape(t, Int64[])
_to_dim_1_1(t::Tensor) = ThC.opt_reshape(t, [1, 1])

function Base.getindex(t::Tensor, I...)
    ts, shape = _tensor_indices(t, I)
    ret = t
    for i in 1:length(ts)
        ret = ThC.opt_index_select(ret, i - 1, ts[i])
    end
    all(x -> x == 1, size(ret)) && shape == Union{}[] && return _to_dim_0(ret)
    ThC.opt_reshape(ret, shape)
end
Base.getindex(t::Tensor{T}) where T = item(t)
Base.getindex(t::Tensor, i::Int64) = t[eachindex(t)[i]]
Base.getindex(t::Tensor{T, 1}, i::Int64) where T =
    ThC.opt_index_select(t, 0, (i - 1)) |> _to_dim_0
function Base.getindex(t::Tensor, I::UnitRange{Int64})
    t = vcat(map(i->_to_dim_1_1(t[i]), eachindex(t)[I])...)
    ThC.opt_reshape(t, [length(t)])
end

function Base.setindex!(t::Tensor{T}, v::Tensor{T}, I...) where T
    @assert length(I) > 0  "no indices given"
    @assert(!any(i -> i isa StepRange, I),
            "StepRange indices are not supported in Tensor assignment")
    ts, _1 = _tensor_indices(t, I)
    ret = t
    for i in 1:(length(ts) - 1)
        ret = ThC.narrow(ret, i - 1, ts[i][1], length(ts[i]))
    end
    dshape = length.(ts)
    ThC.index_copy!(ret, length(ts) - 1, Tensor(ts[end]), reshape(v, dshape))
    v
end
Base.setindex!(t::Tensor{T}, v::Array, I...) where T =
    setindex!(t, Tensor{T}(v), I...)
Base.setindex!(t::Tensor{T}, v::TorchNumber, i::Int64) where T =
    setindex!(t, Tensor{T}([v]), (eachindex(t)[i].I)...)
Base.setindex!(t::Tensor{T, 1}, v::TorchNumber, i::Int64) where T =
    setindex!(t, Tensor{T}([v]), i)

function Base.setindex!(t::Tensor{T}, v::Array, I::UnitRange{Int64}) where T
    indices = eachindex(t)[I]
    @assert length(v) == length(indices)
    for idx in 1:length(v)
        setindex!(t, Tensor{T}(v[[idx]]), (indices[idx].I)...)
    end
end

function Base.iterate(t::Tensor, state=(eachindex(t),))
    y = iterate(state...)
    y === nothing && return nothing
    t[y[1]], (state[1], Base.tail(y)...)
end

Base.cat(I::Vararg{Tensor}; dims) = cat(collect(I), dims)
Base.vcat(I::Vararg{Tensor}) = cat(collect(I), 0)
Base.hcat(I::Vararg{Tensor}) = cat(collect(I), 1)
function Base.hvcat(rows::Tuple{Vararg{Int}}, I::Vararg{Tensor,N}) where N
    ts = Iterators.Stateful(I)
    hs = map(n -> collect(Iterators.take(ts, n)), rows)
    hs = [hcat(t...) for t in hs]
    vcat(hs...)
end

# methods
function item(t::Tensor{T,N}) where {T,N}
    @assert(N == 0 || prod(size(t)) == 1,
            "The Tensor must contain only one element.")
    data = T[zero(T)]
    ccall((:tensor_method_item, libtorch_capi),
          Cvoid, (Ptr{Cvoid}, Cchar, Ptr{Cvoid}),
          t.pointer, TYPE_MAP[T], data)
    return data[1]
end

# devices

abstract type Device end
struct CPU <: Device end
struct CUDA <: Device
    index::Int
end

Base.convert(::Type{Int}, ::CPU) = -1
Base.convert(::Type{Int}, d::CUDA) = d.index

to(t::Tensor, d::Device) = ThC.to(t, convert(Int, d))
to(t::Tensor, ::Type{T}) where T <: TorchNumber = ThC.to2(t, eltype_id(T), 0, 0)
to(t::Tensor, ::Type{T}, d::Device) where T <: TorchNumber = to(t, d, T)
to(t::Tensor, d::Device, ::Type{T}) where T <: TorchNumber =
    ThC.to4(t, convert(Int, d), eltype_id(T), 0, 0)

function on(t::Tensor)
    data = Int64[0, 0]
    ccall((:tensor_method_device, libtorch_capi),
          Cvoid, (Ptr{Cvoid}, Ptr{Cvoid}), t.pointer, data)

    data[1] == -1 && return CPU()
    return CUDA(data[2])
end


# out-of-place initialization
init(𝒜::GLGM06, 𝑆::AbstractSystem, 𝑂::Options) = init!(𝒜, 𝑆, copy(𝑂))

function options_GLGM06()

    𝑂spec = Vector{OptionSpec}()

    # step size
    push!(𝑂spec, OptionSpec(:δ, 1e-2, domain=Float64, aliases=[:sampling_time],
                            domain_check=(v  ->  v > 0.), info="time step"))

    # discretization
    push!(𝑂spec, OptionSpec(:discretization, "forward", domain=String,
                            info="model for bloating/continuous time analysis"))

    push!(𝑂spec, OptionSpec(:sih_method, "concrete", domain=String,
                            info="method to compute the symmetric interval hull in discretization"))

    push!(𝑂spec, OptionSpec(:exp_method, "base", domain=String,
                            info="method to compute the matrix exponential"))

    # approximation options
    push!(𝑂spec, OptionSpec(:max_order, 10, domain=Int,
                            info="maximum allowed order of zonotopes"))

    return 𝑂spec
end

# in-place initialization
function init!(::GLGM06, 𝑆::AbstractSystem, 𝑂::Options)

    # state dimension
    𝑂[:n] = statedim(𝑆)

    # adds default values for unspecified options
    𝑂init = validate_solver_options_and_add_default_values!(𝑂)

    return 𝑂init
end


export add_type_B!, remove_type_B!, change_type_B!

"""Returns True if left_kink and right_kink are entangled in a Type-B way.
This does not check wether the two kinks are neighbouring"""
function is_type_B(left_kink::T4, right_kink::T4)
  if (Set([left_kink.i, left_kink.j]) == Set([right_kink.k, right_kink.l])) &
        (Set([left_kink.k, left_kink.l]) == Set([right_kink.i, right_kink.j]))
    return true
  else
    return false
  end
end

"""Return pairs with indices of 'neighbouring' kinks that are Type-B-Entangled.
This function has no use in the current update set. Removability of a Kink will therefore not be considered here.
'neighbouring' refers to that only Tuples of Kinks that are the closest Kink to act on an orbital of the
other kink in the corresponding direktion are looked at.
The Tuples are always arranged in a way that the Kink who gets neighboured by
the opther stands first. (for type-B-entanglement that always imples the
vice versa case)
The Set consists of the pairs where the Type-B-entanglement is oriented
to the left of the first τ."""
function left_type_B_pairs(ck)
    pairs_left = Set{Tuple{Int,Int}}()
    for (i,(τ,kink)) in enumerate(ck)
        i_left = prev_affecting(ck, orbs(kink), τ)
        if is_type_B(last(ck[τ_left]), kink)
            push!(pairs_left, (i, i_left))
        end
    end
    return pairs_left
end

"""Return pairs with indices of 'neighbouring' kinks that are Type-B-Entangeld. Removablility of Kinks will
no be looked at. 'neighbouring' refers to that only Tuples of Kinks that are the closest Kink to act on an orbital of the
other kink in the corresponding direktion are looked at.
The Tuples are always arranged in a way that the Kink who gets neighboured by
the opther stands first. (for type-B-entanglement that always imples the
vice versa case)
The Set consists of the pairs where the Type-B-entanglement is oriented
#to the right of the first τ."""
function right_type_B_pairs(ck)
    pairs_right = Set{Tuple{Int,Int}}()
    for (i,(τ,kink)) in enumerate(ck)
        i_right = next_affecting(ck, orbs(kink), τ)
        if is_type_B(kink, last(ck[i_right]))
            push!(pairs_right, (i, i_right))
        end
    end
    return pairs_right
end

"""Return pairs with indices of 'neighbouring' Kinks that are Type-B-Entangeld AND removable.
'neighbouring' refers to that only Tuples of Kinks that are the closest Kink to act on an orbital of the
other kink in the corresponding direktion are looked at.
The Tuples are always arranged in a way that the Kink who gets neighboured by
the other and is removable stands first. (for type-B-entanglement that always imples the
vice versa case)
The Set consists of the pairs where the Type-B-entanglement is oriented
#to the right of the first τ."""
function right_type_B_removable_pairs(ck)
    pairs_right = Set{Tuple{Int,Int}}()
    for (i,(τ,kink)) in enumerate(ck)
        i_right = next_affecting(ck, orbs(kink), τ)
        if is_type_B(kink, last(ck[i_right]))
            if norm(kink.i.vec - kink.k.vec) <= ex_radius
                if kink.i.spin == kink.k.spin
                    push!(pairs_right, (i, i_right))
                end
            end
        end
    end
    return pairs_right
end

"""
    possible_new_orb_a(occs, orb_c, orb_d)
This function will find possibilites for the choice of the first annihilator for an add_type_B-update when the two creators are already selected.
To check possibility this will in particular check conservation laws and the occupation of the resulting fourth orb.
"""
function possible_new_orb_a(occs, orb_c, orb_d)
    filter(orb_a -> !in(find_fourth_orb_for_kink(orb_a, orb_c, orb_d),occs) && (orb_a != find_fourth_orb_for_kink(orb_a, orb_c, orb_d)),
                    setdiff!(sphere_with_same_spin(orb_c, dk = ex_radius), occs))
end

function add_type_B!(m::Model, e::Ensemble, c::Configuration) :: Tuple{Float64, Step}
    #sampling propability
    prop_prob = 1
    #get first τ
    τ1 = ImgTime(rand())
    #do not allow two kinks at the same time
    while haskey(c.kinks, τ1)
        τ1 = ImgTime(rand())
    end
    occs = occupations_at(c, τ1)
    orb_c = rand(occs)
    prop_prob *= 1.0/e.N
    orb_d = rand(drop_orbs(occs, (orb_c,)))
    prop_prob *= 1.0/(e.N-1)
    opportunities_orb_a = possible_new_orb_a(occs, orb_c, orb_d)
    if isempty(opportunities_orb_a)
        return 1.0, Step()
    end
    orb_a = rand(opportunities_orb_a)
    orb_b = find_fourth_orb_for_kink(orb_a, orb_c, orb_d)
    @assert !((orb_a == orb_d) | (orb_b == orb_c))
    @assert (!in(orb_b,occs) & (orb_a != orb_b))
    #We will change the proposal probability after we get τ2

    #get τ2
    borders = τ_borders(c.kinks, Set([orb_a,orb_b,orb_c,orb_d]),τ1)
    possible_τ2_interval = borders[2]-borders[1]
    if possible_τ2_interval < 0
        possible_τ2_interval += 1
    end
    τ2 = ImgTime(rand()*(possible_τ2_interval) + borders[1])
    if τ2 > 1
        τ2 -= 1
    end
    #do not allow two kinks at the same time
    while haskey(c.kinks, τ2)
        τ2 = ImgTime(rand()*(possible_τ2_interval) + borders[1])
        if τ2 > 1
            τ2 -= 1
        end
    end
    #See which of the two imiginary time point is the "left border of the intervall"
    #If there are no kinks effecting any of the new kinks orbitals then anyone
    #of the two τs can be first τ
    if borders[2] == 1
        if rand() < 0.5
            firstτ = τ1
            lastτ = τ2
        else
            firstτ = τ2
            lastτ = τ1
        end
        prop_prob *= 0.5
    #If there are kinks which τ is firstτ depends on which is the
    #left one inside the given Interval
    elseif τ1 > borders[1]
        if τ2 > borders[1]
            firstτ = min(τ1,τ2)
            lastτ = max(τ1,τ2)
        else
            firstτ = τ1
            lastτ = τ2
        end
    else
        if τ2 < borders[1]
            firstτ = min(τ1,τ2)
            lastτ = max(τ1,τ2)
        else
            firstτ = τ2
            lastτ = τ1
        end
    end
    delta_τ = lastτ - firstτ
    if delta_τ < 0
        delta_τ += 1
    end
    @assert(delta_τ > 0)
    @assert(delta_τ <= 1)

    #We do consider states that differ only threw the order off indices of kinks
    #as different states, that contribute all with the same weight with is already
    #blocked over all permutations (see function “abs_Woffdiag_element”),
    #therefore the updates where we end up with the same kinks but start building
    #the kink with a different Excitation will result in a different order off indices
    #and therefore considered a different Update (to compensate that we use a factor ¼
    #in the off diagonal Matrix element contribution) .

    #However we have to consider the different ways off getting to the same
    #update by choosing the imaginary times in a different order.

    #Therefore we modify the proposal_probability in the following way
    occs_τ2 = occupations_at(c, τ2)
    opportunities_orb_a_τ2 = possible_new_orb_a(occs_τ2, orb_c, orb_d)
    @assert !isempty(opportunities_orb_a_τ2)
    prop_prob *= (1.0/length(opportunities_orb_a) + 1.0/length(opportunities_orb_a_τ2)) * 1.0/float(possible_τ2_interval)

    kinks_add = (
                firstτ => T4(orb_a,orb_b,orb_c,orb_d),
                # shuffle creators and shuffle annihilators
                lastτ => T4( random_shuffle(orb_d,orb_c)..., random_shuffle(orb_b, orb_a)... )
                )
    prop_prob *= 1/4

    #Look at wether the new pair of kinks modifies the Occupations at τ = 0
    if firstτ > lastτ
        orbs_drop = (orb_c, orb_d)
        orbs_add = (orb_a, orb_b)
    else
        orbs_drop = nothing
        orbs_add = nothing
    end

    # MC step generated by this update
    Δ = Step(orbs_drop, nothing, orbs_add, kinks_add)

    #calculate change in diagonal interaction energy
    delta_di = ΔWdiag_element(m, e, c, orb_a, orb_b, orb_c, orb_d, firstτ, lastτ)

    apply_step!(c,Δ)
    # quotient of proposal probabilities
    dv = ( 1.0/ length( right_type_B_removable_pairs(c.kinks) ) ) / prop_prob

    # weight factor
    dw = ((e.β)^2) * abs(Woffdiag_element(m, e, orb_a, orb_b,orb_c,orb_d))^2 * exp(-((delta_τ)*e.β * (energy(m, orb_a) + energy(m, orb_b) - energy(m, orb_c) - energy(m, orb_d)) + delta_di*e.β))
    @assert (dv*dw) >= 0
    @assert !isinf(dv*dw)
    return dv*dw, Δ
end

function remove_type_B!(m::Model, e::Ensemble, c::Configuration) :: Tuple{Float64, Step}
    if isempty(c.kinks)
        return 1.0, Step()
    end
    opportunities = right_type_B_removable_pairs(c.kinks)
    if isempty(opportunities)
        return 1.0, Step()
    end
    #If a kink1 is entangeld to the right with the nearest kink, that
    # acts on one of his orbs, in a type-B-way that implies the vice versa case
    # therefor there is no value in distingusihing between left and right entanglement
    i1, i2 = rand(opportunities)
    kink1 = c.kinks[i1]
    kink2 = c.kinks[i2]
    prop_prob = 1.0/length(opportunities)

    #change configuration
    #see if occupations at τ=0 are modified
    if first(kink1) > first(kink2)
        # change_occupations_at(c.occupations, last(kink2))
        orbs_add = (last(kink2).i, last(kink2).j)
        orbs_drop = (last(kink2).k, last(kink2).l)
        delta_τ = first(kink2)-first(kink1) + 1.0
    else
        orbs_add = nothing
        orbs_drop = nothing
        delta_τ = first(kink2)-first(kink1)
    end

    # MC step generated by this update
    Δ = Step(orbs_drop, (kink1, kink2), orbs_add, nothing)
    @assert(delta_τ > 0)
    @assert(delta_τ <= 1)

    apply_step!(c,Δ)

    # calculate inverse prop_prob (see add_type_B)
    ijkl = Set([last(kink1).i, last(kink1).j, last(kink1).k, last(kink1).l])
    borders = τ_borders(c.kinks, ijkl, first(kink1))
    possible_τ2_interval = borders[2]-borders[1]
    if possible_τ2_interval < 0
        possible_τ2_interval = 1 + possible_τ2_interval
    end
    occs_τ_kink1 = occupations_at(c, first(kink1))
    occs_τ_kink2 = occupations_at(c, first(kink2))
    orb_a = last(kink1).i
    orb_b = last(kink1).j
    orb_c = last(kink1).k
    orb_d = last(kink1).l

    #See how prop_prob changes in the function add_type_B to understand this expression
    inverse_prop_prob = (1/e.N)*(1/(e.N-1)) * (1.0/length(possible_new_orb_a(occs_τ_kink1, orb_c, orb_d)) + 1/length(possible_new_orb_a(occs_τ_kink2, orb_c, orb_d))) * 1.0/float(possible_τ2_interval) * (1/4)

    if borders[2] == 1
        inverse_prop_prob *= 0.5
    end
    # quotient of proposal probabilities
    dv = inverse_prop_prob/prop_prob

    # calculate change in diagonal interaction energy
    delta_di = ΔWdiag_element(m, e, c, last(kink1).i, last(kink1).j, last(kink1).k, last(kink1).l, first(kink1), first(kink2))
    # weight factor
    dw = (1.0/(e.β)^2) * (1.0/(abs(Woffdiag_element(m, e, last(kink1))))^2) * exp((delta_τ)*e.β * (energy(m, orb_a) + energy(m, orb_b) - energy(m, orb_c) - energy(m, orb_d)) + e.β * delta_di)

    @assert (dv*dw) >= 0
    @assert !isinf(dv*dw)
    return dv*dw, Δ
end

function change_type_B!(m::Model, e::Ensemble, c::Configuration) :: Tuple{Float64,Step} #This update is redundant wif we have add- and remove-Type-C-Updates
    if isempty(c.kinks)
        return 1.0, Step()
    end

    kink_opportunities = right_type_B_pairs(c.kinks)
    if isempty(kink_opportunities)
        return 1.0, Step()
    end
    i1, i2 = rand(kink_opportunities)
    kink1 = c.kinks[i1]
    kink2 = c.kinks[i2]
    occs = occupations_at(c, first(kink1))

    opportunities = filter( x -> isunaffected_in_interval(c.kinks, x, first(kink1), first(kink2))
                        ,possible_new_orb_a(occs, last(kink1).i, last(kink1).j))
    delete!(opportunities, last(kink1).k)
    delete!(opportunities, last(kink1).l)

    if isempty(opportunities)
        return 1.0, Step()
    end

    new_orb_i = rand(opportunities)
    new_orb_j = find_fourth_orb_for_kink(new_orb_i, last(kink1).i, last(kink1).j)
    if new_orb_i == new_orb_j
        return 1.0, Step()
    end
    if !isunaffected_in_interval(c.kinks,new_orb_j,first(kink1),first(kink2)) | in(new_orb_j,occs)
        return 1.0, Step()
    end

    # see if occupations change
    if first(kink1) > first(kink2)
        orbs_drop = creators(last(kink1))
        orbs_add = (new_orb_i, new_orb_j)
        delta_τ = first(kink2)-first(kink1) + 1.0
    else
        orbs_add = nothing
        orbs_drop = nothing
        delta_τ = first(kink2)-first(kink1)
    end

    #calculate change in diagonal interaction energy
    delta_di = ΔWdiag_element(m, e, c, new_orb_i, new_orb_j, last(kink1).i, last(kink1).j, first(kink1), first(kink2))

    # shuffle indices of the new orbs in the second kink
    kinks_drop = (kink1,kink2)
    kinks_add = (
                first(kink1) => T4(new_orb_i, new_orb_j, last(kink1).k, last(kink1).l),
                # shuffle creators and shuffle annihilators
                first(kink2) => T4( random_shuffle(last(kink2).i, last(kink2).j)..., random_shuffle(new_orb_i, new_orb_j)...)
                )
    # do not add factor 1/4 to proposal porbability (prop_prob *= 1/4)
    # since this cancels with the reverse update in the quotient of proposal propabilities

    excite!(occs, new_orb_i, new_orb_j, last(kink1).i, last(kink1).j)

    # MC Step generated by this update
    Δ = Step(orbs_drop, kinks_drop, orbs_add, kinks_add)

    apply_step!(c,Δ)

    kink_opportunities_reverse = right_type_B_pairs(c.kinks)
    opportunities_reverse = filter( x -> isunaffected_in_interval(c.kinks, x, first(kink1), first(kink2))
                                ,possible_new_orb_a(occs, new_orb_i, new_orb_j))
    delete!(opportunities_reverse, last(kink1).k)
    delete!(opportunities_reverse, last(kink1).l)

    # quotient of proposal probabilities
    dv = length(opportunities)*length(kink_opportunities)/(length(opportunities_reverse)*length(kink_opportunities_reverse))

    # weight factor
    dw = exp(-(e.β*delta_τ*(energy(m, new_orb_i) + energy(m, new_orb_j) - energy(m, last(kink1).i) - energy(m, last(kink1).j)) + e.β*delta_di)) * ( Woffdiag_element(m, e, new_orb_i, new_orb_j, last(kink1).k, last(kink1).l ) / Woffdiag_element(m, e, last(kink1)) )^2

    @assert (dw * dv) >= 0
    @assert !isinf(dw * dv)
    return dw*dv, Δ
end

function isuseful(c::Configuration, up::typeof(remove_type_B!))
    if isempty(c.kinks) || length(c.kinks) == 3
        return false
    else
        return true
    end
end

function isuseful(c::Configuration, up::typeof(change_type_B!))
    if isempty(c.kinks) || length(c.kinks) == 3
        return false
    else
        return true
    end
end


using Pardiso: Pardiso, set_solver!, pardisoinit, set_phase!, set_iparm!, set_msglvl!, get_iparm, pardiso, PardisoSolver, MKLPardisoSolver, get_nprocs, set_nprocs!, set_matrixtype!
export PardisoDirectKKTSolver, PardisoIndirectKKTSolver, MKLPardisoSolver

abstract type AbstractPardisoKKTSolver <: AbstractKKTSolver end

#---------------------------
#Pardiso 5.0 Direct Solver Configuration
#---------------------------

struct PardisoDirectKKTSolver{Tv, Ti} <: AbstractPardisoKKTSolver

    ps::PardisoSolver
    K::SparseMatrixCSC{Tv, Ti}
    m::Integer
    n::Integer
    work::Vector{Tv}  #working memory for Pardiso

    function PardisoDirectKKTSolver(P::SparseMatrixCSC{Tv, Ti}, A::SparseMatrixCSC{Tv, Ti}, sigma, rho;
        iparm::Dict{Int64, Int64} = Dict{Int64, Int64}(), msg_level_on::Bool = false) where {Tv, Ti}

        m, n, K, ps, work = _pardiso_common_init(P, A, sigma, rho, PardisoSolver, iparm, msg_level_on)

        set_solver!(ps,Pardiso.DIRECT_SOLVER)
        _kktutils_direct_solver_init(ps, K, work, m, n)

        return new{Tv, Ti}(ps, K, m, n, work)
    end
end

#---------------------------
#Pardiso 5.0 Indirect Solver Configuration
#---------------------------

struct PardisoIndirectKKTSolver{Tv, Ti} <: AbstractPardisoKKTSolver

    ps::PardisoSolver
    K::SparseMatrixCSC{Tv, Ti}
    m::Integer
    n::Integer
    work::Vector{Tv}  #working memory for Pardiso

    function PardisoIndirectKKTSolver(P::SparseMatrixCSC{Tv, Ti}, A::SparseMatrixCSC{Tv, Ti}, sigma, rho;
        iparm::Dict{Int64, Int64} = Dict{Int64, Int64}(), msg_level_on::Bool = false) where {Tv, Ti}

        m, n, K, ps, work = _pardiso_common_init(P, A, sigma, rho, PardisoSolver, iparm, msg_level_on)

        set_solver!(ps, Pardiso.ITERATIVE_SOLVER)
        pardisoinit(ps)

        ## set phase to solving for iterations
        set_phase!(ps, Pardiso.SOLVE_ITERATIVE_REFINE)

        return new{Tv, Ti}(ps, K, m, n, work)
    end
end

#---------------------------
#MKL Pardiso Solver Configuration
#---------------------------

struct MKLPardisoKKTSolver{Tv, Ti} <: AbstractPardisoKKTSolver

    ps::MKLPardisoSolver
    K::SparseMatrixCSC{Tv, Ti}
    m::Integer
    n::Integer
    work::Vector{Tv}  #working memory for Pardiso

    function MKLPardisoKKTSolver(P::SparseMatrixCSC{Tv, Ti}, A::SparseMatrixCSC{Tv, Ti}, sigma, rho;
        iparm::Dict{Int64, Int64} = Dict{Int64, Int64}(), msg_level_on::Bool = false, num_threads::Int64 = 1) where {Tv, Ti}

        m, n, K, ps, work = _pardiso_common_init(P, A, sigma, rho, MKLPardisoSolver, iparm, msg_level_on)

        _kktutils_direct_solver_init(ps, K, work, m, n)
        if num_threads >= 1
            set_nprocs!(ps, num_threads)
        else
            error("The number of CPU threads has to be greater than 0.")
        end

        return new{Tv, Ti}(ps ,K, m, n, work)
    end
end

#-------------------
#Pardiso common utils

function _pardiso_common_init(P, A, sigma, rho, Solver::Type, iparm::Dict{Int64, Int64}, msg_level_on::Bool)

    #allow Pardiso solvers do this first

    m,n  = _kktutils_check_dims(P, A, sigma, rho)
    K    = _kktutils_make_kkt(P, A, sigma, rho, :L)
    work = zeros(eltype(A), m + n)
    ps   = Solver()

    #set to symmetric indefinite
    set_matrixtype!(ps, Pardiso.REAL_SYM_INDEF)

    if msg_level_on
        set_msglvl!(ps, Pardiso.MESSAGE_LEVEL_ON)
    else
        set_msglvl!(ps, Pardiso.MESSAGE_LEVEL_OFF)
    end

    # set to allow non-default iparams
    set_iparm!(ps, 1, 1)

    # set user specific parameters
    for (i, v) in iparm
        set_iparm!(ps, i, v)
    end

    return m, n, K, ps, work
end

function _kktutils_direct_solver_init(ps, K, work, m, n)

    # set the transpose flag (Pardiso: CSR, Julia CSC)
    set_iparm!(ps, 12, 1)

    # Analyze the matrix and compute a symbolic factorization.
    set_phase!(ps, Pardiso.ANALYSIS)
    pardiso(ps, K, work)

    # Compute the numeric factorization.
     set_phase!(ps, Pardiso.NUM_FACT)
     pardiso(ps, K, work)
    _pardiso_check_inertia(ps, m, n)

    ## set phase to solving for iterations
    set_phase!(ps, Pardiso.SOLVE_ITERATIVE_REFINE)
end

solve!(s::AbstractPardisoKKTSolver, lhs, rhs) = pardiso(s.ps, lhs, s.K, rhs)

function update_rho!(s::AbstractPardisoKKTSolver, rho::Union{Vector,AbstractFloat})
    update_kkt_matrix(s.K, s.n, s.m, rho)

    #only refactor for the direct solver
    if s.ps.solver == Pardiso.DIRECT_SOLVER
        # Compute the numeric factorization again,
        #but skipping the analysis phase
        set_phase!(s.ps, Pardiso.NUM_FACT)
        pardiso(s.ps, s.K, s.work)
        _pardiso_check_inertia(s.ps, s.m, s.n)

        ## set back to solving phase
        set_phase!(s.ps, Pardiso.SOLVE_ITERATIVE_REFINE)
    end
end

function _pardiso_check_inertia(ps, m, n)

    num_pos_eigenvalues = get_iparm(ps, 22)
    num_neg_eigenvalues = get_iparm(ps, 23)

    num_neg_eigenvalues == m || throw("P matrix appears to have negative eigenvalues")
end

function free_memory!(s::AbstractPardisoKKTSolver)
    set_phase!(s.ps, Pardiso.RELEASE_ALL)
    pardiso(s.ps)
end

#number of solver threads active
get_number_of_threads(s::AbstractPardisoKKTSolver) = get_nprocs(s.ps)

#MKL can set number of threads directly
set_number_of_threads(s::MKLPardisoKKTSolver, i) = set_nprocs!(s.ps, i)

#Non-MKL can only set number of threads via an environment variable
set_number_of_threads(s::AbstractPardisoKKTSolver, i) =
    error("Pardiso 5.0 (non-MKL) must set thread count via environment variable, e.g. ENV[\"OMP_NUM_THREADS\"] = 4")


# include("../datasets/Vic120MM.jl")

function addLandmarksFactoGraph!(fg::FactorGraph, f, idx, prevn, nextn;
                    lcmode=:mmodal, lsrNoise=diagm([0.1;1.0]), N::Int=100, MM=Union{})
    I = intersect(keys(f[idx-1]),keys(f[idx]))
    for i in I
      pfez = f[idx-1][i]
      fez = f[idx][i]
      if abs(fez[2]) < (pi/2.0+5.0/180.0*pi)   # only add those recently seen by the laser, and not purely propagated way behind car
        lsy = string("l",i)
        if !haskey(MM,i)
          if !haskey(fg.IDs, lsy)    # has landmark
            # add from previous and latest factor here
            projNewLandm!(fg, prevn, lsy, [pfez[2];pfez[1]], lsrNoise , N=N )
            addBRFG!(fg, nextn, lsy, [fez[2];fez[1]], lsrNoise)
          else
            # previous already added, only add the new factor here
            addBRFG!(fg, nextn, lsy, [fez[2];fez[1]], lsrNoise)
          end
        else
          lsymm = string("l",MM[i])
          if lcmode == :mmodal
            println("Adding bimodal factor")
            if !haskey(fg.IDs, lsy)
              vlm = projNewLandm!(fg, prevn, lsy, [pfez[2];pfez[1]], lsrNoise, addfactor=false)
              addMMBRFG!(fg, prevn, [lsymm;lsy], [pfez[2];pfez[1]], lsrNoise, w=[0.5;0.5] )
            end
            addMMBRFG!(fg, nextn, [lsymm;lsy], [fez[2];fez[1]], lsrNoise, w=[0.5;0.5] )
          elseif lcmode == :unimodal
            println("adding unimodal loop closure")
            addBRFG!(fg, nextn, lsymm, [fez[2];fez[1]], lsrNoise)
          end
        end
      end
    end

    # error("addLandmarksFactoGraph! -- not implemented yet")
    nothing
end

#@time d,f,DBG = advOdoByRules(DRS[1:2000,:],LsrFeats,distrule=15.0,yawrule=pi/6.0);
#progressExamplePlot(dOdo, LsrFeats, toT=100)
#plotParVicPrk(DRS)

function appendFactorGraph!(fg::FactorGraph, d, f;
                  toT=Inf, fgpdf=false, lcmode=:mmodal,
                  lsrNoise=diagm([0.1;1.0]), P0=diagm([0.01;0.01;0.001]), Podo=diagm([0.5;0.5;0.005]),
                  idx::Int=1, N::Int=100, MM=Union{})

  len = length(d)
  T = d[idx][4]
  if idx==1
    # init pose
    prevn = initFactorGraph!(fg, init=d[idx][1:3])
  else
    v,X,nextn = getLastPose2D(fg)
    @show prevn = String(v.label)
  end

  while T < toT && idx <= len
    idx += 1
    T = idx <= len ? d[idx][4] : break
    if T > toT
      break
    end

    # add new poses via odom
    prev, X, nextn = getLastPose2D(fg)
    vp, fp = addOdoFG!(fg, nextn, d[idx][1:3], Podo, N=N)

    # add landmarks
    addLandmarksFactoGraph!(fg, f, idx, prevn, nextn, lcmode=lcmode, lsrNoise=lsrNoise, N=N, MM=MM)
    prevn = nextn
  end
  if fgpdf   writeGraphPdf(fg); end
  return idx
end

function doBatchRun(d, f; toT=30)
  fg = initfg()
  appendFactorGraph!(fg, d, f; toT=toT);
  p = drawPosesLandms(fg)
  tree = prepBatchTree!(fg);
  writeGraphPdf(fg); # make pdf of factor graph so we can see
  inferOverTree!(fg, tree)
  return fg, tree, p
end

function saveImgSeq(d::Dict{Int,Dict{Int,Feature}}, lsrFeats::Dict{Int,LaserFeatures}; from::Int=1,to::Int=10,step::Int=1)
  for i in from:step:to
    p = drawFeatTrackers(d[i], lsrFeats[i].feats);
    Gadfly.draw(PNG(string("imgs/img",i,".png"),35cm,25cm),p)
  end
  nothing
end

function drawIROSVicPrkFig(fgl::FactorGraph, d, f, T, isamdict, fgu)
  p1 = progressExamplePlot(d,f,toT=T);
  p2 = drawCompPosesLandm(fg,isamdict, fgu, lbls=false,drawunilm=false);

  vstack(p1,p2)
end


"""
    Layers(layers::Vector{Layer})

Algebraic object encoding a list of [`AlgebraOfGraphics.Layer`](@ref) objects.
`Layers` objects can be added or multiplied, yielding a novel `Layers` object.
"""
struct Layers
    layers::Vector{Layer}
end

Base.convert(::Type{Layers}, s::Layer) = Layers([s])
Base.convert(::Type{Layers}, s::Layers) = s

Base.getindex(v::Layers, i::Int) = v.layers[i]
Base.length(v::Layers) = length(v.layers)
Base.eltype(::Type{Layers}) = Layer
Base.iterate(v::Layers, args...) = iterate(v.layers, args...)

const OneOrMoreLayers = Union{Layers, Layer}

function Base.:+(s1::OneOrMoreLayers, s2::OneOrMoreLayers)
    l1::Layers, l2::Layers = s1, s2
    return Layers(vcat(l1.layers, l2.layers))
end

function Base.:*(s1::OneOrMoreLayers, s2::OneOrMoreLayers)
    l1::Layers, l2::Layers = s1, s2
    return Layers([el1 * el2 for el1 in l1 for el2 in l2])
end

function compute_processedlayers_grid(processedlayers, categoricalscales)
    indices = CartesianIndices(compute_grid_positions(categoricalscales))
    pls_grid = map(_ -> ProcessedLayer[], indices)
    for processedlayer in processedlayers
        append_processedlayers!(pls_grid, processedlayer, categoricalscales)
    end
    return pls_grid
end

function compute_entries_continuousscales(pls_grid, categoricalscales)
    # Here processed layers in `pls_grid` are "sliced",
    # the categorical scales have been applied, but not
    # the continuous scales

    rescaled_pls_grid = map(_ -> ProcessedLayer[], pls_grid)
    continuousscales_grid = map(_ -> MixedArguments(), pls_grid)

    for idx in eachindex(pls_grid), pl in pls_grid[idx]
        # Apply continuous transformations
        positional = map(contextfree_rescale, pl.positional)
        named = map(contextfree_rescale, pl.named)
        plottype = Makie.plottype(pl.plottype, positional...)

        # Compute continuous scales with correct plottype, to figure out role of color
        continuousscales = AlgebraOfGraphics.continuousscales(ProcessedLayer(pl; plottype))
        mergewith!(mergescales, continuousscales_grid[idx], continuousscales)

        # Compute `ProcessedLayer` with rescaled columns
        push!(rescaled_pls_grid[idx], ProcessedLayer(pl; plottype, positional, named))
    end

    # Compute merged continuous scales, as it may be needed to use global extrema
    merged_continuousscales = reduce(mergewith!(mergescales), continuousscales_grid, init=MixedArguments())

    to_entry = function (pl)
        attrs = compute_attributes(pl, categoricalscales, continuousscales_grid, merged_continuousscales)
        return Entry(pl.plottype, pl.positional, attrs)
    end
    entries_grid = map(pls -> map(to_entry, pls), rescaled_pls_grid)

    return entries_grid, continuousscales_grid, merged_continuousscales
end

function compute_palettes(palettes)
    layout = Dictionary((layout=wrap,))
    theme_palettes = map(to_value, Dictionary(Makie.current_default_theme()[:palette]))
    user_palettes = Dictionary(palettes)
    return foldl(merge!, (layout, theme_palettes, user_palettes), init=NamedArguments())
end

function compute_axes_grid(fig, s::OneOrMoreLayers;
                           axis=NamedTuple(), palettes=NamedTuple())

    axes_grid = compute_axes_grid(s; axis, palettes)
    sz = size(axes_grid)
    if sz != (1, 1) && fig isa Axis
        msg = "You can only pass an `Axis` to `draw!` if the calculated layout only contains one element. Elements: $(sz)"
        throw(ArgumentError(msg))
    end

    return map(ae -> AxisEntries(ae, fig), axes_grid)
end

function compute_axes_grid(s::OneOrMoreLayers;
                           axis=NamedTuple(), palettes=NamedTuple())
    layers::Layers = s
    processedlayers = map(ProcessedLayer, layers)

    palettes = compute_palettes(palettes)

    categoricalscales = MixedArguments()
    for processedlayer in processedlayers
        mergewith!(
            mergescales,
            categoricalscales,
            AlgebraOfGraphics.categoricalscales(processedlayer, palettes)
        )
    end
    # fit categorical scales (compute plot values using all data values)
    map!(fitscale, categoricalscales, categoricalscales)

    pls_grid = compute_processedlayers_grid(processedlayers, categoricalscales)
    entries_grid, continuousscales_grid, merged_continuousscales =
        compute_entries_continuousscales(pls_grid, categoricalscales)

    indices = CartesianIndices(pls_grid)
    axes_grid = map(indices) do c
        return AxisSpecEntries(
            AxisSpec(c, axis),
            entries_grid[c],
            categoricalscales,
            continuousscales_grid[c]
        )
    end

    # Axis labels and ticks
    for ae in axes_grid
        ndims = isaxis2d(ae) ? 2 : 3
        for (i, var) in zip(1:ndims, (:x, :y, :z))
            scale = get(ae.categoricalscales, i) do
                return get(ae.continuousscales, i, nothing)
            end
            isnothing(scale) && continue
            label = getlabel(scale)
            # Use global scales for ticks for now
            # TODO: requires a nicer mechanism that takes into account axis linking
            (scale isa ContinuousScale) && (scale = merged_continuousscales[i])
            for (k, v) in pairs((label=to_string(label), ticks=ticks(scale)))
                keyword = Symbol(var, k)
                # Only set attribute if it was not present beforehand
                get!(ae.axis.attributes, keyword, v)
            end
        end
    end

    return axes_grid
end


# import stuff
using Distributed
@everywhere using Pkg
@everywhere Pkg.activate(".")
@everywhere using Statistics
@everywhere using GRASS
@everywhere using SharedArrays
@everywhere using EchelleCCFs
using CSV
using JLD2
using LsqFit
using FileIO
using DataFrames
using LaTeXStrings
using HypothesisTests

# define rms loop function
include(GRASS.moddir * "figures/fig_functions.jl")

# some global stuff
const N = 132
const Nloop = 1200

# get command line args and output directories
run, plot = parse_args(ARGS)
grassdir, plotdir, datadir = check_plot_dirs()

function main()
    # set observing parameters
    N_obs = range(2, 4, step=1)  # number of observations
    exp_time = 600.0             # ~2 p-mode periods
    snr = 5000.0                 # SNR per res element to get 50 cm/s precision
    new_res = 1.17e5

    # dead times
    short_dead = 30.0            # representative for NEID read-out time
    large_dead = 3600.0          # ~60 mins

    # set up parameters for spectrum
    lines = [5434.5]
    depths = [0.8]
    resolution = 7e5

    # allocate shared arrays for short off-target time
    avg_shortdead = SharedArray{Float64}(length(N_obs), Nloop)
    rms_shortdead = SharedArray{Float64}(length(N_obs), Nloop)

    # allocate shared arrays for large off-target time
    avg_largedead = SharedArray{Float64}(length(N_obs), Nloop)
    rms_largedead = SharedArray{Float64}(length(N_obs), Nloop)

    # create spec params instance
    spec1 = SpecParams(lines=lines, depths=depths, resolution=resolution)

    # loop over number of observations
    for i in eachindex(N_obs)
        # create observation plan instances
        obs_shortdead = GRASS.ObservationPlan(N_obs=N_obs[i],
                                              obs_per_night=N_obs[i],
                                              time_per_obs=exp_time,
                                              dead_time=short_dead)
        obs_largedead = GRASS.ObservationPlan(N_obs=N_obs[i],
                                              obs_per_night=N_obs[i],
                                              time_per_obs=exp_time,
                                              dead_time=large_dead)

        # compute vels repeatedly to get good stats
        @sync @distributed for j in 1:Nloop
            # first simulate short off-target time
            rvs1, sigs1 = GRASS.simulate_observations(obs_shortdead, spec1, N=N, snr=snr, new_res=new_res)
            avg_shortdead[i, j] = mean(rvs1)
            rms_shortdead[i, j] = calc_rms(rvs1)

            # then simulate large off-target time
            rvs2, sigs2 = GRASS.simulate_observations(obs_largedead, spec1, N=N, snr=snr, new_res=new_res)
            avg_largedead[i, j] = mean(rvs2)
            rms_largedead[i, j] = calc_rms(rvs2)
        end
    end

    # convert to plain arrays
    avg_shortdead = convert(Array{Float64}, avg_shortdead)
    rms_shortdead = convert(Array{Float64}, rms_shortdead)
    avg_largedead = convert(Array{Float64}, avg_largedead)
    rms_largedead = convert(Array{Float64}, rms_largedead)

    # save the output
    outfile = datadir * "observe_" * string(N) * "_loop_" * string(Nloop) * ".jld2"
    save(outfile,
         "avg_shortdead", avg_shortdead,
         "rms_shortdead", rms_shortdead,
         "avg_largedead", avg_largedead,
         "rms_largedead", rms_largedead,
         "exp_time", exp_time,
         "depths", depths,
         "N_obs", N_obs,
         "snr", snr)
    return nothing
end

# run the simulation
if run
    main()
end

# plotting code block
if plot
    # plotting imports
    import PyPlot; plt = PyPlot; mpl = plt.matplotlib; plt.ioff()
    using PyCall; animation = pyimport("matplotlib.animation")
    mpl.style.use(GRASS.moddir * "figures/fig.mplstyle")

    # read in the data
    file = datadir * "observe_" * string(N) * "_loop_" * string(Nloop) * ".jld2"
    d = load(file)
    avg_shortdead = d["avg_shortdead"]
    rms_shortdead = d["rms_shortdead"]
    avg_largedead = d["avg_largedead"]
    rms_largedead = d["rms_largedead"]
    exp_time = d["exp_time"]
    depths = d["depths"]
    N_obs = d["N_obs"]
    snr = d["snr"]

    # initialize plotting objects
    fig = plt.figure(figsize=(10.5, 6.75))
    ax1 = plt.subplot2grid((2,4),(0,0), colspan=2)
    ax2 = plt.subplot2grid((2,4),(0,2), colspan=2)
    ax3 = plt.subplot2grid((2,4),(1,1), colspan=2)
    axs = [ax1, ax2, ax3]

    # make arrays for limits and loop over plotting windows
    xlims = []
    ylims = []
    for i in eachindex(axs)
        # histogram for short dead time
        axs[i].hist(avg_shortdead[i,:], density=true, histtype="stepfilled", fc="k", ec="k", lw=2, alpha=0.5, label="Short wait")
        axs[i].hist(avg_shortdead[i,:], density=true, histtype="step", ec="k", lw=1.0, alpha=0.8)

        # histogram for long wait time
        axs[i].hist(avg_largedead[i,:], density=true, histtype="stepfilled", fc="tab:blue", ec="tab:blue", lw=1.5, alpha=0.5, label="Long wait")
        axs[i].hist(avg_largedead[i,:], density=true, histtype="step", ec="tab:blue", lw=1.5, alpha=0.8)

        # get axis limits
        append!(xlims, [axs[i].get_xlim()])
        append!(ylims, [axs[i].get_ylim()])

        # report two-sample KS test
        @show(ApproximateTwoSampleKSTest(avg_shortdead[i,:], avg_largedead[i,:]))
        println()
    end

    # set the axes limits
    bbox = Dict("ec"=>"black", "fc"=>"white")
    xy = (maximum(last.(xlims)) - 0.25, maximum(last.(ylims)) - 0.4)
    for i in eachindex(axs)
        # annotate axes and set labels, etc.
        axs[i].annotate(L"N_{\rm obs} =\ " * latexstring(N_obs[i]), xy=xy, bbox=bbox)
        axs[i].set_xlabel(L"{\rm Mean\ velocity\ (m s}^{-1})")
        axs[i].set_ylabel(L"{\rm Probability\ density}")
        axs[i].set_xlim(round(minimum(first.(xlims)), digits=1),
                        round(maximum(last.(xlims)), digits=1))
        axs[i].set_ylim(minimum(first.(ylims)), maximum(last.(ylims)))
        axs[i].xaxis.set_major_locator(mpl.ticker.MaxNLocator(5))
    end

    # set legend in one panel and write figure
    axs[1].legend(loc="upper left")
    fig.savefig(plotdir * "fig7.pdf")
    plt.clf(); plt.close()
    println(">>> Figure written to: " * plotdir * "fig7.pdf")
end


using Dates
printstyled("  sync!\n", color=:light_green)

fs = 100.0
ns = 10
t = floor(Int64, time()-60.0)*sμ
δ3 = 1.0
dtμ = round(Int, 1.0e6/fs)

# for k = 1:10
# Test three seismic and three irregularly sampled channels
S = randSeisData(4, s=1.0)[2:4] + randSeisData(4, c=1.0)[2:4]

# seismic channels will be ns seconds of data sampled at 100 Hz with preset gaps
nx = round(Int, ns * fs)
for i = 1:3
  S.x[i] = rand(nx)       # Set S.x to be 1000 samples each
  S.fs[i] = fs            # Set S.fs to be uniformly 100 Hz
end
S.t[1] = [1 t; nx 0]
S.t[2] = [1 t-ns*1000000; nx 0]
S.t[3] = [1 t; 2 round(Int, δ3*1.0e6); nx 0]

# irregular channels will be 100 points each, randomly sampled between the start and the end
n_irr = div(nx,10)
for i = 4:6
  S.x[i] = rand(n_irr)
  S.fs[i] = 0.0
  r = UnitRange{Int64}(0:ns*1000000)
  S.t[i] = hcat(zeros(Int64, n_irr), t.+sort(rand(r, n_irr)))

  # Ensure the last sample is always <1s from the end
  τ = view(S.t[i], :, 2)
  if last(τ)-first(τ) < (ns-1)*1.0e6
    τ[end] = first(τ) + (ns-0.5)*1.0e6
  end
end
sx = Lx(S)

# Set derived variables
ts, te = get_edge_times(S)
ts₀ = minimum(ts)
ts₁ = maximum(ts)

# TEST 1 =====================================================================
# Sync to last start; don't sync ends
printstyled(stdout,"    sync to last start; don't sync ends\n", color=:light_green)
T = sync(S)
basic_checks(T)
ts_new, te_new = get_edge_times(T)

# Expectation: T.t[2] is deleted
@test T.n == 5
@test S.id[1] == T.id[1]
@test findid(S.id[2], T.id) == 0
for i = 3:6
  @test(S.id[i] == T.id[i-1])
end

# Expectation: the latest time in S.t is now the earliest start time
@test minimum(ts_new .≥ ts₁) == true
@test maximum(ts_new .== ts₁) == true

# TEST 2 =====================================================================
# Sync to last start; don't sync ends

# change trace 2 to begin only 1s earlier; resync
S.t[2] = [1 t-1000000; nx 0]
ts, te = get_edge_times(S)
ts₀ = minimum(ts)
ts₁ = maximum(ts)

T = sync(S)
ts_new, te_new = get_edge_times(T)

# Expectation: T.x[2] is at least 100 pts shorter and exactly 100 shorter than T.x[1]
@test T.n == 6
tx = Lx(T)
@test tx[2] ≤ nx-100
@test tx[2] == tx[1] - 100
@test tx[2] ≤ tx[3]
@test minimum(ts_new .≥ ts₁) == true
@test maximum(ts_new .== ts₁) == true
basic_checks(T)

# TEST 3 =====================================================================
# Sync to last start; sync to first end
printstyled(stdout,"    sync to last start; sync to first end\n", color=:light_green)
te₀ = minimum(te)
te₁ = maximum(te)

T = sync(S, t="first")
basic_checks(T)
ts_new, te_new = get_edge_times(T)

# Expectation: T.x[1] and T.x[2] are same length, T.x[3] is shorter
tx = Lx(T)
@test tx[1] == tx[2]
@test tx[2] > tx[3]
@test minimum(te_new .≤ te₀) == true
@test maximum(te_new .== te₀) == true
@test minimum(sx.-tx.≥0) == true                # vx is always longer

# TEST 4 =====================================================================
# Sync to first start; sync to first end
printstyled(stdout,"    sync to first start; sync to last end\n", color=:light_green)
U = sync(S, s="first", t="first")
basic_checks(U)
ts_new, te_new = get_edge_times(U)

# Expectation:
ux = Lx(U)
@test minimum(ux.-tx.≥0) == true                # vx is always longer
@test tx[1]+100 ≤ ux[1]                        # U.x[1] gains >100 samples
@test minimum(ts_new .≥ ts₀) == true           # Should still be true
@test 2 ∈ findall((ts_new .== ts₀).== true)   # Defined earliest start time
@test ux[2]==ux[1]                            # Due to how we set the gap
@test ux[1]-ux[3] == 100
@test minimum(te_new .≤ te₀) == true
@test maximum(te_new .== te₀) == true
for i = 4:6
  @test ux[i] ≥ tx[i]
end

# TEST 5 =====================================================================
# Sync to first start; sync to last end
V = sync(S, s="first", t="last")
basic_checks(V)
ts_new, te_new = get_edge_times(V)
vx = Lx(V)

# Expectation:
@test minimum(vx.-sx.≥0) == true                # vx is always longer
@test vx[1]-sx[1] == 199
@test vx[1]-ux[1] == 199
@test minimum(ts_new .≥ ts₀) == true           # Should still be true
@test 2 ∈ findall((ts_new .== ts₀).== true)   # Defined earliest start time
@test vx[2] == vx[1]                           # Due to how we set the gap
@test vx[2] - vx[3] == 99
@test minimum(te_new .≤ te₁) == true
@test maximum(te_new .== te₁) == true
for i = 4:6
  @test vx[i] ≥ tx[i]
  @test vx[i] == sx[i]
end

# TEST 6 =====================================================================
# Sync to s = DateTime (first trace start), t = "none"
# trace 3 should be 100 samples shorter
# so should trace 2
printstyled(stdout,"    sync to DateTime; don't sync ends\n", color=:light_green)
ts₆ = S.t[1][1,2]
te₆ = S.t[1][1,2] + round(Int, 1.0e6*nx/fs)
ds₆ = u2d(ts₆*1.0e-6)
de₆ =  u2d(te₆*1.0e-6)
W = sync(S, s=ds₆)
basic_checks(W)
ts_new, te_new = get_edge_times(W)
wx = Lx(W)

# Expectations:
@test sx[2]-wx[2] == 100                      # Trace 2 is 100 samples shorter
for i in [1,3,4,5,6]
  @test sx[i] == wx[i]                        # No change in other trace lengths
end
@test minimum(ts_new .≥ ts₆) == true           # We start at ts₆, not before
@test findfirst(ts_new .== ts₆).== 1          # Defined start time

# Repeat with an end time
W = sync(S, s=ds₆, t=de₆)
basic_checks(W)
ts_new, te_new = get_edge_times(W)
wx = Lx(W)

# Expectations:
@test sx[3]-wx[3] == 99                       # Trace 3 is 99 samples shorter
for i in [1,2,4,5,6]
  @test sx[i] == wx[i]                        # No change in other trace lengths; 2 gets padded
end
@test minimum(ts_new .≥ ts₆) == true           # We start at ts₆, not before
@test findfirst(ts_new .== ts₆).== 1          # Defined start time

# TEST 7 =====================================================================
# Sync to DateTime 1s before first trace, 0.01s after; is it 101 pts longer?

# Repeat with an increased window range
ds₆ -= Second(1)
de₆ += Millisecond(10)
X = sync(S, s=ds₆, t=de₆)
basic_checks(X)
ts_new, te_new = get_edge_times(X)
xx = Lx(X)

# Expectations:
@test xx[1]-sx[1] == 101    # Trace 1 is 101 samples longer
@test xx[2]-sx[2] == 101    # Trace 2 is also 101 samples longer
@test xx[3]-sx[3] == 2      # Trace 3 loses 99 samples at the end, but gains 100 at end and 1 at start ... net gain +2
for i = 4:6
  @test xx[i] == sx[i]
end

# Sync to DateTime 2s after first trace, t="first"
ds₆ += Second(3)
X = sync(S, s=ds₆, t=de₆)
basic_checks(X)
ts_new, te_new = get_edge_times(X)
xx = Lx(X)

# Expectations:
@test xx[1] == 801          # Trace 1 should be 801 samples
@test xx[1] == xx[2]        # Should be the same due to padding
# In fact trace 2 should have 101 points appended
found = findall([occursin("appended 101", i) for i in X.notes[2]])
@test isempty(found) == false

# TEST 8 =====================================================================
# A few simple combinations; do these work?
printstyled(stdout,"    sync start to DateTime; sync first end\n", color=:light_green)
X = sync(S, s=ds₆, t="first"); basic_checks(X)

printstyled(stdout,"    sync start to DateTime; sync to last end\n", color=:light_green)
X = sync(S, s=ds₆, t="last"); basic_checks(X)

printstyled(stdout,"    sync start to string time; sync to last end\n", color=:light_green)
ss = string(ds₆)
Y = sync(S, s=ds₆, t="last", v=3); basic_checks(Y)

# Expect: X != Y due to notes, but all other fields equal
for f in datafields
  if f != :notes
    @test isequal(getfield(X,f), getfield(Y,f))
  end
end

# TEST 9 =====================================================================
# Do we actually prune campaign data when all times are out of range?
printstyled(stdout,"    prune all irregular data when all times are out of range\n", color=:light_green)

ss = string(ds₆)
Z = deepcopy(S)
t = deepcopy(Z.t[5])
t = hcat(t[:,1:1], vcat(0, diff(t[:,2:2], dims=1)))
Z.t[5] = deepcopy(t)

redirect_stdout(out) do
  sync!(Z, v=3); basic_checks(Z)
end

# Expect: Z[5] is gone
for i in [1,2,3,4,6]
  @test any(Z.id.==S.id[i])
end
@test (any(Z.id.==S.id[5]) == false)

# ===========================================================================
# method extenson to SeisChannel
printstyled(stdout,"    SeisChannel method extension\n", color=:light_green)
ts₆ = S.t[1][1,2]
te₆ = S.t[1][1,2] + round(Int, 1.0e6*nx/fs)
ds₆ = u2d(ts₆*1.0e-6)
de₆ =  u2d(te₆*1.0e-6)
C = deepcopy(S[1])
sync!(C, s=ds₆)
W = SeisData(C)
basic_checks(W)
ts_new, te_new = get_edge_times(W)
wx = Lx(W)

# Repeat with an end time
C = deepcopy(S[1])
W = SeisData(sync(C, s=ds₆, t=de₆))
basic_checks(W)
ts_new, te_new = get_edge_times(W)
wx = Lx(W)

# ===========================================================================
# method extenson to SeisEvent
printstyled(stdout,"    SeisEvent method extension\n", color=:light_green)
ts₆ = S.t[1][1,2]
te₆ = S.t[1][1,2] + round(Int, 1.0e6*nx/fs)
ds₆ = u2d(ts₆*1.0e-6)
de₆ =  u2d(te₆*1.0e-6)
Ev = SeisEvent(hdr = randSeisHdr(), data = deepcopy(S))
sync!(Ev.data, s=ds₆)
W = Ev.data
basic_checks(W)
ts_new, te_new = get_edge_times(W)
wx = Lx(W)

# Expectations:
@test sx[2]-wx[2] == 100                      # Trace 2 is 100 samples shorter
for i in [1,3,4,5,6]
  @test sx[i] == wx[i]                        # No change in other trace lengths
end
@test minimum(ts_new .≥ ts₆) == true           # We start at ts₆, not before
@test findfirst(ts_new .== ts₆).== 1          # Defined start time

Ev = SeisEvent(hdr = randSeisHdr(), data = deepcopy(S))
W = sync(Ev.data, s=ds₆)
basic_checks(W)
ts_new, te_new = get_edge_times(W)
wx = Lx(W)

# Expectations:
@test sx[2]-wx[2] == 100                      # Trace 2 is 100 samples shorter
for i in [1,3,4,5,6]
  @test sx[i] == wx[i]                        # No change in other trace lengths
end
@test minimum(ts_new .≥ ts₆) == true           # We start at ts₆, not before
@test findfirst(ts_new .== ts₆).== 1          # Defined start time


using ShaderToy
shadertoy("seascape.frag")


"""
    ManyPencilArray{T,N,M}

Container holding `M` different [`PencilArray`](@ref) views to the same
underlying data buffer. All views share the same element type `T` and
dimensionality `N`.

This can be used to perform in-place data transpositions with
[`transpose!`](@ref Transpositions.transpose!).

---

    ManyPencilArray{T}(undef, pencils...; extra_dims=())

Create a `ManyPencilArray` container that can hold data of type `T` associated
to all the given [`Pencil`](@ref)s.

The optional `extra_dims` argument is the same as for [`PencilArray`](@ref).
"""
struct ManyPencilArray{
        T,  # element type of each array
        N,  # number of dimensions of each array (including extra_dims)
        M,  # number of arrays
        Arrays <: Tuple{Vararg{PencilArray,M}},
    }
    data :: Vector{T}
    arrays :: Arrays

    function ManyPencilArray{T}(
            init, pencils::Vararg{Pencil{Np},M};
            extra_dims::Dims=(),
        ) where {Np,M,T}
        data_length = max(length.(pencils)...) * prod(extra_dims)
        data = Vector{T}(init, data_length)
        arrays = _make_arrays(data, extra_dims, pencils...)
        A = typeof(arrays)
        N = Np + length(extra_dims)
        new{T,N,M,A}(data, arrays)
    end
end

function _make_arrays(data::Vector{T}, extra_dims::Dims, p::Pencil,
                      pens::Vararg{Pencil}) where {T}
    dims = (size_local(p, MemoryOrder())..., extra_dims...)
    n = prod(dims)
    @assert n == length_local(p) * prod(extra_dims)
    vec = view(data, Base.OneTo(n))
    arr = reshape(vec, dims)
    A = PencilArray(p, arr)
    (A, _make_arrays(data, extra_dims, pens...)...)
end

_make_arrays(::Vector, ::Dims) = ()

Base.eltype(A::ManyPencilArray{T}) where {T} = T
Base.ndims(A::ManyPencilArray{T,N}) where {T,N} = N

"""
    length(A::ManyPencilArray)

Returns the number of [`PencilArray`](@ref)s wrapped by `A`.
"""
Base.length(A::ManyPencilArray{T,N,M}) where {T,N,M} = M

"""
    first(A::ManyPencilArray)

Returns the first [`PencilArray`](@ref) wrapped by `A`.
"""
Base.first(A::ManyPencilArray) = first(A.arrays)

"""
    last(A::ManyPencilArray)

Returns the last [`PencilArray`](@ref) wrapped by `A`.
"""
Base.last(A::ManyPencilArray) = last(A.arrays)

"""
    getindex(A::ManyPencilArray, ::Val{i})
    getindex(A::ManyPencilArray, i::Integer)

Returns the i-th [`PencilArray`](@ref) wrapped by `A`.

If possible, the `Val{i}` form should be preferred, as it is more efficient and
it allows the compiler to know the return type.

See also [`first(::ManyPencilArray)`](@ref), [`last(::ManyPencilArray)`](@ref).

# Example

```julia
A = ManyPencilArray(pencil1, pencil2, pencil3)

# Get the PencilArray associated to `pencil2`.
# u2 = A[2]
u2 = A[Val(2)]  # faster!
```
"""
Base.getindex(A::ManyPencilArray, ::Val{i}) where {i} =
    _getindex(Val(i), A.arrays...)
@inline Base.getindex(A::ManyPencilArray, i) = A[Val(i)]

@inline function _getindex(::Val{i}, a, t::Vararg) where {i}
    i :: Integer
    i <= 0 && throw(BoundsError("index must be >= 1"))
    i == 1 && return a
    _getindex(Val(i - 1), t...)
end

# This will happen if the index `i` intially passed is too large.
@inline _getindex(::Val) = throw(BoundsError("invalid index"))


import SeaMass
using Base.Test
using Plots
pyplot()

cd(ENV["PWD"])

# spectrum ID of spectrum we will be plotting
spectrumID = 1

# load mzMLb input spectrum
mzmlbSpectrumIn = SeaMass.MzmlbSpectrum(
  "data/ubiquitin_5uM_1mTorr_1uscan.mzMLb",
  spectrumID
)

# load SMB input spectrum
smbSpectrumIn = SeaMass.SmbSpectrum(
  "data/out/ubiquitin_5uM_1mTorr_1uscan" *
    "/1.mzmlb2smb/ubiquitin_5uM_1mTorr_1uscan.p-0-005377748.smb",
  spectrumID
)
smbSpectrumInBinWidths = smbSpectrumIn.locations[2:end] -
  smbSpectrumIn.locations[1:end-1]

# load SMB output spectrum
smbSpectrumOut = SeaMass.SmbSpectrum(
  "data/out/ubiquitin_5uM_1mTorr_1uscan" *
    "/4.seamass-td/ubiquitin_5uM_1mTorr_1uscan.p-0-005377748.smb",
  spectrumID
)
smbSpectrumOutBinWidths = smbSpectrumOut.locations[2:end] -
  smbSpectrumOut.locations[1:end-1]

# Compare mzMLb input to 'mzmlb2smb' smb input (binned)
plot(
  mzmlbSpectrumIn.mzs,
  mzmlbSpectrumIn.intensities,
  m = 1,
  label = "mzMLb input",
  title = "Compare mzMLb input (sampled) to 'mzmlb2smb' smb input (binned)",
  xlabel = "m/z (Th)",
  ylabel = "ion count density",
  reuse = false,
)
plot!(
  smbSpectrumIn.locations,
  vcat(
    smbSpectrumIn.counts ./ (smbSpectrumIn.exposure .* smbSpectrumInBinWidths),
    0.0
  ),
  line = :steppost,
  label = "SMB input",
)
gui()

# Compare smb input to 'seamass' ⇨ 'seamass-restore' smb output
plot(
  smbSpectrumIn.locations,
  vcat(
    smbSpectrumIn.counts ./ (smbSpectrumIn.exposure .* smbSpectrumInBinWidths),
    0.0
    ),
  line = :steppost,
  label = "SMB input",
  m = 1,
  title = "Compare smb input to 'seamass' ⇨ 'seamass-restore' smb output)",
  xlabel = "m/z (Th)",
  ylabel = "ion count",
  reuse = false,
)
plot!(
  smbSpectrumOut.locations,
  vcat(
    smbSpectrumOut.counts ./ (smbSpectrumOut.exposure .* smbSpectrumOutBinWidths),
    0.0
  ),
  line = :steppost,
  label = "seaMass-restore SMB output",
)
gui()

println("Press <Enter> to finish")
readline()


using BenchmarkTools
using Folds
using FoldsThreads

const SUITE = BenchmarkGroup()
const DATA_LENGTH = 2^23

for log2_basesize in 12:14
    basesize = 2^log2_basesize
    SUITE[basesize] = s1 = BenchmarkGroup()
    for log2_needleloc in 11:23
        needleloc = 2^log2_needleloc
        s1[needleloc] = s2 = BenchmarkGroup()
        xs = rand(DATA_LENGTH)
        xs[needleloc] = 2
        s2["ThreadedEx"] =
            @benchmarkable( Folds.findfirst(>(1), $xs, ThreadedEx(basesize = $basesize)))
        s2["WorkStealingEx"] =
            @benchmarkable(Folds.findfirst(>(1), $xs, WorkStealingEx(basesize = $basesize)))
        s2["DepthFirstEx"] =
            @benchmarkable(Folds.findfirst(>(1), $xs, DepthFirstEx(basesize = $basesize)))
        #=
        s2["SpawnAllEx"] =
            @benchmarkable(Folds.findfirst(>(1), $xs, SpawnAllEx(basesize = $basesize)))
        =#
    end
end


VERSION >= v"0.4.0-dev+6521" && __precompile__()

module KDTrees

    import Base.show

    using Compat
    using Devectorize

    export KDTree
    export knn, inball

    # Deprecated!
    export k_nearest_neighbour, query_ball_point

    warn(string("KDTrees.jl is deprecated in favor of NearestNeighbors.jl.",
            " If you still want to use KDTrees.jl or rely on a package that",
            " uses it, please run Pkg.pin(\"KDTrees\", v\"0.5.13\") and this warning will go away"))

    include("hyper_rec.jl")
    include("kd_tree.jl")
    include("deprecations.jl")

end


# Julia wrapper for header: /usr/include/scip/pub_bandit_epsgreedy.h
# Automatically generated using Clang.jl wrap_c

function SCIPcreateBanditEpsgreedy(scip, epsgreedy, priorities, eps, preferrecent, decayfactor, avglim, nactions, initseed)
    ccall((:SCIPcreateBanditEpsgreedy, libscip), SCIP_RETCODE, (Ptr{SCIP_}, Ptr{Ptr{SCIP_BANDIT}}, Ptr{Cdouble}, Cdouble, UInt32, Cdouble, Cint, Cint, UInt32), scip, epsgreedy, priorities, eps, preferrecent, decayfactor, avglim, nactions, initseed)
end

function SCIPgetWeightsEpsgreedy(epsgreedy)
    ccall((:SCIPgetWeightsEpsgreedy, libscip), Ptr{Cdouble}, (Ptr{SCIP_BANDIT},), epsgreedy)
end

function SCIPsetEpsilonEpsgreedy(epsgreedy, eps)
    ccall((:SCIPsetEpsilonEpsgreedy, libscip), Cvoid, (Ptr{SCIP_BANDIT}, Cdouble), epsgreedy, eps)
end


function CR3BPSystem(arg0::CelestialBody, arg1::CelestialBody, arg2::jdouble)
    return CR3BPSystem((CelestialBody, CelestialBody, jdouble), arg0, arg1, arg2)
end

function CR3BPSystem(arg0::CelestialBody, arg1::CelestialBody, arg2::jdouble, arg3::jdouble)
    return CR3BPSystem((CelestialBody, CelestialBody, jdouble, jdouble), arg0, arg1, arg2, arg3)
end

function get_ddim(obj::CR3BPSystem)
    return jcall(obj, "getDdim", jdouble, ())
end

function get_gamma(obj::CR3BPSystem, arg0::LagrangianPoints)
    return jcall(obj, "getGamma", jdouble, (LagrangianPoints,), arg0)
end

function get_l_position(obj::CR3BPSystem, arg0::LagrangianPoints)
    return jcall(obj, "getLPosition", Vector3D, (LagrangianPoints,), arg0)
end

function get_mass_ratio(obj::CR3BPSystem)
    return jcall(obj, "getMassRatio", jdouble, ())
end

function get_name(obj::CR3BPSystem)
    return jcall(obj, "getName", JString, ())
end

function get_primary(obj::CR3BPSystem)
    return jcall(obj, "getPrimary", CelestialBody, ())
end

function get_real_apv(obj::CR3BPSystem, arg0::AbsolutePVCoordinates, arg1::AbsoluteDate, arg2::Frame)
    return jcall(obj, "getRealAPV", AbsolutePVCoordinates, (AbsolutePVCoordinates, AbsoluteDate, Frame), arg0, arg1, arg2)
end

function get_rotating_frame(obj::CR3BPSystem)
    return jcall(obj, "getRotatingFrame", Frame, ())
end

function get_secondary(obj::CR3BPSystem)
    return jcall(obj, "getSecondary", CelestialBody, ())
end

function get_tdim(obj::CR3BPSystem)
    return jcall(obj, "getTdim", jdouble, ())
end

function get_vdim(obj::CR3BPSystem)
    return jcall(obj, "getVdim", jdouble, ())
end