#
# This is a package to convert to use zero origin array in a function.
# Usage:
#  @zeroorigin [single symbol or list/tuple for variables with zeroorigin] [definition of function]
#
# When we definie a function, the code `@zeroorigin (x,y)` is added to the head of definition.
# Then in the function, the ref of x and y, i.e., x[i] and y[i] will be changed to x[i+1] and y[i+1].
#
# Example:
#
# using Origin
# using Test
#
# @testset "Origin.jl" begin
#     @origin (a=>0, b=>2) function test()
#         a = collect(0:10)
#         b = collect(1:5)
#         @test a[0] == 0
#         @test a[10] == 10
#         @test b[2] == 1
#         @test b[4] == 3
#         @test b[end] == 5
#         @test b[2:end] == [1, 2, 3, 4, 5]
#         @test b[a[5]] == 4
#     end

#     test()
# end

export @origin

function peek(stack)
    return stack[end]
end

function _changeindex(expr, u)
    if Meta.isexpr(expr, :call) && expr.args[1] == :(:)
        if length(expr.args) == 3
            Expr(:call, :(:), Expr(:call, :+, expr.args[2], u), Expr(:call, :+, expr.args[3], u))
        elseif length(expr.args) == 4
            Expr(:call, :(:), Expr(:call, :+, expr.args[2], u), expr.args[3], Expr(:call, :+, expr.args[4], u))
        else
            error("The inner of index should be a simple expr or call :(:)")
        end
    else
        Expr(:call, :+, expr, u)
    end
end

function _replace(expr::Expr, symlist, stack)
    if Meta.isexpr(expr, :ref)
        push!(stack, expr.args[1])
        args = Any[_replace(x, symlist, stack) for x = expr.args]
        if haskey(symlist, peek(stack))
            m = symlist[peek(stack)]
            u = Expr(:call, :-, 1, m)
            for i = 2:length(args)
                args[i] = _changeindex(args[i], u)
            end
        end
        pop!(stack)
        Expr(expr.head, args...)
    elseif Meta.isexpr(expr, :macrocall) && expr.args[1] == Symbol("@origin")
        macroexpand(Origin, expr, recursive=false)
    else
        if Meta.isexpr(expr, :call) && expr.args[1] == :eachindex
            if _hassym(expr.args[2], symlist)
                println("Warning: This macro has not been applied to the block including 'eachindex' yet.")
            end
        end
        args = Any[_replace(x, symlist, stack) for x = expr.args]
        Expr(expr.head, args...)
    end
end

function _replace(expr::Symbol, symlist, stack)
    if expr == :end && haskey(symlist, peek(stack))
        m = symlist[peek(stack)]
        Expr(:call, :-, :end, Expr(:call, :-, 1, m))
    else
        expr
    end
end

function _replace(expr::Any, symlist, stack)
    expr
end

function _hassym(expr::Expr, symlist)
    any(Bool[_hassym(x, symlist) for x = expr.args])
end

function _hassym(expr::Symbol, symlist)
    haskey(symlist, expr)
end

function _add!(symlist, x)
    if typeof(x) <: Expr && x.head == :call && x.args[1] == :(=>)
        symlist[x.args[2]] = x.args[3]
    end
end

macro origin(vars, expr)
    symlist = Dict{Symbol,Any}()
    stack = []
    if typeof(vars) <: Expr && (vars.head == :vect || vars.head == :tuple)
        for x = vars.args
            _add!(symlist, x)
        end
    else
        _add!(symlist, vars)
    end
    ex = _replace(expr, symlist, stack)
    esc(ex)
end


# errors from LBL, remove post interpolation outliers

using Statistics

## Load necessary code

using Pkg
Pkg.activate(@__DIR__)
pkg"instantiate"
pkg"precompile"

using FileIO

include(joinpath(@__DIR__, "CommonUtils.jl"))
# include(joinpath(@__DIR__, "Plotting.jl"))

## load specific case

pargs = if !isdefined(Main, :parsed_args)
  parse_commandline()
else
  parsed_args
end

##

pargs["reportDir"] = if false && !haskey(pargs, "reportDir") && isdefined(Main, :fg)
  getLogPath(fg)
else
  # "/tmp/caesar/2020-02-23T01:43:32.222/fg_before_x291.tar.gz"
  # "/tmp/caesar/2020-02-24T04:22:23.282/fg_after_x1391.tar.gz"
  "/tmp/caesar/2020-02-24T04:19:39.943/fg_after_x1391.tar.gz"
end

# @show pargs["reportDir"]
# @show splitpath(pargs["reportDir"])

# load the factor graph needed
fg = if true
  println("going to load fg")
  fg = LightDFG{SolverParams}(params=SolverParams())
  @show pathElem = splitpath(pargs["reportDir"])
  @show getSolverParams(fg).logpath = joinpath(pathElem[1:end-1]...)
  loadDFG(pargs["reportDir"], Main, fg)
  fg
else
  fg
end

# random temp data
drt_data, TTm, XXm, YYm, XXf, YYf = loadResultsDRT(fg)

##

posesyms = ls(fg, r"x\d") |> sortDFG
filter!(x->isInitialized(fg, x), posesyms)
filter!(x->solverData(getVariable(fg, x), :lbl) != nothing, posesyms)
Xlbl = (x->(solverData(getVariable(fg, x), :lbl).val[1,1])).(posesyms)
Ylbl = (x->(solverData(getVariable(fg, x), :lbl).val[2,1])).(posesyms)

# pose PPE
Xppe = (x->(getVariablePPE(getVariable(fg, x)).suggested[1])).(posesyms)
Yppe = (x->(getVariablePPE(getVariable(fg, x)).suggested[2])).(posesyms)

# timestamp
ts = (x->getTimestamp(getVariable(fg, x))).(posesyms) .|> datetime2unix
T0 = ts[1]
ts .-= ts[1]
ts[end]

# filter on timestamps
# remove 420-450s
mask = 420 .< ts .< 450
imask = xor.(mask, true)

Xlbl[mask] .= Inf
Ylbl[mask] .= Inf

Gadfly.plot(x=ts, y=Xppe, Geom.line)
Gadfly.plot(x=ts, y=Yppe, Geom.line)

lblErr2 = (Xppe-Xlbl).^2 + (Yppe-Ylbl).^2
@show Statistics.mean(lblErr2[imask]) |> sqrt
@show cov(sqrt.(lblErr2[imask])) |> sqrt

##

Gadfly.plot(x=Xppe-Xlbl, Geom.histogram)
Gadfly.plot(x=ts, y=Xppe-Xlbl, Geom.line)
Gadfly.plot(x=ts, y=Yppe-Ylbl, Geom.line)

Gadfly.plot(x=ts, y=sqrt.(lblErr2), Geom.line)

findVariableNearTimestamp(fg,unix2datetime(T0+580))

## ranges nearby x580

ls(fg, :x581)

reportFactors(fg, Pose2Point2Range) #, [:x581l1f1; :x454l1f1])

ls(fg, :x454)

##  Plot LBL and belief only

drt_data, TTm, XXm, YYm, XXf, YYf = loadResultsDRT(fg)

mask = YYf .< -32
XXfm = XXf[mask]
YYfm = YYf[mask]
# only start does not match
TTmm = sum(mask)<length(TTm) ? TTm[end-sum(mask)+1:end] : TTm

# plot DRT
pl = Gadfly.plot(x=XXm[1:10:end], y=YYm[1:10:end], Geom.path) #, Theme(default_color=colorant"khaki"))

drtt = datetime2unix.(TTmm)
drtt .-= drtt[1]

pl = Gadfly.plot(
  Gadfly.layer(x=ts, y=Xppe, Geom.line, Theme(default_color=colorant"black")),
  Gadfly.layer(x=drtt, y=XXfm, Geom.line, Theme(default_color=colorant"red"))
)

#


using Swizzles.StylishArrays
using Swizzles.StylishArrays: square, root, power
using LinearAlgebra

@testset "StylishArrays" begin
    @testset "Squares" begin
        a = sqrt(prevfloat(typemax(Float64)))
        x = square(a)
        @test root(x) ≈ a
        @test root(x + x + x + x) ≈ 2 * a
        @test root(square(0) + x) ≈ a
        @test root(x + square(0)) ≈ a
        @test root(square(0) + square(0)) ≈ 0
        y = square(Inf)
        z = square(-Inf)
        @test root(y) == Inf
        @test root(y + z) == Inf
    end

    @testset "Powers" begin
        a = sqrt(prevfloat(typemax(Float64)))
        x = power(a, 2)
        @test root(x) ≈ a
        @test root(x + x + x + x) ≈ 2 * a
        @test root(power(0, 2) + x) ≈ a
        @test root(x + power(0, 2)) ≈ a
        @test root(power(0, 2) + power(0, 2)) ≈ 0
        y = power(Inf, 2)
        z = power(-Inf, 2)
        @test root(y) == Inf
        @test root(y + z) == Inf
    end

    struct TestStylishArray{T, N, Arr<:AbstractArray{T, N}} <: StylishArray{T, N}
        parent::Arr
    end
    Base.size(arr::TestStylishArray) = size(arr.parent)
    Base.axes(arr::TestStylishArray) = axes(arr.parent)
    Base.getindex(arr::TestStylishArray, i...) = arr.parent[i...]
    Base.getindex(arr::TestStylishArray, i::Int) = arr.parent[i]
    Base.getindex(arr::TestStylishArray, i::CartesianIndex) = arr.parent[i]
    Base.setindex!(arr::TestStylishArray, x, i...) = arr.parent[i...] = x
    Base.setindex!(arr::TestStylishArray, x, i::Int) = arr.parent[i] = x
    Base.setindex!(arr::TestStylishArray, x, i::CartesianIndex) = arr.parent[i] = x

    @testset "Base" begin

        @testset "copyto!" begin
            for A in ([1 2 3; 4 5 6; 7 8 9], TestStylishArray([1 2 3; 4 5 6; 7 8 9]))
                B = TestStylishArray(zeros(3,3))
                @test A == copyto!(B, A)
                @test A == B
                B = TestStylishArray(zeros(9))
                #@test reshape(A, :) == copyto!(B, A) #FIXME
                #@test reshape(A, :) == B #FIXME
            end

            A = TestStylishArray([1 2 3; 4 5 6; 7 8 9])
            B = zeros(3,3)
            @test A == copyto!(B, A)
            @test A == B
            B = zeros(9)
            #@test reshape(A, :) == copyto!(B, A) #FIXME
            #@test reshape(A, :) == B #FIXME
        end

        @testset "getindex and setindex!" begin
            A = [1 2 3; 4 5 6; 7 8 9]
            @test TestStylishArray(A)[1] == A[1]
            @test TestStylishArray(A)[1:3] == A[1:3]
            @test TestStylishArray(A)[2:3, 1:2] == A[2:3, 1:2]
            @test TestStylishArray(A)[2, 1:3] == A[2, 1:3]

            A = [1 2 3; 4 5 6; 7 8 9]
            B = [1 2 3; 4 5 6; 7 8 9]
            TestStylishArray(A)[1] = -1
            B[1] = -1
            @test A == B

            A = [1 2 3; 4 5 6; 7 8 9]
            B = [1 2 3; 4 5 6; 7 8 9]
            TestStylishArray(A)[1:3] = [-1 -1 -1]
            B[1:3] = [-1 -1 -1]
            @test A == B

            A = [1 2 3; 4 5 6; 7 8 9]
            B = [1 2 3; 4 5 6; 7 8 9]
            TestStylishArray(A)[2:3, 1:2] = [-1 -1; -1 -1]
            B[2:3, 1:2] = [-1 -1; -1 -1]
            @test A == B

            A = [1 2 3; 4 5 6; 7 8 9]
            B = [1 2 3; 4 5 6; 7 8 9]
            TestStylishArray(A)[2, 1:3] = [-1 -1 -1]
            B[2, 1:3] = [-1 -1 -1]
            @test A == B
        end

        @testset "map" begin
            A = [1 2 3; 4 5 6; 7 8 9]
            @test map(x -> x == 2, TestStylishArray(A)) == map(x -> x == 2, A)

            A = [1 2 3; 4 5 6; 7 8 9]
            @test map((a, b) -> a + b == 4, TestStylishArray(A), TestStylishArray(A)) == map((a, b) -> a + b == 4, A, A)
        end

        @testset "foreach" begin
            A = [1 2 3; 4 5 6; 7 8 9]
            B = [1 2 3; 4 5 6; 7 8 9]
            C = [1 2 3; 4 5 6; 7 8 9]
            @test foreach(i -> B[i] = -i, TestStylishArray(A)) == foreach(i -> C[i] = -i, A)
            @test B == C

            A = [1 2 3; 4 5 6; 7 8 9]
            B = [1 2 3; 4 5 6; 7 8 9]
            C = [1 2 3; 4 5 6; 7 8 9]
            @test foreach((a, b) -> B[a] = b, TestStylishArray(A), -A) == foreach((a, b) -> C[a] = b, A, -A)
            @test B == C

            A = [1 2 3; 4 5 6; 7 8 9]
            B = [1 2 3; 4 5 6; 7 8 9]
            C = [1 2 3; 4 5 6; 7 8 9]
            @test foreach((a, b) -> B[a] = b, TestStylishArray(A), -A) == foreach((a, b) -> C[a] = b, A, -A)
            @test B == C
        end

        @testset "reductions" begin
            A = [1 2 3; 4 5 6; 7 8 9]

            @test mapreduce(-, +, TestStylishArray(A)) == mapreduce(-, +, A)
            @test mapreduce(-, +, TestStylishArray(A), dims=2) == mapreduce(-, +, A, dims=2)
            @test mapreduce(-, +, TestStylishArray(A), dims=(1, 2)) == mapreduce(-, +, A, dims=(1, 2))
            @test mapreduce(-, +, TestStylishArray(A), dims=:) == mapreduce(-, +, A, dims=:)
            @test mapreduce(-, +, TestStylishArray(A), dims=()) == mapreduce(-, +, A, dims=())
            @test mapreduce(-, +, TestStylishArray(A), init=10) == mapreduce(-, +, A, init=10)

            @test reduce(+, TestStylishArray(A)) == reduce(+, A)
            @test reduce(+, TestStylishArray(A), dims=2) == reduce(+, A, dims=2)
            @test reduce(+, TestStylishArray(A), dims=(1, 2)) == reduce(+, A, dims=(1, 2))
            @test reduce(+, TestStylishArray(A), dims=:) == reduce(+, A, dims=:)
            @test reduce(+, TestStylishArray(A), dims=()) == reduce(+, A, dims=())
            @test reduce(+, TestStylishArray(A), init=10) == reduce(+, A, init=10)

            @test sum(TestStylishArray(A)) == sum(A)
            @test sum(TestStylishArray(A), dims=2) == sum(A, dims=2)
            @test sum(TestStylishArray(A), dims=(1, 2)) == sum(A, dims=(1, 2))
            @test sum(TestStylishArray(A), dims=:) == sum(A, dims=:)
            @test sum(TestStylishArray(A), dims=()) == sum(A, dims=())
            @test sum(TestStylishArray(A), init=10) == reduce(+, A, init=10)
        end
    end

    @testset "LinearAlgebra" begin
        @testset "norm" begin
            for (X, Y) in ((rand(3, 4), rand(3,4)), (rand(3), rand(3)))
                @test dot(TestStylishArray(X), TestStylishArray(Y)) ≈ dot(X, Y)
            end

            big = prevfloat(typemax(Float64))/16
            small = nextfloat(zero(Float64))

            for ps in ((-Inf,), (-3.0,), (-1,), (-0.5,), (0,), (0.5,), (1,), (2,), (3.0,), (Inf), ())
                for X in ([1, 2], [big, big], [small, small], rand(3, 3), rand(3))
                    @test norm(TestStylishArray(X), ps...) ≈ norm(X, ps...)
                end
            end
        end

        @testset "mul!" begin
            A = rand(3,3)
            u = rand(3,1)
            v = rand(1,3)
            w = rand(3)

            for (a, b) in Iterators.product((A, u, v, w), (A, u, v, w))
                valid = false
                y = nothing
                try
                    y = a * b
                    valid = true
                catch MethodError
                end
                if valid
                    @test TestStylishArray(a) * b ≈ a * b
                    @test a * TestStylishArray(b) ≈ a * b
                    @test TestStylishArray(a) * TestStylishArray(b) ≈ a * b

                    @test mul!(identity.(y), TestStylishArray(a), b) ≈ mul!(identity.(y), a, b)
                    @test mul!(identity.(y), a, TestStylishArray(b)) ≈ mul!(identity.(y), a, b)
                    @test mul!(identity.(y), TestStylishArray(a), TestStylishArray(b)) ≈ mul!(identity.(y), a, b)
                    @test mul!(TestStylishArray(identity.(y)), a, b) ≈ mul!(identity.(y), a, b)
                    @test mul!(TestStylishArray(identity.(y)), TestStylishArray(a), b) ≈ mul!(identity.(y), a, b)
                    @test mul!(TestStylishArray(identity.(y)), a, TestStylishArray(b)) ≈ mul!(identity.(y), a, b)
                    @test mul!(TestStylishArray(identity.(y)), TestStylishArray(a), TestStylishArray(b)) ≈ mul!(identity.(y), a, b)
                end
            end
        end

        @testset "transpose and adjoint" begin
            A = rand(Complex{Float64}, 3,3)
            u = rand(Complex{Float64}, 3,1)
            v = rand(Complex{Float64}, 1,3)
            w = rand(Complex{Float64}, 3)

            for (a) in (A, u, v, w)
                @test transpose(TestStylishArray(a)) == transpose(a)
                @test transpose(TestStylishArray(a)) isa Swizzles.SwizzledArray
                @test adjoint(TestStylishArray(a)) == adjoint(a)
                @test adjoint(TestStylishArray(a)) isa Swizzles.SwizzledArray
            end
        end

        @testset "permutedims" begin
            A = rand(3,3)
            u = rand(3,1)
            v = rand(1,3)
            w = rand(3)

            for a in (A, u, v, w)
                @test permutedims(TestStylishArray(a)) == permutedims(a)
                @test permutedims(TestStylishArray(a)) isa Swizzles.SwizzledArray
            end

            A = rand(3,3,3)

            for perm in [(1, 2, 3), (1, 3, 2), (2, 1, 3), (2, 3, 1), (3, 1, 2), (3, 2, 1)]
                @test permutedims(TestStylishArray(A), perm) == permutedims(A, perm)
                @test permutedims(TestStylishArray(A), perm) isa Swizzles.SwizzledArray
            end
        end
    end
end


@testset "StepRange" begin
    for R in (StepMRange, StepSRange)
        @testset "$R" begin
            r = R(1, 1, 10)
            b = StepRange(1, 1, 10)
            @test R(1:10) == 1:1:10
            @test eltype(R(UInt(1), UInt(1), UInt(10))) == UInt
            @test R{Int,Int}(r) === r
            @test eltype(R{UInt,UInt}(r)) == UInt
            @test first(r) == r.start
            @test last(r) == r.stop

            @test intersect(r, r[2]) == intersect(b, b[2])
            @test intersect(r, r[2]) == intersect(b, b[2])

            @test_throws ErrorException r.notfield
            if R == StepMRange
                setproperty!(r, :step, 2)
                @test r == StepMRange(1, 2, 9)

                setproperty!(r, :start, 2)
                @test r == StepMRange(2, 2, 8)

                setproperty!(r, :stop, 10)
                @test r == StepMRange(2, 2, 10)

                @test_throws ErrorException setproperty!(r, :anything, 3)
            end
        end
    end
end


using Distributions, JLD, CSV, DataFrames,Pkg, DataStructures

austin_stations_pre = CSV.File("raw/austin-hospitals.csv") |> DataFrame

# Yeesian has his coords in 3-stations.csv in this string format:
# "GeoInterface.Point([-77.00514742400335,38.83089499410552])"
formatted_coordinates = [string("GeoInterface.Point([", austin_stations_pre["longitude"][i], ", ", austin_stations_pre["latitude"][i], "])") for i in 1:size(austin_stations_pre["longitude"], 1)]

austin_stations_post = DataFrame(geometry = formatted_coordinates,
                                 name = austin_stations_pre["hospital_names"]
                                 )

CSV.write("formatted/austin_hospitals_formatted.csv", austin_stations_post)


function inverse(side)
  return HorizonSide(mod(Int(side) + 2, 4))
end

function go_to(rob, side)
  a = 0
  while !isborder(rob, side)
      putmarker!(rob)
      move!(rob, side)
      a+=1
  end
  putmarker!(rob)
  for i in (1:a)
      move!(rob, inverse(side))
  end
end

for i in (0:3)
  go_to(r, HorizonSide(i))
end

show(r)


struct BIO
    dict::Dict
    tags::Vector
end
BIO() = BIO(Dict("O"=>1), ["O"])

Base.length(tagset::BIO) = length(tagset.tags)

function encode(tagset::BIO, tags::Vector{String})
    basetag = ""
    map(tags) do tag
        tag == "O" && return 1
        if tag == "_"
            tag = "I-" * basetag
        else
            basetag = tag
            tag = "B-" * basetag
        end
        get!(tagset.dict, tag) do
            id = length(tagset.dict) + 1
            push!(tagset.tags, tag)
            id
        end
    end
end

function decode(tagset::BIO, ids::Vector{Int})
    spans = Tuple{Int,Int,String}[]
    bpos = 0
    for i = 1:length(ids)
        tag = tagset.tags[ids[i]]
        tag == "O" && continue
        startswith(tag, "B-") && (bpos = i)
        nexttag = i == length(ids) ? "O" : tagset.tags[ids[i+1]]
        if !startswith(nexttag,"I-") && bpos > 0
            basetag = tagset.tags[ids[bpos]][3:end]
            push!(spans, (bpos,i,basetag))
            bpos = 0
        end
    end
    spans
end

struct BIO2
    dict::Dict
    tags::Vector
end
BIO2() = BIO2(Dict(), String[])

Base.length(tagset::BIO2) = length(tagset.tags)

function encode(tagset::BIO2, tags::Vector{String})
    basetag = ""
    ids = Int[]
    for tag in tags
        if tag == "O"
        elseif tag == "_"
            tag = "I-" * basetag
        else
            basetag = tag
            tag = "B-" * basetag
        end
        for suffix in ("-1","-2")
            t = tag * suffix
            id = get!(tagset.dict, t) do
                id = length(tagset.dict) + 1
                push!(tagset.tags, t)
                id
            end
            suffix == "-1" && push!(ids,id)
        end
    end
    ids
end

function decode(tagset::BIO2, ids::Vector{Int})
    spans = Tuple{Int,Int,String}[]
    bpos = 0
    for i = 1:length(ids)
        tag = tagset.tags[ids[i]]
        startswith(tag, "O-") && continue
        startswith(tag, "B-") && (bpos = i)
        nexttag = i == length(ids) ? "O" : tagset.tags[ids[i+1]]
        if !startswith(nexttag,"I-") && bpos > 0
            strs = Vector{String}(split(tagset.tags[ids[bpos]], "-"))
            basetag = strs[2]
            push!(spans, (bpos,i,basetag))
            bpos = 0
        end
    end
    spans
end


module InfiniteOpt

# Import and export JuMP
import Reexport
Reexport.@reexport using JuMP

# Import the necessary packages.
import Distributions
import DataStructures
import FastGaussQuadrature
import AbstractTrees
import LeftChildRightSiblingTrees
import LinearAlgebra
import MutableArithmetics
using Base.Meta

# Import and export SpecialFunctions for the NLP interface
Reexport.@reexport using SpecialFunctions

# Make useful aliases (note we get MOI and MOIU from JuMP)
const JuMPC = JuMP.Containers
const MOIUC = MOIU.CleverDicts
const _LCRST = LeftChildRightSiblingTrees
const _MA = MutableArithmetics
export JuMPC, MOIUC # this makes these accessible to the submodules

# Import the Collections module
include("Collections/Collections.jl")

# Import core methods
include("datatypes.jl")
include("infinite_domains.jl")
include("scalar_parameters.jl")
include("array_parameters.jl")
include("variable_basics.jl")
include("infinite_variables.jl")
include("semi_infinite_variables.jl")
include("point_variables.jl")
include("finite_variables.jl")
include("nlp.jl")
include("expressions.jl")
include("measures.jl")

# Import and export MeasureToolbox
include("MeasureToolbox/MeasureToolbox.jl")
Reexport.@reexport using .MeasureToolbox

# import more core methods
include("derivatives.jl")
include("constraints.jl")
include("macros.jl")
include("objective.jl")
include("measure_expansions.jl")
include("derivative_evaluations.jl")
include("optimize.jl")
include("results.jl")
include("show.jl")
include("utilities.jl")
include("general_variables.jl")

# Import and export TranscriptionOpt
include("TranscriptionOpt/TranscriptionOpt.jl")
Reexport.@reexport using .TranscriptionOpt

# Define additional stuff that should not be exported
const _EXCLUDE_SYMBOLS = [Symbol(@__MODULE__), :eval, :include]

# Following JuMP, export everything that doesn't start with a _
for sym in names(@__MODULE__, all = true)
    sym_string = string(sym)
    if sym in _EXCLUDE_SYMBOLS || startswith(sym_string, "_") || startswith(sym_string, "@_")
        continue
    end
    if !(Base.isidentifier(sym) || (startswith(sym_string, "@") && Base.isidentifier(sym_string[2:end])))
        continue
    end
    @eval export $sym
end

end # module


"""
    dump_gate(blk::AbstractBlock) -> Expr

convert a gate to a YaoScript expression for serization.
The fallback is `GateTypeName(fields...)`
"""
function dump_gate end

function dump_gate(blk::ConstantGate)
    Symbol("$(typeof(blk).name)"[1:end-4])
end

function dump_gate(blk::ControlBlock)
    pairs = [:($b => C($c)) for (b, c) in zip(blk.ctrl_locs, blk.ctrl_config)]
    :($(pairs...), $(blk.locs) => $(dump_gate(blk.content)))
end

function dump_gate(blk::ChainBlock)
    Expr(:block, [dump_gate(b) for b in blk]...)
end

function dump_gate(blk::RotationGate)
    :(rot($(dump_gate(blk.block)), $(tokenize_param(blk.theta))))
end

function dump_gate(blk::TimeEvolution)
    :(time($(tokenize_param(blk.dt))) => $(dump_gate(blk.H)))
end

function dump_gate(blk::PutBlock)
    :($(blk.locs) => $(dump_gate(blk.content)))
end

function dump_gate(blk::KronBlock{N}) where {N}
    if any(x -> nqubits(x) != 1, subblocks(blk))
        error("unsupported multi-qubit in kron while dumping to Yao script.")
    end
    if length(occupied_locs(blk)) == N
        :(kron($([dump_gate(blk[i]) for i in 1:N]...)))
    else
        :(($([:($i => $(dump_gate(g))) for (i, g) in blk]...),))
    end
end

function dump_gate(blk::RepeatedBlock)
    :(repeat($(blk.locs...)) => $(dump_gate(blk.content)))
end

function dump_gate(blk::Add)
    :(+($(dump_gate.(subblocks(blk))...)))
end

function dump_gate(blk::Daggered)
    :($(dump_gate(blk.content))')
end

function dump_gate(blk::CachedBlock)
    :(cache($(dump_gate(blk.content))))
end

function dump_gate(blk::Scale)
    :($(factor(blk)) * $(dump_gate(blk.content)))
end

function dump_gate(blk::Measure{N,M}) where {M,N}
    if blk.operator == ComputationalBasis()
        MOP = :(Measure)
    else
        MOP = :(Measure($(dump_gate(blk.operator))))
    end
    locs = blk.locations isa AllLocs ? :ALL : blk.locations
    if blk.postprocess isa NoPostProcess
        :($locs => $MOP)
    elseif blk.postprocess isa ResetTo
        :($locs => $MOP => resetto($(blk.postprocess.x...)))
    elseif blk.postprocess isa RemoveMeasured
        :($locs => $MOP => remove)
    end
end

function dump_gate(blk::Subroutine)
    :(focus($(blk.locs...)) => $(dump_gate(blk.content)))
end

tokenize_param(param::Number) = param

yaotoscript(block::AbstractBlock{N}) where {N} =
    Expr(:block, :(nqubits = $N), dump_gate(block)) |> rmlines
function yaotoscript(block::ChainBlock{N}) where {N}
    ex = dump_gate(block)
    Expr(:let, Expr(:block, :(nqubits = $N), :(version = "0.6")), ex)
end
yaotofile(filename::String, block) = write(filename, string(yaotoscript(block)))

macro dumpload_fallback(blocktype, fname)
    quote
        function YaoBlocks.dump_gate(blk::$(esc(blocktype)))
            vars = [getproperty(blk, x) for x in fieldnames($(esc(blocktype)))]
            Expr(:call, $(QuoteNode(fname)), vars...)
        end
        function YaoBlocks.gate_expr(::Val{$(QuoteNode(fname))}, args, info)
            Expr(:call, $(QuoteNode(fname)), render_arg.(args, Ref(info))...)
        end
    end
end


include("DADA\\errormodels.jl")
ρₚₒᵢₛ = StatsFuns.poispdf
get_pval(nⱼλᵢⱼ,aⱼ) = poisccdf(nⱼλᵢⱼ,aⱼ-1)

function get_norm(nⱼ,TAIL_APPROX_CUTOFF =1e-7)
    norm = (1.0 - exp(-nⱼ))
    if norm < TAIL_APPROX_CUTOFF
      norm = nⱼ - 0.5*nⱼ*nⱼ
    end
    return norm
end

get_normed_pval(nⱼ,λᵢⱼ,aⱼ) = get_pval(nⱼ*λᵢⱼ,aⱼ)/get_norm(nⱼ*λᵢⱼ) #get_norm(nⱼ*λᵢⱼ)?

function compare!(seqs,quals,counts,ind,centres,partition,pvals,n,reads,compI,compV, err,mers,hdists,E_minmax; band_size = 16,kdist_cut = 0.42,gapless = false)
    affinegap = AffineGapScoreModel(match=5,
                                mismatch=-4,
                                 gap_open=-0,
                                 gap_extend=-8)
    base_vals = Base.ImmutableDict(DNA_A=>1,DNA_C=>2,DNA_G=>3,DNA_T=>4)
    l = keys(base_vals)
    mer_dist = 0.0
    λᵢⱼ = 0.0
    nⱼ = sum(counts[partition .== ind])
    N = sum(counts)
    #Threads.@threads for i in 1:n
     for i in 1:n
            hdist = 0.0
                        if !in(i,centres)
                            mer_dist = kmer_distance(mers[:,ind],mers[:,i],length(seqs[ind]),length(seqs[i]))
                            if mer_dist < kdist_cut
                                if gapless & (kord_distance(seqs[ind], seqs[i]) == mer_dist)
                                    aln = Tuple([(seqs[ind][j],seqs[i][j]) for j in 1:length(seqs[i])])
                                else
                                    if band_size >0
                                        pa = pairalign(OverlapAlignment(), seqs[ind], seqs[i],affinegap,true,band_size,band_size)
                                    else
                                        pa = pairalign(OverlapAlignment(), seqs[ind], seqs[i],affinegap)
                                    end
                                aln = (collect(pa.aln))
                                end
                                qs = quals[i]
                                q = 0
                                λᵢⱼ = 1.0
                                for j in 1:length(aln)

                                    if in(aln[j][2],l)
                                        q+=1
                                            j_ind = base_vals[aln[j][2]]
                                            i_ind = in(aln[j][1],l) ? base_vals[aln[j][1]] : base_vals[aln[j][2]]
                                            λᵢⱼ*=  err[(i_ind-1)*4+j_ind,qs[q]+1]
                                        hdist += 1
                                    end
                                end
                            else
                                λᵢⱼ = 0.0
                                hdist = Inf
                            end
                            if λᵢⱼ * N >= E_minmax[i]
                                push!(compI[i],length(centres))
                                push!(compV[i],λᵢⱼ)
                            end
                            if λᵢⱼ * counts[ind] > E_minmax[i]
                                E_minmax[i] = λᵢⱼ * counts[ind]
                            end
                            #λ[i] = λᵢⱼ
                            hdists[i] = hdist
                        end
                    end
end

boolfunction(pvals,n,ωₐ) = any(pvals .* n .< ωₐ)

function minp(pvals,counts,centres,partition,n = length(pvals))
    minval = 1.0
    min_ind = 1
        @inbounds for i in 1:n
        if !in(i,centres)
            if pvals[i] < minval
                minval = pvals[i]
                min_ind = i
            elseif (pvals[i] == minval) & (counts[i]> counts[min_ind])
                minval = pvals[i]
                min_ind = i
            end
        end
    end
    return minval, min_ind
end

function shuffle(centres,partition,compI,compV,reads,pvals,counts,n,shuff,max_shuffle)
    change = false
    @inbounds for i in (1:n)
        if !in(i,centres)
            _,k = findmax(compV[i] .* reads[compI[i]])
            j = compI[i][k]
            prev_j = findfirst(centres .== [partition[i]])
            if j !== prev_j
                change = true
                reads[j] += counts[i]
                reads[prev_j] -= counts[i]
                partition[i] = centres[j]
            end
            if shuff == max_shuffle
                pvals[i] = get_normed_pval(reads[j],compV[i][k],counts[i])
            end
        end
    end
    return change
end

function clustering(seqs, counts, quals,mers,err,ωₐ;band_size =16,log_p = false,log_λ=false, kdist_cut = 0.42,
    gapless = false, max_shuffle = 10)

    n = length(counts)
    pvals = zeros(n)
    centres = []
    E_minmax = zeros(n)
    hdists = zeros(n)
    compI = [Vector{Int}(undef,0) for i in 1:n]
    compV = [Vector{Float64}(undef,0) for i in 1:n]
    ind = findmax(counts)[2]
    centres = [ind]
    pvals[ind] = 1.0
    partition = fill(ind,n)
    compare!(seqs,quals,counts,ind,centres,partition,pvals,n,[],compI,compV, err,mers,hdists,E_minmax,band_size = band_size,kdist_cut = Inf,gapless = gapless)
    base_vals = Base.ImmutableDict(DNA_A=>1,DNA_C=>2,DNA_G=>3,DNA_T=>4)
    l = keys(base_vals)
    reads=[sum(counts)]

    for i in 1:n
        if !in(i,centres)
            partition[i] = centres[1]
            pvals[i] = get_normed_pval(reads[1],compV[i][1],counts[i])
        end
    end
    while true

        val, ind = minp(pvals,counts,centres,partition)
        if  boolfunction(val,n,ωₐ) & !in(ind, centres)
            partition[ind] = ind
            pvals[ind] = 1.0
            partition[centres] .= centres
            push!(centres,ind)
            hdists = hcat(hdists,zeros(n))
            compare!(seqs,quals,counts,ind,centres,partition,pvals,n,reads,compI,compV,
            err,mers,view(hdists,:,length(centres)),E_minmax,
            band_size = band_size,kdist_cut = kdist_cut,gapless = gapless)
        else
            break
        end
        reads = [sum(counts[partition .== i]) for i in centres]'
        change = true
        shuff = 1
        while shuff < max_shuffle
            shuff +=1
            if change ==false
               shuff = max_shuffle
            end
            change = false
            change = shuffle(centres,partition,compI,compV,reads,pvals,counts,n,shuff,max_shuffle)
        end
    end
    reads = [sum(counts[partition .== i]) for i in centres]'
    trans_mat = get_trans(centres,seqs,quals,counts,partition,n)
    cluster_map = Vector{Int}(undef,n)
    pc = copy(partition)
    center_raws = []
    for i in 1:length(centres)
        boolmask = partition .== centres[i]
        raws = seqs[boolmask]
        abunds = counts[boolmask]
        inds = collect(1:n)[boolmask]
        push!(center_raws,raws[findmax(abunds)[2]])
        centres[i] = inds[findmax(abunds)[2]]
        partition[boolmask] .= centres[i]
    end
    partition_map = Dict([centres[i] => i for i in 1:length(centres)]...)
    for i in 1:n
        cluster_map[i] = partition_map[partition[i]]
    end
    #trans_mat = get_trans(centres,seqs,quals,counts,partition,n)

    return (df =DataFrame([center_raws,vec(reads)],[:sequence,:abundance]),trans =  trans_mat,pvals = pvals,cluster_map =cluster_map,
    new_partition = partition,old_partition = pc)
end

function get_trans(centres,seqs,quals,counts,partition,n)
    affinegap = AffineGapScoreModel(match=5,
                                mismatch=-4,
                                 gap_open=-0,
                                 gap_extend=-8)
    base_vals = Base.ImmutableDict(DNA_A=>1,DNA_C=>2,DNA_G=>3,DNA_T=>4)
    l = keys(base_vals)
    trans_mat = zeros(Int,16,41)
    for i in 1:n
        if 1==1#!in(i,centres)
            pa = pairalign(OverlapAlignment(), seqs[partition[i]], seqs[i],affinegap)
            #qs2 = quals[partition[i]]#Int.(round.(derep.quality[i]))
            qs = quals[i]#Int.(round.(derep.quality[i]))
            q = 0
            aln = (collect(pa.aln))
            for j in 1:length(aln)# need to make this start from start of seq[i] in alignment e.g. firt !== DNA_GAP
                    if in(aln[j][2],l)
                        q+=1
                        if in(aln[j][1],l)
                            i_ind = base_vals[aln[j][1]]
                            j_ind =  base_vals[aln[j][2]]
                            #i_ind = in(aln[j][1],l) ? base_vals[aln[j][1]] : base_vals[aln[j][2]]
                            #Q = !in(i,centres) ? qs[q]+1 : qs[q]
                            #trans_mat[(i_ind-1)*4+j_ind, Q] +=counts[i]
                            trans_mat[(i_ind-1)*4+j_ind, qs[q]+1] +=counts[i]
                            #trans_mat[(i_ind-1)*4+j_ind,Int(round(mean([qs[q],qs2[q]])))+1] +=counts[i]
                        end
                    end
            end
        end
    end
    return trans_mat
end
function learnErrors(dereps ::Vector{DataFrame}, max_tries = 10;nbases = 1e8, band_size = 16,ωₐ = 1e-40,kdist_cut = 0.42)

    st = time()
    base_count = 0
    dereps_for_errs = []
    mers = []
    for (j,derep) in enumerate(dereps)
        derep.sequence =LongDNASeq.(derep.sequence)
        push!(dereps_for_errs, derep)
        push!(mers,kmer_count(derep))
        base_count += sum(derep.count .* length.(derep.sequence))
        if base_count > nbases
            break
        end
    end
    println("$(base_count) bases in $(length(dereps_for_errs)) samples")
    #counts = derep.count
    err = ones(16,41)
    #err = fill(.25,16,41)
    trans_mat = zeros(16,41)
    errs = [err]
    trans_mats =Vector{Array}(undef,length(dereps_for_errs))
    for i in 2:12
        #Threads.@threads for j in 1:length(dereps_for_errs)
            for j in 1:length(dereps_for_errs)
            derep = dereps_for_errs[j]
            trans_mats[j]= clustering(derep.sequence,derep.count,derep.quality,mers[j],err,ωₐ,
             band_size = band_size,log_p = true, kdist_cut = kdist_cut).trans
        end
        trans_mat = sum(trans_mats)
        err = loess_errors(trans_mat)
        push!(errs,err)
        println("Time elapsed: $(time() - st) seconds")
       println(UnicodePlots.heatmap(errs[i] .- errs[i-1], title = "Change from previous error estimate"))
       if any([errs[i][.!isnan.(errs[i])] ≈ errs[j][.!isnan.(errs[j])] for j in 1:i-1])
            println("Successfully converged")
            break
       end
    end
    return errs,trans_mat, trans_mats
end
function learnErrors(dereps ::Vector{String};nbases = 1e8, band_size = 16,ωₐ = 1e-40,kdist_cut = 0.42)
    dereps = dereplicate.(dereps)
    return learnErrors(dereps,nbases = nbases , band_size = band_size,ωₐ = ωₐ,kdist_cut = kdist_cut)
end

function learnErrors(derep ::DataFrame;nbases = 1e8, band_size = 16,ωₐ = 1e-40,kdist_cut = 0.42)
    return learnErrors([derep],nbases = nbases , band_size = band_size,ωₐ = ωₐ,kdist_cut = kdist_cut)
end

function dada(derep,err; band_size = 16,ωₐ = 1e-40,kdist_cut = 0.42,log_λ = false, log_p = true, gapless = false)
    println(derep.sample)
    counts = derep.count
    derep.df.sequence =LongDNASeq.(derep.sequence)
    seqs = LongDNASeq.(derep.sequence)
    qs = derep.quality
    partition = Vector{Int}(undef,length(counts))
    mers = kmer_count(derep.df)
    seq_abund = []
    pvals = []
    partition=[]
    λ= []
    hdists = []
    reads = []
    trans_mat = []
        clust = clustering(seqs,counts,qs,mers,err,ωₐ,
        band_size = band_size,log_λ = log_λ, log_p = log_p, kdist_cut = kdist_cut, gapless = gapless)

    return clust
end

function mergePairs(dadaF,derepF,dadaR,derepR)

    rf = dadaF.cluster_map[derepF.read_map]
    rr = dadaR.cluster_map[derepR.read_map]
    pairsArr = hcat(rf,rr)
    unqs = unique(pairsArr, 1)

    Fseqs = dadaF.df.sequence[unqs[:,1]]
    Rseqs = dadaR.df.sequence[unqs[:,2]]
    affinegap = AffineGapScoreModel(match=5,
    mismatch=-4,
     gap_open=-64,
     gap_extend=-64)
     alignments = pairalign.(OverlapAlignment(), Fseqs,Rseqs,affinegap)
    counts = countmap(eachrow(unqs))
    abundance = [counts[unq] for unq in eachrow(unqs)]
    df = Dataframe(hcat(pairsArr,aln.(alignments),abundance),[:rev,:fwd,:aln,:abundance])
    return df
end

function pairalign(::OverlapAlignment, a::S1, b::S2, score::AffineGapScoreModel{T}, banded::Bool,lower_offset,upper_offset;
    score_only::Bool=false) where {S1,S2,T}
    m = length(a)
    n = length(b)
    if banded
        if m > n
            L = m - n + lower_offset
            U = upper_offset
        else
            L = lower_offset
            U = n - m + upper_offset
        end
        bnw = BioAlignments.BandedNeedlemanWunsch{T}(m, n, L, U)
        score = BioAlignments.run!(bnw, a, b, score.submat,
        T(0), T(0), score.gap_open, score.gap_extend, T(0), T(0),
        T(0), T(0), score.gap_open, score.gap_extend, T(0), T(0),
        )
        if score_only
            return BioAlignments.PairwiseAlignmentResult{S1,S2}(score, true)
        else
            a′ = BioAlignments.traceback(bnw, a, b, (m, n))
            return BioAlignments.PairwiseAlignmentResult(score, true, a′, b)
        end
    end
end


# """
# rmp-treeで所望の加速度を計算
# """
# module RMPTree

# export update_nodes
# export calc_desired_ddq

#push!(LOAD_PATH, ".")

using LinearAlgebra

include("utils.jl")

include("rmp.jl")
include("kinematics.jl")

using .Kinematics: calc_all, q_max, q_min, q_neutral, cpoints_local
# using .RMP
#using .Utilis

# """所望の加速度を計算（resolve演算結果を返す）

# # args
# - nodes :
# - rmp_param :
# - goal :
# - obs :
# """
# function calc_desired_ddq(
#     nodes::Vector{Vector{Node{T}}},
#     rmp_param::NamedTuple,
#     goal::State{T},
#     obs::Vector{State{T}}
#     ) where T

#     root_f = zeros(T, 7)
#     root_M = zeros(T, 7, 7)

#     for i in 1:9
#         if i == 1  # ジョイント制限rmp
#             _f, _M = get_natural(
#                 rmp_param.joint_limit_avoidance, nodes[i][1].x, nodes[i][1].dx,
#                 q_max, q_min, q_neutral
#             )
#             #_f = zeros(T, 7)
#             #_M = zeros(T, 7, 7)
#             @. root_f += _f
#             @. root_M += _M

#         else
#             if i == 9  # 目標王達rmpを追加
#             _f, _M = get_natural(
#                 rmp_param.attractor, nodes[i][1].x, nodes[i][1].dx, goal.x
#             )
#             _pulled_f, _pulled_M = pullbacked_rmp(_f, _M, nodes[i][1].Jo,)
#             @. root_f += _pulled_f
#             @. root_M += _pulled_M
#             end

#             # 障害物回避rmpを追加
#             for j in 1:length(nodes[i])
#                 for k in 1:length(obs)
#                     _f, _M = get_natural(
#                         rmp_param.obs_avoidance[i-1], nodes[i][j].x, nodes[i][j].dx, obs[k].x
#                     )
#                     _pulled_f, _pulled_M = pullbacked_rmp(_f, _M, nodes[i][j].Jo,)
#                     @. root_f += _pulled_f
#                     @. root_M += _pulled_M
#                 end
#             end
#         end
#     end

#     #root_f += zeros(T, 7)
#     #root_M += zeros(T, 7, 7)

#     #println("root_f = ", root_f)
#     #println("root_M = ", root_M)

#     ddq = pinv(root_M) * root_f
#     #println("ddq = ", ddq)
#     #ddq = np.linalg.pinv(root_M) * root_f

#     #ddq = zeros(T, 7)
#     return ddq
# end

"""nodeを新規作成"""
function update_nodes(nodes::Nothing, q::Vector{T}, dq::Vector{T}) where T
    # 更新
    _, _,
    _, _, _, _,
    _, Jos_cpoint_all,
    cpoints_x_global, cpoints_dx_global,
    joints_x_global, joints_dx_global = calc_all(q, dq)
    nodes = Vector{Vector{Node{T}}}(undef, 9)
    for i in 1:9
        #println("i = ", i)
        if i == 1  # rootノード
            _children_node = Vector{Node{T}}(undef, 1)
            _children_node[1] = Node(
                x = q,
                dx = dq,
                Jo = Matrix{T}(I, 7, 7),
                f = zeros(T, 7),
                M = zeros(T, 7, 7)
                )
            #println("wow")
        else
            n = length(cpoints_local[i-1]) + 1
            _children_node = Vector{Node{T}}(undef, n)
            _children_node[1] = Node(
                x = joints_x_global[i+2],
                dx = joints_dx_global[i+2],
                Jo = nothing,
                f = nothing,
                M = nothing
            )  # ジョイント点

            for j in 2:n  # 制御点
                _children_node[j] = Node(
                    x = cpoints_x_global[i-1][j-1],
                    dx = cpoints_dx_global[i-1][j-1],
                    Jo = Jos_cpoint_all[i-1][j-1],
                    f = zeros(T, 3),
                    M = zeros(T, 3, 3),
                )
            end
        end
        nodes[i] = _children_node
    end
    return nodes
end

"""nodeを更新"""
function update_nodes!(;
    nodes::Vector{Vector{Node{T}}}, q::Vector{T}, dq::Vector{T},
    rmp_param::NamedTuple, goal::State{T}, obs::Vector{State{T}}
    ) where T
    # 更新
    _, _,
    _, _, _, _,
    _, Jos_cpoint_all,
    cpoints_x_global, cpoints_dx_global,
    joints_x_global, joints_dx_global = calc_all(q, dq)

    #hoge = nodes[1][1].x
    #nodes = copy(nodes)
    #println(old[1][1].x)

    for i in 1:9
        for j in 1:length(nodes[i])
            if i == 1  # root
                nodes[i][j].x = q
                nodes[i][j].dx = dq
                nodes[i][j].Jo = Matrix{T}(I, 7, 7)
                nodes[i][j].f, nodes[i][j].M = get_natural(
                    rmp_param.joint_limit_avoidance, nodes[i][1].x, nodes[i][1].dx,
                    q_max, q_min, q_neutral
                )

                # # ジョイント制限を考慮しない
                # nodes[i][j].f = zeros(T, 7)
                # nodes[i][j].M = zeros(T, 7, 7)

            else  # root以外
                if j == 1   # ジョイント位置
                    nodes[i][j].x = joints_x_global[i+2]
                    nodes[i][j].dx = joints_dx_global[i+2]
                    nodes[i][j].Jo = nothing
                    nodes[i][j].f = nothing
                    nodes[i][j].M = nothing
                else
                    nodes[i][j].x = cpoints_x_global[i-1][j-1]
                    nodes[i][j].dx = cpoints_dx_global[i-1][j-1]
                    nodes[i][j].Jo = Jos_cpoint_all[i-1][j-1]

                    f = zeros(T, 3)
                    M = zeros(T, 3, 3)
                    for k in 1:length(obs)
                        _f, _M = get_natural(
                            rmp_param.obs_avoidance[i-1], nodes[i][j].x, nodes[i][j].dx, obs[k].x
                        )
                        @. f += _f
                        @. M += _M
                    end
                    nodes[i][j].f = f
                    nodes[i][j].M = M

                    if i==9  # エンドエフェクタのときはさらに目標rmpを追加
                        _f, _M = get_natural(
                            rmp_param.attractor, nodes[i][j].x, nodes[i][j].dx, goal.x
                        )
                        @. nodes[i][j].f += _f
                        @. nodes[i][j].M += _M
                    end
                end
            end
        end
    end
    # println(nodes[1][1].x)
    # println(nodes[1][1].x == old[1][1].x)
    return nodes
end

"""nodesからf, Mを集めて所望の加速度を生成

新手法
"""
function calc_desired_ddq(nodes::Vector{Vector{Node{T}}}) where T
    root_f = zeros(T, 7)
    root_M = zeros(T, 7, 7)

    for i in 1:9
        for j in 1:length(nodes[i])  # 目標rmpと障害物rmpの合算
            if i == 1 && !isnothing(nodes[i][j].f)
                _pulled_f, _pulled_M = pullbacked_rmp(
                    nodes[i][j].f, nodes[i][j].M, nodes[i][j].Jo,
                )
                @. root_f += _pulled_f
                @. root_M += _pulled_M
            elseif i > 1 && !isnothing(nodes[i][j].f)
                _pulled_f, _pulled_M = pullbacked_rmp(
                    nodes[i][j].f, nodes[i][j].M, nodes[i][j].Jo,
                )
                @. root_f += _pulled_f
                @. root_M += _pulled_M
            end
        end
    end

    #root_f += zeros(T, 7)
    #root_M += zeros(T, 7, 7)

    #println("root_f = ", root_f)
    #println("root_M = ", root_M)
    ddq = pinv(root_M) * root_f
    #println("ddq = ", ddq)
    #ddq = np.linalg.pinv(root_M) * root_f

    #ddq = zeros(T, 7)
    return ddq
end

"""nodesからジョイント原点位置を取得（アニメ用）"""
function get_joint_positions_from_nodes(nodes::Vector{Vector{Node{T}}}) where T

    xs = Vector{Vector{T}}(undef, 8)

    for i in 2:9
        xs[i-1] = nodes[i][1].x
    end
        return xs
end

"""nodesから制御点位置を取得（アニメ用）"""
function get_cpoints_from_nodes(nodes::Vector{Vector{Node{T}}}) where T

    xs = Vector{Vector{Vector{T}}}(undef, 8)
    for i in 2:9
        _xs = Vector{Vector{T}}(undef, length(nodes[i])-1)
        for j in 2:length(nodes[i])
            _xs[j-1] = nodes[i][j].x
        end
        xs[i-1] = _xs
    end

    return xs

end

"""nodesからrmpを取得（グラフアニメ用）"""
function get_rmps_from_nodes(nodes::Vector{Vector{Node{T}}}) where T
    fs = Vector{Vector{Vector{T}}}(undef, 9)
    Ms = Vector{Vector{Matrix{T}}}(undef, 9)

    # fs[1][1] = nodes[1][1].f
    # Ms[1][1] = nodes[1][1].M

    for i in 1:9
        if i == 1
            _fs = Vector{Vector{T}}(undef, length(nodes[i]))
            _Ms = Vector{Matrix{T}}(undef, length(nodes[i]))
            _fs[i] = nodes[1][1].f
            _Ms[i] = nodes[1][1].M

        else

            _fs = Vector{Vector{T}}(undef, length(nodes[i]) - 1)
            _Ms = Vector{Matrix{T}}(undef, length(nodes[i]) - 1)
            for j in 2:length(nodes[i])
                _fs[j-1] = nodes[i][j].f
                _Ms[j-1] = nodes[i][j].M
            end
        end
        fs[i] = _fs
        Ms[i] = _Ms
    end

    return fs, Ms
end

"""fsからfのノルムだけ取得"""
function get_f_norms_from_fs(fs::Vector{Vector{Vector{T}}}) where T
    f_norms = Vector{Vector{T}}(undef, 9)

    for (i, _fs) in enumerate(fs)
        _f_norms = Vector{T}(undef, length(_fs))
        for (j, f) in enumerate(_fs)
            _f_norms[j] = norm(f)
        end
        f_norms[i] = _f_norms
    end
    return f_norms
end

"""fノルムsを時刻でまとめる"""
function get_f_norms_from_nodes_list(
    t_max::Int64,
    nodes_list::Vector{Vector{Vector{Node{T}}}}
    ) where T

    f_norms_list = Vector{Vector{Vector{T}}}(undef, 9)

    for (t, nodes) in enumerate(nodes_list)
        fs, _ = get_rmps_from_nodes(nodes)
        f_norms = get_f_norms_from_fs(fs)

        if t == 1  # 1回目は配列を宣言する

            for (i, f) in enumerate(f_norms)
                _f_norms_list = Vector{Vector{T}}(undef, length(f))
                for (j, _f) in enumerate(f)
                    _f_norms_list[j] = Vector{T}(undef, t_max)
                    _f_norms_list[j][1] = _f
                end
                f_norms_list[i] = _f_norms_list
            end
        else
            for (i, f) in enumerate(f_norms)
                for (j, _f) in enumerate(f)
                    f_norms_list[i][j][t] = _f
                end
            end
        end
    end
    return f_norms_list
end

#end


## Low level gemm! call with pointers
## Borrowed from Knet.jl

using Base.LinAlg
using Base.LinAlg.BLAS: libblas, BlasInt,  @blasfunc

# C := alpha*op(A)*op(B) + beta*C, where:
# op(X) is one of op(X) = X, or op(X) = XT, or op(X) = XH,
# alpha and beta are scalars,
# A, B and C are matrices:
# op(A) is an m-by-k matrix,
# op(B) is a k-by-n matrix,
# C is an m-by-n matrix.

for (gemm, elty) in ((:dgemm_,:Float64), (:sgemm_,:Float32))
    @eval begin
        function gemm!(transA::Char, transB::Char, M::Int, N::Int, K::Int, alpha::($elty), A::Ptr{$elty}, B::Ptr{$elty}, beta::($elty), C::Ptr{$elty})
            if transA=='N'; lda=M; else; lda=K; end
            if transB=='N'; ldb=K; else; ldb=N; end
            ldc = M;
            ccall((@blasfunc($gemm), libblas), Void,
                  (Ptr{UInt8}, Ptr{UInt8}, Ptr{BlasInt}, Ptr{BlasInt},
                   Ptr{BlasInt}, Ptr{$elty}, Ptr{$elty}, Ptr{BlasInt},
                   Ptr{$elty}, Ptr{BlasInt}, Ptr{$elty}, Ptr{$elty},
                   Ptr{BlasInt}),
                  &transA, &transB, &M, &N, &K,
                  &alpha, A, &lda, B, &ldb, &beta, C, &ldc)
        end
    end
end


using Test, MLToolkit

@testset "Neural SBC" begin
    i_dim = 20
    k_init = 10
    batch_size = 50

    @testset "$sbc" for sbc in [MeanFieldSBC(i_dim, k_init), StructuredSBC(i_dim, k_init)]
        is_all_true = true
        @test begin
            for k_out = 1:k_init
                x = AT(rand(FT, i_dim, batch_size))
                dist_kuma, dist_gumbel = sbc(x, k_out)
                is_all_true = is_all_true && dist_kuma isa BatchKumaraswamy
                is_all_true = is_all_true && dist_gumbel isa BatchGumbelBernoulliLogit
            end
            is_all_true
        end
    end

    @warn "`neural/sbc.jl` is not tested for online initialisation yet."
end


rotatereferenceatlas = x -> reverse(permutedims(x, (1, 3, 2)), dims=(3,))
# Plot the reference volume and probe locations
function plotreferencevolume(S; dotext=true, dostructures = true, ids=:targets, resolution=(1920, 1080), kwargs...)
    channels = AN.getchannels(S)
    vol, info = AN.gettemplatevolume()
    vol = Array{Float16}(vol)
    grad = cgrad(cgrad([RGBA(1.0, 1.0, 1.0, 0.0), RGBA(0.0, 0.0, 0.0, 1.0)]), 100)
    #coords = [(1:s).*25 for s ∈ size(vol)] # 25 μm resolution
    #vol = reverse(permutedims(vol, [1 3 2]), dims=(1,3))
    #coords = [1:x for x ∈ size(vol)].*50
    s = Scene(backgroundcolor = RGBA(1.0, 1.0, 1.0, 0.0), resolution = resolution, transparency=true)
    coords = [1:s for s ∈ size(rotatereferenceatlas(vol))./2]
    coords[3] = .-coords[3]

    markercolor = :gray
    if ids isa Symbol
        if ids == :all
            # Get and plot a bunch of structures
            #ids = AN.getallstructureids()
            anns = unique(["FRP", "MO", "SS", "GU", "VISC", "AUD", "ACA", "PL", "ILA", "ORB", "AI", "RSP", "PTLp", "TEa", "PERI", "ECT", "OLF", "VISp", "VISl", "VISrl", "VISam", "VISpm", "VIS", "VISal","VISmma","VISmmp","VISli", "LGd","LD", "LP", "VPM", "TH", "MGm","MGv","MGd","PO","LGv","VL",
            "VPL","POL","Eth","PoT","PP","PIL","IntG","IGL","SGN","VPL","PF","RT", "CA1", "CA2","CA3", "DG", "SUB", "POST","PRE","ProS","HPF", "MB","SCig","SCiw","SCsg","SCzo","PPT","APN","NOT","MRN","OP","LT","RPF","CP"])
        elseif ids == :corticaltargets
            anns = unique(["VISp", "VISl", "VISrl", "VISal", "VISpm", "VISam"])
        elseif ids == :targets # As listed in the visual coding white paper
            anns =  unique(["VISp", "VISl", "VISrl", "VISal", "VISpm", "VISam", "CA1", "CA3", "DG", "SUB", "ProS", "LGd", "LP", "APN"])
        else
            @error "`ids` keyword argument not valid "
        end
        t = AN.getstructuretree()
        D = t.get_id_acronym_map()
        ids = getindex.((D,), anns)
    end

    f = volume!(s, coords..., rotatereferenceatlas(vol)[1:2:end, 1:2:end, 1:2:end]; algorithm=:mip, colorrange=extrema(vol), colormap=collect(grad), ticks=nothing, fxaa=true, transparency=true, kwargs...)
    #s.plots[1].attributes[:fxaa] = true

    chrome = FileIO.load(download("https://raw.githubusercontent.com/nidorx/matcaps/master/1024/E6BF3C_5A4719_977726_FCFC82.png"))
    for probeid in unique(channels.probe_id)
        (x, y, z) = AN.getprobecoordinates(S, probeid)./50
        if !any(length.((x, y, z)).==0)
            f = meshscatter!(s, x, z, -y; markersize=1.0, fxaa=true, color=markercolor, matcap=chrome, shading=true, transparency=true, kwargs...)

            if dotext
                _, idx = findmax(z) # The highest unit
                text!(s, string(probeid), position=Vec3f0(x[idx], z[idx], -y[idx]), space=:data, textsize=5, align=(:right, :bottom), rotation=Quaternion((-0.3390201, 0.33899, 0.6205722, -0.620517)), transparency=true)
            end
        end
    end
    #scale!(s, 1, 1, -1)
    if dostructures
        channels = subset(AN.getchannels(),    :ecephys_probe_id           => ByRow(!ismissing),
                                                :ecephys_structure_id       => ByRow(!ismissing),
                                                :ecephys_structure_acronym  => ByRow(x->x!=="grey"))
        if isnothing(ids)
            ids = unique(channels[!, :ecephys_structure_id])
            #ids = ids[AN.getstructuretreedepth.(ids) .< 9]
        end
        for id ∈ ids
            try
                try
                    mask, _ = AN.getstructuremask(id)
                catch
                    mask = AN.buildstructuremask(id)
                end
                mask = Array{Float64}(rotatereferenceatlas(mask)[1:2:end, 1:2:end, 1:2:end])
                mc_algo = NaiveSurfaceNets(iso=0, insidepositive=true)
                m = GeometryBasics.Mesh(mask, mc_algo; origin=[min(coords[i]...) for i ∈ 1:length(coords)], widths=[abs(-(extrema(coords[i])...)) for i ∈ 1:length(coords)])
                c = AN.getstructurecolor(id)
                f = mesh!(s, m; color=RGBA(c.r, c.g, c.b, 0.41), fxaa=true, shading=true, transparency=true, kwargs...)
            catch y
                @warn y
            end
        end
    end
    return s
end

function exportreferencevolume(S, file::String="plot.html"; ids=:targets, kwargs...)
    open(file, "w") do io
            println(io, """
            <html>
                <head>
                </head>
                <body>
            """)
            show(io, MIME"text/html"(), Page(exportable=true, offline=true))
            fig = plotreferencevolume(S; dostructures=true, ids, kwargs...)
            rotate_cam!(fig, Vec3f0(0, 2.35, 0))
            show(io, MIME"text/html"(), fig)
            println(io, """
                </body>
            </html>
            """)
    end
end

function formattedreferencevolume(S, file::String="plot.html")
    exportreferencevolume(S, file;
                                    dostructures=true,
                                    ids=:targets,
                                    show_axis=false,
                                    shading=true, resolution=(1080, 1080));
end


@enum(MatchingStrategy::Int,COMPLETE=1,COMPLETE2x2=2,CONSTRAINTS=3)

@kwdef mutable struct Options <: AbstractOptions
    pardiso_matching_strategy::MatchingStrategy = COMPLETE2x2
    pardiso_max_inner_refinement_steps::Int = 1
    pardiso_msglvl::Int = 0
    pardiso_order::Int = 2
    pardiso_log_level::String = ""
end

mutable struct Solver <: AbstractLinearSolver
    pt::Vector{Int}
    iparm::Vector{Int32}
    dparm::Vector{Float64}
    perm::Vector{Int32}
    msglvl::Ref{Int32}
    err::Ref{Int32}
    csc::SparseMatrixCSC{Float64,Int32}
    w::Vector{Float64}
    opt::Options
    logger::Logger
end

_pardisoinit(
    pt::Vector{Int},mtype::Ref{Cint},solver::Ref{Cint},iparm::Vector{Cint},
    dparm::Vector{Cdouble},err::Ref{Cint}) =
        ccall(
            (:pardisoinit,libpardiso),
            Cvoid,
            (Ptr{Int},Ptr{Cint},Ptr{Cint},Ptr{Cint},Ptr{Cdouble},Ptr{Cint}),
            pt,mtype,solver,iparm,dparm,err)
_pardiso(
    pt::Vector{Int},maxfct::Ref{Cint},mnum::Ref{Cint},mtype::Ref{Cint},
    phase::Ref{Cint},n::Ref{Cint},a::Vector{Cdouble},ia::Vector{Cint},ja::Vector{Cint},
    perm::Vector{Cint},nrhs::Ref{Cint},iparm::Vector{Cint},msglvl::Ref{Cint},
    b::StrideOneVector{Cdouble},x::StrideOneVector{Cdouble},err::Ref{Cint},dparm::Vector{Cdouble}) =
        ccall(
            (:pardiso,libpardiso),
            Cvoid,
            (Ptr{Int},Ptr{Cint},Ptr{Cint},Ptr{Cint},Ptr{Cint},Ptr{Cint},Ptr{Cdouble},Ptr{Cint},Ptr{Cint},Ptr{Cint},
             Ptr{Cint},Ptr{Cint},Ptr{Cint},Ptr{Cdouble},Ptr{Cdouble},Ptr{Cint},Ptr{Cdouble}),
            pt,maxfct,mnum,mtype,phase,n,a,ia,ja,perm,nrhs,iparm,msglvl,b,x,err,dparm)

function Solver(csc::SparseMatrixCSC{Float64,Int32};
                opt=Options(),logger=Logger(),
                option_dict::Dict{Symbol,Any}=Dict{Symbol,Any}(),
                kwargs...)
    !isempty(kwargs) && (for (key,val) in kwargs; option_dict[key]=val; end)
    set_options!(opt,option_dict)

    w   = Vector{Float64}(undef,csc.n)

    pt = Vector{Int}(undef,64)
    iparm = Vector{Int32}(undef,64)
    dparm = Vector{Float64}(undef,64)
    perm = Vector{Int32}(undef,csc.n)
    msglvl = Ref{Int32}(0)
    err = Ref{Int32}(0)

    pt.=0
    iparm[1]=0

    _pardisoinit(pt,Ref{Int32}(-2),Ref{Int32}(0),iparm,dparm,err)
    err.x < 0  && throw(SymbolicException())

    iparm[1]=1
    iparm[2]=opt.pardiso_order # METIS
    iparm[3]=haskey(ENV,"OMP_NUM_THREADS") ? parse(Int32,ENV["OMP_NUM_THREADS"]) : 1
    iparm[6]=1
    iparm[8]=opt.pardiso_max_inner_refinement_steps
    iparm[10]=12 # pivot perturbation
    iparm[11]=0 # disable scaling
    iparm[13]=1 # matching strategy
    iparm[21]=3 # bunch-kaufman pivotin
    iparm[24]=1 # parallel factorization
    iparm[25]=1 # parallel solv

    _pardiso(pt,Ref{Int32}(1),Ref{Int32}(1),Ref{Int32}(-2),Ref{Int32}(11),
             Ref{Int32}(csc.n),csc.nzval,csc.colptr,csc.rowval,perm,
             Ref{Int32}(1),iparm,msglvl,Float64[],Float64[],err,dparm)
    err.x < 0  && throw(SymbolicException())

    M = Solver(pt,iparm,dparm,perm,msglvl,err,csc,w,opt,logger)

    finalizer(finalize,M)

    return M
end
function factorize!(M::Solver)
    _pardiso(M.pt,Ref{Int32}(1),Ref{Int32}(1),Ref{Int32}(-2),Ref{Int32}(22),
             Ref{Int32}(M.csc.n),M.csc.nzval,M.csc.colptr,M.csc.rowval,M.perm,
             Ref{Int32}(1),M.iparm,M.msglvl,Float64[],Float64[],M.err,M.dparm)
    M.err.x < 0  && throw(FactorizationException())
    M.iparm[14] == 0 && return M
    _pardiso(M.pt,Ref{Int32}(1),Ref{Int32}(1),Ref{Int32}(-2),Ref{Int32}(12),
             Ref{Int32}(M.csc.n),M.csc.nzval,M.csc.colptr,M.csc.rowval,M.perm,
             Ref{Int32}(1),M.iparm,M.msglvl,Float64[],Float64[],M.err,M.dparm)
    M.err.x < 0  && throw(FactorizationException())
    return M
end
function solve!(M::Solver,rhs::StrideOneVector{Float64})
    _pardiso(M.pt,Ref{Int32}(1),Ref{Int32}(1),Ref{Int32}(-2),Ref{Int32}(33),
             Ref{Int32}(M.csc.n),M.csc.nzval,M.csc.colptr,M.csc.rowval,M.perm,
             Ref{Int32}(1),M.iparm,M.msglvl,rhs,M.w,M.err,M.dparm)
    M.err.x < 0  && throw(SolveException())
    return rhs
end
function finalize(M::Solver)
    _pardiso(M.pt,Ref{Int32}(1),Ref{Int32}(1),Ref{Int32}(-2),Ref{Int32}(-1),
             Ref{Int32}(M.csc.n),M.csc.nzval,M.csc.colptr,M.csc.rowval,M.perm,
             Ref{Int32}(1),M.iparm,M.msglvl,Float64[],M.w,M.err,M.dparm)
end
is_inertia(::Solver)=true
function inertia(M::Solver)
    pos = M.iparm[22]
    neg = M.iparm[23]
    return (pos,M.csc.n-pos-neg,neg)
end
function improve!(M::Solver)
    @debug(M.logger,"improve quality failed.")
    return false
end

introduce(::Solver)="pardiso"


function fixxing(f::Function, model::MetNet, ider, val::Real;
        btol = 0.0
    )
    idx = rxnindex(model, ider)
    bk_lb, bk_ub = model.lb[idx], model.ub[idx]
    try
        dflx = abs(val * btol)
        model.lb[idx] = val - dflx
        model.ub[idx] = val + dflx
        return f()
    finally
        model.lb[idx], model.ub[idx] = bk_lb, bk_ub
    end
end


@testset "Node population reports" begin

@testset "function nodePopReport" begin
    @test isempty( nodePopReport( mpSim, [-12.0, -6.0], "active", "Career",
        "Blort", "A junior" ) )
    @test isempty( nodePopReport( mpSim, [7.5, 3.2], "Blort" ) )
    @test isempty( nodePopReport( mpSim, -12, "active", "Career", "Blort",
        "A junior" ) )
    report = nodePopReport( mpSim, 12, "active", "Career", "Blort", "A junior" )
    @test ( :Blort ∉ names( report ) ) &&
        ( ["active", "Career", "A junior"] ⊆ names( report ) )
    @test all( report[:, "active"] .== vcat( 30, 60, 84, 108, 124, 140, 156,
        172, 188, 204, 196, fill( 188, 15 ) ) )
    @test all( report[:, "A junior"] .== vcat( 10, fill( 20, 25 ) ) )
    @test all( report[:, "Career"] .== vcat( 0, 0, 24, 48, 64, 80, 96, 112,
        128, 144, 136, fill( 128, 15 ) ) )
end  # @testset "function nodePopReport"

@testset "function nodeEvolutionReport" begin
    report = nodeEvolutionReport( mpSim, [-12.0, -6.0], "active", "Career",
        "Blort", "A junior" )
    @test isempty( report[1] ) && isempty( report[2] )
    report = nodeEvolutionReport( mpSim, [7.5, 3.2], "Blort" )
    @test isempty( report[1] ) && isempty( report[2] )
    report = nodeEvolutionReport( mpSim, -12, "active", "Career", "Blort",
        "A junior" )
    @test isempty( report[1] ) && isempty( report[2] )
    report = nodeEvolutionReport( mpSim, 12, "active", "Career", "Blort",
        "A junior" )
    @test ["active", "Career", "A junior"] ⊆ names( report[1] )
    @test all( haskey.( Ref( report[2] ),
        ["active", "Career", "A junior"] ) )
    @test all( [length( report[2]["active"] ) == 2,
        length( report[2]["Career"] ) == 4,
        length( report[2]["A junior"] ) == 2] )
end  # @testset "function nodeEvolutionReport"

end  # @testset "Node population reports"


facts("Random search") do

context("ask") do
  for(i in 1:NumTestRepetitions)
    dims = rand(1:20)
    min = rand(1:123)
    range = (min * rand(), min + rand() * min)

    ss = BlackBoxOptim.symmetric_search_space(dims, range)
    opt = BlackBoxOptim.random_search(ss)

    res1 = BlackBoxOptim.ask(opt)

    @fact length(res1) => 1
    candidate, index = res1[1]
    @fact size(candidate,2) => dims
    @fact isinspace(candidate, ss) => true

    # Fake fitness
    better = BlackBoxOptim.tell(opt, [(candidate, index, 1.0)])

    @fact better => 1
    @fact opt.best => candidate
    @fact opt.best_fitness => 1.0

    # Get one more and fake that it has better fitness.
    res2 = BlackBoxOptim.ask(opt)
    @fact length(res2) => 1
    candidate2, index2 = res2[1]
    @fact size(candidate2,2) => dims
    @fact isinspace(candidate2, ss) => true
    better2 = BlackBoxOptim.tell(opt, [(candidate, index, 0.5)])
  end
end

end


# RUN: julia -e "import Brutus; Brutus.lit(:emit_optimized)" --startup-file=no %s 2>&1 | FileCheck %s

# Intentionally bad code
function matmul!(C, A, B)
    for i in 1:size(A, 1)
        for k in 1:size(A, 2)
            for j in 1:size(B, 2)
                C[i, j] += A[i, k] * B[k, j]
            end
        end
    end
   return nothing
end

emit(matmul!, Matrix{Float64}, Matrix{Float64}, Matrix{Float64})

# CHECK:   func nested @"Tuple{typeof(Main.matmul!), Array{Float64, 2}, Array{Float64, 2}, Array{Float64, 2}}"(%arg0: !jlir<"typeof(Main.matmul!)">, %arg1: !jlir<"Array{Float64, 2}">, %arg2: !jlir<"Array{Float64, 2}">, %arg3: !jlir<"Array{Float64, 2}">) -> !jlir.Nothing attributes {llvm.emit_c_interface} {
# CHECK-NEXT:     "jlir.goto"()[^bb1] : () -> ()
# CHECK-NEXT:   ^bb1:  // pred: ^bb0
# CHECK-NEXT:     %0 = "jlir.constant"() {value = #jlir<"1">} : () -> !jlir.Int64
# CHECK-NEXT:     %1 = "jlir.arraysize"(%arg2, %0) : (!jlir<"Array{Float64, 2}">, !jlir.Int64) -> !jlir.Int64
# CHECK-NEXT:     %2 = "jlir.sle_int"(%0, %1) : (!jlir.Int64, !jlir.Int64) -> !jlir.Bool
# CHECK-NEXT:     %3 = "jlir.constant"() {value = #jlir<"0">} : () -> !jlir.Int64
# CHECK-NEXT:     %4 = "jlir.ifelse"(%2, %1, %3) : (!jlir.Bool, !jlir.Int64, !jlir.Int64) -> !jlir.Int64
# CHECK-NEXT:     %5 = "jlir.slt_int"(%4, %0) : (!jlir.Int64, !jlir.Int64) -> !jlir.Bool
# CHECK-NEXT:     "jlir.gotoifnot"(%5)[^bb3, ^bb2] {operand_segment_sizes = dense<[1, 0, 0]> : vector<3xi32>} : (!jlir.Bool) -> ()
# CHECK-NEXT:   ^bb2:  // pred: ^bb1
# CHECK-NEXT:     %6 = "jlir.constant"() {value = #jlir.true} : () -> !jlir.Bool
# CHECK-NEXT:     %7 = "jlir.undef"() : () -> !jlir.Int64
# CHECK-NEXT:     %8 = "jlir.undef"() : () -> !jlir.Int64
# CHECK-NEXT:     "jlir.goto"(%6, %7, %8)[^bb4] : (!jlir.Bool, !jlir.Int64, !jlir.Int64) -> ()
# CHECK-NEXT:   ^bb3:  // pred: ^bb1
# CHECK-NEXT:     %9 = "jlir.constant"() {value = #jlir.false} : () -> !jlir.Bool
# CHECK-NEXT:     "jlir.goto"(%9, %0, %0)[^bb4] : (!jlir.Bool, !jlir.Int64, !jlir.Int64) -> ()
# CHECK-NEXT:   ^bb4(%10: !jlir.Bool, %11: !jlir.Int64, %12: !jlir.Int64):  // 2 preds: ^bb2, ^bb3
# CHECK-NEXT:     %13 = "jlir.not_int"(%10) : (!jlir.Bool) -> !jlir.Bool
# CHECK-NEXT:     "jlir.gotoifnot"(%13, %11, %12)[^bb28, ^bb5] {operand_segment_sizes = dense<[1, 0, 2]> : vector<3xi32>} : (!jlir.Bool, !jlir.Int64, !jlir.Int64) -> ()
# CHECK-NEXT:   ^bb5(%14: !jlir.Int64, %15: !jlir.Int64):  // 2 preds: ^bb4, ^bb27
# CHECK-NEXT:     %16 = "jlir.constant"() {value = #jlir<"2">} : () -> !jlir.Int64
# CHECK-NEXT:     %17 = "jlir.arraysize"(%arg2, %16) : (!jlir<"Array{Float64, 2}">, !jlir.Int64) -> !jlir.Int64
# CHECK-NEXT:     %18 = "jlir.sle_int"(%0, %17) : (!jlir.Int64, !jlir.Int64) -> !jlir.Bool
# CHECK-NEXT:     %19 = "jlir.ifelse"(%18, %17, %3) : (!jlir.Bool, !jlir.Int64, !jlir.Int64) -> !jlir.Int64
# CHECK-NEXT:     %20 = "jlir.slt_int"(%19, %0) : (!jlir.Int64, !jlir.Int64) -> !jlir.Bool
# CHECK-NEXT:     "jlir.gotoifnot"(%20)[^bb7, ^bb6] {operand_segment_sizes = dense<[1, 0, 0]> : vector<3xi32>} : (!jlir.Bool) -> ()
# CHECK-NEXT:   ^bb6:  // pred: ^bb5
# CHECK-NEXT:     %21 = "jlir.constant"() {value = #jlir.true} : () -> !jlir.Bool
# CHECK-NEXT:     %22 = "jlir.undef"() : () -> !jlir.Int64
# CHECK-NEXT:     %23 = "jlir.undef"() : () -> !jlir.Int64
# CHECK-NEXT:     "jlir.goto"(%21, %22, %23)[^bb8] : (!jlir.Bool, !jlir.Int64, !jlir.Int64) -> ()
# CHECK-NEXT:   ^bb7:  // pred: ^bb5
# CHECK-NEXT:     %24 = "jlir.constant"() {value = #jlir.false} : () -> !jlir.Bool
# CHECK-NEXT:     "jlir.goto"(%24, %0, %0)[^bb8] : (!jlir.Bool, !jlir.Int64, !jlir.Int64) -> ()
# CHECK-NEXT:   ^bb8(%25: !jlir.Bool, %26: !jlir.Int64, %27: !jlir.Int64):  // 2 preds: ^bb6, ^bb7
# CHECK-NEXT:     %28 = "jlir.not_int"(%25) : (!jlir.Bool) -> !jlir.Bool
# CHECK-NEXT:     "jlir.gotoifnot"(%28, %26, %27)[^bb23, ^bb9] {operand_segment_sizes = dense<[1, 0, 2]> : vector<3xi32>} : (!jlir.Bool, !jlir.Int64, !jlir.Int64) -> ()
# CHECK-NEXT:   ^bb9(%29: !jlir.Int64, %30: !jlir.Int64):  // 2 preds: ^bb8, ^bb22
# CHECK-NEXT:     %31 = "jlir.arraysize"(%arg3, %16) : (!jlir<"Array{Float64, 2}">, !jlir.Int64) -> !jlir.Int64
# CHECK-NEXT:     %32 = "jlir.sle_int"(%0, %31) : (!jlir.Int64, !jlir.Int64) -> !jlir.Bool
# CHECK-NEXT:     %33 = "jlir.ifelse"(%32, %31, %3) : (!jlir.Bool, !jlir.Int64, !jlir.Int64) -> !jlir.Int64
# CHECK-NEXT:     %34 = "jlir.slt_int"(%33, %0) : (!jlir.Int64, !jlir.Int64) -> !jlir.Bool
# CHECK-NEXT:     "jlir.gotoifnot"(%34)[^bb11, ^bb10] {operand_segment_sizes = dense<[1, 0, 0]> : vector<3xi32>} : (!jlir.Bool) -> ()
# CHECK-NEXT:   ^bb10:  // pred: ^bb9
# CHECK-NEXT:     %35 = "jlir.constant"() {value = #jlir.true} : () -> !jlir.Bool
# CHECK-NEXT:     %36 = "jlir.undef"() : () -> !jlir.Int64
# CHECK-NEXT:     %37 = "jlir.undef"() : () -> !jlir.Int64
# CHECK-NEXT:     "jlir.goto"(%35, %36, %37)[^bb12] : (!jlir.Bool, !jlir.Int64, !jlir.Int64) -> ()
# CHECK-NEXT:   ^bb11:  // pred: ^bb9
# CHECK-NEXT:     %38 = "jlir.constant"() {value = #jlir.false} : () -> !jlir.Bool
# CHECK-NEXT:     "jlir.goto"(%38, %0, %0)[^bb12] : (!jlir.Bool, !jlir.Int64, !jlir.Int64) -> ()
# CHECK-NEXT:   ^bb12(%39: !jlir.Bool, %40: !jlir.Int64, %41: !jlir.Int64):  // 2 preds: ^bb10, ^bb11
# CHECK-NEXT:     %42 = "jlir.not_int"(%39) : (!jlir.Bool) -> !jlir.Bool
# CHECK-NEXT:     "jlir.gotoifnot"(%42, %40, %41)[^bb18, ^bb13] {operand_segment_sizes = dense<[1, 0, 2]> : vector<3xi32>} : (!jlir.Bool, !jlir.Int64, !jlir.Int64) -> ()
# CHECK-NEXT:   ^bb13(%43: !jlir.Int64, %44: !jlir.Int64):  // 2 preds: ^bb12, ^bb17
# CHECK-NEXT:     %45 = "jlir.constant"() {value = #jlir.true} : () -> !jlir.Bool
# CHECK-NEXT:     %46 = "jlir.arrayref"(%45, %arg1, %14, %43) : (!jlir.Bool, !jlir<"Array{Float64, 2}">, !jlir.Int64, !jlir.Int64) -> !jlir.Float64
# CHECK-NEXT:     %47 = "jlir.arrayref"(%45, %arg2, %14, %29) : (!jlir.Bool, !jlir<"Array{Float64, 2}">, !jlir.Int64, !jlir.Int64) -> !jlir.Float64
# CHECK-NEXT:     %48 = "jlir.arrayref"(%45, %arg3, %29, %43) : (!jlir.Bool, !jlir<"Array{Float64, 2}">, !jlir.Int64, !jlir.Int64) -> !jlir.Float64
# CHECK-NEXT:     %49 = "jlir.mul_float"(%47, %48) : (!jlir.Float64, !jlir.Float64) -> !jlir.Float64
# CHECK-NEXT:     %50 = "jlir.add_float"(%46, %49) : (!jlir.Float64, !jlir.Float64) -> !jlir.Float64
# CHECK-NEXT:     %51 = "jlir.arrayset"(%45, %arg1, %50, %14, %43) : (!jlir.Bool, !jlir<"Array{Float64, 2}">, !jlir.Float64, !jlir.Int64, !jlir.Int64) -> !jlir<"Array{Float64, 2}">
# CHECK-NEXT:     %52 = "jlir.==="(%44, %33) : (!jlir.Int64, !jlir.Int64) -> !jlir.Bool
# CHECK-NEXT:     "jlir.gotoifnot"(%52)[^bb15, ^bb14] {operand_segment_sizes = dense<[1, 0, 0]> : vector<3xi32>} : (!jlir.Bool) -> ()
# CHECK-NEXT:   ^bb14:  // pred: ^bb13
# CHECK-NEXT:     %53 = "jlir.undef"() : () -> !jlir.Int64
# CHECK-NEXT:     %54 = "jlir.undef"() : () -> !jlir.Int64
# CHECK-NEXT:     "jlir.goto"(%53, %54, %45)[^bb16] : (!jlir.Int64, !jlir.Int64, !jlir.Bool) -> ()
# CHECK-NEXT:   ^bb15:  // pred: ^bb13
# CHECK-NEXT:     %55 = "jlir.add_int"(%44, %0) : (!jlir.Int64, !jlir.Int64) -> !jlir.Int64
# CHECK-NEXT:     %56 = "jlir.constant"() {value = #jlir.false} : () -> !jlir.Bool
# CHECK-NEXT:     "jlir.goto"(%55, %55, %56)[^bb16] : (!jlir.Int64, !jlir.Int64, !jlir.Bool) -> ()
# CHECK-NEXT:   ^bb16(%57: !jlir.Int64, %58: !jlir.Int64, %59: !jlir.Bool):  // 2 preds: ^bb14, ^bb15
# CHECK-NEXT:     %60 = "jlir.not_int"(%59) : (!jlir.Bool) -> !jlir.Bool
# CHECK-NEXT:     "jlir.gotoifnot"(%60)[^bb18, ^bb17] {operand_segment_sizes = dense<[1, 0, 0]> : vector<3xi32>} : (!jlir.Bool) -> ()
# CHECK-NEXT:   ^bb17:  // pred: ^bb16
# CHECK-NEXT:     "jlir.goto"(%57, %58)[^bb13] : (!jlir.Int64, !jlir.Int64) -> ()
# CHECK-NEXT:   ^bb18:  // 2 preds: ^bb12, ^bb16
# CHECK-NEXT:     %61 = "jlir.==="(%30, %19) : (!jlir.Int64, !jlir.Int64) -> !jlir.Bool
# CHECK-NEXT:     "jlir.gotoifnot"(%61)[^bb20, ^bb19] {operand_segment_sizes = dense<[1, 0, 0]> : vector<3xi32>} : (!jlir.Bool) -> ()
# CHECK-NEXT:   ^bb19:  // pred: ^bb18
# CHECK-NEXT:     %62 = "jlir.undef"() : () -> !jlir.Int64
# CHECK-NEXT:     %63 = "jlir.undef"() : () -> !jlir.Int64
# CHECK-NEXT:     %64 = "jlir.constant"() {value = #jlir.true} : () -> !jlir.Bool
# CHECK-NEXT:     "jlir.goto"(%62, %63, %64)[^bb21] : (!jlir.Int64, !jlir.Int64, !jlir.Bool) -> ()
# CHECK-NEXT:   ^bb20:  // pred: ^bb18
# CHECK-NEXT:     %65 = "jlir.add_int"(%30, %0) : (!jlir.Int64, !jlir.Int64) -> !jlir.Int64
# CHECK-NEXT:     %66 = "jlir.constant"() {value = #jlir.false} : () -> !jlir.Bool
# CHECK-NEXT:     "jlir.goto"(%65, %65, %66)[^bb21] : (!jlir.Int64, !jlir.Int64, !jlir.Bool) -> ()
# CHECK-NEXT:   ^bb21(%67: !jlir.Int64, %68: !jlir.Int64, %69: !jlir.Bool):  // 2 preds: ^bb19, ^bb20
# CHECK-NEXT:     %70 = "jlir.not_int"(%69) : (!jlir.Bool) -> !jlir.Bool
# CHECK-NEXT:     "jlir.gotoifnot"(%70)[^bb23, ^bb22] {operand_segment_sizes = dense<[1, 0, 0]> : vector<3xi32>} : (!jlir.Bool) -> ()
# CHECK-NEXT:   ^bb22:  // pred: ^bb21
# CHECK-NEXT:     "jlir.goto"(%67, %68)[^bb9] : (!jlir.Int64, !jlir.Int64) -> ()
# CHECK-NEXT:   ^bb23:  // 2 preds: ^bb8, ^bb21
# CHECK-NEXT:     %71 = "jlir.==="(%15, %4) : (!jlir.Int64, !jlir.Int64) -> !jlir.Bool
# CHECK-NEXT:     "jlir.gotoifnot"(%71)[^bb25, ^bb24] {operand_segment_sizes = dense<[1, 0, 0]> : vector<3xi32>} : (!jlir.Bool) -> ()
# CHECK-NEXT:   ^bb24:  // pred: ^bb23
# CHECK-NEXT:     %72 = "jlir.undef"() : () -> !jlir.Int64
# CHECK-NEXT:     %73 = "jlir.undef"() : () -> !jlir.Int64
# CHECK-NEXT:     %74 = "jlir.constant"() {value = #jlir.true} : () -> !jlir.Bool
# CHECK-NEXT:     "jlir.goto"(%72, %73, %74)[^bb26] : (!jlir.Int64, !jlir.Int64, !jlir.Bool) -> ()
# CHECK-NEXT:   ^bb25:  // pred: ^bb23
# CHECK-NEXT:     %75 = "jlir.add_int"(%15, %0) : (!jlir.Int64, !jlir.Int64) -> !jlir.Int64
# CHECK-NEXT:     %76 = "jlir.constant"() {value = #jlir.false} : () -> !jlir.Bool
# CHECK-NEXT:     "jlir.goto"(%75, %75, %76)[^bb26] : (!jlir.Int64, !jlir.Int64, !jlir.Bool) -> ()
# CHECK-NEXT:   ^bb26(%77: !jlir.Int64, %78: !jlir.Int64, %79: !jlir.Bool):  // 2 preds: ^bb24, ^bb25
# CHECK-NEXT:     %80 = "jlir.not_int"(%79) : (!jlir.Bool) -> !jlir.Bool
# CHECK-NEXT:     "jlir.gotoifnot"(%80)[^bb28, ^bb27] {operand_segment_sizes = dense<[1, 0, 0]> : vector<3xi32>} : (!jlir.Bool) -> ()
# CHECK-NEXT:   ^bb27:  // pred: ^bb26
# CHECK-NEXT:     "jlir.goto"(%77, %78)[^bb5] : (!jlir.Int64, !jlir.Int64) -> ()
# CHECK-NEXT:   ^bb28:  // 2 preds: ^bb4, ^bb26
# CHECK-NEXT:     %81 = "jlir.constant"() {value = #jlir.nothing} : () -> !jlir.Nothing
# CHECK-NEXT:     "jlir.return"(%81) : (!jlir.Nothing) -> ()
# CHECK-NEXT:   }
# CHECK-NEXT: }


"""
    simple_extr_val_sel(data::ClustData,
                        extreme_value_descr_ar::Array{SimpleExtremeValueDescr,1};
                        rep_mod_method::String="feasibility")

Selects simple extreme values and returns modified data, extreme values, and the corresponding indices.

Inputs options for `rep_mod_method`:
- `rep_mod_method`::String : `feasibility`,`append`
"""
function simple_extr_val_sel(data::ClustData,
                             extr_value_descr_ar::Array{SimpleExtremeValueDescr,1};
                             rep_mod_method::String="feasibility"
                             )
  idcs = simple_extr_val_ident(data,extr_value_descr_ar)
  extr_vals = extreme_val_output(data,idcs;rep_mod_method=rep_mod_method)
  # for append method: modify data to be clustered to only contain the values that are not extreme values
  if rep_mod_method=="feasibility"
    data_mod = data
  elseif rep_mod_method=="append"
    data_mod = input_data_modification(data,idcs)
  else
    error("rep_mod_method - "*rep_mod_method*" - does not exist")
  end
  return data_mod,extr_vals,idcs
end

"""
    simple_extr_val_sel(data::ClustData,
                        extreme_value_descr::SimpleExtremeValueDescr;
                        rep_mod_method::String="feasibility")
Wrapper function for only one simple extreme value.
Selects simple extreme values and returns modified data, extreme values, and the corresponding indices.
"""
function simple_extr_val_sel(data::ClustData,
                             extr_value_descr::SimpleExtremeValueDescr;
                             rep_mod_method::String="feasibility"
                             )
  return simple_extr_val_sel(data,[extr_value_descr];rep_mod_method=rep_mod_method)
end

"""
    simple_extr_val_ident(data::ClustData,
                               extreme_value_descr_ar::Array{SimpleExtremeValueDescr,1})

Identifies multiple simple extreme values from the data and returns array of column indices of extreme value within data
- data_type: any attribute from the attributes contained within *data*
- extremum: "min" or "max"
- peak_def: "absolute" or "integral"
"""
function simple_extr_val_ident(data::ClustData,
                               extreme_value_descr_ar::Array{SimpleExtremeValueDescr,1})
  idcs = Array{Int,1}()
  # for each desired extreme value description, finds index of that extreme value within data
  for i=1:length(extreme_value_descr_ar)
    append!(idcs,simple_extr_val_ident(data,extreme_value_descr_ar[i]))
  end
  return idcs
end

"""
    simple_extr_val_ident(data::ClustData,
                               extreme_value_descr::SimpleExtremeValueDescr)

Wrapper function for only one simple extreme value:
identifies a single simple extreme value from the data and returns column index of extreme value
- `data_type`: any attribute from the attributes contained within *data*
- `extremum`: "min" or "max"
- `peak_def`: "absolute" or "integral"
- `consecutive_periods`: number of consecutive_periods combined to analyze
"""
function simple_extr_val_ident(data::ClustData,
                               extreme_value_descr::SimpleExtremeValueDescr)
  return simple_extr_val_ident(data, extreme_value_descr.data_type; extremum=extreme_value_descr.extremum, peak_def=extreme_value_descr.peak_def, consecutive_periods=extreme_value_descr.consecutive_periods)
end

"""
    simple_extr_val_ident(clust_data::ClustData,
                               data_type::String;
                               extremum::String="max",
                               peak_def::String="absolute",
                               consecutive_periods::Int=1)

Identifies a single simple extreme period from the data and returns column index of extreme period
- `data_type`: any attribute from the attributes contained within *data*
- `extremum`: "min" or "max"
- `peak_def`: "absolute" or "integral"
- `consecutive_periods`: The number of consecutive periods that are summed to identify a maximum or minimum. A rolling approach is used: E.g. for a value of `consecutive_periods`=2: 1) 1st & 2nd periods summed, 2) 2nd & 3rd period summed, 3) 3rd & 4th ... The min/max of the 1), 2), 3)... is determined and the two periods indices, where the  min/max were identified, are returned
"""
function simple_extr_val_ident(clust_data::ClustData,
                               data_type::String;
                               extremum::String="max",
                               peak_def::String="absolute",
                               consecutive_periods::Int=1)
  data=clust_data.data[data_type]
  delta_period=consecutive_periods-1
  # set data to be compared
  if peak_def=="absolute" && consecutive_periods==1
      data_eval = data
  elseif peak_def=="integral"
    # The number of consecutive_periods is substracted by one as k:k+period
    data_eval=zeros(1,(size(data,2)-delta_period))
    for k in 1:(size(data,2)-delta_period)
      data_eval[1,k] = sum(data[:,k:(k+delta_period)])
    end
  else
    error("peak_def - "*peak_def*" and consecutive_periods $consecutive_periods - not defined")
  end
  # find minimum or maximum index. Second argument returns cartesian indices, second argument of that is the column (period) index
  if extremum=="max"
    idx_k = findmax(data_eval)[2][2]
  elseif extremum=="min"
    idx_k = findmin(data_eval)[2][2]
  else
    error("extremum - "*extremum*" - not defined")
  end
  idx=collect(idx_k:(idx_k+delta_period))
  return idx
end

"""
    input_data_modification(data::ClustData,
                                 extr_val_idcs::Array{Int,1})

Returns ClustData structs with extreme vals and with remaining input data [data-extreme_vals].
Gives extreme vals the weight that they had in data.
This function is needed for the append method for representation modification
! the k-ids have to be monoton increasing - don't modify clustered data !
"""
function input_data_modification(data::ClustData,
                                 extr_val_idcs::Array{Int,1})
  unique_extr_val_idcs = unique(extr_val_idcs)
  K_dn = data.K- length(unique_extr_val_idcs)
  data_dn=Dict{String,Array}()
  index=setdiff(1:data.K,extr_val_idcs)
  for dt in keys(data.data)
    data_dn[dt] = data.data[dt][:,index] #take all columns but the ones that are extreme vals. If index occurs multiple times, setdiff only treats it as one.
  end
  weights_dn = data.weights[index]
  #take all columns but the ones that are extreme vals
  deltas_dn= data.delta_t[:,index]
  #deepcopy to change k_ids
  k_ids_dn=deepcopy(data.k_ids)
  #check for uniqueness and right sorting (however just those one representing)
  k_ids_check=k_ids_dn[findall(k_ids_dn.!=0)]
  allunique(k_ids_check) || error("the provided clust_data.k_ids are not unique - The clust_data is probably the result of a clustering already.")
  sort(k_ids_check)==k_ids_check || error("the provided clust_data.k_ids are not monoton increasing - The clust_data is probably the result of a clustering already.")
  #get all k-ids that are represented within this clust-data
  k_ids_dn_data=k_ids_dn[findall(data.k_ids.!=0)]
  for k in sort(extr_val_idcs)
    #reduce the following k_ids by one for all of the following k-ids (the deleted column will reduce the following column-indices by one for each deleted column)
    k_ids_dn_data[k:end].-=1
    #set this k_id to zero, as it corresponding column is being removed from the data
    k_ids_dn_data[k]=0
  end
  #just modify the k_ids that are also represented within this clust-data (don't reduce 0 to -1...)
  k_ids_dn[findall(data.k_ids.!=0)]=k_ids_dn_data
  #return the new Clust Data
  return ClustData(data.region,data.years,K_dn,data.T,data_dn,weights_dn,k_ids_dn;delta_t=deltas_dn,mean=data.mean,sdv=data.sdv)
end

"""
    input_data_modification(data::ClustData,extr_val_idcs::Int)

wrapper function for a single extreme val.
returns ClustData structs with extreme vals and with remaining input data [data-extreme_vals].
Gives extreme vals the weight that they had in data.
"""
function input_data_modification(data::ClustData,extr_val_idcs::Int)
  return input_data_modification(data,[extr_val_idcs])
end

"""
    extreme_val_output(data::ClustData,
                            extr_val_idcs::Array{Int,1};
                            rep_mod_method="feasibility")

Takes indices as input and returns ClustData struct that contains the extreme vals from within data.
"""
function extreme_val_output(data::ClustData,
                            extr_val_idcs::Array{Int,1};
                            rep_mod_method="feasibility")
  unique_extr_val_idcs = unique(extr_val_idcs)
  K_ed = length(unique_extr_val_idcs)
  data_ed=Dict{String,Array}()
  for dt in keys(data.data)
    data_ed[dt] = data.data[dt][:,unique_extr_val_idcs]
  end
  weights_ed=[]
  #initiate new k-ids-ed that don't represent any original time-period
  k_ids_ed=zeros(Int,size(data.k_ids))
  if rep_mod_method == "feasibility"
    weights_ed = zeros(length(unique_extr_val_idcs))
    #no representation is done of the original time-period, it's just for feasibility
  elseif rep_mod_method == "append"
    weights_ed = data.weights[unique_extr_val_idcs]
    # if original time series period isn't represented by any extreme period it has value 0
    # get all the indices that acutally represent original time-series
    index_k_ids_data=findall(data.k_ids.!=0)
    # each original time series period which is represented recieves the number of it's extreme period in this extreme value output
    k_ids_ed_data=zeros(size(index_k_ids_data))
    k_ids_ed_data[unique_extr_val_idcs]=collect(1:K_ed)
    # assign it to the full original time-series
    k_ids_ed[index_k_ids_data]=k_ids_ed_data
  else
    error("rep_mod_method - "*rep_mod_method*" - does not exist")
  end
  delta_t_ed=data.delta_t[:,unique_extr_val_idcs]
  extr_vals = ClustData(data.region,data.years,K_ed,data.T,data_ed,weights_ed,k_ids_ed;delta_t=delta_t_ed,mean=data.mean,sdv=data.sdv)
  return extr_vals
end

"""
   extreme_val_output(data::ClustData,extr_val_idcs::Array{Int,1};rep_mod_method="feasibility")

wrapper function for a single extreme val.
Takes indices as input and returns ClustData struct that contains the extreme vals from within data.
"""
function extreme_val_output(data::ClustData,
                            extr_val_idcs::Int;
                            rep_mod_method="feasibility")
  return extreme_val_output(data,[extr_val_idcs];rep_mod_method=rep_mod_method)
end

"""
    representation_modification(extr_vals::ClustData,clust_data::ClustData)

Merges the clustered data and extreme vals into one ClustData struct. Weights are chosen according to the rep_mod_method
"""
function representation_modification(extr_vals::ClustData,
                                     clust_data::ClustData)
  K_mod = clust_data.K + extr_vals.K
  data_mod=Dict{String,Array}()
  for dt in keys(clust_data.data)
    data_mod[dt] = [clust_data.data[dt] extr_vals.data[dt]]
  end
  weights_mod = [clust_data.weights; extr_vals.weights]
  # Add extra columns to delta_t
  delta_t_mod = [clust_data.delta_t extr_vals.delta_t]
  # originial time series periods are represented by periods in clust_data
  k_ids_mod=deepcopy(clust_data.k_ids)
  # if this particular original time series period is though represented in the extreme values, the new period number of the extreme value (clust_data.K+old number) is assigned to this original time series period - in case of feasibility they are all zero and nothing is changed
  k_ids_mod[findall(extr_vals.k_ids.!=0)]=extr_vals.k_ids[findall(extr_vals.k_ids.!=0)].+clust_data.K
  return ClustData(clust_data.region,clust_data.years,K_mod,clust_data.T,data_mod,weights_mod,k_ids_mod;delta_t=delta_t_mod,mean=clust_data.mean,sdv=clust_data.sdv)
end

"""
    representation_modification(extr_vals::ClustData,clust_data::ClustData)

Merges the clustered data and extreme vals into one ClustData struct. Weights are chosen according to the rep_mod_method
"""
function representation_modification(extr_vals_array::Array{ClustData,1},
                                     clust_data::ClustData,
                                     )
    for extr_vals in extr_vals_array
      clust_data=representation_modification(extr_vals,clust_data)
    end
    return clust_data
end

"""
    representation_modification(full_data::ClustData,clust_data::ClustData,extr_val_idcs::Array{Int,1};rep_mod_method::String="feasibility")

Merges the clustered data and extreme vals into one ClustData struct. Weights are chosen according to the rep_mod_method
"""
function representation_modification(full_data::ClustData,
                                     clust_data::ClustData,
                                     extr_val_idcs::Array{Int,1};
                                     rep_mod_method::String="feasibility")
  extr_vals = extreme_val_output(full_data,extr_val_idcs;rep_mod_method=rep_mod_method)
  return representation_modification(extr_vals,clust_data;rep_mod_method=rep_mod_method)
end

"""
    representation_modification(full_data::ClustData,clust_data::ClustData,extr_val_idcs::Int;rep_mod_method::String="feasibility")

wrapper function for a single extreme val.
Merges the clustered data and extreme vals into one ClustData struct. Weights are chosen according to the rep_mod_method
"""
function representation_modification(full_data::ClustData,
                                     clust_data::ClustData,
                                     extr_val_idcs::Int;
                                     rep_mod_method::String="feasibility")
  return representation_modification(full_data,clust_data,[extr_val_idcs];rep_mod_method=rep_mod_method)
end


struct BoolDomainViewNot <: BoolDomainView
    orig            ::AbstractBoolDomain
end

struct BoolVarViewNot <: BoolVarView
    x               ::AbstractBoolVar
    domain          ::BoolDomainViewNot
    id              ::String

    """
        BoolVarViewNot(x::AbstractBoolVar, id::String)

    Create a *fake* variable `y`, such that `y = ¬x`. This variable behaves like an usual one.
    """
    function BoolVarViewNot(x::AbstractBoolVar, id::String)
        dom = BoolDomainViewNot(x.domain)
        return new(x, dom, id)
    end
end

assignedValue(x::BoolVarViewNot) = !assignedValue(x.x)

"""
    isempty(dom::BoolDomainView)

Return `true` iff `dom` is an empty set.
"""
Base.isempty(dom::BoolDomainView) = isempty(dom.orig)

reset_domain!(dom::BoolDomainView) = reset_domain!(dom.orig)

"""
    rootVariable(x::BoolVarView)

Return the "true" variable behind `x`.
"""
rootVariable(x::BoolVarView) = rootVariable(x.x)

"""
    length(dom::BoolDomainView)

Return the size of `dom`.
"""
Base.length(dom::BoolDomainView) = length(dom.orig)

"""
    Base.in(value::Bool, dom::BoolDomainViewNot)

Check if a boolean is in the domain.
"""
Base.in(value::Bool, dom::BoolDomainViewNot) = (!value) in dom.orig

"""
    remove!(dom::BoolDomainViewNot, value::Bool)

Remove `value` from `dom`.
"""
remove!(dom::BoolDomainViewNot, value::Bool) = map(x -> !x, remove!(dom.orig, !value))

"""
    removeAll!(dom::BoolDomainViewNot)

Remove every value from `dom`. Return the removed values.
"""
removeAll!(dom::BoolDomainViewNot) = map(x -> !x, removeAll!(dom.orig))

"""
    assign!(dom::BoolDomainViewNot, value::Bool)

Remove everything from the domain but `value`. Return the removed values. Return the pruned values.
"""
assign!(dom::BoolDomainViewNot, value::Bool) = map(x -> !x, assign!(dom.orig, !value))

"""
    Base.iterate(dom::BoolDomainViewNot, state=1)

Iterate over the domain in an efficient way. The order may not be consistent.
WARNING: Do **NOT** update the domain you are iterating on.
"""
function Base.iterate(dom::BoolDomainViewNot, state=1)
    returned = iterate(dom.orig, state)
    if isnothing(returned)
        return nothing
    end

    value, newState = returned
    return !value, newState
end

parentValue(::BoolVarViewNot, v::Bool) = ! v


import PhysicsEngine

@testset "Simulation" begin
  sim = PhysicsEngine.newSimulation()

  # Time test
  initialTime = sim.time
  PhysicsEngine.simulationStep!(sim)
  finalTime = sim.time
  @test finalTime == initialTime + 1

  # Test add particle
  particle = PhysicsEngine.Electron([1, 1, 1], [2, 2, 2])
  @test (particle in sim.objects) == false
  PhysicsEngine.addParticle!(sim, particle)
  @test (particle in sim.objects) == true

  # Remove particle
  PhysicsEngine.removeParticle!(sim, particle)
  @test (particle in sim.objects) == false
end


abstract AtomicUnit{T}

atoms = [
    #     Name   Symbol  Valence
    (:Hydrogen,  :H,     1),
    (  :Carbon,  :C,     4)
]

for (atom, sym, valence) in atoms
    @eval begin
        type $atom{T<:Number} <: AtomicUnit{T}
            value::Complex{T}    # atomic value
            valence::Int         # number of valence electrons
            degree::Int          # number of shared valence electons
            free::Int            # number of free valence electrons

            $atom(value) = new(value, $valence, 0, $valence)
        end

        $atom{T<:Real}(value::Complex{T}) = $atom{T}(value)
        $atom{T<:Real}(value::T) = $atom(complex(value))
        $atom() = $atom(0.0)

        typealias $sym $atom
    end
end

function Base.show(io::IO, a::AtomicUnit)
    print(io, "$(typeof(a))(value = $(a.value), valence = $(a.valence), degree = $(a.degree), free = $(a.free))")
end

Base.(:(==))(a₁::AtomicUnit, a₂::AtomicUnit) = a₁.valence == a₂.valence


using PiGPIOMEM
using LLVM
#using BaremetalPi

nop() = LLVM.Interop.@asmcall("nop")
spin_ns(n) = for i in 1:n nop() end

function test(pin)

    for i in 1:10_000_000
        clear(pin)
        spin_ns(80)
        set(pin)
        spin_ns(80)
    end
end

#function test2()
#    init_gpio()
#    gpio_set_mode(18, :out)
#
#    while true
#        gpio_clear(18)
#        spin_ns(100)
#        gpio_set(18)
#        spin_ns(100)
#    end
#end

#@time spin_ns(100000 / 3)
#@time spin_ns(10000 / 3)
#@time spin_ns(1000 / 3)

test(GPIOPin(18, output=true))

#using InteractiveUtils
#code_native(test, [GPIOPin], debuginfo=:none)

"""
2 * (spin( 80) + d) =  520ns
2 * (spin(100) + d) =  632ns
2 * (spin(150) + d) =  936ns
2 * (spin(200) + d) = 1240ns

ns = 3 x 2 * spin + 34

2 * spin(100): 632ns
2 * spin(100): 632-34 = 598ns
spin(100): 598 / 2 = 300 ns
spin = 3ns
"""

# 2.1MHz
#test2()


# This file was generated, do not modify it. # hide
coefs, intercept = fitted_params(mtm.fitresult).linear_regressor
@show coefs
@show intercept


module DSPHelp
export ev, od

function ev(x::Array{Float64,2})
  return (x + fliplr(x))/2
end

function od(x::Array{Float64,2})
  return (x - fliplr(x))/2
end

end # module


using Humanize
using Base.Test
if VERSION < v"0.4-"
    import Dates
else
    import Base.Dates
end

function test_datasize()
    println("test_datasize")

    tests = [300, 3000, 3000000, 3000000000, 3000000000000, 1e26, 3141592]
    results = [:dec => ["450.000 B", "4.500 kB", "4.500 MB", "4.500 GB", "4.500 TB", "150.000 YB", "4.712 MB"],
               :bin => ["630.000 B", "6.152 KiB", "6.008 MiB", "5.867 GiB", "5.730 TiB", "173.708 YiB", "6.292 MiB"],
               :gnu => ["480.000", "4.688K", "4.578M", "4.470G", "4.366T", "132.349Y", "4.794M"]]
    for (style,mult) in [(:dec,1.5), (:bin,2.1), (:gnu, 1.6)]
        println("  $style")
        for i in 1:length(tests)
            size = tests[i] * mult
            @test datasize(size,style=style,format="%.3f") == results[style][i]
        end
    end
end

function test_timedelta()
    println("test_timedelta")

    test = [(0,0,0,0,0,0), "a moment",
            (0,0,0,0,0,1), "a second",
            (0,0,0,0,0,30), "30 seconds",
            (0,0,0,0,1,30), "a minute",
            (0,0,0,0,2,0), "2 minutes",
            (0,0,0,1,30,30), "an hour",
            (0,0,0,23,50,50), "23 hours",
            (0,0,1,0,0,0), "a day",
            (0,0,500,0,0,0), "1 year, 4 months",
            (0,0,365*2+35,0,0,0), "2 years",
            (0,0,10000,0,0,0), "27 years",
            (0,0,365+30,0,0,0), "1 year, 1 month",
            (0,0,365+4,0,0,0), "a year",
            (0,0,35,0,0,0), "a month",
            (0,0,65,0,0,0), "2 months",
            (0,0,9,0,0,0), "9 days",
            (0,0,365,0,0,0), "a year"]
    n_test = div(length(test),2)

    #timedelta(years::Int,months::Int,days::Int,hours::Int,mins::Int,secs::Int)
    println("  direct")
    for i in 1:n_test
        @test timedelta(test[2i-1]...) == test[2i]
    end
    #timedelta(dt_diff::Dates.Millisecond)
    println("  DateTime diff")
    for i in 1:n_test
        offset = test[2i-1]
        base_datetime = Dates.DateTime(2014,1,1,0,0,0)
        new_datetime  = Dates.Year(offset[1]) + base_datetime
        new_datetime += Dates.Month(offset[2])
        new_datetime += Dates.Day(offset[3])
        new_datetime += Dates.Hour(offset[4])
        new_datetime += Dates.Minute(offset[5])
        new_datetime += Dates.Second(offset[6])
        @test timedelta(new_datetime-base_datetime) == test[2i]
    end
    #timedelta(d_diff::Dates.Day)
    println("  Date diff")
    for i in 1:n_test
        offset = test[2i-1]
        sum(offset[1:3]) == 0 && continue
        base_date = Dates.Date(2014,1,1)
        new_date  = Dates.Year(offset[1]) + base_date
        new_date += Dates.Month(offset[2])
        new_date += Dates.Day(offset[3])
        @test timedelta(new_date-base_date) == test[2i]
    end
end

function test_digitsep()
    println("test_digitsep")

    test = (
            (1, "1"),
            (12, "12"),
            (123, "123"),
            (1234, "1,234"),
            (12345, "12,345"),
            (123456, "123,456"),
            (1234567, "1,234,567"),
            (12345678, "12,345,678")
            )

    n_test = length(test)

    #digitsep(value::Integer)
    println("  direct")
    for t in test
        @test digitsep(t[1]) == t[2]
    end

end

test_datasize()
test_timedelta()
test_digitsep()


mutable struct SimpleArc{T<:Node,U<:Real,V<:Int}
    uuid                    ::UUID
    origin                  ::T
    destination             ::T
    assignment              ::V
    value                   ::U
end

Base.show(io::IO,arc::SimpleArc) = Base.print(
    io,
    "\n   Assignment: ", arc.assignment, "\n",
    "   Arc Value: ", arc.value, "\n"
)

#change hashing of nodes to use uuids
Base.hash(arc::SimpleArc) = hash(arc.uuid)
Base.:(==)(x::SimpleArc,y::SimpleArc) = x.uuid == y.uuid

#do UUIDs automatically
function SimpleArc(origin::T, destination::T, assignment::U, value::V)where{T<:Node, U<:Int, V<:Real}
    return SimpleArc(uuid4(), origin, destination, assignment, value)
end

"""
    deepCopy(arc::SimpleArc)

Deep copy an arc

# Arguments
- `arcs::SimpleArc`: the arc
"""
function deepCopy(arc::SimpleArc)
    return SimpleArc(arc.uuid, arc.origin,arc.destination,arc.assignment,arc.value)
end

"""
    deepCopy(arcs::Vector{SimpleArc})

Deep copy a list of arcs

# Arguments
- `arcs::Array{SimpleArc}`: a list of arcs
"""
function deepCopy(arcs::Vector{SimpleArc})
    newArcs = Vector{SimpleArc}()
    for arc in arcs
        push!(newArcs, deepCopy(arc))
    end
    return newArcs
end

"""
    shallowCopy(arc::SimpleArc)

Shallow copy an arc

# Arguments
- `arcs::SimpleArc`: the arc
"""
function shallowCopy(arc::SimpleArc)
    return SimpleArc(arc.origin,arc.destination,arc.assignment,arc.value)
end

"""
    shallowCopy(arcs::Vector{SimpleArc})

functionally the same as deepCopy a list of arcs, but with different uuids

# Arguments
- `arcs::Array{SimpleArc}`: a list of arcs
"""
function shallowCopy(arcs::Vector{SimpleArc})
    newArcs = Vector{SimpleArc}()
    for arc in arcs
        push!(newArcs, shallowCopy(arc))
    end
    return newArcs
end

"""
    findBestArc(model::SequencingModel, arcs::Vector{SimpleArc})

Find the best arc in a list, where best means containing the most optimal path

# Arguments
- `model::SequencingModel`: The Sequencing problem
- `arcs::Vector{SimpleArc}`: The list of arcs
"""
function findBestArc(model::SequencingModel, arcs::Vector{SimpleArc})
    bestArc = arcs[1]
    bestArcValue = bestArc.origin.lengthToRoot + bestArc.value
    for arc in arcs
        if isBetterSolutionValue(model, arc.origin.lengthToRoot + arc.value, bestArcValue)
            bestArc = arc
            bestArcValue = arc.origin.lengthToRoot + arc.value
        end
    end
    return bestArc, bestArcValue
end

"""
    getArcValue(model::SequencingModel, arc::SimpleArc)

Get the best arc value by checking the states it can be coming from, arc.origin.state must be a BitVector

# Arguments
- `model::SequencingModel`: The Sequencing problem
- `arc::SimpleArc`: The arc
"""
function getArcValue(model::SequencingModel, arc::SimpleArc)
    arcValue = nothing
    for (i, bit) in enumerate(arc.origin.state)
        if bit
            if hasKey(model.objective, i, arc.assignment)
                if isnothing(arcValue) || isBetterSolutionValue(model, evaluateDecision(model.objective, i, arc.assignment),arcValue)
                    arcValue = evaluateDecision(model.objective, i, arc.assignment)
                end
            end
        end
    end
    return arcValue
end

"""
    deleteArc!(arc::SimpleArc)
Remove an arc from both relevant nodes

# Arguments
- `arc::SimpleArc`: the arc to remove
"""
function deleteArc!(arc::SimpleArc)
    filter!(x->x!=arc, arc.origin.arcsOut)
    filter!(x->x!=arc, arc.destination.arcsIn)
end

"""
    deleteArcs!(arcsToRemove::Vector{SimpleArc})
Delete a list of arcs

# Arguments
- `arcsToRemove::Vector{SimpleArc}`: the arcs to remove
"""
function deleteArcs!(arcsToRemove::Vector{SimpleArc})
    toRemove = Vector{SimpleArc}()
    for arc in arcsToRemove
        push!(toRemove, arc)
    end
    for arc in toRemove
        deleteArc!(arc)
    end
end


using KmerIndexes
using BaseTestNext
using Bio.Seq

@testset "short sequence" begin
    seq = dna"ACGTAGACGT"
    index = KmerIndex(DNAKmer{4}, seq)
    @test count(kmer(dna"ACGT"), index) === 2
    @test locate(kmer(dna"ACGT"), index) == [1, 7]
    @test count(kmer(dna"AGAC"), index) === 1
    @test locate(kmer(dna"AGAC"), index) == [5]
end