export connect_system, rename_vars, remove_superfluous_states, substitute_algebraic_states, substitute_derivatives, set_p, simplify_eqs

"""
$(SIGNATURES)

Recursively transform `IOSystems` to `IOBlocks`.

- substitute inputs with connected outputs
- try to eliminate equations for internal states which are not used to calculate the specified outputs of the system.
- try to eliminate explicit algebraic equations (i.e. outputs of internal blocks) by substituting each occurrence
  with their rhs. Explicit algebraic states which are marked as system outputs won't be removed.

Arguments:
- `ios`: system to connect
- `verbose=false`: toggle verbosity (show equations at different steps)
- `remove_superflous_states=true`: toggle whether the system should try to get rid of unused states
- `substitute_algebraic_states=true`: toggle whether the algorithm tries to get rid of explicit algebraic equations
- `substitute_derivatives=true`: toggle whether to expand all derivatives and try to substitute them
- `simplify_eqs=true`: toggle simplification of all equations at the end
"""
function connect_system(ios::IOSystem;
                        verbose=false,
                        simplify_eqs=true,
                        remove_superflous_states=false,
                        substitute_algebraic_states=true,
                        substitute_derivatives=true,
                        warn=true)
    # recursive connect all subsystems
    for (i, subsys) in enumerate(ios.systems)
        if subsys isa IOSystem
            ios.systems[i] = connect_system(subsys, verbose=verbose)
        end
    end
    eqs = vcat([namespace_equations(iob.system) for iob in ios.systems]...)
    removed_eqs = vcat([namespace_rem_eqs(iob) for iob in ios.systems]...)

    verbose && @info "Transform IOSystem $(ios.name) to IOBlock" ios.name ios.inputs ios.outputs ios.connections eqs

    # get rid of closed inputs by substituting output states
    substitutions = reverse.(ios.connections)
    for (i, eq) in enumerate(eqs)
        eqs[i] = eq.lhs ~ substitute(eq.rhs, substitutions)
    end
    for (i, eq) in enumerate(removed_eqs)
        removed_eqs[i] = eq.lhs ~ substitute(eq.rhs, substitutions)
    end

    verbose && @info "substitute inputs with outputs" eqs

    # apply the namespace transformations
    promotion_rules = ios.namespace_map
    eqs = map(eq->eqsubstitute(eq, promotion_rules), eqs)
    removed_eqs  = map(eq->eqsubstitute(eq, promotion_rules), removed_eqs)

    block = IOBlock(ios.name, eqs, ios.inputs, ios.outputs, removed_eqs; iv=get_iv(ios), warn)

    if remove_superflous_states
        block = BlockSystems.remove_superflous_states(block; verbose, warn)
    end

    if substitute_algebraic_states
        block = BlockSystems.substitute_algebraic_states(block; verbose, warn)
    end

    if substitute_derivatives
        block = BlockSystems.substitute_derivatives(block; verbose, warn)
    end

    if simplify_eqs
        block = BlockSystems.simplify_eqs(block; warn)
    end

    return block
end

"""
    remove_superfluous_states(iob::IOBlock; verbose=false, warn=true)

This function removes equations from block, which are not used in order to
generate the outputs. It looks for equations which have no path to the outputs
equations in the dependency graph. Returns a new IOBlock.

The removed equations will be not avaiblable as removed quations of the new IOBlock

TODO: Maybe we should try to reduce the inputs to.
"""
function remove_superfluous_states(iob::IOBlock; verbose=false, warn=true)
    iv = get_iv(iob)
    outputs = iob.outputs

    neweqs = deepcopy(equations(iob))
    sys = ODESystem(neweqs, iv; name=:tmp) # will be used for the dependency graph
    neweqs = get_eqs(sys) # the ODESystem might reorder the equations
    # generate dependency graph
    graph = eqeq_dependencies(asgraph(sys), variable_dependencies(sys))
    # find 'main' eq for each output
    output_idx = [findfirst(x->o ∈ Set(get_variables(x.lhs)), neweqs) for o in outputs]

    if any(isnothing, output_idx)
        verbose && @info "Can't remove souperflous states if outputs implicitly defined."
        return neweqs
    end

    # if there is no path from equation to output equation is not necessary
    removable = []
    for eq_node in 1:length(neweqs)
        if !any(has_path(graph, eq_node, out_node) for out_node in output_idx)
            push!(removable, eq_node)
        end
    end

    removed_eqs = neweqs[removable]
    deleteat!(neweqs, sort(removable))

    verbose && @info "Removed superflous states with equations" removed_eqs

    IOBlock(iob.name, neweqs, iob.inputs, iob.outputs, iob.removed_eqs; iv, warn)
end

"""
    substitute_algebraic_states(iob::IOBlock; verbose=false, warn=true)

Reduces the number of equations by substituting explicit algebraic equations.
Returns a new IOBlock with the reduced equations. The removed eqs are stored
together with the previous `removed_eqs` in the new IOBlock.
Won't reduce algebraic states which are labeld as `output`.
"""
function substitute_algebraic_states(iob::IOBlock; verbose=false, warn=true)
    reduced_eqs = deepcopy(equations(iob))

    (rules, removable) = _algebraic_substitution_rules(reduced_eqs; skip=Set(iob.outputs))

    # subsitute all the equations, remove substituted
    for (i, eq) in enumerate(reduced_eqs)
        neweqs = eq.lhs ~ recursive_substitute(eq.rhs, rules)
        type, lhs_var = eq_type(neweqs)
        if type == :implicit_algebraic && !isnothing(lhs_var)
            verbose && print("Algebraic substitution resulted in implicit equation...")
            # TODO: solve_for implicit algebraic equations
            # try
            #     neweqs = lhs_var ~ Symbolics.solve_for(neweqs, lhs_var)
            #     verbose && println("which could be solved: $neweqs")
            # catch e
            #   e isa AssertionError || rethrow(e)
                neweqs = 0 ~ neweqs.rhs - neweqs.lhs
                verbose && println("which was transformed to `0 ~ f(...)` form: $neweqs")
            # end
        end
        reduced_eqs[i] = neweqs
    end

    # also substitute in the allready removed_eqs
    removed_eqs = deepcopy(iob.removed_eqs)
    for (i, eq) in enumerate(removed_eqs)
        removed_eqs[i] = eq.lhs ~ recursive_substitute(eq.rhs, rules)
    end

    verbose && @info "Substituted algebraic states:" rules

    # append the knows removed wqs with the newly removed eqs
    append!(removed_eqs, reduced_eqs[removable])
    # remove removable equations from reduced_eqs
    deleteat!(reduced_eqs, sort(removable))

    IOBlock(iob.name, reduced_eqs, iob.inputs, iob.outputs, removed_eqs; iv=get_iv(iob), warn)
end

"""
    _algebraic_substitution_rules(eqs; skip=nothing)

Extract substition rules for explicit algebraic equations in given equations.
Makes sure that those substitutiosn are pairwise cycle free.

Returns Tuple of substitution dict and corresponding indices of algebraic equations
in the given equations.
"""
function _algebraic_substitution_rules(eqs; skip=nothing)
    # only consider states for reduction which are explicit algebraic and not in outputs
    condition = eq -> begin
        (type, var) = eq_type(eq)
        type == :explicit_algebraic && (skip===nothing || var ∉ skip)
    end
    algebraic_idx = findall(condition, eqs)

    # symbols of all algebraic eqs
    symbols = [eq.lhs for eq ∈ eqs[algebraic_idx]]

    # generate dependency graph
    g = SimpleDiGraph(length(algebraic_idx))
    for (i, eq) in enumerate(eqs[algebraic_idx])
        rhs_vars = get_variables(eq.rhs)
        for (isym, sym) in enumerate(symbols)
            if Set([sym]) ⊆ Set(rhs_vars)
                add_edge!(g, isym => i)
            end
        end
    end
    removable = algebraic_idx[_pairwise_cycle_free(g)]
    @assert allunique(removable)

    rules = Dict(eq.lhs => eq.rhs for eq in eqs[removable])
    return (rules, removable)
end

"""
    _pairwise_cycle_free(g:SimpleDiGraph)

Returns an array of vertices, which pairwise do not belong to any cycle in `g`.
Uses `simplecycles` from `Graphs`. The algorithm starts with all vertices
and iteratively removes the vertices is part of most cycles.
"""
function _pairwise_cycle_free(g::SimpleDiGraph)
    idx = collect(vertices(g))
    cycles = simplecycles(g)
    while true
        cycles_per_idx = zeros(Int, length(idx))
        for (i, id1) ∈ enumerate(idx)
            for (j, id2) ∈ enumerate(@view idx[i+1 : end])
                for c in cycles
                    if id1 ∈ c && id2 ∈ c
                        cycles_per_idx[i] += 1
                        cycles_per_idx[i+j] += 1
                    end
                end
            end
        end
        if sum(cycles_per_idx) > 0
            worst_idx = findmax(cycles_per_idx)[2]
            deleteat!(idx, worst_idx)
        else
            return idx
        end
    end
end

"""
    substitute_derivatives(iob::IOBlock; verbose=false, warn=true)

Expand all derivatives in the RHS of the system. Try to substitute
in the lhs with their definition.

I.e.

    D(o) ~ 1 + D(i)   =>  D(o) ~ 2 + a
    D(i) ~ 1 + a          D(i) ~ 1 + a

Process happens in multiple steps:
- try to find explicit equation for differential
- if none found try to recursivly substitute inside differential with known algebraic states
- expand derivatives and try again to substitute with known differentials

"""
function substitute_derivatives(iob::IOBlock; verbose=false, warn=true)
    diffs = rhs_differentials(iob)

    # if there are none just return the old block
    if isempty(diffs)
        verbose && @info "No rhs derivatives in block!"
        return iob
    end
    verbose && @info "Substitute rhs derivatives..."

    eqs = deepcopy(equations(iob))

    algebraic_subs = Dict{Symbolic, Any}()
    function lazy_alg_subs()
        if isempty(algebraic_subs)
            verbose && println("Lazily initilized algebraic substitutions for substituion of derivatives!")
            merge!(algebraic_subs, _algebraic_substitution_rules(eqs)[1])
        end
        return algebraic_subs
    end

    rules = Dict()
    known_differentials = Dict(eq.lhs => eq.rhs for eq in eqs if istree(eq.lhs) && operation(eq.lhs) isa Differential)
    for diff in diffs
        if haskey(known_differentials, diff)
            rules[diff] = known_differentials[diff]
            verbose && println("    ", diff, " => ", rules[diff])
        else
            substituted = recursive_substitute(diff, lazy_alg_subs())
            expanded = expand_derivatives(substituted)
            sub_known = substitute(expanded, known_differentials)

            rules[diff] = sub_known
            verbose && println("    ", diff, " => ", rules[diff])
            if verbose && !isempty(_collect_differentials(sub_known))
                println("        └ could not resolve this one!")
            end
        end
    end

    eqs = deepcopy(equations(iob))
    rem_eqs = deepcopy(iob.removed_eqs)

    for (i, eq) in enumerate(eqs)
        eqs[i] = eq.lhs ~ substitute(eq.rhs, rules)
    end
    for (i, eq) in enumerate(rem_eqs)
        rem_eqs[i] = eq.lhs ~ substitute(eq.rhs, rules)
    end

    newblock = IOBlock(iob.name, eqs, iob.inputs, iob.outputs, rem_eqs; iv=get_iv(iob), warn)

    return newblock
end

"""
    simplify_eqs(iob::IOBlock; verbose=false, warn=true)

Simplify eqs and removed eqs and return new IOBlock.
"""
function simplify_eqs(iob::IOBlock; verbose=false, warn=true)
    verbose && @info "Simplify iob equations..."
    simplified_eqs = simplify.(equations(iob))
    simplified_rem_eqs = simplify.(iob.removed_eqs)
    IOBlock(iob.name, simplified_eqs, iob.inputs, iob.outputs, simplified_rem_eqs; iv=get_iv(iob), warn)
end

"""
    rename_vars(blk::IOBLock; warn=true, kwargs...)
    rename_vars(blk::IOBlock, subs::Dict{Symbolic,Symbolic}; warn=true)

Returns new IOBlock which is similar to blk but with new variable names.
Variable renamings should be provided as keyword arguments, i.e.

    rename_vars(blk; x=:newx, k=:knew)

to rename `x(t)=>newx(t)` and `k=>knew`. Subsitutions can be also provided as
dict of `Symbolic` types (`Sym`s and `Term`s).
"""
function rename_vars(blk::IOBlock; warn=true, kwargs...)
    substitutions = Dict{Symbolic, Symbolic}()
    for pair in kwargs
        key = remove_namespace(blk.name, getproperty(blk, pair.first))
        val = rename(key, pair.second)
        substitutions[key] = val
    end
    rename_vars(blk, substitutions; warn)
end

function rename_vars(blk::IOBlock, subs::Dict{Symbolic,Symbolic}; warn=true)
    eqs     = map(eq->eqsubstitute(eq, subs), get_eqs(blk.system))
    rem_eqs = map(eq->eqsubstitute(eq, subs), blk.removed_eqs)
    inputs  = map(x->substitute(x, subs), blk.inputs)
    outputs = map(x->substitute(x, subs), blk.outputs)
    IOBlock(blk.name, eqs, inputs, outputs, rem_eqs; iv=get_iv(blk), warn)
end

"""
    set_p(blk::IOBlock, p::Dict; warn=true)
    set_p(blk::IOBlock, p::Pair; warn=true)

Substitutes certain parameters by actual Float values. Returns an IOBlock without those parameters.

Keys of dict can be either `Symbols` or the `Sybolic` subtypes. I.e. `blk.u => 1.0` is as valid as `:u => 1.0`.
"""
function set_p(blk::IOBlock, p::Dict; warn=true)
    subs = Dict{Symbolic, Float64}()
    validp = Set(blk.iparams)
    for k in keys(p)
        if k isa Symbol
            sym = getproperty(blk, k)
        elseif k isa Num
            sym = value(k)
        elseif k isa Symbolic
            sym = k
        else
            error("Can't use $k, wrong type $(typeof(k))")
        end
        sym = remove_namespace(blk.name, sym)
        @check sym ∈ validp "Symbol $sym is not iparam of block"
        @check p[k] isa Number "p value has to be a number! Maybe you are looking for `rename_vars` instead?"
        subs[sym] = p[k]
    end
    eqs     = map(eq->eqsubstitute(eq, subs), equations(blk))
    rem_eqs = map(eq->eqsubstitute(eq, subs), blk.removed_eqs)
    IOBlock(blk.name, eqs, blk.inputs, blk.outputs, rem_eqs; iv=get_iv(blk), warn)
end

set_p(blk::IOBlock, p...; warn=true) = length(p) > 1 ? set_p(blk, Dict(p); warn) : set_p(blk, Dict(only(p)); warn)


#Recursively traverses an expression and removes things like X^1, 1*X. Will not actually have any effect on the expression when used as a function, but will make it much easier to look at it for debugging, as well as if it is transformed to LaTeX code.
function recursive_clean!(expr)
    (expr isa Symbol) && (expr == :no___noise___scaling) && (return 1)
    (typeof(expr)!=Expr) && (return expr)
    for i = 1:length(expr.args)
        expr.args[i] = recursive_clean!(expr.args[i])
    end
    (expr.args[1] == :^) && (expr.args[3] == 1) && (return expr.args[2])
    if expr.args[1] == :*
        in(0,expr.args) && (return 0)
        i = 1
        while (i = i + 1) <= length(expr.args)
             if (typeof(expr.args[i]) == Expr) && (expr.args[i].head == :call) && (expr.args[i].args[1] == :*)
                 for arg in expr.args[i].args
                     (arg != :*) && push!(expr.args, arg)
                 end
             end
        end
        for i = length(expr.args):-1:2
            (typeof(expr.args[i]) == Expr) && (expr.args[i].head == :call) && (expr.args[i].args[1] == :*) && deleteat!(expr.args,i)
            (expr.args[i] == 1) && deleteat!(expr.args,i)
        end
        (length(expr.args) == 2) && (return expr.args[2])                   # We have a multiplication of only one thing, return only that thing.
        (length(expr.args) == 1) && (return 1)                              # We have only * and no real arguments.
        (length(expr.args) == 3) && (expr.args[2] == -1) && return :(-$(expr.args[3]))
        (length(expr.args) == 3) && (expr.args[3] == -1) && return :(-$(expr.args[2]))
    end
    if expr.head == :call
        (expr.args[1] == :/) && (expr.args[3] == 1) && (return expr.args[2])
        haskey(funcdict, expr.args[1]) && return funcdict[expr.args[1]](expr.args[2:end])
        in(expr.args[1],hill_name) && return hill(expr)
        in(expr.args[1],hillR_name) && return hillR(expr)
        in(expr.args[1],mm_name) && return mm(expr)
        in(expr.args[1],mmR_name) && return mmR(expr)
        (expr.args[1] == :binomial) && (expr.args[3] == 1) && return expr.args[2]
        #@isdefined($(expr.args[1])) || error("Function $(expr.args[1]) not defined.")
    end
    return expr
end

function chemical_arrows(rn::ModelingToolkit.ReactionSystem;
    expand = true, double_linebreak=false, mathjax=true, starred=false, kwargs...)
    str = starred ? "\\begin{align*}\n" : "\\begin{align}\n"
    eol = double_linebreak ? "\\\\\\\\\n" : "\\\\\n"

    mathjax && (str *= "\\require{mhchem}\n")
    backwards_reaction = false
    rxs = ModelingToolkit.equations(rn)
    @variables t

    # this should replace A(t) with A in equations however, currently substituter rewrites
    # things like x/y as inv(y)*x^1, which looks worse... for now we leave a stub that can
    # be updated when substitution preserves expressions better.
    # subber = ModelingToolkit.substituter([s(t) => s() for s in states(rn)])
    subber = x -> x

    for (i, r) in enumerate(rxs)
        if backwards_reaction
            backwards_reaction = false
            continue
        end
        str *= "\\ce{ "

        ### Expand functions to maths expressions
        rate = r.rate isa Operation ? Expr(subber(r.rate)) : r.rate
        expand && (rate = recursive_clean!(rate))
        expand && (rate = recursive_clean!(rate))

        ### Generate formatted string of substrates
        substrates = [latexraw("$(substrate[2]== 1 ? "" : "$(substrate[2]) * ") $(substrate[1].op.name)"; kwargs...) for substrate in zip(r.substrates,r.substoich)]
        isempty(substrates) && (substrates = ["\\varnothing"])

        str *= join(substrates, " + ")

        ### Generate reaction arrows
        prestr  = mathjax ? "[" : "[\$"
        poststr = mathjax ? "]" : "\$]"
        if i + 1 <= length(rxs) && issetequal(r.products,rxs[i+1].substrates) && issetequal(r.substrates,rxs[i+1].products)
            ### Bi-directional arrows
            rate_backwards = rxs[i+1].rate isa Operation ? Expr(subber(rxs[i+1].rate)) : rxs[i+1].rate
            expand && (rate_backwards = recursive_clean!(rate_backwards))
            expand && (rate_backwards = recursive_clean!(rate_backwards))
            str *= " &<=>"
            str *= prestr * latexraw(rate; kwargs...) * poststr
            str *= prestr * latexraw(rate_backwards; kwargs...) * poststr * " "
            backwards_reaction = true
        else
            ### Uni-directional arrows
            str *= " &->"
            str *= prestr * latexraw(rate; kwargs...) * poststr * " "
        end

        ### Generate formatted string of products
        products = [latexraw("$(product[2]== 1 ? "" : "$(product[2]) * ") $(product[1].op.name)"; kwargs...) for product in zip(r.products,r.prodstoich) ]
        isempty(products) && (products = ["\\varnothing"])
        str *= join(products, " + ")
        str *= "}$eol"
    end
    str = str[1:end-length(eol)] * "\n"

    str *= starred ? "\\end{align*}\n" : "\\end{align}\n"

    latexstr = Latexify.LaTeXString(str)
    Latexify.COPY_TO_CLIPBOARD && clipboard(latexstr)
    return latexstr
end

@latexrecipe function f(sys::ModelingToolkit.ReactionSystem)
    # Set default option values.
    env --> :chem
    cdot --> false

    return sys
end

function Latexify.infer_output(env, rs::ReactionSystem, args...)
    env in [:arrows, :chem, :chemical, :arrow] && return chemical_arrows

    error("The environment $env is not defined.")
    latex_function = Latexify.get_latex_function(rs, args...)

    return latex_function
end


using TensorOperations

include("RCCSDHelper.jl")

function RCCSD(moints::IntegralHelper{T,E,O}, newT1::AbstractArray{T,2}, newT2::AbstractArray{T,4},
                    alg::RCCSDa) where {T<:AbstractFloat, E<:AbstractERI, O<:AbstractRestrictedOrbitals}

    # Print intro
    cc_header()
    Eref = moints.orbitals.sd_energy
    ndocc = moints.molecule.Nα
    nvir = size(moints.orbitals.C,1) - ndocc
    core = Options.get("drop_occ")
    inac = Options.get("drop_vir")

    if core ≥ ndocc
        throw(FermiException("invalid number of frozen orbitals ($core) for $ndocc doubly occupied orbitals"))
    end
    if inac ≥ nvir
        throw(FermiException("invalid number of inactive virtual orbitals ($inac) for $nvir total virtual orbitals"))
    end

    # Get relevant Options
    cc_max_iter = Options.get("cc_max_iter")
    cc_e_conv = Options.get("cc_e_conv")
    cc_max_rms = Options.get("cc_max_rms")
    precision_override = Options.get("precision_override")

    dp = Options.get("cc_damp_ratio")
    do_diis = Options.get("cc_diis")
    ndiis = Options.get("cc_ndiis")
    diis_start = Options.get("diis_start")
    diis_relax = Options.get("cc_diis_relax")

    diis_prec = Options.get("diis_prec")
    diis_prec = diis_prec == "double" ? Float64 : diis_prec == "single" ? Float32 : Float16
    do_diis ? DM_T1 = Fermi.DIIS.DIISManager{T, diis_prec}(size=ndiis) : nothing
    do_diis ? DM_T2 = Fermi.DIIS.DIISManager{T, diis_prec}(size=ndiis) : nothing

    output("\tDropped Occupied Orbitals →  {:3.0f}", core)
    output("\tDropped Virtual Orbitals  →  {:3.0f}", inac)
    output("\n"*repeat("-",80))
    output("\tOptions:")
    output("\tPrecision? {}", T)
    output("\tDIIS? {}", do_diis)
    output("\tDIIS Vectors? {}", (do_diis ? ndiis : 0))
    output("\tDIIS Precision? {}", diis_prec)
    output("\tDamping percentage ? {}", dp)
    output("\t\tcc_max_iter →  {:3.0d}", cc_max_iter)
    output("\t\tcc_e_conv   →  {:2.0e}", cc_e_conv)
    output("\t\tcc_max_rms  →  {:2.0e}", cc_max_rms)
    if (cc_e_conv < eps(T) || cc_max_rms < eps(T)) && !(precision_override)
        output("\t⚠ WARNING ⚠  cc_e_conv set to less than ϵ ({}) for precision {}", eps(T), T)
        output("\t             CCSD is unlikely to converge to your standards.")
        output("\t             OVERRIDING set convergence criterion.")
        output("\t             Use @set `precision_override` true to perform the computation as entered.")
        cc_e_conv = eps(T)
        cc_max_rms = eps(T)
    end

    # Initialize Loop parameters
    r1 = 1
    r2 = 1
    dE = 1
    rms = 1
    ite = 1
    T1 = deepcopy(newT1)
    T2 = deepcopy(newT2)

    output(repeat("-", 80))

    # Compute Guess Energy
    Ecc = cc_update_energy(newT1, newT2, moints, alg)
    Eguess = Ecc + Eref

    output("\tGuess Correlation Energy:   {:15.10f}", Ecc)
    output("\tGuess Total Energy:         {:15.10f}\n", Eguess)
    output("    Starting CC Iterations\n")

    dE = 1
    rms = 1
    ite = 1

    if do_diis
        DM_T1 = Fermi.DIIS.DIISManager{T, diis_prec}(size=ndiis)
        DM_T2 = Fermi.DIIS.DIISManager{T, diis_prec}(size=ndiis)
    end

    main_time = 0
    output("{:10s}    {: 15s}    {: 12s}    {:12s}    {:10s}    {:s5}","Iteration","CC Energy","ΔE","Max RMS","time (s)", "diis")
    while (abs(dE) > cc_e_conv || rms > cc_max_rms)
        if ite > cc_max_iter
            output("\n⚠  CC Equations did not converge in {:1.0d} iterations.", cc_max_iter)
            break
        end
        t = @elapsed begin

            T1 .= newT1
            T2 .= newT2
            oldE = Ecc

            update_amp!(newT1, newT2, T1, T2, moints, alg)

            # Compute residues
            r1 = sqrt(sum((newT1 .- T1).^2)/length(T1))
            r2 = sqrt(sum((newT2 .- T2).^2)/length(T2))

            diis = false
            if do_diis
                # Update DIIS vector
                e1 = (newT1 - T1)
                e2 = (newT2 - T2)
                push!(DM_T1,newT1,e1)
                push!(DM_T2,newT2,e2)
                # Using DIIS at every iteration is BAD! So do it every three steps
                if ite ≥ diis_start && ite % diis_relax == 0
                    # Get new extrapolated amplitudes from DIIS
                    newT2 = Fermi.DIIS.extrapolate(DM_T2;add_res=true)
                    newT1 = Fermi.DIIS.extrapolate(DM_T1;add_res=true)
                    diis = true
                end
            end

            # Apply damping
            if dp > 0
                newT1 .= (1-dp)*newT1 .+ dp*T1
                newT2 .= (1-dp)*newT2 .+ dp*T2
            end

            Ecc = cc_update_energy(newT1, newT2, moints, alg)
        end
        rms = max(r1,r2)
        dE = Ecc - oldE
        main_time += t
        output("    {:<5.0d}    {:<15.10f}    {:>+12.10f}    {:<12.10f}    {:<10.5f}    {:s5}", ite, Ecc, dE, rms, t, diis)
        ite += 1
    end
    output("\nMain CCSD iterations done in {:5.5f} s", main_time)
    output("Average time per iteration {:5.5f}", main_time/(ite-1))

    # Converged?
    conv = false
    if abs(dE) < cc_e_conv && rms < cc_max_rms
        output("\n 🍾 Equations Converged!")
        conv = true
    end

    output(" @Final CCSD Correlation Energy:     {:15.10f}", Ecc)
    output(" @Final CCSD Energy:                 {:15.10f}", Ecc+Eref)
    output(repeat("-",80))

    return RCCSD(Eguess, Ecc+Eref, Ecc, newT1, newT2, dE, rms)
end


using YAML
using Statistics
using Cambrian
import Cambrian.mutate

# mutate must be overriden in the global scope (or use eval)
cfg = get_config("../cfg/oneplus.yaml")
mutate(i::Cambrian.FloatIndividual) = Cambrian.mutate(i, cfg.m_rate)

function test_oneplus_evo(fitness::Function, d_fitness::Int)
    cfg = get_config("../cfg/oneplus.yaml"; d_fitness=d_fitness, id="test")
    e = OnePlusEvo{Cambrian.FloatIndividual}(cfg, fitness)

    @test length(e.population) == cfg.population.size
    for i in e.population
        @test all(i.fitness .== -Inf)
    end

    evaluate!(e)

    for i in e.population
        @test i.fitness == fitness(i)
    end
    best = sort(e.population)[end]

    populate!(e)
    new_best = sort(e.population)[end]
    @test !(new_best < best)
    max_count = 0
    for i in e.population
        if i.fitness == new_best.fitness
            max_count += 1
        else
            for f in i.fitness
                @test f == -Inf
            end
        end
    end
    @test max_count == 1
    e.gen += 1

    step!(e)
    @test length(e.population) == cfg.population.size
    for i in e.population
        @test i.fitness == fitness(i)
    end
    new_best = sort(e.population)[end]
    @test !(new_best < best)
    @test e.gen == 2

    print("1+λ step ", string(fitness))
    @timev step!(e)

    run!(e)
    @test length(e.population) == cfg.population.size
    for i in e.population
        @test i.fitness == fitness(i)
    end
    new_best = sort(e.population)[end]
    @test !(new_best < best)
    @test e.gen == cfg.n_gen
end

@testset "1+λ" begin
    @testset "Sum of genes" begin
        function sum_eval(i::Individual)
            [sum(i.genes)]
        end
        test_oneplus_evo(sum_eval, 1)
    end

    @testset "Rosenbrock" begin
        function rosenbrock(i::Individual)
            x = i.genes
            y = -(sum([(1.0 - x[i])^2 + 100.0 * (x[i+1] - x[i]^2)^2
                      for i in 1:(length(x)-1)]))
            [y]
        end
        test_oneplus_evo(rosenbrock, 1)
    end

    @testset "Multi-objective" begin
        function moo(i::Individual)
            [mean(i.genes), std(i.genes), minimum(i.genes)]
        end
        test_oneplus_evo(moo, 3)
    end
end


# This file includes testset for RODESystem

import DifferentialEquations.RandomEM

@testset "RODESystemTestSet" begin
    @info "Running RODESystemTestSet ..."

    # RODESystem construction
    function statefunc(dx, x, u, t, W)
        dx[1] = 2x[1]*sin(W[1] - W[2])
        dx[2] = -2x[2]*cos(W[1] + W[2])
    end
    outputfunc(x, u, t) = x
    ds = RODESystem(righthandside=statefunc, readout=outputfunc, state=ones(2), input=nothing, output=Outport(2),
        alg=RandomEM())
    ds = RODESystem(righthandside=statefunc, readout=outputfunc, state=ones(2), input=nothing, output=Outport(2),
        alg=RandomEM(), modelkwargs=(rand_prototype=zeros(2),))
    @test typeof(ds.trigger) == Inpin{Float64}
    @test typeof(ds.handshake) == Outpin{Bool}
    @test ds.input === nothing
    @test isa(ds.output, Outport)
    @test length(ds.output) == 2
    @test ds.state == ones(2)
    @test ds.integrator.sol.prob.p === nothing

    # Driving RODESystem
    iport = Inport(2)
    trg = Outpin()
    hnd = Inpin{Bool}()
    connect!(ds.output, iport)
    connect!(trg, ds.trigger)
    connect!(ds.handshake, hnd)
    tsk = launch(ds)
    tsk2 = @async while true
        all(take!(iport) .=== NaN) && break
    end
    for t in 1 : 10
        put!(trg, t)
        take!(hnd)
        @test ds.t == t
        @test [read(pin.link.buffer) for pin in iport] == ds.state
    end
    put!(trg, NaN)
    sleep(0.1)
    @test istaskdone(tsk)
    put!(ds.output, NaN * ones(length(ds.output)))
    sleep(0.1)
    @test istaskdone(tsk2)

    # Driving RODESystem with input
    function sfunc(dx, x, u, t, W)
        dx[1] = 2x[1]*sin(W[1] - W[2]) + cos(u[1](t))
        dx[2] = -2x[2]*cos(W[1] + W[2]) - sin(u[1](t))
    end
    ofunc(x, u, t) = x
    ds = RODESystem(righthandside=sfunc, readout=ofunc, state=ones(2), input=Inport(2), output=Outport(2),
        modelkwargs=(rand_prototype=zeros(2),))
    @test typeof(ds.integrator.sol.prob.p) <: Interpolant
    @test size(ds.integrator.sol.prob.p.timebuf) == (3,)
    @test size(ds.integrator.sol.prob.p.databuf) == (2,3)
    oport = Outport(2)
    iport = Inport(2)
    trg = Outpin()
    hnd = Inpin{Bool}()
    connect!(oport, ds.input)
    connect!(ds.output, iport)
    connect!(trg, ds.trigger)
    connect!(ds.handshake, hnd)
    tsk = launch(ds)
    tsk2 = @async while true
        all(take!(iport) .=== NaN) && break
    end
    for t in 1 : 10
        put!(trg, t)
        put!(oport, [t, 2t])
        take!(hnd)
        @test ds.t == t
        @test [read(pin.link.buffer) for pin in iport] == ds.state
    end
    put!(trg, NaN)
    sleep(0.1)
    @test istaskdone(tsk)
    put!(ds.output, NaN * ones(length(ds.output)))
    sleep(0.1)
    @test istaskdone(tsk2)

    # Test defining new RODESystem types
    # The type must be mutable
    @test_throws Exception @eval @def_rode_system struct RODESystem{RH, RO, ST, IP, OP}
        righthandside::RH
        readout::RO
        state::ST
        input::IP
        output::OP
    end

    # The type must be of type AbstractRODESystem
    @test_throws Exception @eval @def_rode_system mutable struct RODESystem{RH, RO, ST, IP, OP}
        righthandside::RH
        readout::RO
        state::ST
        input::IP
        output::OP
    end

    # The type must be of type AbstractRODESystem
    @test_throws Exception @eval @def_rode_system mutable struct RODESystem{RH, RO, ST, IP, OP} <: MyDummyAbstractRODESystem
        righthandside::RH
        readout::RO
        state::ST
        input::IP
        output::OP
    end

    @info "Done RODESystemTestSet ..."
end  # testset


export BasicDirectory

"""
    BasicDirectory(map::BlockMap)
Creates a BasicDirectory, which implements the methods of Directory with
basic implmentations
"""
mutable struct BasicDirectory{GID <: Integer, PID <:Integer, LID <: Integer} <: Directory{GID, PID, LID}
    map::BlockMap{GID, PID, LID}

    directoryMap::Union{BlockMap, Nothing}

    procList::Vector{PID}
    procListLists::Vector{Vector{PID}}

    entryOnMultipleProcs::Bool

    localIndexList::Vector{LID}
    allMinGIDs::Vector{GID}

    function BasicDirectory{GID, PID, LID}(map::BlockMap{GID, PID, LID}) where {GID, PID, LID}

        if !(distributedGlobal(map))
            new(map, nothing, [], [], numProc(getComm(map))!=1, [], [])
        elseif linearMap(map)
            commObj = getComm(map)

            allMinGIDs = gatherAll(commObj, minMyGID(map))
            allMinGIDs = vcat(allMinGIDs, [1+maxAllGID(map)])

            entryOnMultipleProcs = length(Set(allMinGIDs)) != length(allMinGIDs)

            new(map, nothing, [], [], entryOnMultipleProcs, [], allMinGIDs)
        else
            generateContent(
                new(map, nothing, [], [], false, [], []),
                map)
        end
    end
end

"""
internal method to assist constructor
"""
function generateContent(dir::BasicDirectory{GID, PID, LID}, map::BlockMap{GID, PID, LID}) where GID <: Integer where PID <: Integer where LID <: Integer
    minAllGID = minAllGID(map)
    maxAllGID = maxAllGID(map)

    dirNumGlobalElements = maxAllGID - minAllGID + 1

    directoryMap = BlockMap(dirNumGlobalElements, minAllGID, commObj)
    dir.directoryMap = directoryMap

    dirNumMyElements = numMyElements(dir.directoryMap)

    if dirNumMyElements > 0
        dir.procList = Vector{PID}(undef, dirNumMyElements)
        dir.localIndexList = Vector{LID}(undef, dirNumMyElements)

        fill!(dir.procList, -1)
        fill!(dir.localIndexList, -1)
    else
        dir.procList = []
        dir.localIndexList = []
    end

    map_numMyElements = numMyElements(map)
    map_myGlobalElements = myGlobalElements(map)

    sendProcs = remoteIDList(directoryMap, map_numMyElements, myGlobalElements(map))

    distributor = createDistributor(commObj)
    numRecvs = createFromSends(distributor, map_numMyElements, sendProcs)

    exportElements = Array{Tuple{GID, PID, LID}}(numMyElements)

    myPIDVal = myPID(commObj)
    for i = 1:numMyElements
        exportElements[i] = (map_myGlobalElements[i], myPIDVal, i)
    end

    importElements = resolve(distributor, exportElements)

    for i = 1:numRecvs
        currLID = lid(directoryMap, importElements[i][1])
        @assert currLID > 0 //internal error

        proc = importElements[i][2]
        if dir.procList[currLID] >= 0
            if dir.procList[currLID] != proc
                if dir.procListLists == []
                    numProcLists = numMyElements(directoryMap)
                    procListLists = Vector{Vector{PID}}(undef, numProcLists)
                    fill!(procListLists, PID[])
                end

                l = procListLists[currLID]

                index = searchsortedfirst(l, procList[currLID])
                insert(l, index, procList[currLID])

                index = searchsortedfirst(l, proc)
                insert(l, index, proc)

                dir.procList[currLID] = dir.procListLists[curr_LID][1]
            end
        else
            dir.procList[currLID] = proc
        end

        dir.localIndexList[currLID] = importElements[i][3]
    end

    globalVal = maxAll(commObj, numProcLists)
    dir.entryOnMultipleProcs = globalval > 0 ? true : false;

    dir
end

function getDirectoryEntries(directory::BasicDirectory{GID, PID, LID}, map::BlockMap{GID, PID, LID}, globalEntries::AbstractVector{GID},
        high_rank_sharing_procs::Bool)::Tuple{Vector{PID}, Vector{LID}} where GID <: Integer where PID <: Integer where LID <: Integer
    numEntries = length(globalEntries)
    procs = Vector{PID}(undef, numEntries)
    localEntries = Vector{LID}(undef, numEntries)

    if !distributedGlobal(map)
        myPIDVal = myPid(getComm(map))

        for i = 1:numEntries
            lidVal = lid(map, globalEntries[i])

            if lidVal == 0
                procs[i] = 0
                warn("GID $(globalEntries[i]) is not part of this map")
            else
                procs[i] = myPIDVal
            end
            localEntries[i] = lidVal
        end
    elseif linearMap(map)
        minAllGIDVal = minAllGID(map)
        maxAllGIDVal = maxAllGID(map)

        numProcVal = numProc(getComm(map))

        n_over_p = numGlobalElements(map)/numProcVal

        allMinGIDs_list = copy(directory.allMinGIDs)
        order = sortperm(allMinGIDs_list)
        permute!(allMinGIDs_list, order)

        for i = 1:numEntries
            lid  = 0
            proc = 0

            gid = globalEntries[i]
            if gid < minAllGIDVal || gid > maxAllGIDVal
                throw(InvalidArgumentError("GID=$gid out of valid range [$minAllGIDVal, $maxAllGIDVal]"))
            end
            #guess uniform distribution and start a little above it
            proc1 = min(GID(fld(gid, max(n_over_p, 1)) + 2), numProcVal)
            proc1 = 1
            found = false

            while proc1 >= 1 && proc1 <= numProcVal
                if allMinGIDs_list[proc1] <= gid
                    if (gid < allMinGIDs_list[proc1+1])
                        found = true
                        break
                    else
                        proc1 += 1
                    end
                else
                    proc1 -= 1
                end
            end
            if found
                proc = order[proc1]
                lid = gid - allMinGIDs_list[proc1] + 1
            end

            procs[i] = proc
            localEntries[i] = lid
        end
    else #general case
        distributor = createDistributor(getComm(map))

        dirProcs = remoteIDList(map, numEntries, globalEntries)

        numMissing = 0
        for i = 1:numEntries
            if dirProcs[i] == 0
                procs[i] = 0
                localEntries[i] = 0
                numMissing += 1
            end
        end

        (sendGIDs, sendPIDs) = createFromRecvs(distrbutor, globalEntries, dirProcs)
        numSends = length(sendGIDs)

        if numSends > 0
            exports = Array{Tuple{GID, PID, LID}}(numSends)
            for i = 1:numSends
                currGID = sendGIDs[i]
                exports[i][1] = currGID

                currLID = lid(map, currGID)
                @assert currLID > 0 #internal error
                if !high_rank_sharing_procs
                    exports[i][2] = procList[currLID]
                else
                    numProcLists = numMyElements(directory.directoryMap)
                    if numProcLists > 0
                        num = length(directory.procListLists[currLID])
                        if num > 1
                            exports[i][2] = directory.procListLists[currLID][num]
                        else
                            exports[i][2] = directory.procList[currLID]
                        end
                    else
                        exports[i][2] = directory.procList[currLID]
                    end
                end
                exports[i][3] = directory.localIndexList[currLID]
            end
        end

        numRecv = numEntries - numMissing
        imports = resolve(distributor, exports)

        offsets = sortperm(globalEntries)
        sortedGE = globalEntries[offsets]

        for i = 1:numRecv
            currLID = imports[i][1]
            j = searchsortedfirst(sortedGE, currLID)
            if j > 0
                procs[offsets[j]] = imports[i][2]
                localEntries[offsets[j]] = imports[i][3]
            end
        end
    end
    (procs, localEntries)
end

function gidsAllUniquelyOwned(directory::BasicDirectory)
    !directory.entryOnMultipleProcs
end


function update_feed!(
    state::Tuple{AbstractGraph, AbstractArray}, agent_idx::Integer,
    config::Config
)
    graph, agent_list = state
    this_agent = agent_list[agent_idx]
    unique!(this_agent.feed)
    deleted_posts = Integer[]
    for (index, post) in enumerate(this_agent.feed)
        if post.weight == -1 || !(post.source_agent in inneighbors(graph, agent_idx))
            push!(deleted_posts, index)
        else
            post.weight = 0.5 * post.weight
        end
    end
    deleteat!(this_agent.feed, deleted_posts)
    sort!(this_agent.feed, lt=<, rev=true)
    if length(this_agent.feed) > 15
        this_agent.feed = this_agent.feed[1:15]
    end

    for post in this_agent.feed
        if !(agent_idx in post.seen_by)
            push!(post.seen_by, agent_idx)
        end
    end

    return state
end

# suppress output of include()
;


"""
    abstract type ProjectiveTransform <: Transform

Abstract supertype for projective transformations. See
[Projective transformations](../docs/projective/interface.md).
"""
abstract type ProjectiveTransform <: Transform end

"""
    getprojection(tfm, bounds; randstate)

Create a projection for an item with spatial bounds `bounds`.
The projection should be a `CoordinateTransformations.Transformation`.
See [CoordinateTransformations.jl](https://github.com/JuliaGeometry/CoordinateTransformations.jl)
"""
function getprojection end

struct Bounds{N}
    rs::NTuple{N, UnitRange{Int}}
end

function Base.show(io::IO, bounds::Bounds{N}) where N
    print(io, "Bounds(")
    for (i, r) in enumerate(bounds.rs)
        print(io, first(r), ':', last(r))
        i == N || print(io, '×')
    end
    print(io, ")")
end

Bounds(sz::NTuple{N, <:Integer}) where N = Bounds(Tuple(1:n for n in sz))
Bounds(axes::NTuple{N, <:Base.OneTo}) where N = Bounds(convert.(UnitRange{Int}, axes))

Base.:(==)(bs1::Bounds, bs2::Bounds) = false
function Base.:(==)(bs1::Bounds{N}, bs2::Bounds{N}) where N
    c1 = all((first(bs1.rs[i]) == first(bs2.rs[i])) for i in 1:N)
    c2 = all((last(bs1.rs[i]) == last(bs2.rs[i])) for i in 1:N)
    return c1 && c2
end

"""
    transformbounds(bounds, P)

Apply `CoordinateTransformations.Transformation` to `bounds`.
"""
function transformbounds(bounds::Bounds, P::CoordinateTransformations.Transformation)
    return Bounds(ImageTransformations.autorange(CartesianIndices(bounds.rs), P))
end

"""
    getbounds(item)

Return the spatial bounds of `item`. For a 2D-image (`Image{2}`)
the bounds are the 4 corners of the bounding rectangle. In general,
for an N-dimensional item, the bounds are a vector of the N^2 corners
of the N-dimensional hypercube bounding the data.
"""
function getbounds end

getbounds(wrapper::ItemWrapper) = getbounds(getwrapped(wrapper))

"""
    project(P, item, indices)

Project `item` using projection `P` and crop to `indices` if given.
"""
function project end

"""
    project!(bufitem, P, item, indices)

Project `item` using projection `P` and crop to `indices` if given.
Store result in `bufitem`. Inplace version of [`project`](#).

Default implementation falls back to `project`.
"""
function project!(bufitem, P, item, indices)
    titem = project(P, item, indices)
    copyitemdata!(bufitem, titem)

    return bufitem
end

"""
    projectionbounds(tfm, P, bounds, randstate)
"""
function projectionbounds(tfm, P, bounds; randstate = getrandstate(tfm))
    return transformbounds(bounds, P)
end

# With the interface defined, any `ProjectiveTransform` can be `apply`ed
# like this:

function apply(tfm::ProjectiveTransform, item::Item; randstate = getrandstate(tfm))
    bounds = getbounds(item)
    P = getprojection(tfm, bounds; randstate = randstate)
    bounds_ = projectionbounds(tfm, P, bounds; randstate = randstate)
    return P isa IdentityTransformation ? item : project(P, item, bounds_)
end

# For the buffered version, `project!` is used. Of course the size
# of the data produced by `tfm` needs to be the same every time it
# is applied for this to work.

function apply!(
        bufitem::AbstractItem,
        tfm::ProjectiveTransform,
        item::AbstractItem;
        randstate = getrandstate(tfm))
    bounds = getbounds(item)
    P = getprojection(tfm, bounds; randstate = randstate)
    bounds_ = projectionbounds(tfm, P, bounds; randstate = randstate)
    res = project!(bufitem, P, item, bounds_)
    return res
end

# The simplest `ProjectiveTransform` is the static `Project`.

struct Project{T<:CoordinateTransformations.Transformation} <: ProjectiveTransform
    P::T
end

getprojection(tfm::Project, bounds; randstate = nothing) = tfm.P


import UnicodePlots
using StaticArrays

const CountType = UInt32
struct UnicodePlotter{T<:Number} <: Plotter
    fps::T
    counter::Base.RefValue{CountType}
    skip::CountType
end

function UnicodePlotter(; fps = 20, counter = 0, skip = 1 )
    UnicodePlotter{typeof(fps)}(
        fps,
        Ref(convert(CountType, counter)),
        convert(CountType, skip)
        )
end # function

function plot( ::UnicodePlotter, world::World, time )
    active_dogs = filter( d->(d.state==ACTIVE), world.dogs.member )
    sleepy_dogs = typeof(world.dogs.member)[]
    occupied_dogs = typeof(world.dogs.member)[]
    if length( active_dogs ) < length( world.dogs )
        sleepy_dogs = filter( d->(d.state==SLEEPY), world.dogs.member )
        occupied_dogs = filter( d->(d.state==OCCUPIED), world.dogs.member )
    end
    p = UnicodePlots.scatterplot( Float64[],
            xlim = @SVector([0, world.boxsize[]]),
            ylim = @SVector([0, world.boxsize[]])
            )
    for data in ( (active_dogs,:green), (sleepy_dogs,:red), (occupied_dogs,:white) )
        positions = getproperty.( data[1], :position )
        xs = map( p->p[1], positions )
        ys = map( p->p[2], positions )
        p = UnicodePlots.scatterplot!( p, xs, ys, color = data[2] )
    end # for
    sheeps = collect( world.sheeps )
    sheep_xs = map( s->s[2][1], sheeps )
    sheep_ys = map( s->s[2][2], sheeps )
    UnicodePlots.scatterplot!( p, sheep_xs, sheep_ys )
    return p
end # function

function plot!( io, p, plotter::UnicodePlotter, world::World, time )
    if plotter.counter[] % plotter.skip == zero(CountType)
        for _ in 1:(UnicodePlots.nrows( p.graphics )+p.margin)
            print(io, "\e[2K\e[1F")
        end # for
        p = plot( plotter, world, time )
        show(IOContext(io, :color => true), p)
        print(io,"\n\t\tPress Enter twice to end simulation.")
        print(stdout, String(take!(io)))
        sleep( 1 / plotter.fps )
    end # if
    plotter.counter[] += 1
    return nothing
end # function


module Wilsonloops_module

    mutable struct Wilson_loop
        wilson_loop::Array{Tuple{Int8,Int8},1}
        origin::NTuple{4,Int8}
        Wilson_loop() = new([],(0,0,0,0))
        #Wilson_loop(loop::Array{Tuple{Int,Int},1},origin) = new(loop,origin)
        #Wilson_loop(loop,origin) = new([Tuple(loop)],origin)
        #Wilson_loop(loop::Array{Tuple{Int,Int},1}) = new(loop,(0,0,0,0))

        function Wilson_loop(loop::Array{Tuple{T,T},1}) where T <: Integer
            return new(make_links(loop),(0,0,0,0))
        end
        function  Wilson_loop(loop::Array{Tuple{T,T},1},origin) where T<: Integer
            return new(make_links(loop),origin)
        end

        Wilson_loop(loop) = new([Tuple(loop)],(0,0,0,0))
    end

    mutable struct Tensor_wilson_lines
        left::Wilson_loop
        #Array{Tuple{Int8,Int8},1}
        right::Wilson_loop
        #Array{Tuple{Int8,Int8},1}
        position::NTuple{4,Int8}
        origin::NTuple{4,Int8}
        dUμ::NTuple{2,Int8}
        rightstart::NTuple{4,Int8}
    end

    function get_leftstartposition(w::Tensor_wilson_lines)
        return w.origin
    end

    function get_rightstartposition(w::Tensor_wilson_lines)
        return w.rightstart
    end

    mutable struct Tensor_wilson_lines_set
        loopset::Array{Tensor_wilson_lines,1}
        Tensor_wilson_lines_set() = new([])
    end

    function Base.getindex(g::Tensor_wilson_lines_set,i)
        return g.loopset[i]
    end

    function Base.length(x::Tensor_wilson_lines_set)
        return length(x.loopset)
    end

    function Base.push!(g::Tensor_wilson_lines_set,w::Tensor_wilson_lines)
        push!(g.loopset,w)
        return
    end

    function Base.display(w::Tensor_wilson_lines)
        println("tensor differenciated by ",w.dUμ)
        println("relative origin = ",w.origin)
        println("position = ",w.position)
        println("startpoint for leftpart: ",w.origin)
        println("startpoint for rightpart: ",w.rightstart)
        print("[")
        for i=1:length(w.left)
            print(w.left[i],",")
        end
        print("] ")
        print(" ⊗ ")
        print(" [")
        for i=1:length(w.right)
            print(w.right[i],",")
        end
        println("]\t")
    end

    function Base.display(w::Wilson_loop)
        println("origin = ",w.origin)
        print("[")
        for i=1:length(w)
            print(w.wilson_loop[i],",")
        end
        println("]\t")
    end

    function set_origin!(w::Wilson_loop,origin)
        w.origin = origin
    end

    mutable struct Wilson_loop_set
        #loopset::Set{Wilson_loop}
        #loopset::Array{Wilson,1}
        loopset::Array{Wilson_loop,1}
        Wilson_loop_set() = new([])
    end

    mutable struct Tensor_derivative_set
        tensors::Array{Array{Dict{Tuple{Int8,Int8},Tensor_wilson_lines_set},1},1}
        function Tensor_derivative_set(staples::Array{Wilson_loop_set,1})
            tensor_derivative = Array{Array{Dict{Tuple{Int8,Int8},Tensor_wilson_lines_set},1},1}(undef,4)
            for mu=1:4
                #println("$mu -direction")
                #display(staples[i][mu])
                tensor_derivative[mu] = Array{Dict{Tuple{Int8,Int8},Tensor_wilson_lines_set},1}(undef,length(staples[mu].loopset))

                for (k,link) in enumerate(staples[mu].loopset)
                    tensor_derivative[mu][k] = make_tensor_derivative(link)
                    #display(tensor_derivative[i][mu][k])
                end
            end
            return new(tensor_derivative)
        end
    end

    function Base.getindex(g::Tensor_derivative_set,i)
        return g.tensors[i]
    end

    function Base.display(w::Wilson_loop_set)
        for i=1:length(w)
            println("$i-th loops")
            display(w[i])
        end
        println("\t")
    end

    function Base.length(x::Wilson_loop_set)
        return length(x.loopset)
    end

    function Base.length(x::Wilson_loop)
        return length(x.wilson_loop)
    end

    function Base.getindex(g::Wilson_loop_set,i)
        return g.loopset[i]
    end

    function Base.getindex(g::Wilson_loop,i)
        return g.wilson_loop[i]
    end

    function Base.push!(g::Wilson_loop,link)
        push!(g.wilson_loop,Tuple(link))
        return
    end

    function Base.push!(g::Wilson_loop_set,w::Wilson_loop)
        push!(g.loopset,w)
        return
    end

    function make_cloverloops(μ,ν)
        loops = Wilson_loop_set()
        loop_righttop = Wilson_loop([(μ,1),(ν,1),(μ,-1),(ν,-1)])
        loop_lefttop = Wilson_loop([(ν,1),(μ,-1),(ν,-1),(μ,1)])
        loop_rightbottom = Wilson_loop([(ν,-1),(μ,1),(ν,1),(μ,-1)])
        loop_leftbottom= Wilson_loop([(μ,-1),(ν,-1),(μ,1),(ν,1)])
        push!(loops,loop_righttop)
        push!(loops,loop_lefttop)
        push!(loops,loop_rightbottom)
        push!(loops,loop_leftbottom)
        return loops
    end

    function make_originalactions_fromloops(coupling_loops)
        actions = Array{Wilson_loop_set,1}(undef,length(coupling_loops))
        for (i,loopset) in enumerate(coupling_loops)
            actions[i] = make_originalaction(loopset)
        end
        return actions
    end

    function make_loopforactions(couplinglist,L)
        loops = Array{Wilson_loop_set,1}(undef,length(couplinglist))
        for (i,name) in enumerate(couplinglist)
            if  name == "plaq"
                loops[i] = make_plaq()
            elseif name == "rect"
                loops[i] = make_rect()
            elseif name == "polyt"
                loops[i] = make_polyakov(4,L[4])
            elseif name == "polyx"
                loops[i] = make_polyakov(3,L[3])
            elseif name == "polyy"
                loops[i] = make_polyakov(2,L[2])
            elseif name == "polyz"
                loops[i] = make_polyakov(1,L[1])
            else
                error("$name is not supported!")
            end
        end
        return loops
    end

    function make_originalaction(loopset::Array{T,1}) where T <: Any
        loop_wilson = Wilson_loop_set()
        for loops in loopset
            @assert eltype(loops) == Tuple{<: Int,<: Int} "The loop should be like [(1, 1), (2, 1), (1, -1), (2, -1)]"
            loop1 = Wilson_loop(loops)
            push!(loop_wilson,loop1)
        end
        return loop_wilson
    end

    function make_plaqloops()
        loopset = []
        for μ=1:4
            for ν=μ:4
                #for ν=1:4
                if ν == μ
                    continue
                end
                push!(loopset,[(μ,1),(ν,1),(μ,-1),(ν,-1)])
            end
        end
        return loopset
    end

    function make_rectloops(N)
        loopset = []
        for μ=1:4
            for ν=μ:4
                if ν == μ
                    continue
                end
                push!(loopset,[(μ,1),(ν,N),(μ,-1),(ν,-N)])
                push!(loopset,[(μ,N),(ν,1),(μ,-N),(ν,-1)])
            end
        end
        return loopset
    end

    function make_polyakovloops(μ,Nμ)
        loopset = []
        push!(loopset,[(μ,Nμ)])
        return loopset
    end

    function make_plaq()
        loopset = make_plaqloops()
        loops = make_originalaction(loopset)
        return loops
    end

    function make_plaq(origin)
        loops = Wilson_loop_set()
        for μ=1:4
            #for ν=1:4
            for ν=μ:4
                if ν == μ
                    continue
                end
                loop = make_links([(μ,1),(ν,1),(μ,-1),(ν,-1)])
                loop1 = Wilson_loop(loop,Tuple(origin))
                push!(loops,loop1)
            end
        end

        return loops
    end

    function make_plaq(μ,ν)
        loops = Wilson_loop_set()
        origin = zeros(Int8,4)

        loop = make_links([(μ,1),(ν,1),(μ,-1),(ν,-1)])
        loop1 = Wilson_loop(loop,Tuple(origin))
        push!(loops,loop1)

        return loops
    end

    function make_plaq(μ,ν,origin)
        loops = Wilson_loop_set()
        #origin = zeros(Int8,4)

        loop = make_links([(μ,1),(ν,1),(μ,-1),(ν,-1)])
        loop1 = Wilson_loop(loop,Tuple(origin))
        push!(loops,loop1)

        return loops
    end

    function make_rect()
        loops = Wilson_loop_set()
        origin = zeros(Int8,4)

        for μ=1:4
            for ν=μ:4
                if ν == μ
                    continue
                end
                #loop = make_links([(μ,1),(ν,2),(μ,-1),(ν,-2)])

                loop1 = Wilson_loop([(μ,1),(ν,2),(μ,-1),(ν,-2)])
                #loop1 = Wilson_loop(loop,Tuple(origin))
                push!(loops,loop1)

                loop1 = Wilson_loop([(μ,2),(ν,1),(μ,-2),(ν,-1)])
                push!(loops,loop1)
            end
        end

        return loops
    end

    function make_polyakov(μ,Nμ)
        loops = Wilson_loop_set()
        origin = zeros(Int8,4)
        loop = make_links([(μ,Nμ)])
        loop1 = Wilson_loop(loop,Tuple(origin))
        push!(loops,loop1)
        return loops
    end

    function make_polyakov_xyz(Nμ)
        loops = Wilson_loop_set()
        for μ=1:3
            origin = zeros(Int8,4)
            loop = make_links([(μ,Nμ)])
            loop1 = Wilson_loop(loop,Tuple(origin))
            push!(loops,loop1)
        end

        return loops
    end

    function make_chair()
        #loopset = []
        loopset = Wilson_loop_set()
        set1 = (1,2,3)
        set2 = (1,2,4)
        set3 = (2,3,4)
        set4 = (1,3,4)

        for set in (set1,set2,set3,set4)
            mu,nu,rho = set
            origin = zeros(Int8,4)
            loop = [(mu,1),(nu,1),(rho,1),(mu,-1),(rho,-1),(nu,-1)]
            loop1 = Wilson_loop(loop,Tuple(origin))
            push!(loopset,loop1)

            mu,rho,nu = set
            loop = [(mu,1),(nu,1),(rho,1),(mu,-1),(rho,-1),(nu,-1)]
            loop1 = Wilson_loop(loop,Tuple(origin))
            push!(loopset,loop1)

            nu,rho,mu = set
            loop = [(mu,1),(nu,1),(rho,1),(mu,-1),(rho,-1),(nu,-1)]
            loop1 = Wilson_loop(loop,Tuple(origin))
            push!(loopset,loop1)

            nu,mu,rho = set
            loop = [(mu,1),(nu,1),(rho,1),(mu,-1),(rho,-1),(nu,-1)]
            loop1 = Wilson_loop(loop,Tuple(origin))
            push!(loopset,loop1)

            rho,mu,nu = set
            loop = [(mu,1),(nu,1),(rho,1),(mu,-1),(rho,-1),(nu,-1)]
            loop1 = Wilson_loop(loop,Tuple(origin))
            push!(loopset,loop1)

            rho,nu,mu = set
            loop = [(mu,1),(nu,1),(rho,1),(mu,-1),(rho,-1),(nu,-1)]
            loop1 = Wilson_loop(loop,Tuple(origin))
            push!(loopset,loop1)

        end
        return loopset
    end

    function make_plaq_staple(μ)
        loops = Wilson_loop_set()
        origin = zeros(Int8,4)
        origin[μ] = 1
        for ν=1:4
            if ν == μ
                continue
            end
            loop = make_links([(ν,1),(μ,-1),(ν,-1)])
            loop1 = Wilson_loop(loop,Tuple(origin))
            push!(loops,loop1)

            loop = make_links([(ν,-1),(μ,-1),(ν,1)])
            loop2 = Wilson_loop(loop,Tuple(origin))
            push!(loops,loop2)
        end
        return loops
    end

    function Base.adjoint(x::Wilson_loop_set)
        xout = Wilson_loop_set()
        num = length(x)

        for i=1:num
            xi = x[i]
            #display(xi)
            numloops = length(xi)
            coordinates = calc_coordinate(xi)

            #println("endpoint")
            endpoint = collect(coordinates[end])
            lastlink = xi[numloops]
            #println(lastlink)
            endpoint[lastlink[1]] += lastlink[2]
            #println(endpoint)

            loop = Tuple{Int64,Int64}[]
            for k=numloops:-1:1
                link = xi[k]
                push!(loop,(link[1],-link[2]))
            end
            #println(loop)
            loopw = make_links(loop)

            loop2 = Wilson_loop(loopw,Tuple(endpoint))

            #println(xi)
            #println("inv")
            #display(loop2)
            push!(xout,loop2)
        end
        #exit()
        return xout
    end

    function make_plaq_staple_prime(μ)
        loops = Wilson_loop_set()
        origin = zeros(Int8,4)
        #origin[μ] = 1
        for ν=1:4
            if ν == μ
                continue
            end
            loop = make_links([(ν,1),(μ,1),(ν,-1)])
            loop1 = Wilson_loop(loop,Tuple(origin))
            push!(loops,loop1)

            loop2 = Wilson_loop([(ν,-1),(μ,1),(ν,1)],Tuple(origin))
            push!(loops,loop2)
        end
        return loops
    end

    function make_plaq_staple_prime(μ,Dim)
        loops = Wilson_loop_set()
        origin = zeros(Int8,4)
        #origin[μ] = 1
        for ν=1:Dim
            if ν == μ
                continue
            end
            loop = make_links([(ν,1),(μ,1),(ν,-1)])
            loop1 = Wilson_loop(loop,Tuple(origin))
            push!(loops,loop1)

            loop2 = Wilson_loop([(ν,-1),(μ,1),(ν,1)],Tuple(origin))
            push!(loops,loop2)
        end
        return loops
    end

    function calc_coordinate(w::Wilson_loop)
        numloops = length(w)
        coordinates = Array{NTuple{4,Int8},1}(undef,numloops)

        for k=1:numloops-1
            if k == 1
                coordinates[k] = w.origin
            end
            loopk = w[k]
            if loopk[1] == 1
                coordinates[k+1] = (coordinates[k][1]+loopk[2],coordinates[k][2],coordinates[k][3],coordinates[k][4])
            elseif loopk[1] == 2
                coordinates[k+1] = (coordinates[k][1],coordinates[k][2]+loopk[2],coordinates[k][3],coordinates[k][4])
            elseif loopk[1] == 3
                coordinates[k+1] = (coordinates[k][1],coordinates[k][2],coordinates[k][3]+loopk[2],coordinates[k][4])
            elseif loopk[1] == 4
                coordinates[k+1] = (coordinates[k][1],coordinates[k][2],coordinates[k][3],coordinates[k][4]+loopk[2])
            end
        end
        return coordinates
    end

    function calc_shift(w::Wilson_loop)
        numloops = length(w)
        coordinates = calc_coordinate(w)
        shifts = Array{NTuple{4,Int8},1}(undef,numloops)
        for k=1:numloops
            loopk = w[k]
            if loopk[2] == 1
                shifts[k] = coordinates[k]
            elseif loopk[2] == -1
                if loopk[1] == 1
                    shifts[k] = (coordinates[k][1]+loopk[2],coordinates[k][2],coordinates[k][3],coordinates[k][4])
                elseif loopk[1] == 2
                    shifts[k] = (coordinates[k][1],coordinates[k][2]+loopk[2],coordinates[k][3],coordinates[k][4])
                elseif loopk[1] == 3
                    shifts[k] = (coordinates[k][1],coordinates[k][2],coordinates[k][3]+loopk[2],coordinates[k][4])
                elseif loopk[1] == 4
                    shifts[k] = (coordinates[k][1],coordinates[k][2],coordinates[k][3],coordinates[k][4]+loopk[2])
                end
            end
        end
        return shifts
    end

    const kindsof_loops = ["plaquette","rectangular","chair","polyakov_x","polyakov_y","polyakov_z","polyakov_t"]

    function make_loops(loops_list,L)
        loops = []
        for loopname in loops_list
            if loopname == "plaquette"
                loop = make_plaq()
            elseif loopname == "rectangular"
                loop = make_rect()
            elseif loopname == "chair"
                loop = make_chair()
            elseif loopname == "polyakov_x"
                loop = make_polyakov(1,L[1])
            elseif loopname == "polyakov_y"
                loop = make_polyakov(2,L[2])
            elseif loopname == "polyakov_z"
                loop = make_polyakov(3,L[3])
            elseif loopname == "polyakov_t"
                loop = make_polyakov(4,L[4])
            else
                error("loop name $(loopname) is not supported!")
            end
            push!(loops,loop)
        end
        return loops
    end

    function make_links(segments::Array{Tuple{T,T},1}) where T <: Integer
        links = Tuple{Int8,Int8}[]
        for segment in segments
            s = sign(segment[2])
            if segment[2] == 0
                push!(links,(segment[1],0))
            else
            #@assert segment[2] != 0
                for i=1:abs(segment[2])
                    push!(links,(segment[1],s*1))
                end
            end
        end
        return links
    end

    function make_staples(w::Wilson_loop_set)
        num = length(w)
        staplesset = Array{Wilson_loop_set,1}(undef,4)
        tempset = Array{Array{Any,1},1}(undef,4)
        #tempset = Array{Set,1}(undef,4)
        for mu=1:4
            staplesset[mu] = Wilson_loop_set()
            #tempset[mu] = Set()
            tempset[mu] = []
        end

        for i=1:num
            staples = make_staples(w[i])
            for mu=1:4
                origin = zeros(Int8,4)
                origin[mu] = 1

                for k=1:length(staples[mu])
                    push!(tempset[mu],staples[mu][k])
                    #Wilson_loop(staples[mu][k],Tuple(origin)))
                end

                #for k=1:length(staples[mu])
                #    push!(staplesset[mu],Wilson_loop(staples[mu][k],Tuple(origin))  )
                #end
            end
        end

        for mu=1:4
            #display(tempset[mu])
            #exit()
            origin = zeros(Int8,4)
            origin[mu] = 1
            for temp in tempset[mu]
                push!(staplesset[mu],Wilson_loop(temp,Tuple(origin)))
            end
        end

        return staplesset
    end

    function get_coordinate(w::Wilson_loop,position)
        coor = collect(w.origin)
        if position != 1
            for i=1:position-1
                link = w.wilson_loop[i]
                coor[link[1]] += link[2]
            end
        end

        return coor
    end

    #=
    mutable struct Tensor_wilson_lines
        left::Array{Tuple{Int8,Int8},1}
        right::Array{Tuple{Int8,Int8},1}
        position::NTuple{4,Int8}
    end
    =#

    function make_tensor_derivative(w::Wilson_loop)
        dVdU = Dict{Tuple{Int8,Int8},Tensor_wilson_lines_set}()
        #display(w)
        for μ=1:4
            for sign = [+1,-1]
                u = (μ,sign)
                #println("u = $u")
                dVdU[u] = make_tensor_Wilson_line(w,u)
            end
        end
        return dVdU
    end

    function make_tensor_Wilson_line(w::Wilson_loop,u::Tuple{T,T}) where T <: Int
        left = Tuple{Int8,Int8}[]
        right = Tuple{Int8,Int8}[]
        count,positions = count_terms(w,u)
        len = length(w)

        tw = Tensor_wilson_lines_set()
        #println("u = $u count = ",count," position = $positions")
        if count != 0
            #println("u = $u")
            for i=1:count
                left = Tuple{Int8,Int8}[]
                right = Tuple{Int8,Int8}[]

                position = positions[i]
                coor = get_coordinate(w,position)
                #println("position ", coor)
                #insertpoint = Tuple(collect(w.origin) .- coor)
                insertpoint = Tuple(- collect(coor))
                #println("position2 ", insertpoint )

                #println(positions[i])
                if position == 1
                    #println(u)
                    push!(left,(0,0))
                    #display(w)
                else
                    for k=1:position-1
                        push!(left,w.wilson_loop[k])
                    end
                end
                #println("left = ",left)

                if position == len
                    #if u == (1,1)
                    #    println(right)
                    #end
                    push!(right,(0,0))
                else
                    for k=position+1:len
                        push!(right,w.wilson_loop[k])
                    end
                end
                #println("right = ",right)
                relativeorigin = [0,0,0,0]
                if left[1] == (0,0)
                else
                    for i=length(left):-1:1
                        link = left[i]
                        relativeorigin[link[1]] -= link[2]
                    end
                end

                temp_right = [0,0,0,0]
                temp_right[u[1]] += u[2]

                left_wilson = Wilson_loop(make_links(left),Tuple(relativeorigin))
                right_wilson = Wilson_loop(make_links(right),Tuple(temp_right))

                #tensorlink = Tensor_wilson_lines(left,right,insertpoint,Tuple(relativeorigin),u,Tuple(temp_right))
                tensorlink = Tensor_wilson_lines(left_wilson,right_wilson,insertpoint,Tuple(relativeorigin),u,Tuple(temp_right))

                #=
                if position == len
                    if u == (1,1)
                        display(tensorlink)
                    end
                end
                =#

                #display(tensorlink)
                push!(tw,tensorlink)
            end

        end
        return tw
    end

    function count_terms(w::Wilson_loop,u)
        count = 0
        positions = []
        #origin = w.origin
        #x = collect(origin)
        for (i,link) in enumerate(w.wilson_loop)

            if link == u
                count += 1
                push!(positions,i)
            end
            #x[link[1]] += link[2]
            #println(x)
            #println("$link,$count,$positions")
        end
        return count,positions
    end

    function make_staple_staple(staples::Array{Wilson_loop,1})
        staplestaple = Array{Wilson_loop,2}(undef,4,4)
        for ν=1:4
            staple = make_staples(staples[ν])
            for μ=1:4
                staplestaple[ν,μ] = staple[μ]
            end
        end
        return staplestaple
    end

    function make_staples(w::Wilson_loop)
        staples = make_staples(w.wilson_loop)
        return staples[1],staples[2],staples[3],staples[4]
    end

    function make_staples(links)
        numlinks = length(links)
        #println("num. of links: $numlinks")
        staples = Array{Array{Array{Tuple{Int,Int},1}}}(undef,4)
        for mu=1:4
            linkmu = []
            for (i,link) in enumerate(links)
                if link[1] == mu
                    append!(linkmu,i)
                end
            end
            #println(mu,"\t",linkmu)

            staple_mu = []

            if length(linkmu) == 0
                staple = Tuple{Int,Int}[]
                staple_mu = Array{Tuple{Int,Int},1}[]
            end

            staple_mu = Array{Array{Tuple{Int,Int},1}}(undef,length(linkmu))
            count = 0
            for i in linkmu
                count += 1
                staple = Array{Tuple{Int,Int}}(undef,numlinks-1)
                if i != numlinks
                    staple[1:numlinks-i] = links[i+1:end]
                end

                if i != 1
                    staple[numlinks-i+1:end] = links[1:i-1]
                end

                if links[i][2] == -1
                    staple = staple[end:-1:1]
                    for j = 1:numlinks-1
                        staple[j] = (staple[j][1],-staple[j][2])
                    end
                end
                #println(staple)
                staple_mu[count] = staple[:]
            end
            staples[mu] = staple_mu
            #println(staple_mu)
        end

        #println(staples)
        return staples
    end

    function calc_loopset_μν_name(name)
        loops_μν= Array{Wilson_loop_set,2}(undef,4,4)
        if name == "plaq"
            numofloops = 1
            for μ=1:4
                for ν=1:4
                    if ν == μ
                        continue
                    end
                    loops = Wilson_loop_set()
                    loop = Wilson_loop([(μ,1),(ν,1),(μ,-1),(ν,-1)])
                    push!(loops,loop)
                    loops_μν[μ,ν] = loops
                end
            end
        elseif name == "clover"
            numofloops = 4
            for μ=1:4
                for ν=1:4
                    if ν == μ
                        continue
                    end
                    loops = Wilson_loop_set()

                    loop_righttop = Wilson_loop([(μ,1),(ν,1),(μ,-1),(ν,-1)])
                    loop_lefttop = Wilson_loop([(ν,1),(μ,-1),(ν,-1),(μ,1)])
                    loop_rightbottom = Wilson_loop([(ν,-1),(μ,1),(ν,1),(μ,-1)])
                    loop_leftbottom= Wilson_loop([(μ,-1),(ν,-1),(μ,1),(ν,1)])
                    push!(loops,loop_righttop)
                    push!(loops,loop_lefttop)
                    push!(loops,loop_rightbottom)
                    push!(loops,loop_leftbottom)

                    loops_μν[μ,ν] = loops
                end
            end
        elseif name == "rect"
            numofloops = 8
            for μ=1:4
                for ν=1:4
                    if ν == μ
                        continue
                    end
                    loops = Wilson_loop_set()

                    loop_righttop = Wilson_loop([(μ,2),(ν,1),(μ,-2),(ν,-1)])
                    loop_lefttop = Wilson_loop([(ν,1),(μ,-2),(ν,-1),(μ,2)])
                    loop_rightbottom = Wilson_loop([(ν,-1),(μ,2),(ν,1),(μ,-2)])
                    loop_leftbottom= Wilson_loop([(μ,-2),(ν,-1),(μ,2),(ν,1)])
                    push!(loops,loop_righttop)
                    push!(loops,loop_lefttop)
                    push!(loops,loop_rightbottom)
                    push!(loops,loop_leftbottom)

                    loop_righttop = Wilson_loop([(μ,1),(ν,2),(μ,-1),(ν,-2)])
                    loop_lefttop = Wilson_loop([(ν,2),(μ,-1),(ν,-2),(μ,1)])
                    loop_rightbottom = Wilson_loop([(ν,-2),(μ,1),(ν,2),(μ,-1)])
                    loop_leftbottom= Wilson_loop([(μ,-1),(ν,-2),(μ,1),(ν,2)])
                    push!(loops,loop_righttop)
                    push!(loops,loop_lefttop)
                    push!(loops,loop_rightbottom)
                    push!(loops,loop_leftbottom)

                    loops_μν[μ,ν] = loops
                end
            end
        else
            error("$name is not supported")
        end
        return loops_μν,numofloops
    end

end


module Random

using Base.LibRandom

export srand,
       rand, rand!,
       randn, randn!,
       randbool, randbool!,
       AbstractRNG, RNG, MersenneTwister

abstract AbstractRNG

type MersenneTwister <: AbstractRNG
    state::DSFMT_state
    seed::Union(Uint32,Vector{Uint32})

    function MersenneTwister(seed::Vector{Uint32})
        state = DSFMT_state()
        dsfmt_init_by_array(state, seed)
        return new(state, seed)
    end

    MersenneTwister(seed=0) = MersenneTwister(make_seed(seed))
end

function srand(r::MersenneTwister, seed)
    r.seed = seed
    dsfmt_init_gen_rand(r.state, seed)
    return r
end

## initialization

function __init__()

@unix_only begin
    try
        srand("/dev/urandom")
    catch
        println(STDERR, "Entropy pool not available to seed RNG; using ad-hoc entropy sources.")
        seed = reinterpret(Uint64, time())
        seed = bitmix(seed, uint64(getpid()))
        try
            seed = bitmix(seed, parseint(Uint64, readall(`ifconfig` |> `sha1sum`)[1:40], 16))
        catch
            # ignore
        end
        srand(seed)
    end
end

@windows_only begin
    a = zeros(Uint32, 2)
    win32_SystemFunction036!(a)
    srand(a)
end
end

## srand()

function srand(seed::Vector{Uint32})
    global RANDOM_SEED = seed
    dsfmt_gv_init_by_array(seed)
end
srand(n::Integer) = srand(make_seed(n))

function make_seed(n::Integer)
    n < 0 && throw(DomainError())
    seed = Uint32[]
    while true
        push!(seed, n & 0xffffffff)
        n >>= 32
        if n == 0
            return seed
        end
    end
end

function srand(filename::String, n::Integer)
    open(filename) do io
        a = Array(Uint32, int(n))
        read!(io, a)
        srand(a)
    end
end
srand(filename::String) = srand(filename, 4)

## random floating point values

rand(::Type{Float64}) = dsfmt_gv_genrand_close_open()
rand() = dsfmt_gv_genrand_close_open()

rand(::Type{Float32}) = float32(rand())
rand(::Type{Float16}) = float16(rand())

rand{T<:Real}(::Type{Complex{T}}) = complex(rand(T),rand(T))

rand(r::MersenneTwister) = dsfmt_genrand_close_open(r.state)

## random integers

dsfmt_randui32() = dsfmt_gv_genrand_uint32()
dsfmt_randui64() = uint64(dsfmt_randui32()) | (uint64(dsfmt_randui32())<<32)

rand(::Type{Uint8})   = uint8(rand(Uint32))
rand(::Type{Uint16})  = uint16(rand(Uint32))
rand(::Type{Uint32})  = dsfmt_randui32()
rand(::Type{Uint64})  = dsfmt_randui64()
rand(::Type{Uint128}) = uint128(rand(Uint64))<<64 | rand(Uint64)

rand(::Type{Int8})    = int8(rand(Uint8))
rand(::Type{Int16})   = int16(rand(Uint16))
rand(::Type{Int32})   = int32(rand(Uint32))
rand(::Type{Int64})   = int64(rand(Uint64))
rand(::Type{Int128})  = int128(rand(Uint128))

# Arrays of random numbers

rand(::Type{Float64}, dims::Dims) = rand!(Array(Float64, dims))
rand(::Type{Float64}, dims::Int...) = rand(Float64, dims)

rand(dims::Dims) = rand(Float64, dims)
rand(dims::Int...) = rand(Float64, dims)

rand(r::AbstractRNG, dims::Dims) = rand!(r, Array(Float64, dims))
rand(r::AbstractRNG, dims::Int...) = rand(r, dims)

function rand!{T}(A::Array{T})
    for i=1:length(A)
        A[i] = rand(T)
    end
    A
end
rand(T::Type, dims::Dims) = rand!(Array(T, dims))
rand{T<:Number}(::Type{T}) = error("no random number generator for type $T; try a more specific type")
rand{T<:Number}(::Type{T}, dims::Int...) = rand(T, dims)

## Generate random integer within a range

# remainder function according to Knuth, where rem_knuth(a, 0) = a
rem_knuth(a::Uint, b::Uint) = a % (b + (b == 0)) + a * (b == 0)
rem_knuth{T<:Unsigned}(a::T, b::T) = b != 0 ? a % b : a

# maximum multiple of k <= 2^bits(T) decremented by one,
# that is 0xFFFFFFFF if k = typemax(T) - typemin(T) with intentional underflow
maxmultiple(k::Uint32) = convert(Uint32, div(0x0000000100000000,k + (k == 0))*k - 1)
maxmultiple(k::Uint64) = convert(Uint64, div(0x00000000000000010000000000000000, k + (k == 0))*k - 1)
# maximum multiple of k within 1:typemax(Uint128)
maxmultiple(k::Uint128) = div(typemax(Uint128), k + (k == 0))*k - 1
# maximum multiple of k within 1:2^32 or 1:2^64, depending on size
maxmultiplemix(k::Uint64) = convert(Uint64, div((k >> 32 != 0)*0x0000000000000000FFFFFFFF00000000 + 0x0000000100000000, k + (k == 0))*k - 1)

immutable RandIntGen{T<:Integer, U<:Unsigned}
    a::T   # first element of the range
    k::U   # range length or zero for full range
    u::U   # rejection threshold
end
# generators with 32, 128 bits entropy
RandIntGen{T, U<:Union(Uint32, Uint128)}(a::T, k::U) = RandIntGen{T, U}(a, k, maxmultiple(k))
# mixed 32/64 bits entropy generator
RandIntGen{T}(a::T, k::Uint64) = RandIntGen{T,Uint64}(a, k, maxmultiplemix(k))

# generator for ranges
RandIntGen{T<:Unsigned}(r::UnitRange{T}) = isempty(r) ? error("range must be non-empty") : RandIntGen(first(r), convert(T,last(r) - first(r) + 1))
# specialized versions
for (T, U) in [(Uint8, Uint32), (Uint16, Uint32),
               (Int8, Uint32), (Int16, Uint32), (Int32, Uint32), (Int64, Uint64), (Int128, Uint128),
               (Bool, Uint32), (Char, Uint32)]

    @eval RandIntGen(r::UnitRange{$T}) = isempty(r) ? error("range must be non-empty") : RandIntGen(first(r), convert($U, last(r) - first(r) + 1)) # overflow ok
end

# this function uses 32 bit entropy for small ranges of length <= typemax(Uint32) + 1
# RandIntGen is responsible for providing the right value of k
function rand{T<:Union(Uint64, Int64)}(g::RandIntGen{T,Uint64})
    local x::Uint64
    if (g.k - 1) >> 32 == 0
        x = rand(Uint32)
        while x > g.u
            x = rand(Uint32)
        end
    else
        x = rand(Uint64)
        while x > g.u
            x = rand(Uint64)
        end
    end
    return convert(T, g.a + rem_knuth(x, g.k))
end

function rand{T<:Integer, U<:Unsigned}(g::RandIntGen{T,U})
    x = rand(U)
    while x > g.u
        x = rand(U)
    end
    convert(T, g.a + rem_knuth(x, g.k))
end

rand{T<:Union(Signed,Unsigned,Bool,Char)}(r::UnitRange{T}) = rand(RandIntGen(r))
rand{T<:Real}(r::Range{T}) = convert(T, first(r) + rand(0:(length(r)-1)) * step(r))

function rand!(g::RandIntGen, A::AbstractArray)
    for i = 1 : length(A)
        @inbounds A[i] = rand(g)
    end
    return A
end

rand!{T<:Union(Signed,Unsigned,Bool,Char)}(r::UnitRange{T}, A::AbstractArray) = rand!(RandIntGen(r), A)

function rand!{T<:Real}(r::Range{T}, A::AbstractArray)
    g = RandIntGen(0:(length(r)-1))
    f = first(r)
    s = step(r)
    if s == 1
        for i = 1 : length(A)
            @inbounds A[i] = convert(T, f + rand(g))
        end
    else
        for i = 1 : length(A)
            @inbounds A[i] = convert(T, f + rand(g) * s)
        end
    end
    return A
end

rand{T<:Real}(r::Range{T}, dims::Dims) = rand!(r, Array(T, dims))
rand(r::Range, dims::Int...) = rand(r, dims)

## random Bools

rand!(B::BitArray) = Base.bitarray_rand_fill!(B)

randbool(dims::Dims) = rand!(BitArray(dims))
randbool(dims::Int...) = rand!(BitArray(dims))

randbool() = ((dsfmt_randui32() & 1) == 1)
randbool!(B::BitArray) = rand!(B)

## randn() - Normally distributed random numbers using Ziggurat algorithm

# The Ziggurat Method for generating random variables - Marsaglia and Tsang
# Paper and reference code: http://www.jstatsoft.org/v05/i08/

randn() = randmtzig_randn()
randn(rng::MersenneTwister) = randmtzig_randn(rng.state)
randn!(A::Array{Float64}) = (for i = 1:length(A);A[i] = randmtzig_randn();end;A)
randn!(rng::MersenneTwister, A::Array{Float64}) = (for i = 1:length(A);A[i] = randmtzig_randn(rng.state);end;A)
randn(dims::Dims) = randn!(Array(Float64, dims))
randn(dims::Int...) = randn!(Array(Float64, dims...))

## random UUID generation

immutable UUID
    value::Uint128
end

function uuid4()
    u = rand(Uint128)
    u &= 0xffffffffffff0fff3fffffffffffffff
    u |= 0x00000000000040008000000000000000
    UUID(u)
end

function Base.convert(::Type{Vector{Uint8}}, u::UUID)
    u = u.value
    a = Array(Uint8,36)
    for i = [36:-1:25; 23:-1:20; 18:-1:15; 13:-1:10; 8:-1:1]
        d = u & 0xf
        a[i] = '0'+d+39*(d>9)
        u >>= 4
    end
    a[[24,19,14,9]] = '-'
    return a
end

Base.show(io::IO, u::UUID) = write(io,convert(Vector{Uint8},u))
Base.repr(u::UUID) = ASCIIString(convert(Vector{Uint8},u))

end # module


using CUDA, Dates, Glob, SeisNoise, Serialization
include("MPI-CC-impl.jl")

# files and such
BASEDIR = "/media/four/"
FFTDIR = joinpath(BASEDIR,"LASSO/FFT/")
CORRDIR = joinpath(BASEDIR,"LASSO/CORR/")
if !isdir(CORRDIR)
    mkpath(CORRDIR)
end
files = glob("*",FFTDIR)
stas = [split(basename(s),'.')[2] |> (y -> parse(Int,y)) for s in files]
ind = sortperm(stas)
files = files[ind]

# create directory structure for saving CORRs
for sta in stas
    mkpath(joinpath(CORRDIR,"2A.$sta..DPZ"))
end

# parameters
maxlag = 200.
Nper = 18

# warm up functions
splits = splitFFT(files,Nper)
splitCC(splits[1:3],maxlag,CORRDIR)
testfiles = glob("*/*",CORRDIR)
rm.(testfiles)

t1 = now()
## xcorr on the GPU minimizing I/O
splitCC(splits,maxlag,CORRDIR)
t2 = now()
total = Dates.canonicalize(Dates.CompoundPeriod(t2-t1))
println("Total computation took $total")