"""
Object with a *term* and an applicable *BasisFunction* and a eventfield that are later passed to the basisfunction.

$(TYPEDEF)
$(TYPEDSIGNATURES)
$(FIELDS)
# Examples
```julia-repl
julia>  b = TimeExpandedTerm(term,kernel,[:latencyTR,:durationTR])
```
"""
struct TimeExpandedTerm{T<:AbstractTerm} <: AbstractTerm
        "Term that the basis function is applied to. This is regularly called in other functions to get e.g. term-coefnames and timeexpand those"
        term::T
        "Kernel that determines what should happen to the designmatrix of the term"
        basisfunction::BasisFunction
        "Which fields of the event-table should be passed to the basisfunction.Important: The first entry has to be the event-latency in samples!"
        eventfields::Array{Symbol}
end

function TimeExpandedTerm(term::NTuple{N,Union{<:AbstractTerm,<:MixedModels.RandomEffectsTerm}},basisfunction,eventfields::Array{Symbol,1}) where {N}
        # for mixed models, apply it to each Term
        TimeExpandedTerm.(term,Ref(basisfunction),Ref(eventfields))
end
function TimeExpandedTerm(term,basisfunction,eventfields::Nothing)
        # if the eventfield is nothing, call default
        TimeExpandedTerm(term, basisfunction)
end

function TimeExpandedTerm(term,basisfunction,eventfields::Symbol)
        # call with symbol, convert to array of symbol
        TimeExpandedTerm(term, basisfunction,[eventfields])
end

function TimeExpandedTerm(term,basisfunction;eventfields=[:latency])
        # default call, use latency
        TimeExpandedTerm(term, basisfunction,eventfields)
end

function Base.show(io::IO, p::TimeExpandedTerm)
        #print(io, "timeexpand($(p.term), $(p.basisfunction.type),$(p.basisfunction.times))")
        println(io,"$(coefnames(p))")
end

"""
To be explored abstractTerm structure to automatically detect randomeffects. Not in use
"""
struct ZeroCorr2{T<:RandomEffectsTerm} <: AbstractTerm
    term::T
end

"""
Object with a *term* and an applicable *BasisFunction* and a eventfield that are later passed to the basisfunction.

$(TYPEDEF)
$(TYPEDSIGNATURES)
$(FIELDS)
# Examples
```julia-repl
julia>  b = TimeExpandedTerm(term,kernel,[:latencyTR,:durationTR])
```
"""
struct DesignMatrix
        "Array of formulas"
        formulas
        "A concatenated designmatric. In case of Mixed Models an array, where the first one is a FeMat, later ones ReMats. "
        Xs
        "Event table with all events"
        events
end

"""
$(SIGNATURES)
Combine two UnfoldDesignmatrices. This allows combination of multiple events.

This also allows to define events with different lengths.

Not supported for models without timebasis, as it is not needed there (one can simply run multiple models)
# Examples
```julia-repl
julia>  basisfunction1 = firbasis(τ=(0,1),sfreq = 10,name="basis1")
julia>  basisfunction2 = firbasis(τ=(0,0.5),sfreq = 10,name="basis2")
julia>  Xdc1          = designmatrix(UnfoldLinearModel,@formula 0~1,tbl_1,basisfunction1)
julia>  Xdc2          = designmatrix(UnfoldLinearModel,@formula 0~1,tbl_2,basisfunction2)
julia>  combineDesignmatrices(Xdc1,Xdc2)
julia>  Xdc = Xdc1+Xdc2 # equivalently
```

"""
function combineDesignmatrices(X1::DesignMatrix,X2::DesignMatrix)
        X1 = deepcopy(X1)
        X2 = deepcopy(X2)
        Xs1 = get_Xs(X1.Xs)
        Xs2 = get_Xs(X2.Xs)

        sX1 = size(Xs1,1)
        sX2 = size(Xs2,1)

        # append 0 to the shorter designmat
        if sX1 < sX2
                Xs1 = SparseMatrixCSC(sX2, Xs1.n, Xs1.colptr, Xs1.rowval, Xs1.nzval)
        elseif sX2 < sX1
                Xs2 = SparseMatrixCSC(sX1, Xs2.n, Xs2.colptr, Xs2.rowval, Xs2.nzval)
        end
        Xcomb = hcat(Xs1,Xs2)

        #if !(Float64(X1.formulas.rhs.basisfunction.times.step) ≈ Float64(X2.formulas.rhs.basisfunction.times.step))
        #        @warn("Concatenating formulas with different sampling rates. Be sure that this is what you want.")
        #end
        if typeof(X1.Xs) <: Tuple
                # we have random effects
                # combine REMats in single-eventtpe formulas ala y ~ (1|x) + (a|x)
                Xs1 = MixedModels._amalgamate([X1.Xs[2:end]...],Float64)
                Xs2 = MixedModels._amalgamate([X2.Xs[2:end]...],Float64)

                Xcomb = (Xcomb,Xs1...,Xs2...)

                # Next we make the ranefs all equal size
                equalizeReMatLengths!(Xcomb[2:end])

                # check if ranefs can be amalgamated. If this fails, then MixedModels tried to amalgamate over different eventtypes and we should throw the warning
                # if it success, we have to check if the size before and after is identical. If it is not, it tried to amalgamize over different eventtypes which were of the same length

                try
                reterms = MixedModels._amalgamate([Xcomb[2:end]...],Float64)
                if length(reterms) != length(Xcomb[2:end])
                        throw("IncompatibleRandomGroupings")
                end
                catch e
                        @error "Error, you seem to have two different eventtypes with the same random-effect grouping variable. \n
                        This is not allowed, you have to rename one. Example:\n
                        eventA: y~1+(1|item) \n
                        eventB: y~1+(1|item)  \n
                        This leads to this error. Rename the later one\n
                        eventB: y~1+(1|itemB) "
                        throw("IncompatibleRandomGroupings")
                end

        end

        if typeof(X1.formulas) <: FormulaTerm
                DesignMatrix([X1.formulas X2.formulas],Xcomb,[X1.events, X2.events])
        else
                DesignMatrix([X1.formulas... X2.formulas],Xcomb,[X1.events, X2.events])
        end
end

function changeMatSize!(m,fe,remats)
        changeReMatSize!.(remats,Ref(m))
        fe = SparseMatrixCSC(m, fe.n, fe.colptr,fe.rowval, fe.nzval)
        return (fe,remats...,)
end
function changeReMatSize!(remat::MixedModels.AbstractReMat,m ::Integer)

        n = m-length(remat.refs) # missing elements
        if n<0
                deleteat!(remat.refs,range(m+1,stop=length(remat.refs)))
                remat.adjA = remat.adjA[:,1:(m+1)]
                remat.wtz = remat.wtz[:,1:(m+1)]
                remat.z = remat.z[:,1:(m+1)]
        elseif n>0
                append!(remat.refs,repeat([remat.refs[end]],n))
                remat.adjA = [remat.adjA sparse(repeat([0],size(remat.adjA)[1],n))]
                remat.wtz = [remat.wtz zeros((size(remat.wtz)[1],n))]
                remat.z = [remat.z zeros((size(remat.z)[1],n))]
        end

        #ReMat{T,S}(remat.trm, refs, levels, cnames, z, wtz, λ, inds, adjA, scratch)
end
function equalizeReMatLengths!(remats::NTuple{A,MixedModels.AbstractReMat}) where {A}
        # find max length
        m = maximum([x[1] for x in size.(remats)])
        @debug print("combining lengths: $m")
        # for each reMat
        for k = range(1,length=length(remats))
                remat = remats[k]
                if size(remat)[1] == m
                        continue
                end
                # prolong if necessary
                changeReMatSize!(remat,m)

        end
end

Base.:+(X1::DesignMatrix, X2::DesignMatrix) = combineDesignmatrices(X1,X2)

function get_Xs(Xs::Tuple)
        return Xs[1]
end
function get_Xs(Xs::SparseMatrixCSC)
        return Xs
end

"""
$(SIGNATURES)
designmatrix(type, f, tbl; kwargs...)
Return a *DesignMatrix* used to fit the models.
# Arguments
- type::Union{UnfoldLinearMixedModel,UnfoldLinearModel}
- f::FormulaTerm: Formula to be used in this designmatrix
- tbl: Events (usually a data frame) to be modelled
- basisfunction::BasisFunction: basisfunction to be used in modeling (if specified)
- contrasts::Dict: (optional) contrast to be applied to formula
- eventfields::Array: (optional) Array of symbols which are passed to basisfunction event-wise.
First field of array always defines eventonset in samples. Default is [:latency]

# Examples
```julia-repl
julia>  designmatrix(UnfoldLinearModel,f,tbl,basisfunction1)
```

"""
function designmatrix(type,f,tbl,basisfunction;contrasts= Dict{Symbol,Any}(), kwargs...)
        @debug("generating DesignMatrix")
        form = apply_schema(f, schema(f, tbl, contrasts), MixedModels.LinearMixedModel)
        form = apply_basisfunction(form,basisfunction,get(Dict(kwargs),:eventfields,nothing))

        # Evaluate the designmatrix
        if (!isnothing(basisfunction)) & (type<:UnfoldLinearMixedModel)
                X = modelcols.(form.rhs, Ref(tbl))
        else
                X = modelcols(form.rhs, tbl)
        end
        return DesignMatrix(form,X,tbl)
end
function designmatrix(type,f,tbl;kwargs...)
        return designmatrix(type,f,tbl,nothing;kwargs...)
end

"""
$(SIGNATURES)
timeexpand the rhs-term of the formula with the basisfunction

"""

function apply_basisfunction(form,basisfunction::BasisFunction,eventfields)
        @debug("apply_basisfunction")
        return FormulaTerm(form.lhs, TimeExpandedTerm(form.rhs,basisfunction,eventfields))
end

function apply_basisfunction(form,basisfunction::Nothing,eventfields)
        # in case of no basisfunctin, do nothing
        return form
end

"""
$(SIGNATURES)
calculates the actual designmatrix for a timeexpandedterm. Multiple dispatch on StatsModels.modelcols
"""
# Timeexpand the fixed effect part
function StatsModels.modelcols(term::TimeExpandedTerm,tbl)

        X = modelcols(term.term,tbl)
        time_expand(X,term,tbl)
end

mutable struct SparseReMat{T,S} <: MixedModels.AbstractReMat{T}
        trm
        refs::Vector{Int32}
        levels
        cnames::Vector{String}
        z::SparseMatrixCSC{T}
        wtz::SparseMatrixCSC{T}
        λ::LowerTriangular{T,Matrix{T}}
        inds::Vector{Int}
        adjA::SparseMatrixCSC{T,Int32}
        scratch::Matrix{T}
    end
# This function timeexpands the random effects and generates a ReMat object
function StatsModels.modelcols(term::TimeExpandedTerm{<:RandomEffectsTerm},tbl)
#function StatsModels.modelcols(term::TimeExpandedTerm{<:Union{<:RandomEffectsTerm,<:AbstractTerm{<:RandomEffectsTerm}}},tbl)
# exchange this to get ZeroCorr to work

        # get the non-timeexpanded reMat
        reMat = modelcols(term.term,tbl)

        # Timeexpand the designmatrix
        z = transpose(time_expand(transpose(reMat.z),term,tbl))

        # First we check if there is overlap in the timeexpanded term. If so, we cannot continue. Later implementations will remedy that
        #println(dump(term,))
        if hasfield(typeof(term.term),:rhs)
                rhs = term.term.rhs

        elseif hasfield(typeof(term.term.term),:rhs)
                # we probably have something like zerocorr, which does not need to show a .rhs necessarily
                rhs = term.term.term.rhs
        else

                println("term.term: $(dump(term.term))")
                error("unknown RE structure, has no field .rhs:$(typeof(term.term))")

        end

        group = tbl[!,rhs.sym]
        time = tbl[!,term.eventfields[1]]

        # get the from-to onsets of the grouping varibales
        onsets = time_expand_getRandomGrouping(group,time,term.basisfunction)
        #print(size(reMat.z))
        refs = zeros(size(z)[2]).+1
        for (i,o) in enumerate(onsets[2:end])
                # check for overlap
                if (minimum(o) <= maximum(onsets[i+1])) & (maximum(o) <= minimum(onsets[i+1]))
                        error("overlap in random effects structure detected, not currently supported")
                end
        end

        # From now we can assume no overlap
        # We want to fnd which subject is active when
        refs = zeros(size(z)[2]).+1
        uGroup = unique(group)

        for (i,g) = enumerate(uGroup[1:end])

                ix_start = findfirst(g.==group)
                ix_end = findlast(g.==group)
                if i == 1
                        time_start = 1
                else
                        time_start = time[ix_start]
                        time_start = time_start - sum(term.basisfunction.times.<=0)
                end
                if i == length(uGroup)
                        time_stop = size(refs,1)
                else
                        time_stop = time[ix_end]
                        time_stop = time_stop + sum(term.basisfunction.times.>0)
                end
                if time_start < 0
                        time_start = 1
                end

                if time_stop > size(refs,1)
                        time_stop = size(refs,1)
                end

                #println("$g,$time_start,$time_stop")
                refs[Int64(time_start):Int64(time_stop)] .= g
        end

        # Other variables with implementaions taken from the LinerMixedModel function
        wtz = z
        trm = term

        S = size(z, 1)
        T = eltype(z)
        λ  = LowerTriangular(Matrix{T}(I, S, S))

        inds = MixedModels.sizehint!(Int[], (S * (S + 1)) >> 1)
        m = reshape(1:abs2(S), (S, S))
        inds = sizehint!(Int[], (S * (S + 1)) >> 1)
        for j in 1:S
                for i in j:S
                        # We currently restrict to diagonal entries
                        # Once mixedmodels#293 is pushed, we can relax this and use zerocorr()
                        if i == j # for diagonal
                                push!(inds, m[i, j])
                        end
                end
        end

        levels = reMat.levels
        refs =refs

        # reMat.levels doesnt change
        cnames = coefnames(term)
        #print(refs)
        adjA = MixedModels.adjA(refs, z)
        scratch = Matrix{T}(undef, (S, length(uGroup)))

        # Once MixedModels.jl supports it, can be replaced with: SparseReMat
        ReMat{T,S}(rhs,
        refs,
        levels,
        cnames,
        z,
        wtz,
        λ,
        inds,
        adjA,
        scratch)
end

"""
$(SIGNATURES)
Get the timeranges where the random grouping variable was applied
"""
function time_expand_getRandomGrouping(tblGroup,tblLatencies,basisfunction)
        ranges = time_expand_getTimeRange.(tblLatencies,Ref(basisfunction))
end

# helper function to get the ranges from where to where the basisfunction is added
function time_expand_getTimeRange(onset,basisfunction)
        npos = sum(basisfunction.times.>=0)
        nneg = sum(basisfunction.times.<0)

        basis = basisfunction.kernel(onset)

        fromRowIx = floor(onset)-nneg
        toRowIx = floor(onset)+npos

        range(fromRowIx,stop=toRowIx)
end

"""
$(SIGNATURES)
performs the actual time-expansion in a sparse way.

 - Get the non-timeexpanded designmatrix X from StatsModels.
 - evaluate the basisfunction kernel at each event
 - calculate the necessary rows, cols and values for the sparse matrix
 Returns SparseMatrixCSC
"""
function time_expand(X,term,tbl)
        ncolsBasis = size(term.basisfunction.kernel(0),2)
        X = reshape(X,size(X,1),:)

        ncolsX = size(X)[2]
        nrowsX = size(X)[1]
        ncolsXdc = ncolsBasis*ncolsX

        # this is the predefined eventfield, usually "latency"
        onsets = tbl[!,term.eventfields[1]]

        if typeof(term.eventfields) <:Array && length(term.eventfields) == 1
                bases = term.basisfunction.kernel.(tbl[!,term.eventfields[1]])
        else
                bases = term.basisfunction.kernel.(eachrow(tbl[!,term.eventfields]))
        end

        # generate rowindices
        rows =  copy(rowvals.(bases))
        # this shift is necessary as some basisfunction time-points can be negative. But a matrix is always from 1:τ. Thus we have to shift it backwards in time.
        # The onsets are onsets-1 XXX not sure why.
        for r in 1:length(rows)
                rows[r] .+= floor(onsets[r]-1)+term.basisfunction.shiftOnset
        end

        rows = vcat(rows...)
        rows = repeat(rows,ncolsX)

        # generate column indices
        cols = []
        for Xcol in 1:ncolsX
                for b in 1:length(bases)
                        for c in 1:ncolsBasis
                                push!(cols,repeat([c+(Xcol-1)*ncolsBasis],length(nzrange(bases[b],c))))
                        end
                end
        end
        cols = vcat(cols...)

        # generate values
        vals = []
        for Xcol in 1:ncolsX
                push!(vals,vcat(nonzeros.(bases).*X[:,Xcol]...))
        end

        vals = vcat(vals...)
        ix = rows.>0
        A = sparse(rows[ix],cols[ix],vals[ix])

        return A
end

"""
$(SIGNATURES)
coefnames of a TimeExpandedTerm concatenates the basis-function name with the kronecker product of the term name and the basis-function colnames. Separator is ' : '
Some examples for a firbasis:
        basis_313 : (Intercept) : 0.1
        basis_313 : (Intercept) : 0.2
        basis_313 : (Intercept) : 0.3
        ...
"""
function StatsModels.coefnames(term::TimeExpandedTerm)
        terms = coefnames(term.term)
        colnames = term.basisfunction.colnames
        name = term.basisfunction.name
        if typeof(terms) == String
                terms = [terms]
        end
        return name.*" : ".*kron(terms.*" : ",string.(colnames))
end

function StatsModels.coefnames(terms::AbstractArray{<:FormulaTerm})
        return coefnames.(Base.getproperty.(terms,:rhs))
end
function StatsModels.coefnames(term::MixedModels.ZeroCorr)
        coefnames(term.term)
end

function StatsModels.coefnames(term::RandomEffectsTerm)
        coefnames(term.lhs)
end


#=
Copyright (c) 2015, Intel Corporation

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

    * Redistributions of source code must retain the above copyright notice,
      this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in the
      documentation and/or other materials provided with the distribution.
    * Neither the name of Intel Corporation nor the names of its contributors
      may be used to endorse or promote products derived from this software
      without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
=#

export Neuron, neuron, @neuron
abstract Neuron

function init(neuron::Neuron)
end

"""
Specify a neuron function using the syntax
```
@neuron forward(neuron::NeuronSubType) do
    ...
end
```
"""
macro neuron(expr)
    @assert isa(expr, Expr) "@neuron macro expects an expression"
    if expr.head == :call
        @eval function $(expr.args[1])($(expr.args[3:end]...))
            return $(Expr(:quote, expr.args[2].args[2]))
        end
    elseif expr.head == :type
        type_defn = expr.args[2]
        expr.args[2] = Expr(:(<:), type_defn, :Neuron)
        original = remove_line_nodes(copy(expr.args[3])).args
        T = Float32
        block = expr.args[3].args
        unshift!(block, :(∇inputs :: DenseArray{$T}))
        unshift!(block, :(inputs  :: DenseArray{$T}))
        unshift!(block, :(∇       :: $T))
        unshift!(block, :(value   :: $T))
        args = [a.args[1] for a in original]
        defn = quote
            function $type_defn($(original...))
                $type_defn(zero($T), zero($T), Array($T, 1), Array($T, 1),
                           $(args...))
            end
        end

        result = quote
            $expr
            $defn
        end
        @eval $result
        return Expr(:escape, quote
            typealias $type_defn Latte.$type_defn
        end)
    else
        throw("Not Implemented")
    end
end


function merge_decomposition!(db::SQLite.DB, id::Int, origin_id::Int, name::AbstractString, scenario::Union{Int, AbstractString},
                              clique_path::AbstractString, cliquetree_path::AbstractString, graph_path::AbstractString,
                              nb_added_edge_dec::Int,
                              heuristic::Vector{String}, heuristic_switch::Vector{Int},
                              treshold_name::AbstractString, merge_kwargs::AbstractDict;
                              rng)
    println("Merging decomposition: ($name, $scenario) $id")

    # Retrieve the cliques array and the cliquetree
    clique = read_clique(clique_path)
    cliquetree = read_cliquetree(cliquetree_path)

    # Merge
    merged_clique, merged_cliquetree = merge_dec(clique, cliquetree, heuristic, heuristic_switch;
                                                 treshold_name=treshold_name, merge_kwargs=merge_kwargs)

    # Extract features
    g = loadgraph(graph_path)
    merged_g = build_graph_from_clique(merged_clique)
    features = get_features_graph(merged_g)
    nb_added_edge = ne(merged_g) - ne(g)
    features["nb_lc"] = get_nb_lc(merged_clique, merged_cliquetree)
    features["nb_added_edge_dec"] = nb_added_edge_dec + nb_added_edge
    merge!(features, get_clique_features(merged_clique))

    # Save cliques, cliquetree and graph
    cliques_path = dirname(clique_path)
    cliquetrees_path = dirname(cliquetree_path)
    graphs_path = dirname(graph_path)
    uuid = uuid1(rng)
    clique_path = joinpath(cliques_path, "$(name)_$(scenario)_$(uuid)_clique.csv")
    cliquetree_path = joinpath(cliquetrees_path, "$(name)_$(scenario)_$(uuid)_cliquetree.csv")
    graph_path_merge = joinpath(graphs_path, "$(name)_$(scenario)_$(uuid)_graph.lgz")
    save_clique(merged_clique, clique_path)
    save_cliquetree(merged_cliquetree, cliquetree_path)
    savegraph(graph_path_merge, merged_g)

    # Other columns
    date = Dates.now()

    features = Dict(Symbol(k) => v for (k, v) in features)
    insert_decomposition!(db, origin_id, uuid, name, scenario, "merge", "", date, clique_path, cliquetree_path, graph_path_merge; features...)

    # Insert in merge table
    out_id = DBInterface.execute(db, "SELECT id FROM decompositions WHERE uuid = '$uuid'") |> DataFrame
    out_id = out_id[1, :id]
    heuristics_db = join(heuristic, ":")
    treshold_percent = merge_kwargs["treshold_percent"]
    insert_merge!(db, id, out_id, heuristics_db, treshold_name, treshold_percent, nb_added_edge)
end

function merge_decomposition_dfrow!(db::SQLite.DB, row,
                                    heuristic::Vector{String}, heuristic_switch::Vector{Int},
                                    treshold_name::AbstractString, merge_kwargs::AbstractDict; rng)
    merge_decomposition!(db, row[:id], row[:origin_id], row[:origin_name], row[:origin_scenario], row[:clique_path], row[:cliquetree_path], row[:graph_path], row[:nb_added_edge_dec],
                         heuristic, heuristic_switch, treshold_name, merge_kwargs, rng=rng)
end

function merge_decompositions!(db::SQLite.DB;
                               heuristic::Vector{String}, heuristic_switch::Vector{Int},
                               treshold_name::AbstractString, merge_kwargs::AbstractDict,
                               min_nv=typemin(Int), max_nv=typemax(Int), subset=nothing,
                               rng=MersenneTwister(42))

    query = "SELECT id, origin_id, origin_name, origin_scenario, clique_path, cliquetree_path, graph_path, nb_added_edge_dec FROM decompositions WHERE nb_vertex >= $min_nv AND nb_vertex <= $max_nv"
    if !isnothing(subset)
        query *= " AND id IN ($(join(subset, ',')))"
    end
    results = DBInterface.execute(db, query) |> DataFrame
    println("Merging config: heuristic=$heuristic heuristic_switch=$heuristic_switch, treshold_name=$treshold_name, min_nv=$min_nv, max_nv=$max_nv, rng=$rng")
    println("merge_kwargs\n$merge_kwargs\nend merge_kwargs")
    println("subset\n$subset\nend subset")
    merge_function!(row) = merge_decomposition_dfrow!(db, row,
                                                      heuristic, heuristic_switch, treshold_name, merge_kwargs;
                                                      rng=rng)
    merge_function!.(eachrow(results[!, [:id, :origin_id, :origin_name, :origin_scenario, :clique_path, :cliquetree_path, :graph_path, :nb_added_edge_dec]]))
end


#///////////////////////////////////////
#// File Name: type_definition_test.jl
#// Author: Haruki Nishimura (hnishimura@stanford.edu)
#// Date Created: 2021/01/15
#// Description: Test code for src/type_definition.jl
#///////////////////////////////////////

using Distributions
using LinearAlgebra
using RobotOS

@testset "Type Definition Test" begin

# WayPoint & Trajectory
p1 = [3., 4.];
p2 = [2., 1.];
wp1 = WayPoint2D(p1, Time(0.0));
wp2 = WayPoint2D(p2, Time(1.0));
traj = Trajectory2D([wp1, wp2]);
@test get_position(traj, Time(0.0)) == p1;
@test get_position(traj, Time(0.1)) ≈ [2.9, 3.7];
@test get_position(traj, Time(1.0)) == p2;

# Robot State
e1 = RobotState([p1; 0.; 0.]);
@test e1.x == [3., 4., 0., 0.];
@test get_position(e1) == [3., 4.];
@test get_velocity(e1) == [0., 0.];
@test e1 == RobotState([p1; 0.; 0;], Time(0.0));
@test e1 ≈ e1;

# World State
ap_dict = Dict("Pedestrian/1" => p2);
w = WorldState(e1, ap_dict);
@test w.t == Time(0.0);
@test isapprox(w, w);

end


module BridgeSDEInference

using Bridge, StaticArrays, Distributions
using Statistics, Random, LinearAlgebra
using ForwardDiff
using ForwardDiff: value

# fitzHughNagumo.jl
export FitzhughDiffusion, FitzhughDiffusionAux
export regularToAlter, alterToRegular, regularToConjug, conjugToRegular, display

# types.jl
export ImproperPrior, NoChangePt, SimpleChangePt

# mcmc.jl
export mcmc, PartObs, FPT, FPTInfo, ConjugateUpdt, MetropolisHastingsUpdt

# ODE solvers:
export Ralston3, RK4, Tsit5, Vern7

# random_walk.jl
export RandomWalk

# priors.jl
export Priors

# blocking_schedule.jl
export NoBlocking, ChequeredBlocking

# starting_pt.jl
export KnownStartingPt, GsnStartingPt

include("fitzHughNagumo.jl")
include("fitzHughNagumo_conjugateUpdt.jl")

include("types.jl")
include("ralston3.jl")
include("rk4.jl")
include("tsit5.jl")
include("vern7.jl")

include("priors.jl")

include("guid_prop_bridge.jl")
include("random_walk.jl")
include("blocking_schedule.jl")
include("starting_pt.jl")
include("mcmc.jl")
include("path_to_wiener.jl")

end


@everywhere using SeisDownload
using Dates, LightXML

#==================================================#
# Input Parameters
MAX_MEM_PER_CPU = 2.0 # [GB] maximum allocated memory for one cpu
DownloadType = "Earthquake" # Choise of "Noise" or "Earthquake"

network     = ["*"] #Currently only one network is available when you want to specify.
station     = ["*"]
location    = ["*"]
channel     = ["BH?"]

#Specify region [lat_min, lat_max, lon_min, lon_max, dep_min, dep_max], with lat, lon in decimal degrees (°) and depth in km with + = down.
#dep_min and dep_max is optional.
IsLocationBox = true
locationbox   = [30.9332, 41.4757, 129.9166, 145.3711]

method      = "FDSN" # Method to download data.
datasource  = "IRIS"

SecondsBeforePick = 10 # [s] start downloading `SecondsBeforePick`[s] before picked time
SecondsAfterPick  = 10 * 60 # [s] end downloading `SecondsBeforePick`[s] after picked time

# Time info for Noise case
catalog  = "./fdsnws-event_JP.xml" #QuakeML format

IsResponseRemove = true #whether instrumental response is removed or not
pre_filt    = (0.001, 0.002, 10.0, 20.0) #prefilter tuple used obspy remove_response: taper between f1 and f2, f3 and f4 with obspy

fodir       = "./dataset"
foname      = "JPquake" # data is saved at ./dataset/$foname.jld2
#==================================================#

# Read QuakeML
# This can be customized by users

xdoc = parse_file(catalog)
xroot = root(xdoc)  # an instance of XMLElement
# print its name
ces = collect(child_elements(xroot))
ev = ces[1]["event"]
Ne = length(ev)

event = [] # event information (starttime, lat, lon, mag...)

for i = 1:Ne

    Npicks = 1
    #println("index of quake ", i, " Number of picks  ",Npicks)

    #event information
    publicid    = attribute(ev[i]["origin"][1], "publicID")
    catalog     = attribute(ev[i]["origin"][1], "catalog")
    time        = content(ev[i]["origin"][1]["time"][1])[2:end-1]
    latitude    = content(ev[i]["origin"][1]["latitude"][1])[2:end-1]
    longitude   = content(ev[i]["origin"][1]["longitude"][1])[2:end-1]
    depth       = content(ev[i]["origin"][1]["depth"][1])[2:end-1]
    magnitude   = content(ev[i]["magnitude"][1]["mag"][1])[2:end-1]
    type        = content(ev[i]["magnitude"][1]["type"][1])
    origin      = Dict("publicid" => publicid, "catalog" => catalog, "time" => time,
    "latitude"=>latitude, "longitude"=>longitude, "depth"=>depth, "magnitude"=>magnitude, "type"=>type)

    # loop through all phase picked
    # create a dictionary for each quake with channel information and time.
    ptime=Vector{String}(undef,Npicks)
    net=Vector{String}(undef,Npicks)
    sta=Vector{String}(undef,Npicks)
    cha=Vector{String}(undef,Npicks)
    loc=Vector{String}(undef,Npicks)
    src=Vector{String}(undef,Npicks)

    pickphase_dict = []

    for j=1:Npicks
        ptime[j] = content(ev[i]["origin"][1]["time"][1])[2:end-1]

        net[j] = network[1] # network code
        sta[j] = station[1] # station code
        loc[j] = location[1] # station code
        cha[j] = channel[1] # channel code
        src[j] = datasource # data center

        starttime = DateTime(ptime[j])
        starttime = starttime - Dates.Second(SecondsBeforePick)
        endtime  = starttime + Dates.Second(SecondsAfterPick)
        # prints for sanity check

        pdict_temp = Dict("starttime" => starttime, "endtime" => endtime,
         "net" => net[j], "sta" => sta[j], "loc" => loc[j], "cha" => cha[j], "src" => src[j])

        push!(pickphase_dict, pdict_temp)

    end

    event_temp = Dict("origin" => origin, "pickphase" => pickphase_dict)
    push!(event, event_temp)
end

free(xdoc)

mkpath(fodir)
fopath=joinpath(fodir, foname*".jld2")

#if lat-log box or not
IsLocationBox ? reg=locationbox : reg=[]
IsResponseRemove ? pre_filt = pre_filt : pre_filt = []

InputDictionary = Dict([
      "DownloadType"    => DownloadType,
      "method"          => method,
      "event"           => event,
      "IsLocationBox"   => IsLocationBox,
      "reg"             => reg,
      "IsResponseRemove"=> IsResponseRemove,
      "pre_filt"        => pre_filt,
      "fopath"          => fopath

    ])

# mass request with input Dictionary
seisdownload(InputDictionary, MAX_MEM_PER_CPU=float(MAX_MEM_PER_CPU))


@testset "reinterpret" begin
    # Gray
    for sz in ((4,), (4,5))
        a = rand(Gray{N0f8}, sz)
        for T in (Gray{N0f8}, Gray{Float32}, Gray{Float64})
            b = @inferred(convert(Array{T}, a))
            rb = @inferred(reinterpretc(eltype(T), b))
            @test eltype(rb) == eltype(T) && ndims(rb) == length(sz)
            @test size(rb) == sz
            c = copy(rb)
            rc = @inferred(reinterpretc(T, c))
            @test eltype(rc) == T && ndims(rc) == length(sz)
            @test size(rc) == sz
        end
    end
    for sz in ((4,), (4,5))
        # Bool/Gray{Bool}
        b = rand(Bool, sz)
        rb = @inferred(reinterpretc(Gray{Bool}, b))
        @test eltype(rb) == Gray{Bool} && ndims(rb) == length(sz)
        @test size(rb) == sz
        c = copy(rb)
        rc = @inferred(reinterpretc(Bool, c))
        @test eltype(rc) == Bool && ndims(rc) == length(sz)
        @test size(rc) == sz
    end
    for sz in ((4,), (4,5))
        b = Gray24.(reinterpret(N0f8, rand(UInt8, sz)))
        for T in (UInt32, RGB24)
            rb = @inferred(reinterpretc(T, b))
            @test eltype(rb) == T && ndims(rb) == length(sz)
            @test size(rb) == sz
            c = copy(rb)
            rc = @inferred(reinterpretc(Gray24, c))
            @test eltype(rc) == Gray24 && ndims(rc) == length(sz)
            @test size(rc) == sz
        end
    end
    # TransparentGray
    a = rand(AGray{N0f8}, (4,5))
    for T in (AGray{N0f8}, GrayA{Float32}, AGray{Float64})
        b = @inferred(convert(Array{T}, a))
        rb = @inferred(reinterpretc(eltype(T), b))
        @test eltype(rb) == eltype(T) && ndims(rb) == 3
        @test size(rb) == (2,4,5)
        c = copy(rb)
        rc = @inferred(reinterpretc(T, c))
        @test eltype(rc) == T && ndims(rc) == 2
        @test size(rc) == (4,5)
    end
    # Color3
    a = rand(RGB{N0f8}, (4,5))
    for T in (RGB{N0f8}, HSV{Float32}, XYZ{Float64})
        b = @inferred(convert(Array{T}, a))
        rb = @inferred(reinterpretc(eltype(T), b))
        @test eltype(rb) == eltype(T) && ndims(rb) == 3
        @test size(rb) == (3,4,5)
        c = copy(rb)
        rc = @inferred(reinterpretc(T, c))
        @test eltype(rc) == T && ndims(rc) == 2
        @test size(rc) == (4,5)
    end
    for a in (rand(RGB{N0f8}, 4), rand(RGB{N0f8}, (4,5)))
        b = @inferred(reinterpretc(HSV{Float32}, float32.(a)))
        @test eltype(b) == HSV{Float32}
        @test ndims(b) == ndims(a)
    end
    # Transparent color
    a = rand(ARGB{N0f8}, (4,5))
    for T in (ARGB{N0f8}, AHSV{Float32}, AXYZ{Float64})
        b = @inferred(convert(Array{T}, a))
        rb = @inferred(reinterpretc(eltype(T), b))
        @test eltype(rb) == eltype(T) && ndims(rb) == 3
        @test size(rb) == (4,4,5)
        c = copy(rb)
        rc = @inferred(reinterpretc(T, c))
        @test eltype(rc) == T && ndims(rc) == 2
        @test size(rc) == (4,5)
    end
    # XRGB/RGBX
    a = rand(RGB{N0f8}, (4,5))
    for T in (XRGB{N0f8},RGBX{Float32})
        b = @inferred(convert(Array{T}, a))
        rb = @inferred(reinterpretc(eltype(T), b))
        @test eltype(rb) == eltype(T) && ndims(rb) == 3
        @test size(rb) == (4,4,5)
        c = copy(rb)
        rc = @inferred(reinterpretc(T, c))
        @test eltype(rc) == T && ndims(rc) == 2
        @test size(rc) == (4,5)
    end
    a = [RGB(1,0,0) RGB(0,0,1);
         RGB(0,1,0) RGB(1,1,1)]
    @test @inferred(reinterpretc(N0f8, a)) == cat([1 0; 0 1; 0 0], [0 1; 0 1; 1 1]; dims=3)
    b = convert(Array{BGR{N0f8}}, a)
    @test @inferred(reinterpretc(N0f8, b)) == cat([0 0; 0 1; 1 0], [1 1; 0 1; 0 1]; dims=3)
    # RGB24, ARGB32
    for sz in ((4,), (4,5))
        a = rand(UInt32, sz)
        for T in (RGB24, ARGB32)
            b = @inferred(reinterpretc(T, a))
            @test eltype(b) == T && ndims(b) == length(sz)
            @test size(b) == sz
            @test eltype(b) == T
            @test @inferred(reinterpret(UInt32, b)) == a
        end
    end

    # 1d
    a = RGB{Float64}[RGB(1,1,0)]
    af = @inferred(reinterpretc(Float64, a))
    anew = @inferred(reinterpretc(RGB, vec(af)))
    @test anew[1] == a[1]
    @test ndims(anew) == 0

    # #33 and its converse
    a = reinterpretc(BGRA{N0f8}, [0xf0884422])
    @test isa(a, AbstractVector) && a == [BGRA{N0f8}(0.533,0.267,0.133,0.941)]
    @test reinterpretc(UInt32, a) == [0xf0884422]
    @test size(reinterpretc(BGRA{N0f8}, rand(UInt32, 5, 5))) == (5,5)
    @test size(color_view(ARGB32, rand(BGRA{N0f8}, 5, 5))) == (5,5)
    a = reinterpretc(BGRA{N0f8}, reshape([0x22, 0x44, 0x88, 0xf0, 0x01, 0x02, 0x03, 0x04], 4, 2))
    @test a == [BGRA{N0f8}(0.533,0.267,0.133,0.941), BGRA{N0f8}(0.012, 0.008, 0.004, 0.016)]
    @test reinterpret(UInt8, a) == [0x22, 0x44, 0x88, 0xf0, 0x01, 0x02, 0x03, 0x04]
    @test color_view(ARGB32, a) == reinterpretc(ARGB32, [0xf0884422,0x04030201])

    # indeterminate type tests
    a = Array{RGB{AbstractFloat}}(undef, 3)
    @test_throws ErrorException reinterpretc(Float64, a)
    a = Vector{RGB}(undef, 3)
    @test_throws ErrorException reinterpretc(Float64, a)

    # Invalid conversions
    a = rand(UInt8, 4,5)
    ret = @test_throws TypeError reinterpretc(Gray, a)
    a = rand(Int8, 4,5)
    ret = @test_throws TypeError reinterpretc(Gray, a)
end

@testset "convert" begin
    a = [RGB(1,0,0) RGB(0,0,1);
         RGB(0,1,0) RGB(1,1,1)]
    c = @inferred(convert(Array{BGR}, a))
    @test eltype(c) == BGR{N0f8}
    c = @inferred(convert(Array{BGR{Float32}}, a))
    @test eltype(c) == BGR{Float32}
    c = @inferred(convert(Array{Lab}, a))
    @test eltype(c) == Lab{Float32}
    for a in (rand(Float32, (4,5)),
              bitrand(4,5))
        b = @inferred(convert(Array{Gray}, a))
        @test eltype(b) == Gray{eltype(a)}
        b = @inferred(convert(Array{Gray{N0f8}}, a))
        @test eltype(b) == Gray{N0f8}
    end

    # Gray images wrapped by an OffsetArray.
    A = rand(8,8)
    for img in ( Gray.(A),
                 Gray.(N0f8.(A)),
                 Gray.(N0f16.(A)) )
        imgo = OffsetArray(img, -2, -1)
        s = @inferred(convert(OffsetArray{Gray{Float32},2,Array{Gray{Float32},2}}, imgo))
        @test eltype(s) == Gray{Float32}
        @test s isa OffsetArray{Gray{Float32},2,Array{Gray{Float32},2}}
        @test permutedims(permutedims(s,(2,1)),(2,1)) == s
        @test axes(s) === axes(imgo)
    end

    for img in ( Gray.(A),
                 Gray.(N0f8.(A)),
                 Gray.(N0f16.(A)) )
        imgo = OffsetArray(img, -2, -1)
        s = @inferred(convert(OffsetArray{Gray{N0f8},2,Array{Gray{N0f8},2}}, imgo))
        @test eltype(s) == Gray{N0f8}
        @test s isa OffsetArray{Gray{N0f8},2,Array{Gray{N0f8},2}}
        @test permutedims(permutedims(s,(2,1)),(2,1)) == s
        @test axes(s) === axes(imgo)
    end

    for img in ( Gray.(A),
                 Gray.(N0f8.(A)),
                 Gray.(N0f16.(A)) )
        imgo = OffsetArray(img, -2, -1)
        s = @inferred(convert(OffsetArray{Gray{N0f16},2,Array{Gray{N0f16},2}},imgo))
        @test eltype(s) == Gray{N0f16}
        @test s isa OffsetArray{Gray{N0f16},2,Array{Gray{N0f16},2}}
        @test permutedims(permutedims(s,(2,1)),(2,1)) == s
        @test axes(s) === axes(imgo)
    end

    # Color images wrapped by an OffsetArray.
    A = rand(RGB{Float32},8,8)
    for img in ( A,
                 n0f8.(A),
                 n6f10.(A),
                 n4f12.(A),
                 n2f14.(A),
                 n0f16.(A))
        imgo = OffsetArray(img, -2, -1)
        s = @inferred(convert(OffsetArray{RGB{N0f8},2,Array{RGB{N0f8},2}},imgo))
        @test eltype(s) == RGB{N0f8}
        @test s isa OffsetArray{RGB{N0f8},2,Array{RGB{N0f8},2}}
        @test permutedims(permutedims(s,(2,1)),(2,1)) == s
        @test axes(s) === axes(imgo)
    end

    A = rand(RGB{Float32},8,8)
    for img in ( A,
                 n0f8.(A),
                 n6f10.(A),
                 n4f12.(A),
                 n2f14.(A),
                 n0f16.(A))
        imgo = OffsetArray(img, -2, -1)
        s = @inferred(convert(OffsetArray{RGB{Float32},2,Array{RGB{Float32},2}},imgo))
        @test eltype(s) == RGB{Float32}
        @test s isa OffsetArray{RGB{Float32},2,Array{RGB{Float32},2}}
        @test permutedims(permutedims(s,(2,1)),(2,1)) == s
        @test axes(s) === axes(imgo)
    end
end

@testset "eltype conversion" begin
    @test float32(Float64) == Float32
    @test float32(N0f8)      == Float32
    @test float64(RGB{N0f8}) == RGB{Float64}

    a = [RGB(1,0,0) RGB(0,0,1);
         RGB(0,1,0) RGB(1,1,1)]
    @test eltype(a) == RGB{N0f8}
    @test eltype(n0f8.(a))       == RGB{N0f8}
    @test eltype(n6f10.(a)) == RGB{N6f10}
    @test eltype(n4f12.(a)) == RGB{N4f12}
    @test eltype(n2f14.(a)) == RGB{N2f14}
    @test eltype(n0f16.(a)) == RGB{N0f16}
#    @test eltype(float16.(a)) == RGB{Float16}
    @test eltype(float32.(a)) == RGB{Float32}
    @test eltype(float64.(a)) == RGB{Float64}

    a = N0f8[0.1,0.2,0.3]
    @test eltype(a) == N0f8
    @test eltype(n0f8.(a))       == N0f8
    @test eltype(n6f10.(a)) == N6f10
    @test eltype(n4f12.(a)) == N4f12
    @test eltype(n2f14.(a)) == N2f14
    @test eltype(n0f16.(a)) == N0f16
#    @test eltype(float16.(a)) == Float16
    @test eltype(float32.(a)) == Float32
    @test eltype(float64.(a)) == Float64

    a = OffsetArray(N0f8[0.1,0.2,0.3], -1:1)
    @test eltype(a) == N0f8
    @test eltype(n0f8.(a))       == N0f8
    @test eltype(n6f10.(a)) == N6f10
    @test eltype(n4f12.(a)) == N4f12
    @test eltype(n2f14.(a)) == N2f14
    @test eltype(n0f16.(a)) == N0f16
#    @test eltype(float16.(a)) == Float16
    @test eltype(float32.(a)) == Float32
    @test eltype(float64.(a)) == Float64
    @test axes(float32.(a)) == (-1:1,)

    @testset "float" begin
        # float(::Type) is equivalent to floattype
        @test @inferred(float(RGBA{Float32})) == RGBA{Float32}
        @test @inferred(float(BGR{N0f8})    ) == BGR{Float32}
        @test @inferred(float(Gray{N0f8})   ) == Gray{Float32}

        # `floattype(::Number)` is inconsistent to `float(::NUmber)`, see issue:
        # https://github.com/JuliaMath/FixedPointNumbers.jl/issues/127
        # @test @inferred(float(N0f8)         ) == Float32
        # @test @inferred(float(Bool)         ) == Float32
        # @test @inferred(float(Float32)      ) == Float32
        # @test @inferred(float(Float64)      ) == Float64

        @test @inferred(float(ARGB32)) == ARGB{Float32}
        @test @inferred(float(AGray32)) == AGray{Float32}
        @test @inferred(float(RGB24)) == RGB{Float32}
        @test @inferred(float(Gray24)) == Gray{Float32}

        # float(x::Colorant)
        @test float(oneunit(Gray{N0f8})) == oneunit(Gray{Float32})
        @test float(RGB(oneunit(Gray{N0f8}))) == RGB(oneunit(Gray{Float32}))
    end
end

@testset "ambiguities" begin
    # issue #40
    d = Dict(:a=>1, :b=>2.0)
    @test isa(Dict(k=>v for (k,v) in d), Dict{Symbol,Real})
end

nothing


using LULESH
using MPI
# using Enzyme

# Enzyme.API.printperf!(true)
# Enzyme.API.printall!(true)
# Enzyme.API.instname!(true)

# Enzyme.API.inlineall!(true)
# Enzyme.API.maxtypeoffset!(32)
# ccall((:EnzymeSetCLInteger, Enzyme.API.libEnzyme), Cvoid, (Ptr{Cvoid}, Int64), cglobal((:MaxTypeOffset, Enzyme.API.libEnzyme)), 32)

function main(nx, structured, num_iters, mpi)
    # TODO: change default nr to 11
    nr = 1
    balance = 1
    cost = 1
    floattype = Float64
    devicetype = Vector

    if mpi
        !MPI.Initialized() && MPI.Init()
        comm = MPI.COMM_WORLD
    else
        comm = nothing
    end

    prob = LuleshProblem(num_iters, structured, nx, nr, balance, cost, devicetype, floattype, comm)

    # Set up the mesh and decompose. Assumes regular cubes for now
    # TODO: modify this constructor to account for new fields
    # TODO: setup communication buffers

    domain = Domain(prob)
    shadowDomain = Domain(prob)

    # getnodalMass = nodalMass(domain)

    # Initial domain boundary communication
    # commRecv(domain, MSG_COMM_SBN, 1,
    #             domain.sizeX + 1, domain.sizeY + 1, domain.sizeZ + 1,
    #             true, false, prob.comm)
    #    CommSend<&Domain::nodalMass>(*domain, MSG_COMM_SBN,
    #             domain->sizeX() + 1, domain->sizeY() + 1, domain->sizeZ() +  1,
    #             true, false)
    #    CommSBN<&Domain::nodalMass>(*domain)

    # End initialization
    if mpi
        MPI.Barrier(prob.comm::MPI.Comm)
    end

    if getMyRank(prob.comm) == 0
        if (structured)
            @info "Running" until=domain.stoptime domain=(nx,nx,nx)
        else
            @info "Running" until=domain.stoptime domain=domain.numElem
            @warn "Unstructured setup not supported"
        end
    end

    # timestep to solution
    start = getWtime(prob.comm)
    while domain.time < domain.stoptime
        # this has been moved after computation of volume forces to hide launch latencies
        timeIncrement!(domain)
        lagrangeLeapFrog(domain)

        # checkErrors(domain, its, myRank)
        if getMyRank(prob.comm) == 0
            @info "Completed" cycle=domain.cycle time=domain.time dt=domain.deltatime
        end
        if domain.cycle >= num_iters
            break
        end
    end

    # Use reduced max elapsed time
    elapsed_time = getWtime(prob.comm) - start
    elapsed_timeG = comm_max(elapsed_time, prob.comm)

    if getMyRank(prob.comm) == 0
        verifyAndWriteFinalOutput(elapsed_timeG, domain, nx, getNumRanks(prob.comm))
    end

    if mpi
        MPI.Finalize()
    end
end

if !isinteractive()
    args = LULESH.parse_cmd()

    # assume cube subdomain geometry for now (nx)
    nx = args["N"]
    structured = args["s"]
    num_iters = args["num_iters"]
    mpi = args["mpi"]
    main(nx, structured, num_iters, mpi)
end


# Generate new interactions to perturb ensembles

export
    repeatpocketpoints,
    perturbensemble,
    clusterligsite

"""
Find new interactions arising from fake modulator atoms and form a new `Constraints` object.
Arguments are atoms to calculate constraints with, `Constraints` object to add to, generated modulator
coordinates, distance cutoff and constraint tolerances for intra-modulator and modulator-protein
interactions, and minimum lower distance constraint.
Returns a new `Constraints` object including the modulator.
"""
function Constraints(atoms::Array{Atom,1},
                    old_constraints::Constraints,
                    mod_coords::Array{Float64};
                    intra_cutoff::Real=defaults["mod_intra_cutoff"], # If this is 0.0 there are no intra-modulator interactions
                    intra_tolerance::Real=defaults["mod_intra_tolerance"],
                    inter_cutoff::Real=defaults["mod_inter_cutoff"],
                    inter_tolerance::Real=defaults["mod_inter_tolerance"],
                    min_constraint_dist::Real=defaults["mod_min_constraint_dist"])
    n_atoms = length(atoms)
    @assert n_atoms > 0 "No atoms in atom list"
    constraint_atoms = old_constraints.atoms
    @assert n_atoms == length(constraint_atoms) "Different number of atoms in atom list and constraints"
    n_constraints = size(old_constraints.pres_inds, 1)
    @assert n_constraints == length(old_constraints.lower) && n_constraints == length(old_constraints.upper) "The number of lower and upper constraints do not correspond to the number of constraints present"
    n_mod_atoms = size(mod_coords, 2)
    @assert n_mod_atoms > 0 "No modulator coordinates in list"
    @assert intra_cutoff >= 0 "intra_cutoff cannot be negative"
    @assert intra_tolerance >= 0 "intra_tolerance cannot be negative"
    @assert inter_cutoff >= 0 "inter_cutoff cannot be negative"
    @assert inter_tolerance >= 0 "inter_tolerance cannot be negative"
    @assert min_constraint_dist >= 0 "min_constraint_dist cannot be negative"
    n_total = n_atoms + n_mod_atoms
    # Copy Constraints object and form larger constraints lists
    new_lower = copy(old_constraints.lower)
    new_upper = copy(old_constraints.upper)
    new_pres_inds = copy(old_constraints.pres_inds)
    new_atoms = deepcopy(constraint_atoms)
    # Add new atoms to atom array
    for i in 1:n_mod_atoms
        push!(new_atoms, Atom(mod_atom_info["atom_name"], mod_atom_info["res_name"], mod_atom_info["chain_id"], mod_atom_info["res_n"], mod_coords[:,i], mod_atom_info["element"]))
    end
    new_inds_i = Int[]
    new_inds_j = Int[]
    # Find interactions within modulator
    intra_counter = 0
    intra_cutoff_sq = intra_cutoff^2
    for i in 1:n_mod_atoms
        for j in 1:(i-1)
            sq_dist = 0.0
            for m = 1:3
                sq_dist += abs2(mod_coords[m,i] - mod_coords[m,j])
            end
            if sq_dist < intra_cutoff_sq
                dist = sqrt(sq_dist)
                push!(new_lower, max(dist-intra_tolerance, min_constraint_dist))
                push!(new_upper, dist+intra_tolerance)
                push!(new_inds_i, n_atoms+i)
                push!(new_inds_j, n_atoms+j)
                intra_counter += 1
            end
        end
    end
    # Find interactions between modulator and protein
    inter_counter = 0
    inter_cutoff_sq = inter_cutoff^2
    for i in 1:n_mod_atoms
        for j in 1:n_atoms
            sq_dist = 0.0
            for m = 1:3
                sq_dist += abs2(mod_coords[m,i] - atoms[j].coords[m])
            end
            if sq_dist < inter_cutoff_sq
                dist = sqrt(sq_dist)
                push!(new_lower, max(dist-inter_tolerance, min_constraint_dist))
                push!(new_upper, dist+inter_tolerance)
                push!(new_inds_i, n_atoms+i)
                push!(new_inds_j, j)
                inter_counter += 1
            end
        end
    end
    if length(new_pres_inds) > 0
        new_pres_inds = vcat(new_pres_inds, hcat(new_inds_i, new_inds_j))
    else
        new_pres_inds = hcat(new_inds_i, new_inds_j)
    end
    #println("Found ", intra_counter, " interactions within the modulator")
    println("Found ", inter_counter, " interactions between the modulator and the protein")
    return Constraints(new_atoms, new_lower, new_upper, new_pres_inds)
end

"""
Repeat pocket point to get a certain number of points.
Points are repeated as long as whole copies can be made, then the remainder are chosen randomly.
Returns coordinate array.
"""
function repeatpocketpoints(coords::Array{Float64}, n_out_points::Integer=defaults["mod_n_points"])
    n_pocket_points = Int(length(coords) / 3)
    inds_to_use = repeat(collect(1:n_pocket_points), outer=[Int(floor(n_out_points / n_pocket_points))])
    append!(inds_to_use, rand(1:n_pocket_points, n_out_points % n_pocket_points))
    return getindex(coords, collect(1:3), inds_to_use)
end

"""
Gets ensembles for multiple modulators.
Use pocket points to generate additional constraints.
"""
function perturbensemble(atoms::Array{Atom,1},
                    constraints::Constraints,
                    n_strucs::Integer,
                    mod_path::Union{AbstractString, Nothing},
                    n_mods::Integer)
    ensemble_mods = ModelledEnsemble[]
    if n_mods > 0 && mod_path != nothing
        pock_points = readpocketpoints(mod_path)
        n_pocks = length(pock_points)
        if n_mods > n_pocks
            println(n_mods, " modulators asked for but only ", n_pocks, " found")
        end
        n_mods_to_use = min(n_mods, n_pocks)
        for i in 1:n_mods_to_use
            println("Generating ensemble with modulator ", i, " of ", n_mods_to_use)
            mod_coords = repeatpocketpoints(pock_points[i])
            new_constraints = Constraints(atoms, constraints, mod_coords)
            push!(ensemble_mods, generateensemble(new_constraints, n_strucs))
        end
    end
    return ensemble_mods
end

"""
Assign pocket numbers to LIGSITEcs pocket_r.pdb file points.
Arguments are pocket_r.pdb file, pocket_all.pdb file and new pocket_r.pdb file.
New file has pocket number in place of residue number.
Apears to agree largely but not completely with the LIGSITEcs clustering.
Unassigned points given number 0 and should be fixed manually.
"""
function clusterligsite(point_filepath::AbstractString,
                    centre_filepath::AbstractString,
                    out_filepath::AbstractString;
                    cluster_dist::Real=defaults["cluster_dist"])
    point_lines = readpdblines(point_filepath)
    n_points = length(point_lines)
    point_coords = zeros(3, n_points)
    for (i, line) in enumerate(point_lines)
        point_coords[1,i] = parse(Float64, line[31:38])
        point_coords[2,i] = parse(Float64, line[39:46])
        point_coords[3,i] = parse(Float64, line[47:54])
    end
    println("Read ", n_points, " pocket points from LIGSITEcs pocket points PDB file")
    centre_coords, centre_vols = readligsite(centre_filepath)
    n_centres = length(centre_vols)
    assignments = zeros(Int, n_points)

    # Assign points within cluster_dist Angstroms of a centre to belong to the largest pocket centre
    for p in 1:n_points
        centre_closest = 0
        dist_closest = cluster_dist
        for c in n_centres:-1:1
            dist = norm(point_coords[:,p] - centre_coords[:,c])
            if dist <= cluster_dist
                centre_closest = c
                dist_closest = dist
            end
        end
        assignments[p] = centre_closest
    end

    # While some are still unassigned and have not stalled
    n_unassigned = 2 # Placeholder
    new_n_unassigned = 1 # Placeholder
    while new_n_unassigned != n_unassigned && new_n_unassigned > 0
        n_unassigned = sum(x -> x == 0, assignments)
        for p in 1:n_points
            # For each unassigned point
            if assignments[p] == 0
                point_closest = 0
                dist_closest = cluster_dist
                for a in 1:n_points
                    # Check each assigned point
                    if assignments[a] > 0
                        dist = norm(point_coords[:,p] - point_coords[:,a])
                        # Assign it if it within cluster_dist Angstroms and not already assigned to a larger pocket
                        if dist <= cluster_dist && (point_closest == 0 || assignments[a] < assignments[point_closest])
                            point_closest = a
                            dist_closest = dist
                        end
                    end
                end
                if point_closest > 0
                    assignments[p] = assignments[point_closest]
                end
            end
        end
        new_n_unassigned = sum(x -> x == 0, assignments)
    end
    if new_n_unassigned > 0
        println("Could not assign ", new_n_unassigned, " points")
    else
        println("Assigned all points")
    end

    writeclusterpoints(out_filepath, point_lines, assignments)
end


#WIP currently runs simulated SimulatedAnnealing in compressed img
function (algorithm::MultiResolutionHybrid)(img::AbstractArray)
    @warn "This function is WIP and will only return zeros."
    # T₀ = algorithm.T₀
    # α = algorithm.α
    # integration_factor = algorithm.integration_factor
    # smoothness = algorithm.smoothness
    # integration_factor2 = algorithm.integration_factor2
    # smoothness2 = algorithm.smoothness2
    # img_size = first(size(img))
    # E = Float64.(img)
    # Eₘ = maximum(E)
    # ρ = sqrt.((Eₘ^2)./(E.^2) .- 1)
    # ρ = imfilter(ρ, Kernel.gaussian(3))
    # E₁ = Eₘ ./ (1 .+ (ρ[1:2:end, 1:2:end,].^2))
    # ρ = sqrt.((Eₘ^2)./(E₁.^2) .- 1)
    # ρ = imfilter(ρ, Kernel.gaussian(3))
    # E₂ = Eₘ ./ (1 .+ (ρ[1:2:end, 1:2:end,].^2))
    # ρ = sqrt.((Eₘ^2)./(E₂.^2) .- 1)
    # ρ = imfilter(ρ, Kernel.gaussian(3))
    # E₃ = Eₘ ./ (1 .+ (ρ[1:2:end, 1:2:end,].^2))
    # p = rand(Float64, size(E₃))
    # q = rand(Float64, size(E₃))
    # pn, qn = SimulatedAnnealing(pin = p, qin = q, T₀ = T₀, α = α, integration_factor = integration_factor, smoothness = smoothness, max_iter = 1)(E₃)
    p = zeros(Float64, size(img))
    q = zeros(Float64, size(img))
    return p, q
end


"""
Used in gp.jl. Data pre- / post-processing. Takes a ProfileData object and prepares it for use in GP.
"""

abstract type Problem end

abstract type SequentialProblem <: Problem end
abstract type ResidualProblem <: Problem end
abstract type SlackProblem <: Problem end

default_modify_predictor_fn(x, 𝒟, time_index) = x

struct Sequential <: SequentialProblem
    type::String # "T" or "dT" or "wT" or "KPP" or "TKE"
    parameters # Parameters(...) (see OceanTurb documentation)
    modify_predictor_fn
    convective_adjust
end
Sequential(type; parameters=nothing, modify_predictor_fn=default_modify_predictor_fn, convective_adjust=false) = Sequential(type, parameters, modify_predictor_fn, convective_adjust)

struct Residual <: ResidualProblem
    type::String # "KPP" or "TKE"
    parameters # Parameters(...) (see OceanTurb documentation)
    modify_predictor_fn
    convective_adjust
end
Residual(type; parameters=nothing, modify_predictor_fn=default_modify_predictor_fn, convective_adjust=false) = Residual(type, parameters, modify_predictor_fn, convective_adjust)

struct Slack <: SlackProblem
    type::String # "KPP" or "TKE"
    parameters # Parameters(...) (see OceanTurb documentation)
    modify_predictor_fn
    convective_adjust
end
Slack(type; parameters=nothing, modify_predictor_fn=default_modify_predictor_fn, convective_adjust=false) = Slack(type, parameters, modify_predictor_fn, convective_adjust)

mutable struct Sequential_dT <: SequentialProblem
    variable::String #"T" or "wT"
    Δt::Number # assumes constant time interval between all timesteps
    scaling::Scaling
    Sequential_dT(variable, Δt) = new(variable, Δt) # incomplete initialization
end

mutable struct Sequential_T <: SequentialProblem
    variable::String # "T" or "wT"
    scaling::Scaling
    Sequential_T(variable) = new(variable) # incomplete initialization
end

mutable struct Sequential_wT <: SequentialProblem
    variable::String # "T" or "wT"
    scaling::Scaling
    Sequential_wT(variable) = new(variable) # incomplete initialization
end

mutable struct Sequential_KPP <: SequentialProblem
    variable::String
    physics_data
    evolve_physics_model_fn # Function
    Δt # assumes constant time interval between all timesteps
    scaling::Scaling
    Sequential_KPP(variable, physics_data, evolve_physics_model_fn, Δt) = new(variable, physics_data, evolve_physics_model_fn, Δt) # incomplete initialization
end

mutable struct Sequential_TKE <: SequentialProblem
    variable::String
    physics_data
    evolve_physics_model_fn # Function
    Δt # assumes constant time interval between all timesteps
    scaling::Scaling
    Sequential_TKE(variable, physics_data, evolve_physics_model_fn, Δt) = new(variable, physics_data, evolve_physics_model_fn, Δt) # incomplete initialization
end

mutable struct Residual_KPP <: ResidualProblem
    variable::String
    physics_data
    evolve_physics_model_fn # Function
    Δt # assumes constant time interval between all timesteps
    scaling::Scaling
    Residual_KPP(variable, physics_data, evolve_physics_model_fn, Δt) = new(variable, physics_data, evolve_physics_model_fn, Δt) # incomplete initialization
end

mutable struct Residual_TKE <: ResidualProblem
    variable::String
    physics_data
    evolve_physics_model_fn # Function
    Δt # assumes constant time interval between all timesteps
    scaling::Scaling
    Residual_TKE(variable, physics_data, evolve_physics_model_fn, Δt) = new(variable, physics_data, evolve_physics_model_fn, Δt) # incomplete initialization
end

mutable struct Slack_KPP <: SlackProblem
    variable::String
    physics_data::Array
    scaling::Scaling
    Slack_KPP(variable, physics_data) = new(variable, physics_data) # incomplete initialization
end

mutable struct Slack_TKE <: SlackProblem
    variable::String
    physics_data::Array
    scaling::Scaling
    Slack_TKE(variable, physics_data) = new(variable, physics_data) # incomplete initialization
end

"""
get_v(problem)
# Description
    Return data array from the OceananigansData object associated with the variable of interest.
# Arguments
- 'problem': (Problem).     what mapping you wish to evaluate with the model. (Sequential("T"), Sequential("wT"))
"""
function get_v(problem, les)
    if problem.type == "wT"
        return les.wT
    elseif problem.type in ["T", "dT", "KPP", "TKE"]
        return les.T
    else
        throw(error())
    end
end

"""
get_problem(problem::Problem, data::OceananigansData, timeseries)
# Description
    Creates an instance of a Problem struct depending on the type of mapping.
    Returns the problem and the correct data array from the OceananigansData objectssociated with the variable of interest.
# Arguments
- 'problem': (Problem).      what mapping you wish to evaluate with the model. (Sequential("T"), Sequential("wT"))
- 'les': (OceananigansData). LES simulation output
- 'v': (Array).              Array of values
- 'N²': (Number)             initial buoyancy stratification
- 'D': (Number)              number of grid points in the vertical
- 'Nt': (Integer)            number of time steps
"""
function get_problem(problem, les, v, N², D, Nt)

    Δt = les.t[2]-les.t[1]

    if typeof(problem) <: SequentialProblem

        if problem.type == "dT"
            Δt /= N² # Δt' = Δt / N²
            return Sequential_dT("T", Δt) # Sequential_dT(variable, Δt)

        elseif problem.type == "T"
            return Sequential_T("T")

        elseif problem.type == "wT"
            return Sequential_wT("wT")

        elseif problem.type == "KPP"
            f = closure_free_convection_kpp(problem.parameters, D, Δt, les, n_steps=1)
            evolve_physics_model_fn(T⁰) = f(; T⁰=T⁰)[:,2]
            kpp_data = Array{Array{Float64,1},1}(UndefInitializer(), Nt)
            kpp_data[1] = custom_avg(v[:,1], D)
            for i in 1:Nt-1
                kpp_data[i+1] = evolve_physics_model_fn(v[:,i])
            end
            return Sequential_KPP("T", kpp_data, evolve_physics_model_fn, Δt)

        elseif problem.type == "TKE"
            # Use the LES profile at time index i to predict time index i+1 using TKE
            f = closure_free_convection_tke(problem.parameters, D, Δt, les, n_steps=1)
            evolve_physics_model_fn2(T⁰) = f(; T⁰=T⁰)[:,2]
            tke_data = Array{Array{Float64,1},1}(UndefInitializer(), Nt)
            tke_data[1] = custom_avg(v[:,1], D)
            for i in 1:Nt-1
                tke_data[i+1] = evolve_physics_model_fn2(v[:,i])
            end
            return Sequential_TKE("T", tke_data, evolve_physics_model_fn2, Δt)

        else; throw(error())
        end

    elseif typeof(problem) <: ResidualProblem

        if problem.type == "KPP"
            # Use the LES profile at time index i to predict time index i+1 using KPP
            f = closure_free_convection_kpp(problem.parameters, D, Δt, les, n_steps=1)
            evolve_physics_model_fn3(T⁰) = f(; T⁰=T⁰)[:,2]
            kpp_data = Array{Array{Float64,1},1}(UndefInitializer(), Nt)
            kpp_data[1] = custom_avg(v[:,1], D)
            for i in 1:Nt-1
                kpp_data[i+1] = evolve_physics_model_fn3(v[:,i])
            end
            return Residual_KPP("T", kpp_data, evolve_physics_model_fn3, Δt)

        elseif problem.type == "TKE"
            # Use the LES profile at time index i to predict time index i+1 using TKE
            f = closure_free_convection_tke(problem.parameters, D, Δt, les, n_steps=1)
            evolve_physics_model_fn4(T⁰) = f(; T⁰=T⁰)[:,2]
            tke_data = Array{Array{Float64,1},1}(UndefInitializer(), Nt)
            tke_data[1] = custom_avg(v[:,1], D)
            for i in 1:Nt-1
                tke_data[i+1] = evolve_physics_model_fn4(v[:,i])
            end
            return Residual_TKE("T", tke_data, evolve_physics_model_fn4, Δt)

        else; throw(error())
        end

    elseif typeof(problem) <: SlackProblem

        if problem.type == "KPP"
            # Predict the full evolution of the temperature profile from the initial time step using KPP
            f = closure_free_convection_kpp_full_evolution(problem.parameters, D, Δt, les)
            kpp_data = f()
            kpp_data = [kpp_data[:,i] for i in 1:Nt]
            return Slack_KPP("T", kpp_data)

        elseif problem.type == "TKE"
            # Predict the full evolution of the temperature profile from the initial time step using TKE
            f = closure_free_convection_tke_full_evolution(problem.parameters, D, Δt, les)
            tke_data = f()
            tke_data = [tke_data[:,i] for i in 1:Nt]
            return Slack_TKE("T", tke_data)

        else; throw(error())
        end

    else; throw(error())
    end

end


for (func, elty) in [(:CLBlastSgemv, Float32), (:CLBlastDgemv, Float64),
                     (:CLBlastCgemv, ComplexF32), (:CLBlastZgemv, ComplexF64)]
    #TODO: (:CLBlastHgemv, Float16)

    @eval function $func(layout::CLBlastLayout, a_transpose::CLBlastTranspose,
                         m::Integer, n::Integer,
                         alpha::$elty,
                         a_buffer::cl.CL_mem, a_offset::Integer, a_ld::Integer,
                         x_buffer::cl.CL_mem, x_offset::Integer, x_inc::Integer,
                         beta::$elty,
                         y_buffer::cl.CL_mem, y_offset::Integer, y_inc::Integer,
                         queue::cl.CmdQueue, event::cl.Event)
        err = ccall(
            ($(string(func)), libCLBlast),
            cl.CL_int,
            (Cint, Cint, Csize_t, Csize_t, $elty, Ptr{Cvoid}, Csize_t, Csize_t, Ptr{Cvoid}, Csize_t, Csize_t,
              $elty, Ptr{Cvoid}, Csize_t, Csize_t, Ptr{Cvoid}, Ptr{Cvoid}),
            Cint(layout), Cint(a_transpose), m, n, alpha, a_buffer, a_offset, a_ld, x_buffer, x_offset, x_inc,
              beta, y_buffer, y_offset, y_inc, Ref(queue), Ref(event)
        )
        if err != cl.CL_SUCCESS
            println(stderr, "Calling function $(string($func)) failed!")
            throw(CLBlastError(err))
        end
        return err
    end

    @eval function gemv!(tA::Char, α::Number, A::cl.CLArray{$elty,2},
                         x::cl.CLArray{$elty}, β::Number, y::cl.CLArray{$elty};
                         queue::cl.CmdQueue=cl.queue(y))
        # check and convert arguments
        m, n = size(A)
        if tA == 'N'
            a_transpose = CLBlastTransposeNo
            if length(x) != n || length(y) != m
                throw(DimensionMismatch("A has dimensions $(size(A)), x has length $(length(x)) and y has length $(length(y))."))
            end
        elseif tA == 'T' || (tA == 'C' && $elty <: Real)
            a_transpose = CLBlastTransposeYes
            if length(x) != m || length(y) != n
                throw(DimensionMismatch("The adjoint of A has dimensions $n, $m, x has length $(length(x)) and y has length $(length(y))"))
            end
        elseif tA == 'C' && $elty <: Complex
            a_transpose = CLBlastTransposeConjugate
            if length(x) != m || length(y) != n
                throw(DimensionMismatch("The transpose of A has dimensions $n, $m, x has length $(length(x)) and y has length $(length(y))."))
            end
        else
            throw(ArgumentError("Transpose marker `tA` is $(tA) but only 'N', 'T', and 'C' are allowed."))
        end
        alpha = convert($elty, α)
        beta  = convert($elty, β)
        layout = CLBlastLayoutColMajor

        # output event
        event::cl.Event = cl.Event(C_NULL)

        $func(layout, a_transpose,
              m, n,
              alpha,
              pointer(A), 0, size(A,1),
              pointer(x), 0, 1,
              beta,
              pointer(y), 0, 1,
              queue, event)

        # wait for kernel
        cl.wait(event)

        y
    end

end


using ArgParse
using util
using ranking
using preprocessing
using Method
using data
using Metrics
using preprocessing
using JSON
using DataStructures

# Include run_method file
dir_name, script_name = splitdir(@__FILE__)
include(joinpath(dir_name,"run_method.jl"))

s = ArgParseSettings()
@add_arg_table s begin
    "bug_dataset"
        help="File that contains all bug report data(categorical, summary,...)"
    "evaluation"
        help="Evaluation dataset"
    "method_name"
    "search_space_script"
    "--max_evals"
            help = ""
            arg_type = Int
            default = 100
    "--test"
        help="Test dataset"
end

import_settings(s, get_basic_args())

parsed_args = parse_args(s, as_symbols=true)

@info parsed_args

# Load validation set. This set is used to find the best parameters.
validationDataset = readDataset(parsed_args[:evaluation])

# Load search space for the experiment
using PyCall

hyperopt = pyimport("hyperopt")
space_script_path = parsed_args[:search_space_script]

py"""
space_script = $(space_script_path)
f = open(space_script, 'r')
code_str = f.read()
f.close()

exec(compile(code_str, space_script, 'exec'))
"""

fixed_values = py"""fixed_values"""
space = py"""space"""

# Objective Function
function objective(params...)
    exp_parameter = Dict(parsed_args)

    for (k,v) in  params[1]
        exp_parameter[Symbol(k)] = v
    end

    for (k,v) in  fixed_values
        exp_parameter[Symbol(k)] = v
    end

    results = Dict()

    timeScore = @elapsed begin
        try
            results = run_method(validationDataset.bugIds, exp_parameter)
        catch e
            @error exception=(e, catch_backtrace())
            throw(e)
        end
    end

    map = 0.0
    auc = 0.0
    rr1 = 0.0
    rr5 = 0.0

    print(typeof(results))
    for r in results
        if r["label"] == "MAP_RecallRate"
            map = r["map"]::Float64
            rr1 = r["rr"][1]::Float64
            rr5 = r["rr"][5]::Float64
        elseif r["label"] == "BinaryPredictionROC"
            auc = r["auc"]::Float64
        end
    end

    loss = 2.0 - (map + auc)

    new_trial = OrderedDict(:x=> params[1],:loss => loss, :map=> map, :auc=>auc,:rr1 => rr1, :rr5=>rr5 , :args => exp_parameter)
    @info new_trial
    @info "Finish a new trial in $(timeScore)"

    return Dict("loss"=> 2.0 - (map + auc), "status"=> hyperopt.STATUS_OK, "results"=> results, "map"=> map, "auc"=> auc,
                "args"=> exp_parameter)
end

# Find the best parameters
trials = hyperopt.Trials()
best = hyperopt.fmin(objective,
            space=space,
            algo=hyperopt.tpe.suggest,
            max_evals=parsed_args[:max_evals],
            trials=trials)

@info "Paremeter search is over!"
@info "Best: $(best)"

@info "Sorted trial:"

all_trials = [ OrderedDict{Symbol, Any}(:vals => t["misc"]["vals"], :loss => t["result"]["loss"], :map => t["result"]["map"], :auc => t["result"]["auc"], :result => t["result"]["results"], :args => t["result"]["args"]) for t in trials]

sort!(all_trials, by=x-> x[:loss])

for t in all_trials
    @info "$(t)"
end

best_result = all_trials[1]
println(best_result)

if !isnothing(get(parsed_args, :test, nothing))
    @info "Start test:"
    testDataset = readDataset(parsed_args[:test])
    best_args = Dict{Symbol, Any}(( Symbol(k) => v for (k, v) in best_result[:args]))

    @info "$(best_args)"

    results = run_method(testDataset.bugIds, best_args)

    @info  JSON.json(results)
end


"""
`module  JAC.RadiativeOpacity`
    ... a submodel of JAC that contains all methods for computing radiative -- spectral & mean -- opacities.
"""
module RadiativeOpacity

    using Printf, JAC, ..Basics, ..Defaults, ..ManyElectron, ..Radial, ..TableStrings

    """
    `struct  RadiativeOpacity.Settings  <:  AbstractPropertySettings`
        ... defines a type for the details and parameters of computing radiative opacities.

        + approach           ::AbstractOpacityApproach
            ... Determines the approach for calculating radiative opacities: {SincleConf(), ...}.
        + NoExcitations      ::Int64
            ... Number of excitations with regard to the given (reference) configurations.
        + NoLevels           ::Int64
            ... maximum number of levels (to terminate long computations in a regular fashion).
        + maxExcitation      ::Float64
            ... maximum excitation energy w.r.t. the ground level for which levels are considered.
        + deltaLambda        ::Float64          ... Delta Lambda in the expansion opacity.
        + time               ::Float64          ... Time after merger in the expansion opacity.
        + densities          ::Array{Float64,1} ... densities [g/cm^3] in the expansion opacity.
        + temperatures       ::Array{Float64,1} ... temperatures [K] in the expansion opacity.
    """
    struct Settings  <:  AbstractPropertySettings
        approach             ::Int64
        NoExcitations        ::Int64
        NoLevels             ::Int64
        maxExcitation        ::Float64
        DeltaLambda          ::Float64
        time                 ::Float64
        densities            ::Array{Float64,1}
        temperatures         ::Array{Float64,1}
    end

    """
    `RadiativeOpacity.Settings()`
        ... constructor for an `empty` instance of RadiativeOpacity.Settings for the computation of radiative opacities.
    """
    function Settings()
        Settings( )
    end

    # `Base.show(io::IO, settings::RadiativeOpacity.Settings)`  ... prepares a proper printout of settings::RadiativeOpacity.Settings.
    function Base.show(io::IO, settings::RadiativeOpacity.Settings)
        println(io, "approach:              $(settings.approach)  ")
        println(io, "NoExcitations:         $(settings.NoExcitations)  ")
        println(io, "NoLevels:              $(settings.NoLevels)  ")
        println(io, "maxExcitation:         $(settings.maxExcitation)  ")
        println(io, "DeltaLambda:           $(settings.DeltaLambda)  ")
        println(io, "time:                  $(settings.time)  ")
        println(io, "densities:             $(settings.densities)  ")
        println(io, "temperatures:          $(settings.temperatures)  ")
    end

    """
    `struct  RadiativeOpacity.Outcome`
        ... defines a type to keep the outcome of a opacity computation as well other results.

        + spectralOpacity       ::Float64            ... Spectral opacity.
        + expansionOpacity      ::Float64            ... Expansion opacity.
        + rosselandOpacity      ::Float64            ... Mean Rosseland opacity.
    """
    struct Outcome
        spectralOpacity         ::Float64
        expansionOpacity        ::Float64
        rosselandOpacity        ::Float64
    end

    """
    `RadiativeOpacity.Outcome()`
        ... constructor for an `empty` instance of RadiativeOpacity.Outcome for the computation of opacities.
    """
    function Outcome()
        Outcome(0., 0., 0.)
    end

    # `Base.show(io::IO, outcome::RadiativeOpacity.Outcome)`  ... prepares a proper printout of outcome::RadiativeOpacity.Outcome.
    function Base.show(io::IO, outcome::RadiativeOpacity.Outcome)
        println(io, "spectralOpacity:         $(outcome.spectralOpacity)  ")
        println(io, "expansionOpacity:        $(outcome.expansionOpacity)  ")
        println(io, "rosselandOpacity:        $(outcome.rosselandOpacity)  ")
    end

    ## Introduce some sort Opacity.Line to keep the data E_i, E_f, level_i, level_f, J_i, J_f, gf ... for all calculated
    ## lines ... but not the full representation of amplitudes and levels
    ## Perhaps, levels::Array{Level,1}  ... to keep a number of levels
    ## calculate multiplet of configurations
    ## generate a list of configurations for which levels are taken into account.

end # module


#=##############################################################################
# DESCRIPTION
    Simulation interface connecting QVLMVehicle vehicles and maneuvers.

# AUTHORSHIP
  * Author    : Eduardo J. Alvarez
  * Email     : Edo.AlvarezR@gmail.com
  * Created   : Apr 2020
  * License   : MIT
=###############################################################################

function solve(self::Simulation{V, M, R}, Vinf::Function,
                pfield::vpm.ParticleField, wake_coupled::Bool,
                dt::Real, rlx::Real, sigma::Real, rho::Real,
                speedofsound::Real, staticpfield::vpm.ParticleField;
                init_sol::Bool=false, vlm_fsgm=1
                ) where {V<:QVLMVehicle, M<:AbstractManeuver, R}

    # Bring the rotors back to their initial positions at beginning of this
    # time step before `precalculations`. This is needed to avoid double
    # accounting for the RPM when the solver is run
    rotate_rotors(self, -dt)

    vhcl = self.vehicle
    mnvr = self.maneuver
    t = self.t

    # TIME-STEPPING PROCEDURE:
    # -1) Solve one particle field time step
    # 0) Translate and rotate systems
    # 1) Recalculate horseshoes with kinematic velocity
    # 2) Paste previous Gamma solution to new system position after translation
    # 3) Shed semi-infinite wake after translation
    # 4) Calculate wake-induced velocity on VLM and Rotor system
    # 5) Solve VLM and Rotor system
    # 6) Shed unsteady-loading wake after new solution
    # 7) Save new solution as prev solution
    # Iterate
    #
    # On the first time step (pfield.nt==0), it only does steps (5) and (7),
    # meaning that the unsteady wake of the first time step is never shed.
    if self.nt==0
        # NOTE: VLMs and Rotors solutions are losely coupled
        # NOTE: VLMs surfaces and their wakes are also losely coupled

        # Set Vinf on system to get horseshoes
        vlm.setVinf(vhcl.system, Vinf)

        # Solve VLMs
        # TODO: Add Rotor-on-VLM induced velocity
        # NOTE: Since there isn't a kinematic velocity in the first time step
        #   (t==0), the semi-infinite wake would start from a back initial
        #   solution if calculated in this step, so this step is rather going
        #   to waste and we wait for the next step to initiate the wake properly
        if init_sol
            # Here I use the freestream-wake for the first step
            vlm.solve(vhcl.vlm_system, Vinf; t=t, keep_sol=true)
        else
            # Here I completely ignore the wake
            vlm.solve(vhcl.vlm_system, Vinf; t=t, keep_sol=true,
                                                    vortexsheet=(X,t)->zeros(3))
        end

        # Calculate induced velocities to use in rotor solver
        ## Points where to calculate induced velocities
        Xs = _get_midXs(vhcl.rotor_systems)

        ## Evaluate VLM-on-Rotor induced velocity
        Vinds = [vlm.Vind(vhcl.vlm_system, X; t=t, ign_infvortex=init_sol)
                                                                    for X in Xs]

        # Solve Rotors
        for (si, rotors) in enumerate(vhcl.rotor_systems)

            # RPM of this rotor system
            RPM = self.RPMref*get_RPM(mnvr, si, t/self.ttot)

            for (ri, rotor) in enumerate(rotors)

                # Get velocities induced at every blade of this rotor
                this_Vinds = _parse_midXs(vhcl.rotor_systems,  Vinds, si, ri)

                VindVkin = _format_blades(this_Vinds, vhcl.rotor_systems, si, ri)

                vlm.solvefromCCBlade(rotor, Vinf, RPM, rho; _Vinds=VindVkin,
                                              t=t, sound_spd=speedofsound)
            end
        end

    else

        # Initialize the wake system as a VLM of only the wake system on the
        # kinematic velocity
        if self.nt==1
            vlm.solve(vhcl.wake_system, Vinf; t=t, extraVinf=_extraVinf1,
                                keep_sol=true, vortexsheet=(X,t)->zeros(3))
        end

        # Previous rotor system for kinematic and induced velocity calculation
        prev_rotor_systems = _get_prev_rotor_systems(vhcl)

        # ---------- 4) Calculate VPM velocity on VLM and Rotor system -----

        # Control points of VLM
        Xs = _get_Xs(vhcl.vlm_system, "CP"; t=t)

        # Wake-induced velocity at every VLM control point
        Vinds = [vlm.Vind(vhcl.wake_system, X; t=t, only_infvortex=true)
                                                                    for X in Xs]

        # Rotor-induced velocity at every VLM control point (only blade-induced
        # velocities---this ignores the rotor slipstream)
        for (si, rotors) in enumerate(vhcl.rotor_systems)
            for (ri, rotor) in enumerate(rotors)

                vlm.getHorseshoe(rotor, 1)          # Force HS calculation
                vlm._addsolution(rotor, "Gamma",    # Give previous solution
                                prev_rotor_systems[si][ri]._wingsystem.sol["Gamma"])

                Vinds .+= [vlm.Vind(rotor, X; t=t, ign_infvortex=true)
                                                                    for X in Xs]
            end
        end

        vlm._addsolution(vhcl.vlm_system, "Vind", Vinds; t=t)

        # Add dummy VPM velocity for `calc_aerodynamicforce()`
        Vdummy = zeros(3)
        Vdummies = [Vdummy for i in 1:vlm.get_m(vhcl.vlm_system)]
        vlm._addsolution(vhcl.vlm_system, "Vvpm_ApA", Vdummies; t=t)
        vlm._addsolution(vhcl.vlm_system, "Vvpm_AB", Vdummies; t=t)
        vlm._addsolution(vhcl.vlm_system, "Vvpm_BBp", Vdummies; t=t)

        # Calculate induced velocities to use in rotor solver
        ## Points where to calculate induced velocities
        Xs = _get_midXs(vhcl.rotor_systems)

        ## Evaluate wake-on-Rotor induced velocity
        Vinds = [vlm.Vind(vhcl.wake_system, X; t=t, only_infvortex=true)
                                                                    for X in Xs]
        ## Evaluate VLM-on-Rotor induced velocity
        Vinds .+= [vlm.Vind(vhcl.vlm_system, X; t=t, ign_infvortex=true)
                                                                    for X in Xs]
        ## Evaluate Rotor-on-Rotor (and self) induced velocity
        # NOTE: This include the blade-induced velocity of the rotor on itself,
        #   which is already accounted for in the momentum-balance part of BEMT.
        #   Remember to substract this later or the solver will get on a
        #   positive-feedback instability.
        for rotors in vhcl.rotor_systems
            for rotor in rotors
                Vinds .+= [vlm.Vind(rotor, X; t=t, ign_infvortex=true)
                                                                    for X in Xs]
            end
        end

        # ---------- 5) Solve VLM system -----------------------------------
        vlm.solve(vhcl.vlm_system, Vinf; t=t, extraVinf=_extraVinf3,
                                keep_sol=true, vortexsheet=(X,t)->zeros(3))

        # Relaxes (rlx->1) or stiffens (rlx->0) the VLM solution
        if rlx >= 0
            rlxd_Gamma = rlx*vhcl.vlm_system.sol["Gamma"] +
                                (1-rlx)*_get_prev_vlm_system(vhcl).sol["Gamma"]
            vlm._addsolution(vhcl.vlm_system, "Gamma", rlxd_Gamma)
        end

        # ---------- 5) Solve Rotor system ---------------------------------
        for (si, rotors) in enumerate(vhcl.rotor_systems)

            # RPM of this rotor system
            RPM = self.RPMref*get_RPM(mnvr, si, t/self.ttot)

            for (ri, rotor) in enumerate(rotors)

                # Calculate kinematic velocities
                Vkin = _Vkinematic_rotor(vhcl.rotor_systems,
                                         prev_rotor_systems, si, ri, dt)

                # Get velocities induced at every blade of this rotor
                this_Vinds = _parse_midXs(vhcl.rotor_systems, Vinds, si, ri)

                # Calculate and substract the blade-induced velocity of the
                # rotor on itself
                this_Xs = _parse_midXs(vhcl.rotor_systems, Xs, si, ri)
                this_Vinds .-= [vlm.Vind(rotor, X; t=t, ign_infvortex=true)
                                                               for X in this_Xs]

                VindVkin = _format_blades(this_Vinds + Vkin, vhcl.rotor_systems,
                                                                        si, ri)

                vlm.solvefromCCBlade(rotor, Vinf, RPM, rho; _Vinds=VindVkin,
                                            t=t, sound_spd=speedofsound)
            end
        end

        # Rotate rotors
        # NOTE: this means that the rotational velocity hadn't been included in
        # the kinematic velocitie up to this point
        rotate_rotors(self, dt)
    end
end


#=
utilities for Pardiso
only works with Float64

TODO add to docs following example:
```julia
import Hypatia

ENV["OMP_NUM_THREADS"] = length(Sys.cpu_info())
import Pardiso

syssolver = Hypatia.Solvers.NaiveElimSparseSystemSolver{Float64}(fact_cache = Hypatia.PardisoNonSymCache())
solver = Hypatia.Solvers.Solver{Float64}(syssolver)

include("test/native.jl")
nonnegative1(Float64, solver = solver)
```
=#

mutable struct PardisoNonSymCache{Float64} <: SparseNonSymCache{Float64}
    analyzed::Bool
    pardiso::Pardiso.MKLPardisoSolver
    function PardisoNonSymCache{Float64}()
        cache = new{Float64}()
        cache.analyzed = false
        cache.pardiso = Pardiso.MKLPardisoSolver()
        Pardiso.set_matrixtype!(cache.pardiso, Pardiso.REAL_NONSYM)
        return cache
    end
end

mutable struct PardisoSymCache{Float64} <: SparseSymCache{Float64}
    analyzed::Bool
    pardiso::Pardiso.MKLPardisoSolver
    function PardisoSymCache{Float64}()
        cache = new{Float64}()
        cache.analyzed = false
        cache.pardiso = Pardiso.MKLPardisoSolver()
        Pardiso.set_matrixtype!(cache.pardiso, Pardiso.REAL_SYM_INDEF)
        return cache
    end
end

PardisoSparseCache = Union{PardisoSymCache{Float64}, PardisoNonSymCache{Float64}}
int_type(::PardisoSparseCache) = Int32

function update_fact(
    cache::PardisoSparseCache{Float64},
    A::SparseMatrixCSC{Float64, Int32},
    )
    pardiso = cache.pardiso

    if !cache.analyzed
        Pardiso.pardisoinit(pardiso)
        # solve transposed problem (Pardiso accepts CSR matrices)
        Pardiso.fix_iparm!(pardiso, :N)
        Pardiso.set_phase!(pardiso, Pardiso.ANALYSIS)
        Pardiso.pardiso(pardiso, A, Float64[])
        cache.analyzed = true
    end

    Pardiso.set_phase!(pardiso, Pardiso.NUM_FACT)
    Pardiso.pardiso(pardiso, A, Float64[])

    return
end

function inv_prod(
    cache::PardisoSparseCache{Float64},
    x::Vector{Float64},
    A::SparseMatrixCSC{Float64, Int32},
    b::Vector{Float64},
    )
    pardiso = cache.pardiso

    Pardiso.set_phase!(pardiso, Pardiso.SOLVE_ITERATIVE_REFINE)
    Pardiso.pardiso(pardiso, x, A, b)

    return x
end

function free_memory(cache::PardisoSparseCache{Float64})
    Pardiso.set_phase!(cache.pardiso, Pardiso.RELEASE_ALL)
    Pardiso.pardiso(cache.pardiso)
    return
end


using VectorizationBase: vnoaliasstore!

@inline vstoreadditivereduce!(args...) = vnoaliasstore!(args...)
@inline vstoremultiplicativevereduce!(args...) = vnoaliasstore!(args...)
@inline function vstoreadditivereduce!(ptr::VectorizationBase.AbstractStridedPointer, v::VectorizationBase.SVec, i::Tuple{Vararg{Union{Integer,Static}}})
    vnoaliasstore!(ptr, SIMDPirates.vsum(v), i)
end
@inline function vstoreadditivereduce!(ptr::VectorizationBase.AbstractStridedPointer, v::VectorizationBase.SVec, i::Tuple{Vararg{Union{Integer,Static}}}, m::VectorizationBase.Mask)
    vnoaliasstore!(ptr, SIMDPirates.vsum(v), i, m)
end
@inline function vstoremultiplicativevereduce!(ptr::VectorizationBase.AbstractStridedPointer, v::VectorizationBase.SVec, i::Tuple{Vararg{Union{Integer,Static}}})
    vnoaliasstore!(ptr, SIMDPirates.vprod(v), i)
end
@inline function vstoremultiplicativevereduce!(ptr::VectorizationBase.AbstractStridedPointer, v::VectorizationBase.SVec, i::Tuple{Vararg{Union{Integer,Static}}}, m::VectorizationBase.Mask)
    vnoaliasstore!(ptr, SIMDPirates.vprod(v), i, m)
end

function storeinstr(op::Operation, vectorized::Symbol)
    # defaultstoreop = :vstore!
    defaultstoreop = :vnoaliasstore!
    vectorized ∉ reduceddependencies(op) && return lv(defaultstoreop)
    vectorized ∈ loopdependencies(op) && return lv(defaultstoreop)
    # vectorized is not a loopdep, but is a reduced dep
    opp = first(parents(op))
    while vectorized ∉ loopdependencies(opp)
        oppold = opp
        for oppp ∈ parents(opp)
            if vectorized ∈ reduceddependencies(oppp)
                @assert opp !== oppp "More than one parent is a reduction over the vectorized variable."
                opp = oppp
            end
        end
        @assert opp !== oppold "Failed to find any parents "
    end
    instr_class = reduction_instruction_class(instruction(opp))
    instr = if instr_class === ADDITIVE_IN_REDUCTIONS
        :vstoreadditivereduce!
    elseif instr_class === MULTIPLICATIVE_IN_REDUCTIONS
        :vstoremultiplicativevereduce!
    else #FIXME
        defaultstoreop
    end
    lv(instr)
end

# const STOREOP = :vstore!
# variable_name(op::Operation, ::Nothing) = mangledvar(op)
# variable_name(op::Operation, suffix) = Symbol(mangledvar(op), suffix, :_)
# # variable_name(op::Operation, suffix, u::Int) = (n = variable_name(op, suffix); u < 0 ? n : Symbol(n, u))
function reduce_range!(q::Expr, toreduct::Symbol, instr::Instruction, Uh::Int, Uh2::Int)
    if 2Uh == Uh2
        for u ∈ 0:2:Uh2-1
            instrexpr = callexpr(instr)
            push!(instrexpr.args, Symbol(toreduct, u))
            push!(instrexpr.args, Symbol(toreduct, u + 1))
            push!(q.args, Expr(:(=), Symbol(toreduct, (u>>>1)), instrexpr))
        end
    else
        for u ∈ Uh:Uh2-2
            tru = Symbol(toreduct, u - Uh)
            instrexpr = callexpr(instr)
            push!(instrexpr.args, tru)
            push!(instrexpr.args, Symbol(toreduct, u))
            push!(q.args, Expr(:(=), tru, instrexpr))
        end
        for u ∈ 2Uh:Uh2-1
            tru = Symbol(toreduct, u - 2Uh)
            instrexpr = callexpr(instr)
            push!(instrexpr.args, tru)
            push!(instrexpr.args, Symbol(toreduct, u))
            push!(q.args, Expr(:(=), tru, instrexpr))
        end
    end
end
function reduce_range!(q::Expr, ls::LoopSet, Ulow::Int, Uhigh::Int)
    for or ∈ ls.outer_reductions
        op = ls.operations[or]
        var = mangledvar(op)
        instr = Instruction(reduction_to_single_vector(op.instruction))
        reduce_range!(q, var, instr, Ulow, Uhigh)
    end
end

function reduce_expr!(q::Expr, toreduct::Symbol, instr::Instruction, U::Int)
    U == 1 && return nothing
    @assert U > 1 "U = $U somehow < 1"
    instr = Instruction(reduction_to_single_vector(instr))
    Uh2 = U
    iter = 0
    while true # combine vectors
        Uh = Uh2 >> 1
        reduce_range!(q, toreduct, instr, Uh, Uh2)
        Uh == 1 && break
        Uh2 = Uh
        iter += 1; iter > 4 && throw("Oops! This seems to be excessive unrolling.")
    end
    nothing
end

# pvariable_name(op::Operation, ::Nothing) = mangledvar(first(parents(op)))
# pvariable_name(op::Operation, ::Nothing, ::Symbol) = mangledvar(first(parents(op)))
# pvariable_name(op::Operation, suffix) = Symbol(pvariable_name(op, nothing), suffix, :_)
# function pvariable_name(op::Operation, suffix, tiled::Symbol)
#     parent = first(parents(op))
#     mname = mangledvar(parent)
#     tiled ∈ loopdependencies(parent) ? Symbol(mname, suffix, :_) : mname
# end

function lower_conditionalstore_scalar!(
    q::Expr, op::Operation, ua::UnrollArgs, mask::Union{Nothing,Symbol,Unsigned}, inds_calc_by_ptr_offset::Vector{Bool}
)
    @unpack u₁, u₁loopsym, u₂loopsym, vectorized, suffix = ua
    mvar, opu₁, opu₂ = variable_name_and_unrolled(first(parents(op)), u₁loopsym, u₂loopsym, suffix)
    # var = pvariable_name(op, suffix, tiled)
    cond = last(parents(op))
    condvar, condu₁ = condvarname_and_unroll(cond, u₁loopsym, u₂loopsym, suffix, opu₂)
    loopdeps = loopdependencies(op)
    opu₁ = u₁loopsym ∈ loopdeps
    ptr = vptr(op)
    for u ∈ 0:u₁-1
        varname = varassignname(mvar, u, opu₁)
        condvarname = varassignname(condvar, u, condu₁)
        td = UnrollArgs(ua, u)
        push!(q.args, Expr(:&&, condvarname, Expr(:call, storeinstr(op, vectorized), ptr, varname, mem_offset_u(op, td, inds_calc_by_ptr_offset))))
    end
    nothing
end
function lower_conditionalstore_vectorized!(
    q::Expr, op::Operation, ua::UnrollArgs, mask::Union{Nothing,Symbol,Unsigned}, isunrolled::Bool, inds_calc_by_ptr_offset::Vector{Bool}
)
    @unpack u₁, u₁loopsym, u₂loopsym, vectorized, suffix = ua
    loopdeps = loopdependencies(op)
    @assert vectorized ∈ loopdeps
    mvar, opu₁, opu₂ = variable_name_and_unrolled(first(parents(op)), u₁loopsym, u₂loopsym, suffix)
    # var = pvariable_name(op, suffix, tiled)
    if isunrolled
        umin = 0
        U = u₁
    else
        umin = -1
        U = 0
    end
    ptr = vptr(op)
    vecnotunrolled = vectorized !== u₁loopsym
    cond = last(parents(op))
    condvar, condu₁ = condvarname_and_unroll(cond, u₁loopsym, u₂loopsym, suffix, opu₂)
    # @show parents(op) cond condvar
    for u ∈ 0:U-1
        td = UnrollArgs(ua, u)
        name, mo = name_memoffset(mvar, op, td, opu₁, inds_calc_by_ptr_offset)
        condvarname = varassignname(condvar, u, condu₁)
        instrcall = Expr(:call, storeinstr(op, vectorized), ptr, name, mo)
        if mask !== nothing && (vecnotunrolled || u == U - 1)
            push!(instrcall.args, Expr(:call, :&, condvarname, mask))
        else
            push!(instrcall.args, condvarname)
        end
        push!(q.args, instrcall)
    end
end

function lower_store_scalar!(
    q::Expr, op::Operation, ua::UnrollArgs, mask::Union{Nothing,Symbol,Unsigned}, inds_calc_by_ptr_offset::Vector{Bool}
)
    @unpack u₁, u₁loopsym, u₂loopsym, vectorized, suffix = ua
    mvar, opu₁, opu₂ = variable_name_and_unrolled(first(parents(op)), u₁loopsym, u₂loopsym, suffix)
    ptr = vptr(op)
    # var = pvariable_name(op, suffix, tiled)
    # parentisunrolled = unrolled ∈ loopdependencies(first(parents(op)))
    for u ∈ 0:u₁-1
        varname = varassignname(mvar, u, opu₁)
        td = UnrollArgs(ua, u)
        push!(q.args, Expr(:call, storeinstr(op, vectorized), ptr, varname, mem_offset_u(op, td, inds_calc_by_ptr_offset)))
    end
    nothing
end
function lower_store_vectorized!(
    q::Expr, op::Operation, ua::UnrollArgs, mask::Union{Nothing,Symbol,Unsigned}, isunrolled::Bool, inds_calc_by_ptr_offset::Vector{Bool}
)
    @unpack u₁, u₁loopsym, u₂loopsym, vectorized, suffix = ua
    loopdeps = loopdependencies(op)
    @assert vectorized ∈ loopdeps
    mvar, opu₁, opu₂ = variable_name_and_unrolled(first(parents(op)), u₁loopsym, u₂loopsym, suffix)
    ptr = vptr(op)
    # var = pvariable_name(op, suffix, tiled)
    # parentisunrolled = unrolled ∈ loopdependencies(first(parents(op)))
    if isunrolled
        umin = 0
        U = u₁
    else
        umin = -1
        U = 0
    end
    vecnotunrolled = vectorized !== u₁loopsym
    for u ∈ umin:U-1
        td = UnrollArgs(ua, u)
        name, mo = name_memoffset(mvar, op, td, opu₁, inds_calc_by_ptr_offset)
        instrcall = Expr(:call, storeinstr(op, vectorized), ptr, name, mo)
        if mask !== nothing && (vecnotunrolled || u == U - 1)
            push!(instrcall.args, mask)
        end
        push!(q.args, instrcall)
    end
end
function lower_store!(
    q::Expr, ls::LoopSet, op::Operation, ua::UnrollArgs, mask::Union{Nothing,Symbol,Unsigned} = nothing
)
    @unpack u₁, u₁loopsym, u₂loopsym, vectorized, suffix = ua
    isunrolled₁ = isu₁unrolled(op) #u₁loopsym ∈ loopdependencies(op)
    # isunrolled₂ = isu₂unrolled(op)
    inds_calc_by_ptr_offset = indices_calculated_by_pointer_offsets(ls, op.ref)
    # if isunrolled₁ & ((!isnothing(suffix)) && isu₂unrolled(op))
    #     u₁ind = findfirst(isequal(u₁loopsym), loopdependencies(op))::Int
    #     u₂ind = findfirst(isequal(u₁loopsym), loopdependencies(op))::Int
    #     if inds_calc_by_ptr_offset[u₁ind] && inds_calc_by_ptr_offset[u₂ind]
    #         if u₁ind < u₂ind
    #             zero_offset = u₂ind
    #             gespind = gensym("gesp")
    #             # gesp by suffix
    #         else
    #             zero_offset = u₁ind
    #             if iszero(suffix) # gesp for each

    #             end
    #         end
    #     end
    # else
    # end
    ua = UnrollArgs(ua, isunrolled₁ ? u₁ : 1)
    if instruction(op).instr !== :conditionalstore!
        if vectorized ∈ loopdependencies(op)
            lower_store_vectorized!(q, op, ua, mask, isunrolled₁, inds_calc_by_ptr_offset)
        else
            lower_store_scalar!(q, op, ua, mask, inds_calc_by_ptr_offset)
        end
    else
        if vectorized ∈ loopdependencies(op)
            lower_conditionalstore_vectorized!(q, op, ua, mask, isunrolled₁, inds_calc_by_ptr_offset)
        else
            lower_conditionalstore_scalar!(q, op, ua, mask, inds_calc_by_ptr_offset)
        end
    end
end


module ConcolicFuzzer

export execute, check, fuzz, fuzz_wargs, fuzz_and_check

# Cassette is a non-standard execution engine for Julia
# It allows for contextualised execution. I use Cassette to generate
# concolic traces of arbitrary Julia programs. Each execution creates
# a particular trace depending on the concrete input arguments.
using Cassette
using Cassette: Tagged, tag, untag, istagged, metadata, hasmetadata,
                enabletagging, overdub, canoverdub, similarcontext, fallback

Cassette.@context TraceCtx

anything(x) = x
anything(x::Some) = something(x)

include("taint.jl")
include("trace.jl")
include("asserts.jl")
include("traceutils.jl")

"""
    execute(f, x)

Executes the function `f` concolicly by tainting the argument `x`.
Returns a tuple with the first element being the concrete value,
the second element a boolean that indicates whether the output is dependend on the input
and the third element is a concolic Trace.
"""
function execute(f, args...; rands = Any[])
    trace = Callsite[]
    ctx = enabletagging(TraceCtx(metadata = trace, pass = InsertAssertsPass), f)
    tagged_args = map(enumerate(args)) do (i, arg)
        sym = Sym(Symbol(:arg_, i), typeof(arg))
        tag(arg, ctx, sym)
    end
    y = try
        Cassette.@overdub(ctx, f(tagged_args...))
    catch err
        err
    end

    # Unpack the trace
    @assert length(trace) == 1
    trace = trace[1]
    @assert trace.f === Core._apply
    @assert length(trace.children) == 1
    trace = first(trace.children)
    verify(trace)

    if istagged(y, ctx)
        vy = untag(y, ctx)
    else
        vy = y
    end

    symb = istagged(y, ctx) && hasmetadata(y, ctx)
    return (vy, symb, trace)
end

include("z3.jl")
include("fuzzer.jl")

"""
    check(f, args...)

Given a `f` that uses manually inserted `assert` and `prove` statements.
Check if the symbolic part of the trace is satisfiable or not.
"""
function check(f, args...; rands = Any[])
    _, _, trace = execute(f, args...; rands = rands)
    stream = filter(trace)
    return checkStream(stream)
end

"""
    fuzz_and_check(f, argtypes...)

Ussing a user provided `prove` stament, proves that
the condition holds across all reachable branches.

Returns a list of `(sat, args)` where sat indicates
the trace was satisfiable and under which arguments.
To prove a statement you want to have branches be
unsatisfiable.

NOTE:
  - Do not use manually inserted `assert` statements, since
    that will throw `fuzz` off the trail. You can use `@assert`
    to the same effect.
"""
function fuzz_and_check(f, argtypes...)
    tested, errored = fuzz(f, argtypes...)
    ntested = Any[]
    for (out, args, rands) in tested
        result = check(f, args...; rands=rands)
        push!(ntested, result)
    end
    return ntested
end
end # module