"""
	SMMOptions

SMMOptions is a struct that contains options related to the optimization.
"""
struct SMMOptions
	globalOptimizer::Symbol #algorithm for finding a global maximum
	localOptimizer::Symbol 	#algorithm for finding a local maximum
	maxFuncEvals::Int64			#maximum number of evaluations
	saveSteps::Int64				#maximum number of steps
	saveName::String				#name under which the optimization should be saved
	showDistance::Bool			#show the distance, everytime the objective function is calculated?
	minBox::Bool						#When looking for a local maximum, use Fminbox ?
	populationSize::Int64		#When using BlackBoxOptim, set the population size
	penaltyValue::Float64 	#Objective function's value when the model fails
	gridType::Symbol				#sampling procedure to use (latin hypercube by default)
	saveStartingValues::Bool #whether or not saving the starting values when using local_to_global
	maxTrialsStartingValues::Int64 #maximum number of attempts when searching for valid starting values
	thresholdStartingValue::Float64 #value under which a point is considered as a valid starting value
end

function SMMOptions( ;globalOptimizer::Symbol=:dxnes,
											localOptimizer::Symbol=:LBFGS,
											maxFuncEvals::Int64=1000,
											saveSteps::Int64 = 100,
											saveName::String = get_now(),
											showDistance::Bool = false,
											minBox::Bool = false,
											populationSize::Int64 = 50,
											penaltyValue::Float64 = 999999.0,
											gridType::Symbol = :latin,
											saveStartingValues::Bool = true,
											maxTrialsStartingValues::Int64 = 1000,
											thresholdStartingValue::Float64 = 99999.0)

	# Safety Checks
	#--------------
	if saveSteps == 0
		error("You cannot set saveSteps equal to 0.")
	end

	if saveSteps > maxFuncEvals
		error("Error in the constructor for SMMOptions. \n saveSteps = $(saveSteps) > maxFuncEvals = $(maxFuncEvals)")
	end

	if mod(maxFuncEvals, saveSteps) != 0
		error("Error in the constructor for SMMOptions. \n maxFuncEvals should be a multiple of saveSteps")
	end

	if thresholdStartingValue > penaltyValue
		error("Please set thresholdStartingValue < penaltyValue.")
	end

	# Warnings
	#----------
	# Saving too often (relative to population size) leads to a poor performance if using BlackBoxOptim
	# see here: https://github.com/robertfeldt/BlackBoxOptim.jl/issues/99
	#--------------------------------------------------------------------
	# [TODO] Give a warning relative to the Population Size instead.
	if saveSteps/populationSize < 6 && is_bb_optimizer(globalOptimizer) == true
		info("WARNING. When saveSteps/populationSize < 6 using BlackBoxOptim, the performance of the global maximizer may deteriorate.")
	end

	SMMOptions(globalOptimizer,
						localOptimizer,
						maxFuncEvals,
						saveSteps,
						saveName,
						showDistance,
						minBox,
						populationSize,
						penaltyValue,
						gridType,
						saveStartingValues,
						maxTrialsStartingValues,
						thresholdStartingValue)

end

"""
	SMMProblem

SMMProblem is a mutable struct that caries all the information needed to
perform the optimization and display the results.
"""
mutable struct SMMProblem
	iter::Int64
	priors::OrderedDict{String,Array{Float64,1}}
	empiricalMoments::OrderedDict{String,Array{Float64,1}}
	simulatedMoments::OrderedDict{String, Float64}
	distanceEmpSimMoments::Float64
	simulate_empirical_moments::Function					#returns an ordered dict
	simulate_empirical_moments_array::Function		#returns an Array
	objective_function::Function
	options::SMMOptions
	bbSetup::BlackBoxOptim.OptController					#set up when using BlackBoxOptim (global minimum)
	bbResults::BlackBoxOptim.OptimizationResults	#results when using BlackBoxOptim (global minimum)
	optimResults::Optim.OptimizationResults				#results when using Optim (local minimum)
	Sigma0::Array{Float64,2}											#distance matrix (in the terminology of Duffie and Singleton (1993))
	Avar::Array{Float64,2}											  #asymptotic variance of the SMM estimate
end

# Constructor for SMMProblem
#------------------------------------------------------------------------------
function SMMProblem(  ;iter::Int64 = 0,
											priors::OrderedDict{String,Array{Float64,1}} = OrderedDict{String,Array{Float64,1}}(),
											empiricalMoments::OrderedDict{String,Array{Float64,1}} = OrderedDict{String,Array{Float64,1}}(),
											simulatedMoments::OrderedDict{String, Float64} = OrderedDict{String,Float64}(),
											distanceEmpSimMoments::Float64 = 0.,
											simulate_empirical_moments::Function = default_function,       #returns an ordered dict
											simulate_empirical_moments_array::Function = default_function, #returns an Array
											objective_function::Function = default_function,
											options::SMMOptions = SMMOptions(),
											bbSetup::BlackBoxOptim.OptController = defaultbbOptimOptController,
											bbResults::BlackBoxOptim.OptimizationResults = defaultbbOptimOptimizationResults,
											optimResults::Optim.OptimizationResults = defaultOptimResults,
											Sigma0::Array{Float64,2} = Array{Float64}(0,0),
											Avar::Array{Float64,2} = Array{Float64}(0,0))

	SMMProblem(iter,
						priors,
						empiricalMoments,
						simulatedMoments,
						distanceEmpSimMoments,
						simulate_empirical_moments,
						simulate_empirical_moments_array,
						objective_function,
						options,
						bbSetup,
						bbResults,
						optimResults,
						Sigma0,
						Avar)

end

"""
	default_function(x)

Function x->x. Used to initialize functions.
"""
function default_function(x)
	println("default_function, returns input")
	x
end

"""
	rosenbrock2d(x)

Rosenbrock function. Used to initialize BlackBoxOptim.OptController().
"""
function rosenbrock2d(x)
  return (1.0 - x[1])^2 + 100.0 * (x[2] - x[1]^2)^2
end

# It is quite useful to have "default" BlackBoxOptim.OptController and
# BlackBoxOptim.OptimizationResults objects, since
# BlackBoxOptim.OptController() and BlackBoxOptim.OptimizationResults()
# do not work
#-------------------------------------------------------------------------------
defaultbbOptimOptController = bbsetup(x -> rosenbrock2d(x);
																		Method=:dxnes,
																		SearchRange = (-5.0, 5.0),
             												NumDimensions = 2, MaxFuncEvals = 2,
																		TraceMode = :silent)

defaultbbOptimOptimizationResults = bboptimize(defaultbbOptimOptController)

defaultOptimResults = optimize(rosenbrock2d, [0.0, 0.0], LBFGS())

"""
	is_global_optimizer(s::Symbol)

function to check that the global optimizer chosen is available.
"""
function is_global_optimizer(s::Symbol)

	# Global Optimizers using BlackBoxOptim
	# source: https://github.com/robertfeldt/BlackBoxOptim.jl/blob/master/examples/benchmarking/latest_toplist.csv
	#------------------------------------------------------------------------------
	listValidGlobalOptimizers = [:dxnes, :adaptive_de_rand_1_bin_radiuslimited, :xnes,
															 :de_rand_1_bin_radiuslimited, :adaptive_de_rand_1_bin,
															 :generating_set_search, :de_rand_1_bin,
															 :separable_nes, :resampling_inheritance_memetic_search,
															 :probabilistic_descent, :resampling_memetic_search,
															 :de_rand_2_bin_radiuslimited, :de_rand_2_bin,
															 :random_search, :simultaneous_perturbation_stochastic_approximation]

	in(s, listValidGlobalOptimizers)

end

"""
	is_bb_optimizer(s::Symbol)

function to check whether the global optimizer is using BlackBoxOptim
"""
function is_bb_optimizer(s::Symbol)

	# source: https://github.com/robertfeldt/BlackBoxOptim.jl/blob/master/examples/benchmarking/latest_toplist.csv
	listbbOptimizers = [:dxnes, :adaptive_de_rand_1_bin_radiuslimited, :xnes,
															 :de_rand_1_bin_radiuslimited, :adaptive_de_rand_1_bin,
															 :generating_set_search, :de_rand_1_bin,
															 :separable_nes, :resampling_inheritance_memetic_search,
															 :probabilistic_descent, :resampling_memetic_search,
															 :de_rand_2_bin_radiuslimited, :de_rand_2_bin,
															 :random_search, :simultaneous_perturbation_stochastic_approximation]

	in(s, listbbOptimizers)

end

"""
	is_local_optimizer(s::Symbol)

function to check that the local optimizer chosen is available.
"""
function is_local_optimizer(s::Symbol)

	# source: https://github.com/robertfeldt/BlackBoxOptim.jl/blob/master/examples/benchmarking/latest_toplist.csv
	listValidLocalOptimizers = [:NelderMead, :SimulatedAnnealing, :ParticleSwarm,
															:BFGS, :LBFGS, :ConjugateGradient, :GradientDescent,
															:MomentumGradientDescent, :AcceleratedGradientDescent]

	in(s, listValidLocalOptimizers)

end

"""
	is_optim_optimizer(s::Symbol)

function to check whether the local minimizer uses the package Optim.
"""
function is_optim_optimizer(s::Symbol)

	# source: https://github.com/robertfeldt/BlackBoxOptim.jl/blob/master/examples/benchmarking/latest_toplist.csv
	listOptimOptimizers = [:NelderMead, :SimulatedAnnealing, :ParticleSwarm,
															:BFGS, :LBFGS, :ConjugateGradient, :GradientDescent,
															:MomentumGradientDescent, :AcceleratedGradientDescent]

	in(s, listOptimOptimizers)

end

"""
	convert_to_optim_algo(s::Symbol)

function to convert local optimizer (of type Symbol) to an Optim algo.
"""
function convert_to_optim_algo(s::Symbol)
	eval(Meta.parse("$(s)()"))
end

"""
	convert_to_fminbox(s::Symbol)

function to convert local optimizer (of type Symbol) to a Fminbox usable
by Optim.
"""
function convert_to_fminbox(s::Symbol)

	# Old API (before v0.15.0)
	# To be changed when switching to Julia v0.7
	eval(Meta.parse("Fminbox{$(s)}()"))

end


module TestRandomIncrements

using Test
using TSXPlayground:
    parallel_random_increments!,
    random_increments_problem_clusters,
    random_increments_problem_uniform,
    serial_random_increments!

ncounters = 2^20
nindices = 2^5

@testset "$label" for (label, (counters, indices)) in [
    ("uniform", random_increments_problem_uniform(ncounters, nindices)),
    ("cluster", random_increments_problem_clusters(ncounters, nindices, 5)),
]
    counters_seq = zero(counters)
    serial_random_increments!(counters_seq, indices)
    @testset for ntasks in 1:Threads.nthreads()
        counters_atomic = zero(counters)
        counters_tsx = zero(counters)
        parallel_random_increments!(counters_atomic, indices, ntasks)
        parallel_random_increments!(counters_tsx, indices, ntasks, Val(true), 10, 4)
        @test counters_atomic == counters_seq
        @test counters_tsx == counters_seq
    end
end

end  # module


#Iterate through samps and circ list, recording the indices at which the samp vector crosses the thresholds in circ_list
#circ_list is expected to be sorted in either forward or reverse order, and the first index of samps is expected to be outside the range spanned by first(circ_list) and last(circ_list)
#After fully iterating through circ_list the search will continue on the reversed circ_list.
#Since the endpoints of circ_list get repeated they can be counted too often due to noise.  For this reason the pad_samps argument determines how long to wait before counting another threshold crossing each time the list is reversed
#Returns a vector-of-vectors where each vector contains the crossing indices of a particular circ_list value
function find_circular(samps::AbstractVector, circ_list::AbstractVector, pad_nsamps::Int)
    is_incr = true
    reversed = false
    rlist = reverse(circ_list)
    if !issorted(circ_list)
        if !issorted(rlist)
            error("circ_list must be in increasing or decreasing order")
        end
        circ_list = rlist
        is_incr = false
        reversed = true
    end
    samp1 = first(samps)
    if (is_incr && (samp1 > circ_list[1])) || (!is_incr && (samp1 < circ_list[end]))
        error("The sample vector begins in the middle of a cycle")
    end
    outputs = Vector{Vector{Int}}(undef, length(circ_list))
    for i = 1:length(circ_list)
        outputs[i] = Int[]
    end
    wait_stop = 0
    circ_i = reversed ? length(circ_list) : 1
    for i = 1:length(samps)
        if i == wait_stop
            if (is_incr && (samps[i] >= circ_list[circ_i])) || (!is_incr && (samps[i] <= circ_list[1]))
                error("The signal had already passed threshold after waiting for pad_nsamps samples")
            end
        elseif i > wait_stop
            if is_incr && (samps[i] >= circ_list[circ_i])
                push!(outputs[circ_i], i)
                if circ_i == length(circ_list)
                    is_incr = false
                    wait_stop = i + pad_nsamps
                else
                    circ_i += 1
                end
            elseif !is_incr && (samps[i] <= circ_list[circ_i])
                push!(outputs[circ_i], i)
                if circ_i == 1
                    is_incr = true
                    wait_stop = i + pad_nsamps
                else
                    circ_i -= 1
                end
            end
        end
    end
    if reversed
        outputs = reverse(outputs)
    end
    return outputs
end

#Returns a vector that is the mean of each cycle in "samps" beginning at start_idx
#Setting start_idx greater than one allows skipping of early samples while the piezo warms up
function mean_cycle(samps::AbstractVector, nsamps_cycle::Int; start_idx = 1)
    lsamps = length(samps)
    @assert start_idx >=1 && start_idx <= lsamps
    @assert (lsamps-start_idx+1) >= nsamps_cycle
    rng_analyzed = start_idx:lsamps
    samps_used = view(samps, rng_analyzed)
    if mod(length(samps_used), nsamps_cycle) != 0
        @warn "Found a fractional number of cycles.  Truncating to analyze a whole number of cycles."
        rng_analyzed = start_idx:start_idx+(nsamps_cycle*div(length(rng_analyzed),nsamps_cycle)-1)
        samps_used = view(samps, rng_analyzed)
    end
    cyc_mat = get_cycles(samps_used, nsamps_cycle)
    print("Averaging $(size(cyc_mat,2)) cycles...\n")
    return reshape(mean(ustrip.(cyc_mat), dims=2), nsamps_cycle)  * unit(samps[1]) #work around limitation of mean function with Unitful Quantities
end

#find integer sample shift to align mod and mon.  (Even with calibration there is often a phase delay between the MOD and MON signal)
#We do this so that we know when the MON signal reaches its minimum
#indmax(xcorr(a,b)) is displaced from the center of the xcorr vector by the amount that b would need to be shifted to align with a
function mon_delay(mod_cyc, mon_cyc)
    xc = xcorr(repeat(ustrip.(mon_cyc), 3), ustrip.(mod_cyc)) #replicate it so that edges don't corrupt results
    ctr_i = div(length(xc),2)+1
    half_window_sz = div(length(mon_cyc),4)
    inner_half = xc[(ctr_i-half_window_sz):(ctr_i+half_window_sz)]
    #amount that mod_cyc needs to be shifted to align with mon_cyc (should be positive)
    #we examine only the inner half of shifts because we don't want to shift more than a half cycle
    return argmax(inner_half) - (div(length(inner_half),2) + 1)
end

#Returns a set of indices marking when the mon signal crosses the values in the crossings vector.
#Useful for deciding when to time camera exposures / laser pulses
#note: if calibration is needed it must be done before calling this function. i.e. sig should be calibrated already
function pulse_timings(mod_cyc::AbstractVector, mon_cyc::AbstractVector, targets::AbstractVector, pad_nsamps::Int)
    @assert length(mod_cyc) == length(mon_cyc)
    nsamps_cycle = length(mod_cyc)
    mon_idxs = find_circular(mon_cyc, targets, pad_nsamps)
    @assert all(map(length, mon_idxs).==2) #should have exactly two crossings for each target with a cyclical waveform
    return mon_idxs
end

function flash_cam_cycs(mod_cyc::AbstractVector, mon_cyc::AbstractVector, slice_zs, pad_nsamps::Int, flash_time::HasTimeUnits, exp_time::HasTimeUnits, sample_rate::HasInverseTimeUnits; is_bidi = true)
    offset = mon_delay(ustrip.(mod_cyc), ustrip.(mon_cyc))
    mon_cyc = circshift(mon_cyc, -offset)
    @assert flash_time <= exp_time
    flash_ctr_is = pulse_timings(mod_cyc, mon_cyc, slice_zs, pad_nsamps)
    flash_ctr_fwd = [x[1] for x in flash_ctr_is]
    flash_ctr_back = reverse([x[2] for x in flash_ctr_is]) #reversed so that corresponding z locations line up
    if is_bidi
        flash_ctr_is = vcat(flash_ctr_fwd, flash_ctr_back)
    else
        flash_ctr_is = flash_ctr_fwd
    end
    flash_cam_cycs(flash_ctr_is, flash_time, exp_time, sample_rate, length(mon_cyc), offset)
end

function flash_cam_cycs(flash_ctr_is, flash_time::HasTimeUnits, exp_time::HasTimeUnits, sample_rate::HasInverseTimeUnits, nsamps_cyc::Int, offset::Int)
    #@assert all(map((x,y) -> isapprox(x, y, rtol = (1/typemax(Int16) * 800μm)), (mean_cyc[flash_ctr_fwd], mean_cyc[flash_ctr_back])))
    min_flash_sep = minimum(diff(flash_ctr_is))
    max_exp_time = ((min_flash_sep-1) / sample_rate) * 0.97
    #@show max_exp_time = (1/stack_rate - (1/samprate(pos) *length(slice_zs))) / length(slice_zs) #leaves one low sample between exposure pulses
    if exp_time >= max_exp_time
        @warn "Could not use the requested exposure time while keeping sufficient separation between exposures.  Using an exposure time of $(max_exp_time) instead.  This means that the maximum framerate required from the camera will be $(100.0 * inv(max_exp_time)/inv(exp_time))% of the requested rate. (choose the ROI size to meet this requirement)"
        exp_time = max_exp_time
    end
    #now place flashes, centered on the indices above
    nsamps_flash = ImagineInterface.calc_num_samps(flash_time, sample_rate)
    if iseven(nsamps_flash)
	    nsamps_flash -= 1 #force odd so that fwd and reverse flashes line up in bidi recordings
    end
    nsamps_from_ctr = div(nsamps_flash,2)
    flash_ivs = map(x-> ClosedInterval(x-nsamps_from_ctr, x+nsamps_from_ctr), flash_ctr_is)
    #now place exposures, aligning exposure ends with flash ends
    nsamps_exp = ImagineInterface.calc_num_samps(exp_time, sample_rate)
    exp_ivs= map(x-> ClosedInterval(x+nsamps_from_ctr-nsamps_exp+1, x+nsamps_from_ctr), flash_ctr_is)
    flash_cyc = ImagineInterface.gen_pulses(nsamps_cyc, flash_ivs)
    cam_cyc = ImagineInterface.gen_pulses(nsamps_cyc, exp_ivs)

    return flash_cyc, cam_cyc, offset
end

#empirical max acceleration
max_acc(v, sr) = maximum(abs.(upreferred.(diff(diff(v)./(1/sr))./(1/sr))))
max_vel(v, sr) = maximum(abs.(upreferred.(diff(v)./(1/sr))))

#Returns a triangle wave equivalent to the input (same max, min, freq), useful for comparisons
function eq_triangle(v, sr::HasInverseTimeUnits)
    mx = maximum(v)
    mn = minimum(v)
    freq = upreferred(sr/length(v))
    fwd, bck = ImagineInterface.gen_bidi_pos(mn, mx, upreferred(1/(2.0*freq)), sr)
    return vcat(fwd,bck)
end

#calculate the fraction of the camera's max framerate that can be utilized with the given sample vector
#by comparing the maximum velocity reached with the ideal constant velocity of a triangle wave
function framerate_efficiency(v, sr::HasInverseTimeUnits)
    tri = eq_triangle(v, sr)
    vin = max_vel(v,sr)
    vtri = max_vel(tri,sr)
    return upreferred(vtri/vin)
end

#calculate the fraction of the camera's max framerate that can be utilized with the given sample vector and slice locations
#This version is more practical than above (above is more of a worst-case efficiency)
function framerate_efficiency(v, sr::HasInverseTimeUnits, slice_zs; pad_nsamps=ImagineInterface.calc_num_samps(0.0005s, sr), toffsets_fwd=fill(0.0s, length(slice_zs)), toffsets_bck=fill(0.0s, length(slice_zs)))
    mexp = max_exp(v,sr,slice_zs; pad_nsamps=pad_nsamps, toffsets_fwd=toffsets_fwd, toffsets_bck=toffsets_bck)
    tri = eq_triangle(v, sr)
    mexp_tri = max_exp(tri, sr, slice_zs; pad_nsamps=2)
    @assert eltype(v) == eltype(slice_zs)
    return upreferred(mexp/mexp_tri)
end

#calculate longest exposure time possible given sample vector v
#Also considers temporal offsets that have been found empirically (toffsets_fwd and toffsets_bck keyword args)
function max_exp(v, sr::HasInverseTimeUnits, slice_zs; pad_nsamps = ImagineInterface.calc_num_samps(0.0005s, sr), toffsets_fwd=fill(0.0s, length(slice_zs)), toffsets_bck = fill(0.0s,length(slice_zs)))
    first_idx = argmin(v)
    v = circshift(v, -first_idx) #assumes slice_zs are sorted and increasing
    timings = ImagineAnalyses.find_circular(v, slice_zs, pad_nsamps)
    fwdt = [x[1] for x in timings]
    bckt = reverse([x[2] for x in timings])
    toffsets_fwd_nsamps = map(x->ImagineInterface.calc_num_samps(x, sr), toffsets_fwd)
    toffsets_bck_nsamps = map(x->ImagineInterface.calc_num_samps(x, sr), toffsets_bck)
    fwdt = fwdt.-toffsets_fwd_nsamps
    bckt = bckt.-reverse(toffsets_bck_nsamps)
    diffi = minimum(diff(vcat(fwdt, bckt))) - 1 #subtract one because we need a sample between exposures (to send TTL down)
    diffi = min(diffi, first(fwdt)+(length(v)-last(bckt)) - 1) #handle first/last timing
    return upreferred(diffi / sr)
end


struct Serial <: ExecutionPolicy
    fval::Float64
    x::Vector{Float64}

    function (::Type{Serial})()
        return new(0., Vector{Float64}())
    end
end

function initialize!(serial::Serial, x₀::AbstractVector)
    resize!(serial.x,length(x₀))
    serial.x .= x₀
end
getx(serial::Serial) = serial.x

function initialize!(proxgrad::ProxGradient{Serial})
    init_c = @cfunction(init_wrapper, Nothing,
                        (Ptr{Cdouble}, Ptr{Cdouble}, Ptr{Cvoid}))
    boost = POLO.boosting(proxgrad)
    boosting_c = @cfunction(boost_wrapper, Ptr{Cdouble},
                              (Cint, Cint, Cint, Ptr{Cdouble}, Ptr{Cdouble}, Ptr{Cdouble}, Ptr{Cvoid}))
    step = POLO.stepsize(proxgrad)
    step_c = @cfunction(step_wrapper, Cdouble,
                          (Cint, Cint, Cdouble, Ptr{Cdouble}, Ptr{Cdouble}, Ptr{Cdouble}, Ptr{Cvoid}))
    smooth = POLO.smoothing(proxgrad)
    smoothing_c = @cfunction(smooth_wrapper, Ptr{Cdouble},
                               (Cint, Cint, Ptr{Cdouble}, Ptr{Cdouble}, Ptr{Cdouble}, Ptr{Cdouble}, Ptr{Cvoid}))
    proxim = POLO.prox(proxgrad)
    prox_c = @cfunction(prox_wrapper, Ptr{Cdouble},
                          (Cdouble, Ptr{Cdouble}, Ptr{Cdouble}, Ptr{Cdouble}, Ptr{Cdouble}, Ptr{Cvoid}))
    proxgrad.ptr = ccall(POLO.proxgradient_s, Ptr{Cvoid},
                         (Ptr{Cvoid},
                          Ptr{Cvoid}, Any,
                          Ptr{Cvoid}, Any,
                          Ptr{Cvoid}, Any,
                          Ptr{Cvoid}, Any),
                         init_c,
                         boosting_c, boost,
                         step_c, step,
                         smoothing_c, smooth,
                         prox_c, proxim)
    return proxgrad
end

execution_handle(::Serial) = POLO.run_serial
delete_handle(::Serial) = POLO.delete_proxgradient_s
getf_handle(::Serial) = POLO.getf_s
getx_handle(::Serial) = POLO.getx_s


@deprecate calibrationerror(estimator, data...) estimator(predictions_targets(data...)...)

@deprecate BiasedSKCE(k1::Kernel, k2::Kernel) BiasedSKCE(k1 ⊗ k2)
@deprecate UnbiasedSKCE(k1::Kernel, k2::Kernel) UnbiasedSKCE(k1 ⊗ k2)
@deprecate BlockUnbiasedSKCE(k1::Kernel, k2::Kernel) BlockUnbiasedSKCE(k1 ⊗ k2)
@deprecate BlockUnbiasedSKCE(k1::Kernel, k2::Kernel, blocksize::Int) BlockUnbiasedSKCE(
    k1 ⊗ k2, blocksize
)
@deprecate UCME(k1::Kernel, k2::Kernel, data...) UCME(k1 ⊗ k2, data...)
@deprecate UCME(kernel::Kernel, testdata...) UCME(
    kernel, predictions_targets(testdata...)...
)
@deprecate UCME(kernel::Kernel, testdata::Tuple{<:Any,<:Any}) UCME(
    kernel, predictions_targets(testdata...)...
)
@deprecate UCME(kernel::Kernel, testdata::AbstractVector{<:Tuple{<:Any,<:Any}}) UCME(
    kernel, predictions_targets(map(first, testdata), map(last, testdata))...
)
@deprecate UCME(
    kernel::Kernel, predictions::AbstractMatrix{<:Real}, targets::AbstractVector
) UCME(kernel, predictions_targets(predictions, targets)...)

@deprecate predictions_targets((predictions, targets)::Tuple{<:Any,<:Any}) predictions_targets(
    predictions, targets
) false
@deprecate predictions_targets(data::AbstractVector{<:Tuple{<:Any,<:Any}}) predictions_targets(
    map(first, data), map(last, data)
) false
@deprecate predictions_targets(predictions::AbstractMatrix{<:Real}, targets::AbstractVector) predictions_targets(
    [predictions[:, i] for i in axes(predictions, 2)], targets
) false
@deprecate predictions_targets(predictions::AbstractVector, targets::AbstractVector) tuple(
    predictions, targets
) false

@deprecate unbiasedskce(kernel, predictions, targets) UnbiasedSKCE(kernel)(
    predictions, targets
) false


# This file is a part of BAT.jl, licensed under the MIT License (MIT).

include("mcmc_weighting.jl")
include("proposaldist.jl")
include("mcmc_sampleid.jl")
include("mcmc_algorithm.jl")
include("mcmc_noop_tuner.jl")
include("mcmc_stats.jl")
include("mcmc_convergence.jl")
include("chain_pool_init.jl")
include("multi_cycle_burnin.jl")
include("mcmc_sample.jl")
include("mh/mh.jl")
include("ahmc/ahmc.jl")


"""
    InitiatorValue{V}(; safe::V, unsafe::V, initiator::V) where V
Composite "walker" with three fields. For use with [`InitiatorDVec`](@ref)s.
"""
struct InitiatorValue{V}
    safe::V
    unsafe::V
    initiator::V
end
function InitiatorValue{V}(;safe=zero(V), unsafe=zero(V), initiator=zero(V)) where {V}
    return InitiatorValue{V}(V(safe), V(unsafe), V(initiator))
end
function InitiatorValue{V}(i::InitiatorValue) where {V}
    return InitiatorValue{V}(V(i.safe), V(i.unsafe), V(i.initiator))
end

function Base.:+(v::InitiatorValue, w::InitiatorValue)
    return InitiatorValue(v.safe + w.safe, v.unsafe + w.unsafe, v.initiator + w.initiator)
end
function Base.:-(v::InitiatorValue, w::InitiatorValue)
    return InitiatorValue(v.safe - w.safe, v.unsafe - w.unsafe, v.initiator - w.initiator)
end
function Base.:-(v::InitiatorValue{V}) where {V}
    return InitiatorValue{V}(-v.safe, -v.unsafe, -v.initiator)
end
Base.zero(::Union{I,Type{I}}) where {V,I<:InitiatorValue{V}} = InitiatorValue{V}()

Base.convert(::Type{<:InitiatorValue{V}}, x::InitiatorValue{V}) where {V} = x
function Base.convert(::Type{<:InitiatorValue{U}}, x::InitiatorValue{V}) where {U,V}
    return InitiatorValue{U}(safe=x.safe, unsafe=x.unsafe, initiator=x.initiator)
end
function Base.convert(::Type{<:InitiatorValue{V}}, x) where {V}
    return InitiatorValue{V}(safe=convert(V, x))
end

"""
    InitiatorRule{V}

Abstract type for defining initiator rules for [`InitiatorDVec`](@ref).
Concrete implementations:

* [`Initiator`](@ref)
* [`SimpleInitiator`](@ref)
* [`CoherentInitiator`](@ref)

When defining a new `InitiatorRule`, also define a corresponding method for [`value`](@ref)!
"""
abstract type InitiatorRule{V} end

"""
    value(i::InitiatorRule, v::InitiatorValue)
Convert the [`InitiatorValue`](@ref) `v` into a scalar value according to the
[`InitiatorRule`](@ref) `i`.

Internal function that implements functionality of [`InitiatorDVec`](@ref).
"""
value

"""
    Initiator(threshold) <: InitiatorRule

Initiator rule to be passed to [`InitiatorDVec`](@ref). An initiator is a configuration
`add` with a coefficient with magnitude `abs(v[add]) > threshold`. Rules:

* Initiators can spawn anywhere.
* Non-initiators can spawn to initiators.

See [`InitiatorRule`](@ref).
"""
struct Initiator{V} <: InitiatorRule{V}
    threshold::V
end

function value(i::Initiator, v::InitiatorValue)
    return v.safe + v.initiator + !iszero(v.initiator) * v.unsafe
end

"""
    SimpleInitiator(threshold) <: InitiatorRule

Simplified initiator rule to be passed to [`InitiatorDVec`](@ref).
An initiator is a configuration `add` with a coefficient with magnitude
`abs(v[add]) > threshold`. Rules:

* Initiators can spawn anywhere.
* Non-initiators cannot spawn.

See [`InitiatorRule`](@ref).
"""
struct SimpleInitiator{V} <: InitiatorRule{V}
    threshold::V
end

function value(i::SimpleInitiator, v::InitiatorValue)
    return v.safe + v.initiator
end

"""
    CoherentInitiator(threshold) <: InitiatorRule

Initiator rule to be passed to [`InitiatorDVec`](@ref).
An initiator is a configuration `add` with a coefficient with magnitude
`abs(v[add]) > threshold`. Rules:

* Initiators can spawn anywhere.
* Non-initiators can spawn to initiators.
* Multiple non-initiators can spawn to a single non-initiator if their contributions add up
  to a value greater than the initiator threshold.

  See [`InitiatorRule`](@ref).
"""
struct CoherentInitiator{V} <: InitiatorRule{V}
    threshold::V
end

function value(i::CoherentInitiator, v::InitiatorValue)
    if !iszero(v.initiator) || abs(v.unsafe) > i.threshold
        return v.initiator + v.safe + v.unsafe
    else
        return v.initiator + v.safe
    end
end

"""
    InitiatorDVec{K,V} <: AbstractDVec{K,V}

Dictionary-based vector-like data structure for use with [`lomc!`](@ref) and
[`KrylovKit.jl`](https://github.com/Jutho/KrylovKit.jl). See [`AbstractDVec`](@ref).
Functionally identical to [`DVec`](@ref), but contains [`InitiatorValue`](@ref)s internally
in order to facilitate initiator methods. How the initiators are handled is controlled by
the `initiator` keyword argument (see below).

## Constructors

* `InitiatorDVec(dict::AbstractDict[; style, initiator, capacity])`: create an
  `InitiatorDVec` with `dict` for storage.  Note that the data may or may not be copied.

* `InitiatorDVec(args...[; style, initiator, capacity])`: `args...` are passed to the `Dict`
  constructor. The `Dict` is used for storage.

* `InitiatorDVec{K,V}([; style, initiator, capacity])`: create an empty `InitiatorDVec{K,V}`.

* `InitiatorDVec(dv::AbstractDVec[; style, initiator, capacity])`: create an `InitiatorDVec`
   with the same contents as `dv`. The `style` is inherited from `dv` by default.

## Keyword  arguments

* `style`: A valid [`StochasticStyle`](@ref).  The default is selected based on the
  `InitiatorDVec`'s `valtype` (see [`default_style`](@ref)). If a style is given and
  the `valtype` does not match the `style`'s `eltype`, the values are converted to an
  appropriate type.

* `initiator = Initiator(1)`: A valid [`InitiatorRule`](@ref). See [`Initiator`](@ref).

* `capacity`: Indicative size as `Int`. Optional. Sets the initial size of the
  `InitiatorDVec` via [`sizehint!`](@ref).

"""
struct InitiatorDVec{
    K,V,D<:AbstractDict{K,InitiatorValue{V}},S<:StochasticStyle{V},I<:InitiatorRule
} <: AbstractDVec{K,V}
    storage::D
    style::S
    initiator::I
end

###
### Constructors
###
# Vararg
function InitiatorDVec(args...; kwargs...)
    storage = Dict(args...)
    return InitiatorDVec(storage; kwargs...)
end
# Dict with InitiatorValues
function InitiatorDVec(
    dict::AbstractDict{K,InitiatorValue{V}};
    style=default_style(V), initiator=Initiator(one(V)), capacity=0
) where {K,V}
    T = eltype(style)
    if T === V
        storage = dict
        sizehint!(storage, capacity)
    else
        storage = Dict{K,InitiatorValue{T}}()
        sizehint!(storage, capacity)
        for (k, v) in pairs(dict)
            storage[k] = InitiatorValue{T}(v.safe, v.unsafe, v.initiator)
        end
    end
    return InitiatorDVec(storage, style, initiator)
end
# Dict with regular values
function InitiatorDVec(
    dict::AbstractDict{K,V}; style=default_style(V), initiator=Initiator(one(V)), capacity=0
) where {K,V}
    T = eltype(style)
    storage = Dict{K,InitiatorValue{T}}()
    sizehint!(storage, capacity)
    for (k, v) in pairs(dict)
        storage[k] = InitiatorValue{T}(safe=v)
    end
    return InitiatorDVec(storage, style, initiator)
end
# Empty
function InitiatorDVec{K,V}(; kwargs...) where {K,V}
    return InitiatorDVec(Dict{K,InitiatorValue{V}}(); kwargs...)
end
# From another DVec
function InitiatorDVec(
    dv::AbstractDVec{K,V}; style=StochasticStyle(dv), initiator=Initiator(one(V)), capacity=0
) where {K,V}
    return InitiatorDVec(copy(storage(dv)); style, initiator)
end

function Base.empty(dvec::InitiatorDVec{K,V}) where {K,V}
    return InitiatorDVec{K,V}(; style=dvec.style, initiator=dvec.initiator)
end
function Base.empty(dvec::InitiatorDVec{K}, ::Type{V}) where {K,V}
    return InitiatorDVec{K,V}(; initiator=dvec.initiator)
end
function Base.empty(dvec::InitiatorDVec, ::Type{K}, ::Type{V}) where {K,V}
    return InitiatorDVec{K,V}(; initiator=dvec.initiator)
end

###
### Show
###
function Base.summary(io::IO, dvec::InitiatorDVec{K,V}) where {K,V}
    len = length(dvec)
    print(io, "InitiatorDVec{$K,$V} with $len entries, style = $(dvec.style), initiator = $(dvec.initiator)")
end

###
### Interface
###
StochasticStyle(dv::InitiatorDVec) = dv.style
storage(dv::InitiatorDVec) = dv.storage

function Base.getindex(dvec::InitiatorDVec{<:Any,V}, add) where {V}
    return value(dvec.initiator, get(dvec.storage, add, zero(InitiatorValue{V})))
end
function Base.setindex!(dvec::InitiatorDVec{<:Any,V}, v, k) where {V}
    if iszero(v)
        delete!(dvec.storage, k)
    else
        dvec.storage[k] = InitiatorValue{V}(safe=v)
    end
    return v
end

function Base.get(dvec::InitiatorDVec, args...)
    return value(dvec.initiator, get(dvec.storage, args...))
end
function Base.get!(dvec::InitiatorDVec{<:Any,V}, key, default) where {V}
    return value(dvec.initiator, get!(dvec.storage, key, InitiatorValue{V}(safe=default)))
end
function Base.get!(f::Function, dvec::InitiatorDVec{<:Any,V}, key) where {V}
    return value(dvec.initiator, get!(dvec.storage, key, InitiatorValue{V}(safe=f())))
end

@delegate InitiatorDVec.storage [haskey, getkey, pop!, isempty, length, keys]
@delegate_return_parent InitiatorDVec.storage [delete!, empty!, sizehint!]

"""
    deposit!(w::InitiatorDVec, add, val, p_add=>p_val)
Add `val` into `w` at address `add` as an [`InitiatorValue`](@ref).
"""
function deposit!(w::InitiatorDVec, add, val, (p_add, p_val))
    V = valtype(w)
    i = w.initiator
    old_val = get(w.storage, add, zero(InitiatorValue{V}))
    if p_add == add # diagonal death
        if abs(p_val) > i.threshold
            new_val = InitiatorValue{V}(initiator=val)
        else
            new_val = InitiatorValue{V}(safe=val)
        end
    else # offdiagonal spawn
        if abs(p_val) > i.threshold
            new_val = InitiatorValue{V}(safe=val)
        else
            new_val = InitiatorValue{V}(unsafe=val)
        end
    end
    new_val += old_val
    if new_val == InitiatorValue{V}(0, 0, 0)
        delete!(w.storage, add)
    else
        w.storage[add] = new_val
    end
    return w
end

function deposit!(w::InitiatorDVec{<:Any,V}, add, val::InitiatorValue{V}, _) where {V}
    dict = storage(w)
    old_val = get(dict, add, zero(valtype(dict)))
    dict[add] = old_val + val
    return w
end

###
### Iterators
###
# These are needed because `Iterators.map` does not infer `eltype` correctly and does not work
# with SplittablesBase.jl.
"""
    InitiatorIterator

Iterator over pairs or values of an `InitiatorDVec`. Supports the `SplittablesBase`
interface.
"""
struct InitiatorIterator{T,D,I}
    iter::D
    initiator::I

    InitiatorIterator{T}(iter::D, initiator::I) where {T,D,I} = new{T,D,I}(iter, initiator)
end
function SplittablesBase.halve(p::InitiatorIterator{T}) where {T}
    left, right = SplittablesBase.halve(p.iter)
    return InitiatorIterator{T}(left, p.initiator), InitiatorIterator{T}(right, p.initiator)
end
SplittablesBase.amount(p::InitiatorIterator) = SplittablesBase.amount(p.iter)

Base.length(p::InitiatorIterator) = length(p.iter)
Base.IteratorSize(::InitiatorIterator) = Base.HasLength()
Base.IteratorEltype(::InitiatorIterator) = Base.HasEltype()
Base.eltype(::InitiatorIterator{T}) where {T} = T

function Base.pairs(dvec::InitiatorDVec{K,V}) where {K,V}
    InitiatorIterator{Pair{K,V}}(pairs(storage(dvec)), dvec.initiator)
end
function Base.iterate(p::InitiatorIterator{<:Pair}, args...)
    it = iterate(p.iter, args...)
    isnothing(it) && return nothing
    (k, v), st = it
    return k => value(p.initiator, v), st
end

function Base.values(dvec::InitiatorDVec{<:Any,V}) where {V}
    InitiatorIterator{V}(values(storage(dvec)), dvec.initiator)
end
function Base.iterate(p::InitiatorIterator, args...)
    it = iterate(p.iter, args...)
    isnothing(it) && return nothing
    v, st = it
    return value(p.initiator, v), st
end


module TestSources

using MLJBase
using Test

X = 7

Xs = source(X)
@test Xs() == X
@test Xs(8) == 8
@test elscitype(Xs) == Count
@test scitype(Xs) == MLJBase.CallableReturning{Count}
rebind!(Xs, nothing)
@test isempty(Xs)
@test Xs.scitype == Nothing

end
true


using ParserCombinator
using GenericRegex.IR.Graph; G = GenericRegex.IR.Graph
using GenericRegex.Parse.Unicode; U = GenericRegex.Parse.Unicode

for (s, r) in [
               ("abc",
                [G.Sequence{Char}([G.Literal('a'), G.Literal('b'), G.Literal('c')])]),
               ("a|b",
                [G.Choice{Char}([G.Literal('a'), G.Literal('b')])]),
               ("aa|b",
                [G.Choice{Char}([G.Sequence{Char}([G.Literal('a'), G.Literal('a')]), G.Literal('b')])]),
               ("a|bb",
                [G.Choice{Char}([G.Literal('a'), G.Sequence{Char}([G.Literal('b'), G.Literal('b')])])]),
               ("(a|b)",
                [G.Group(1, G.Choice{Char}([G.Literal('a'), G.Literal('b')]))]),
               ("(a|(b))",
                [G.Group(1, G.Choice{Char}([G.Literal('a'), G.Group(2, G.Literal('b'))]))]),
               ("(?:a|b)",
                [G.Choice{Char}([G.Literal('a'), G.Literal('b')])]),
               ("a|b(?:c|d)",
                [G.Choice{Char}([G.Literal('a'), G.Sequence{Char}([G.Literal('b'), G.Choice{Char}([G.Literal('c'), G.Literal('d')])])])]),
               ("a*",
                [G.Repeat(G.Literal('a'), 0, typemax(Int))]),
               ("a+",
                [G.Repeat(G.Literal('a'), 1, typemax(Int))]),
               ("ab+",
                [G.Sequence{Char}([G.Literal('a'), G.Repeat(G.Literal('b'), 1, typemax(Int))])]),
               ("a?a|b",
                [G.Choice{Char}([G.Sequence{Char}([G.Repeat(G.Literal('a'), 0, 1), G.Literal('a')]), G.Literal('b')])]),
               ("aa?|b",
                [G.Choice{Char}([G.Sequence{Char}([G.Literal('a'), G.Repeat(G.Literal('a'), 0, 1)]), G.Literal('b')])]),
               ("a|b?b",
                [G.Choice{Char}([G.Literal('a'), G.Sequence{Char}([G.Repeat(G.Literal('b'), 0, 1), G.Literal('b')])])]),
               ("a|bb?",
                [G.Choice{Char}([G.Literal('a'), G.Sequence{Char}([G.Literal('b'), G.Repeat(G.Literal('b'), 0, 1)])])]),
               ("a",
                [G.Literal('a')]),
               ("\\+",
                [G.Literal('+')])
               ]
    print("$s...")
    pattern = U.make_pattern()
    @test parse_dbg(s, pattern; debug=true) == r
    println(" ok")
end


function UnivariateDerivative()
    return UnivariateDerivative(())
end

function get_derivative(obj::UnivariateDerivative, arg0::jint)
    return jcall(obj, "getDerivative", jdouble, (jint,), arg0)
end

function get_free_parameters(obj::UnivariateDerivative)
    return jcall(obj, "getFreeParameters", jint, ())
end

function get_partial_derivative(obj::UnivariateDerivative, arg0::Vector{jint})
    return jcall(obj, "getPartialDerivative", jdouble, (Vector{jint},), arg0)
end

function to_derivative_structure(obj::UnivariateDerivative)
    return jcall(obj, "toDerivativeStructure", DerivativeStructure, ())
end


# To execute all statistical tests execute this file from within this folder
# "julia execute_tests_from_paper.jl"

include("stat_tests_mutation.jl") # Mutation scenario
include("stat_tests_between_scenarios.jl") # Mutation vs. no mutation
include("stat_tests_no_mutation.jl") # No mutation scenario
include("stat_tests_no_mutation_pinf125.jl") # Higer infection probability scenario


"""
    Field()

Wrap inbound fields and define conversions to
`String`, `Int`, `Float64`, `Bool` and `Enums`.
"""
struct Field{T<:AbstractString}
  value::T
end

# String
Base.convert(::Type{String}, x::Field) = String(x.value)

# Int
function Base.convert(::Type{Int}, x::Field)

  x.value ∈ ("", "2147483647", "9223372036854775807") ? nothing :   # typemax(Int32) typemax(Int64)
                                                        parse(Int, x.value)
end

# Float
function Base.convert(::Type{Float64}, x::Field)

  x.value ∈ ("", "1.7976931348623157E308") ? nothing :  # prevfloat(Inf)
                                             parse(Float64, x.value)
end

# Bool
function Base.convert(::Type{Bool}, x::Field)

  res = findfirst(==(x.value), ("1", "true", "0", "false"))

  isnothing(res) && error("invalid Bool: $(x.value)")

  res < 3
end

# Enum
Base.convert(::Type{T}, x::Field) where T<:Enum{Int32} = T(convert(Int, x))


# License for this file: MIT (expat)
# Copyright 2021, DLR Institute of System Dynamics and Control (DLR-SR)
# Developer: Martin Otter, DLR-SR
#
# This file is part of module ModiaResult

"""
    @enum ModiaResult.SignalType

Defines the type of the signal. Supported values:

- `ModiaResult.Independent`: Independent variable (usually the time signal).

- `ModiaResult.Continuous`: Piece-wise continuous signal (typically linearly interpolated).

- `ModiaResult.Clocked`: Clocked signal
  (values are only defined at the corresponding `Time` signal time instants and have
   no value in between; the latter might be signaled by piece-wise constant
   dotted lines).
"""
@enum SignalType Independent=1 Continuous=2 Clocked=3

"""
    (timeSignal, signal, signalType) = ModiaResult.rawSignal(result, name)

Given the result data structure `result` and a variable `name::AbstractString`,
return the result values of the independent variable (= `timeSignal`), the
corresponding result values of the variable (= `signal`) and the type
of the signal `signalType::`[`SignalType`](@ref)). Note, an error shall be raised,
if `name` is not known.

The following figure sketches the returned `timeSignal` and `signal` data structures:

![SignalDefinition](https://modiasim.github.io/ModiaResult.jl/resources/images/signal-definition.png)

Other signal types might be mapped to this basic signal type by introducing views.

# Details of the return arguments

`timeSignal::Vector{Vector{T1}}`:
A result consists of one or more **segments**.
`timeSignal[i][j]` is the value of time instant `j` in segment `i`.
`timeSignal[i][:]` must have monotonically increasing values and
type `T1<:Real` must be a subtype of `Real` for which a conversion to `AbstractFloat`
is defined. For example, `T1::Rational` is fine, but `T1::Complex` is not allowed.

`signal::Vector{Vector{T2}}` or `signal::Vector{Vector{Array{T2,N}}}`:
`signal[i][j]` is the value of the variable at time instant `timeSignal[i][j]`.
This value can be a scalar or an array. Type `T2` can have one of the following values:

- `T2 <: Real`, must be a subtype of `Real` for which a conversion to `AbstractFloat`
   is defined, or
- `T2 <: Measurements.Measurement{T1}`, or
- `T2 <: MonteCarloMeasurements.StaticParticles{T1,N}`, or
- `T2 <: MonteCarloMeasurements.Particles{T1,N}`.

If the signal is a constant with value `value`, return
`([[t_min, t_max]], [[value, value]], ModiaResult.Continuous)`.

If the signal is the time signal, return
`(timeSignal, timeSignal, ModiaResult.independent)`.
The `timeSignal` might be a dummy vector consisting of the first and last time point
in the result (if different timeSignals are present for different signals or
if the signal is constant).

`signal` and `timeSignal` may have units from package `Unitful`.

The information `signalType::SignalType` defines how the signal can be interpolated
and/or plotted.
"""
function rawSignal end

"""
    ModiaResult.signalNames(result)

Return a string vector of the signal names that are present in result.
"""
function signalNames end

"""
    ModiaResult.timeSignalName(result)

Return the name of the independent variable (typically: "time").
"""
function timeSignalName end

"""
    ModiaResult.hasOneTimeSignal(result)

Return true if `result` has one time signal.
Return false, if `result` has two or more time signals.
"""
function hasOneTimeSignal end

"""
    (signal, timeSignal, timeSignalName, signalType, arrayName,
     arrayIndices, nScalarSignals) = getSignalDetails(result, name)

Return the signal defined by `name::AbstractString` as
`signal::Vector{Matrix{<:Real}}`.
`name` may include an array range, such as "a.b.c[2:3,5]".
In this case `arrayName` is the name without the array indices,
such as `"a.b.c"`, `arrayIndices` is a tuple with the array indices,
such as `(2:3, 5)` and `nScalarSignals` is the number of scalar
signals, such as `3` if arrayIndices = `(2:3, 5)`.
Otherwise `arrayName = name, arrayIndices=(), nScalarSignals=1`.

In case the signal is not known or `name` cannot be interpreted,
`(nothing, nothing, nothing, nothing, name, (), 0)` is returned.

It is required that the value of the signal at a time instant
has either `typeof(value) <: Real` or
`typeof(value) = AbstractArray{Real, N}`.
The following `Real` types are currently supported:

1. `convert(Float64, eltype(value)` is supported
   (for example Float32, Float64, DoubleFloat, Rational, Int32, Int64, Bool).

2. Measurements.Measurement{<Type of (1)>}.

3. MonteCarloMeasurements.StaticParticles{<Type of (1)>}.

4. MonteCarloMeasurements.Particles{<Type of (1)>}.
"""
function getSignalDetails end

"""
    ModiaResult.hasSignal(result, name)

Returns `true` if signal `name::AbstractString` is available in `result`.
"""
function hasSignal(result, name::AbstractString)::Bool
    hasName = true
    try
        sigInfo = rawSignal(result,name)
    catch
        hasName = false
    end
    return hasName
end

"""
    ModiaResult.defaultHeading(result)

Return default heading of result as a string
(can be used as default heading for a plot).
"""
defaultHeading(result) = ""


using Pkg

cd("2022/2022_Scandalist/Julia")
Pkg.activate(".")
Pkg.instantiate()

using Revise

include("01_wrangling.jl")
include("../src/tweet_helpers_refac.jl")
include("../src/tweet_graphing.jl")
include("../src/tweet_mining.jl")

using Arrow
using Chain
using CSV
using DataFrames
using DataFramesMeta
using Dates
using SparseArrays
using DataStructures

dfs = read_files("Data");
scandals = read_names("Data");

original_dfs = [@subset(df,:true_type .== "NULL") for df in dfs]

original_dfs = [unique(original_df,[:tweet_id,:user_username]) for original_df in original_dfs]

function compute_ratios(df)
    df = @chain df begin
        @subset(:like_count .> 0, :retweet_count .> 0)
        @transform(:ratio_reply_like = :reply_count .- :like_count, :ratio_quote_retweet = :quote_count .- :retweet_count)
    end
end

original_df_ratios = compute_ratios.(original_dfs)

for (df,filename) in zip(original_df_ratios, scandals)
    Arrow.write("CleanData/" * filename * ".arrow", df, ntasks = 12 )
end

## K most liked/replied/... tweets

function kmost_tweets(df::DataFrame,k::I,what) where I<:Integer
    result_df = @chain df begin
        @orderby(-1 .* $what)
        @select(:user_username,:text,:created_at,:user_url,:user_name,:user_location,:user_verified,
        :retweet_count,:like_count,:quote_count,:reply_count, :ratio_reply_like, :ratio_quote_retweet)
    end
    return result_df[1:k,:]
end

metrics = String.([:retweet_count,:like_count,:quote_count,:reply_count, :ratio_reply_like, :ratio_quote_retweet])

for (df,scandal) in zip(original_df_ratios, scandals)
    for metric in metrics
        mkpath("Tweets/k_top_tweets_by_metric/" * scandal)
        CSV.write( "Tweets/k_top_tweets_by_metric/" * scandal * "/top_200_" * metric * ".csv"  , kmost_tweets(df, 200, metric))
    end
end

## K most active users

function kmost_active_users(df::DataFrame,k::I) where I<:Integer

    result_df = @chain df begin
        groupby([:author_id,:user_username,:true_type])
        combine(nrow => :n)
        @orderby(:user_username)
        unstack(:true_type,:n)
    end

    if "c(\"quoted\", \"replied_to\")" ∈ names(result_df)
        renaming_dict = Dict("user_username" => "username",
        "replied_to" => "n_replies",
        "retweeted" => "n_retweets",
        "NULL" => "n_authored",
        "quoted" => "n_quoted",
        "c(\"quoted\", \"replied_to\")" => "n_quoting_replies")

        out_df = @chain result_df begin
            rename(renaming_dict)
            @rtransform(:total = sum(skipmissing([:n_replies,:n_retweets,:n_authored,:n_quoted,:n_quoting_replies])))
            @orderby(-1 .* :total)
        end

    else
        renaming_dict = Dict("user_username" => "username",
        "replied_to" => "n_replies",
        "retweeted" => "n_retweets",
        "NULL" => "n_authored",
        "quoted" => "n_quoted")

        out_df = @chain result_df begin
            rename(renaming_dict)
            @rtransform(:total = sum(skipmissing([:n_replies,:n_retweets,:n_authored,:n_quoted])))
            @orderby(-1 .* :total)
        end

    end

    return out_df[1:k,:]
end

for (df,scandal) in zip(dfs, scandals)
    mkpath("Tweets/k_top_active_users/" * scandal)
    CSV.write( "Tweets/k_top_active_users/" * scandal * "/top_20_users.csv"  , kmost_active_users(df, 20))
    @show scandal
end

## Peak and out of peak tweets

dfs_peak = readdir("CleanData"; join = true) .|> Arrow.Table .|> DataFrame

for df in dfs_peak
    df.date = Date.(df.date)
end

over_mad(x) = median(x) + mad(x)
get_date(x) = Date(x)

function peak_dates(df,thresh_fun = over_mad)
    uqu = get_column_summary(df, :author_id, get_date, length ∘ unique)
    which_date = @subset uqu :y .> over_mad(uqu.y)
    peak_dates = which_date.x

    return peak_dates
end

for (df,scandal) in zip(dfs_peak, scandals)

    peak = peak_dates(df)

    df_in = @subset df @byrow  Date(:date) ∈ peak
    for metric in metrics
        mkpath("Tweets/peak_k_top_tweets_by_metric/" * scandal)
        CSV.write( "Tweets/peak_k_top_tweets_by_metric/" * scandal * "/top_50_" * metric * ".csv"  , kmost_tweets(df_in, 50, metric))
    end

    df_out = @subset df @byrow  Date(:date) ∉ peak
    for metric in metrics
        mkpath("Tweets/outofpeak_k_top_tweets_by_metric/" * scandal)
        CSV.write( "Tweets/outofpeak_k_top_tweets_by_metric/" * scandal * "/top_20_" * metric * ".csv"  , kmost_tweets(df_out, 20, metric))
    end
end

for df in dfs
    df.date = Date.(df.date)
end

for (df,scandal) in zip(dfs, scandals)

    @show scandal

    peak = peak_dates(df)

    df_in = @subset df @byrow  :date ∈ peak
    mkpath("Tweets/peak_k_top_active_users/" * scandal)
    CSV.write( "Tweets/peak_k_top_active_users/" * scandal * "/top_20_users.csv"  , kmost_active_users(df_in, 20))

    df_out = @subset df @byrow  :date ∉ peak
    mkpath("Tweets/outofpeak_k_top_active_users/" * scandal)
    CSV.write( "Tweets/outofpeak_k_top_active_users/" * scandal * "/top_10_users.csv"  , kmost_active_users(df_out, 10))
    println("done\n")
end


using BioSeq
importall Base, BioSeq

immutable Alphabet{ T <: BioUnit }
  alphabet::Array{T,1}
  indice::Array{UInt8,1}
  case_sensitive::Bool
end

function alphabet{T <: BioUnit}(seq::Array{T,1}, case_sensitive::Bool)
  indices = zeros(UInt8, 256 )
  if case_sensitive
    seq = unique(seq)
    indices[seq] = 1:length(seq)
  else
    seq = unique(lowercase(seq))
    indices[seq] = 1:length(seq)
    seq = uppercase!(seq)
    indices[seq] = 1:length(seq)
  end

  Alphabet{T}(seq,indices,case_sensitive)
end

length{T <: BioUnit}(ab::Alphabet{T}) = length(ab.alphabet)

start{T <: BioUnit}(ab::Alphabet{T}) = 1
next{T <: BioUnit}(ab::Alphabet{T},i::Int) = (ab.alphabet[i],i+1)
done{T <: BioUnit}(ab::Alphabet{T},i::Int) = (i > length(ab))

join{T <: BioUnit}(ab::Alphabet{T}) = join(ab.alphabet)

function show{T <: BioUnit}(io::IO,ab::Alphabet{T})
  println(io,"Case $(ab.case_sensitive ? "S":"Ins")ensitive Alphabet{$(T)} of $(length(ab)) elements:\n")
  println(io," indice   : $(length(ab.indice))-element $UInt8 Array")
  println(io," alphabet : $(length(ab.alphabet))-element $T Array\n")
  println(" alphabet\t\t\tindice[alphabet]")
  println(io," $T ($Int)\t\tUInt8 ($Int)\n")
  for letter in ab
    println(io," $letter ($(int(letter))) \t\t\t$(ab.indice[letter]) ($(int(ab.indice[letter])))")
  end
end

getindex{T <: BioUnit, I <: Union{UInt8,Int}}(ab::Alphabet{T}, i::I ) = ab.alphabet[ i ]
getindex{T <: BioUnit, I <: Union{UInt8,Int}}(ab::Alphabet{T}, i::AbstractArray{I} ) = ab.alphabet[ i ]
getindex{T <: BioUnit}(ab::Alphabet{T}, i::T ) = ab.indice[ i ]
getindex{T <: BioUnit}(ab::Alphabet{T}, i::AbstractArray{T} ) = ab.indice[ i ]

in{T <: BioUnit}(element::T, ab::Alphabet{T}) = ab.indice[element] != 0

## Is In Alphabet ? ##

function isin{T,A<:Union{IntSet,Set,Alphabet}}(s::AbstractArray{T},y::A)
  res = BitArray(size(s))
  for i in 1:length(s)
    res[i] = in(s[i],y)
  end
  res
end

function isin{T,A<:Associative}(s::AbstractArray{T},y::A)
  res = BitArray(size(s))
  for i in 1:length(s)
    res[i] = haskey(y,s[i])
  end
  res
end

isin(s::ASCIIString,y)= isin(s.data,y)

## Check Alphabet ##

function check{T,A<:Union{IntSet,Set,Alphabet}}(s::AbstractArray{T},y::A)
  for x in s
    if !in(x,y)
      println("$(Char(x)) is not in this Alphabet")
      return(false)
    end
  end
  true
end

function check{T,A<:Associative}(s::AbstractArray{T},y::A)
  for x in s
    if !haskey(y,x)
      println("$(Char(x)) is not in this Alphabet")
      return(false)
    end
  end
  true
end

check(s::ASCIIString,y)= check(s.data,y)

## Interchange between Dic and Array values ##

function swap!{T<:Number}(s::AbstractArray{T},y::Dict{T,T})
  for i in 1:length(s)
    s[i] = get(y,s[i],s[i])
  end
  s
end

function swap{T,Tk,Tv}(s::AbstractArray{T},y::Dict{Tk,Tv})
  res = similar(s,Tv)
  for i in 1:length(s)
    res[i] = get(y,s[i],s[i])
  end
  res
end

swap!{T<:Integer,Tv}(s::AbstractArray{T},y::Array{Tv}) = (for i in length(s) s[i] = y[s[i]] end)
swap{T<:Integer,Tv}(s::AbstractArray{T},y::Array{Tv}) = [y[b] for b in s]

swap(s::ASCIIString,y)= ASCIIString(convert(Vector{UInt8},swap(s.data,y)))

## TODO: rand


export MetaBayesTree, BayesTree
export CSMHistoryTuple

## ========================================================================================================================
## Bayes Trees
## ========================================================================================================================

abstract type AbstractBayesTree end

# TODO DEV MetaGraphs bayes tree, will potentially also make a LightBayesTree, CloudBayesTree,
"""
$(TYPEDEF)
Data structure for the Bayes (Junction) tree, which is used for inference and constructed from a given `::AbstractDFG`.
"""
mutable struct MetaBayesTree <: AbstractBayesTree
  bt::MetaDiGraph{Int,Float64}
  btid::Int
  frontals::Dict{Symbol,CliqueId{Int}}
  eliminationOrder::Vector{Symbol}
  buildTime::Float64
end

const BayesTree = MetaBayesTree

MetaBayesTree() = MetaBayesTree(MetaDiGraph{Int,Float64}(), 0, Dict{AbstractString, Int}(), Symbol[], 0.0)

Base.propertynames(x::MetaBayesTree, private::Bool=false) = (:bt, :btid, :cliques, :frontals, :variableOrder, :buildTime)

Base.getproperty(x::MetaBayesTree,f::Symbol) = begin
    if f == :cliques
      @warn "Don't use cliques field directly, use eg. getClique(tree, cliqId)" maxlog=1
      d = Dict{Int,Any}()
      for (k,v) in x.bt.vprops
        d[k] = v[:clique]
      end
      return d
    elseif f == :variableOrder
      @warn "MetaBayesTree.variableOrder has been deprecated in favor of .eliminationOrder"
      return getfield(x,:eliminationOrder)
    else
      getfield(x,f)
    end
  end

function Base.setproperty!(x::MetaBayesTree, f::Symbol, val)
  if f == :cliques
    @warn "`setproperty!(clique)` Don't use cliques field directly, use eg. addClique(tree, cliqId)" maxlog=1
    for (k,v) in val
      set_prop!(x.bt, k, :clique, v)
    end
  else
    setfield!(x,f,val)
  end
end

function getMessageChannels(tree::MetaBayesTree)

  d = Dict{Int, NamedTuple{(:upMsg, :downMsg),Tuple{Channel{LikelihoodMessage},Channel{LikelihoodMessage}}}}()

  for (k,e) in tree.bt.eprops
    d[k.dst] = (upMsg=e[:upMsg], downMsg=e[:downMsg])
  end

  return d

end

function Base.show(io::IO, mbt::MetaBayesTree)
  printstyled(io, "MetaBayesTree\n", color=:blue)
  println(io, "  Nr cliques:  ", length(mbt.cliques))

  # TODO ad dmore stats: max depth, widest point, longest chain, max clique size, average nr children

  nothing
end

Base.show(io::IO, ::MIME"text/plain", mbt::MetaBayesTree) = show(io, mbt)

"""
    $TYPEDEF

Container for upward tree solve / initialization.

DevNotes
- TODO more direct clique access (cliq, parent, children), for multi-process solves
"""
mutable struct CliqStateMachineContainer{BTND, G <: AbstractDFG, InMemG <: InMemoryDFGTypes, BT <: AbstractBayesTree}
  dfg::G
  cliqSubFg::InMemG
  tree::BT
  cliq::TreeClique
  incremental::Bool
  drawtree::Bool
  dodownsolve::Bool
  delay::Bool
  opts::SolverParams
  refactoring::Dict{Symbol, String}
  oldcliqdata::BTND
  logger::SimpleLogger
  cliqId::CliqueId
  algorithm::Symbol
  init_iter::Int
  enableLogging::Bool
  solveKey::Symbol
  _csm_iter::Int
end

#TODO use @NamedTuple if julia compat > 1.5

const CSMHistoryTuple =  NamedTuple{(:timestamp, :id, :f, :csmc), Tuple{DateTime, Int, Function, CliqStateMachineContainer}}
const CSMHistory = Vector{CSMHistoryTuple}

Base.show(io::IO, o::CSMHistoryTuple) = print(io, "$(o[1]), $(o[2]), $(o[3])")

function CliqStateMachineContainer( dfg::G,
                                    cliqSubFg::M,
                                    tree::T,
                                    cliq::TreeClique,
                                    incremental::Bool,
                                    drawtree::Bool,
                                    dodownsolve::Bool,
                                    delay::Bool,
                                    opts::SolverParams,
                                    refactoring::Dict{Symbol,String}=Dict{Symbol,String}(),
                                    oldcliqdata::BTND=BayesTreeNodeData(),
                                    logger::SimpleLogger=SimpleLogger(Base.stdout);
                                    cliqId::CliqueId = cliq.id,
                                    algorithm::Symbol = :default,
                                    init_iter::Int=0,
                                    enableLogging::Bool=true,
                                    solveKey::Symbol=:default,
                                    _csm_iter::Int=0  ) where {BTND, G <: AbstractDFG, M <: InMemoryDFGTypes, T <: AbstractBayesTree}
  #
  CliqStateMachineContainer{BTND, G, M, T}( dfg,
                                            cliqSubFg,
                                            tree,
                                            cliq,
                                            incremental,
                                            drawtree,
                                            dodownsolve,
                                            delay,
                                            opts,
                                            refactoring,
                                            oldcliqdata,
                                            logger,
                                            cliqId,
                                            algorithm,
                                            init_iter,
                                            enableLogging,
                                            solveKey,
                                            _csm_iter )
  #
end

# TODO resolve name conflict
function DFG.compare( cs1::CliqStateMachineContainer{BTND1, T1, InMemG1, BT1},
                      cs2::CliqStateMachineContainer{BTND2, T2, InMemG2, BT2};
                      skip::Vector{Symbol}=Symbol[] ) where {BTND1, T1 <: AbstractDFG, InMemG1 <: InMemoryDFGTypes, BT1 <: AbstractBayesTree, BTND2, T2 <: AbstractDFG, InMemG2 <: InMemoryDFGTypes, BT2 <: AbstractBayesTree}
  #
  BTND1 == BTND2 ? nothing : @warn("oldcliqdata::$BTND1 != oldcliqdata::$BTND2")
  T1 == T2 ? nothing : @warn("dfg::$T1 != dfg::$T2")
  InMemG1 == InMemG2 ? nothing : @warn("cliqSubFg::$InMemG1 != cliqSubFg::$InMemG2")
  BT1 == BT2 ? nothing : @warn("tree::$BQ1 != tree::$BT2")

  TP = true
  @warn "Skipping compare of CSMC.dfg and .cliqSubFg"
  # TP = TP && compare(cs1.dfg,  cs2.dfg)
  # TP = TP && compare(cs1.cliqSubFg,  cs2.cliqSubFg)
  @warn "Skipping compare of CSMC.tree"
  # TP = TP && compare(cs1.tree,  cs2.tree)
  TP = TP && compare(cs1.cliq,  cs2.cliq)
  TP = TP && compare(cs1.cliqId,  cs2.cliqId)
  TP = TP && length(cs1.parentCliq) == length(cs2.parentCliq)
  for i in 1:length(cs1.parentCliq)
    TP = TP && compare(cs1.parentCliq[i],  cs2.parentCliq[i])
  end
  TP = TP && length(cs1.childCliqs) == length(cs2.childCliqs)
  for i in 1:length(cs1.childCliqs)
    TP = TP && compare(cs1.childCliqs[i],  cs2.childCliqs[i])
  end
  TP = TP && compare(cs1.incremental,  cs2.incremental)
  TP = TP && compare(cs1.drawtree,  cs2.drawtree)
  TP = TP && compare(cs1.dodownsolve,  cs2.dodownsolve)
  TP = TP && compare(cs1.delay,  cs2.delay)
  @warn "skipping compare on csmc.opts::SolverParams"
  # TP = TP && compare(cs1.opts,  cs2.opts)
  TP = TP && compare(cs1.refactoring,  cs2.refactoring)
  # TP = TP && compare(cs1.oldcliqdata,  cs2.oldcliqdata)
  # TP = TP && compare(cs1.logger,  cs2.logger)

  return TP
end

#


println("First line of the script")
time_start = time()
using Test
using Logging
using SparseArrays
using ExSup
using Pio3d
using JLD2
@info "Before Distributed"
using Distributed
if nprocs() == 1
    addprocs(3)
end
# using Revise
# using ExSup
using LarSurf
# @everywhere using LarSurf

fn = "exp_surface_extraction_ircad_times.csv"
data = Dict()

@info "before everywhere using"
@info "time from start: $(time()-time_start) [s]"
# using Plasm
# @everywhere using LarSurf
# @everywhere using Distributed

@info "after everywhere using, time from start: $(time()-time_start) [s]"

data_id = 1
stepz = 1
stepxy = 1
# stepxy = 4
block_size = [64, 64, 64]
# block_size = [128, 128, 128]
# block_size = [128, 128, 128]
# block_size = [32, 32, 32]
# data_size1 = 128
# data_size1 = 256
# data_size1 = 512

LarSurf.set_time_data(data)
LarSurf.report_init_row(@__FILE__)

data["block size"] = block_size[1]
data["using done"] = time()-time_start
# segmentation = LarSurf.data234()
@info "Generate data..."
@info "time from start: $(time() - time_start) [s]"
mask_labels=["liver", "portalvein"]
# pth = Pio3d.datasets_join_path("medical/orig/3Dircadb1.$data_id/MASKS_DICOM/liver")

for mask_label in mask_labels
	pth = Pio3d.datasets_join_path("medical/orig/3Dircadb1.$data_id/MASKS_DICOM/$(mask_label)")
	datap = Pio3d.read3d(pth)
	data3d_full = datap["data3d"]
	data3d_full = data3d_full[1:stepz:end, 1:stepxy:end, 1:stepxy:end]
	    # round(size(data3d_full, 1) / target_size1)
	    # return data3d_full
	segmentation = convert(Array{Int8, 3}, data3d_full .> 0)
	voxelsize_mm = datap["voxelsize_mm"]
	voxelsize_mm[1] = voxelsize_mm[1] * stepz
	voxelsize_mm[2] = voxelsize_mm[2] * stepxy
	voxelsize_mm[3] = voxelsize_mm[3] * stepxy
	LarSurf.Experiments.report_add_data_info(data, segmentation, voxelsize_mm)
	@info "voxelsize mm = $(voxelsize_mm), size=$(size(data3d_full))"
	# segmentation = LarSurf.generate_cube(data_size1; remove_one_pixel=true)
	@info "==== using done, data generated time from start: $(time()-time_start) [s]"
	data["data generated"] = time()-time_start

	@info "Setup..."
	setup_time = @elapsed LarSurf.lsp_setup(block_size;reference_time=time_start)
	println("setup time: $setup_time")
	@info "==== setup done, time from start: $(time()-time_start) [s]"
	data["setup done"] = time()-time_start
	# for wid in workers()
	#     # println("testing on $wid")
	#     ftr = @spawnat wid LarSurf._single_boundary3
	#     @test fetch(ftr) != nothing
	# end

	# @debug "Setup done"
	tmd = @timed larmodel = LarSurf.lsp_get_surface(segmentation; voxelsize=voxelsize_mm)
	val, tm, mem, gc = tmd
	println("Total time per $(mask_label): $tm")
	@info "==== time from start: $(time()-time_start) [s]"
	data["finished"] = time()-time_start
	ExSup.add_to_csv(data, fn)

	V, FV = larmodel
	FVtri = LarSurf.triangulate_quads(FV)

	@JLD2.save "ircad_$(mask_label).jld2" V FV
	using ViewerGL
	println("=== ViewerGL readed")
	ViewerGL.VIEW([
	    ViewerGL.GLGrid(V,FVtri,ViewerGL.Point4d(1,1,1,0.1))
		ViewerGL.GLAxis(ViewerGL.Point3d(-1,-1,-1),ViewerGL.Point3d(1,1,1))
	])
	# FV = FVtri
	@JLD2.save "ircad_$(mask_label)_tri.jld2" V FVtri
	# @JLD2.save "liver01tri.jld2" V FVtri
end
# Plasm.view(val)


module TestParseUInts

import GraphIdx.Io: parse_uints
using Test

function record_ints(io::IO)
    args = Vector{Tuple{UInt,Bool}}()
    parse_uints(io) do u::UInt, b::Bool
        push!(args, (u, b))
    end
    return args
end

@testset "parse_uints                    " begin
    @test record_ints(IOBuffer("")) == []
    @test record_ints(IOBuffer("0")) == [(0, true)]
    @test record_ints(IOBuffer("0120 3")) == [(120, false), (3, true)]
    @test record_ints(IOBuffer("0120 \n 3")) == [(120, true), (3, true)]
    @test record_ints(IOBuffer(", 0120 3")) == [(120, false), (3, true)]
    @test record_ints(IOBuffer(", 0120 , 3")) == [(120, false), (3, true)]
    @test record_ints(IOBuffer(", 0120 ,\n 3")) == [(120, true), (3, true)]
    @test record_ints(IOBuffer(", 0120 ,\n 3_000")) == [(120, true), (3000, true)]
end

end


# benchmarking

using ArrayViews

# auxiliary functions

mps(a::AbstractArray, rtimes::Int, et::Float64) = length(a) * 1.0e-6 * rtimes / et

myrepr(r::Range) = repr(r)
myrepr(r::Real) = repr(r)
myrepr(r::Colon) = ":"

# timing functions

function time_view1d{T}(a::AbstractArray{T}, rtimes::Int)
    # warm up
    n::Int = length(a)
    for i = 1:n
        @inbounds a[i]
    end

    # benchmark
    s = zero(T)
    et = @elapsed for t = 1 : rtimes
        for i = 1:n
            @inbounds s += a[i]
        end
    end
    return (et, s)
end

function time_view2d{T}(a::AbstractArray{T}, rtimes::Int)
    # warm up
    m::Int = size(a, 1)
    n::Int = size(a, 2)
    for j = 1:n, i = 1:m
        @inbounds a[i,j]
    end

    # benchmark
    s = zero(T)
    et = @elapsed for t = 1 : rtimes
        for j = 1:n, i = 1:m
            @inbounds s += a[i,j]
        end
    end
    return (et, s)
end

function time_view3d{T}(a::AbstractArray{T}, rtimes::Int)
    # warm up
    m::Int = size(a, 1)
    n::Int = size(a, 2)
    k::Int = size(a, 3)
    for l = 1:k, j = 1:n, i = 1:m
        @inbounds a[i,j,l]
    end

    # benchmark
    s = zero(T)
    et = @elapsed for t = 1 : rtimes
        for l = 1:k, j = 1:n, i = 1:m
            @inbounds s += a[i,j,l]
        end
    end
    return (et, s)
end

function perf_view(a::Array, i1; rtimes::Int=100000)
    et_s, _ = time_view1d(sub(a, i1), rtimes)
    et_v, _ = time_view1d(view(a, i1), rtimes)
    et_u, _ = time_view1d(unsafe_view(a, i1), rtimes)

    v = view(a, i1)
    mps_s = mps(v, rtimes, et_s)
    mps_v = mps(v, rtimes, et_v)
    mps_u = mps(v, rtimes, et_u)

    @printf("%-24s:   %10.3f MPS   %10.3f MPS (%6.3fx)   %10.3f MPS (%6.3fx)\n",
        "[$(myrepr(i1))]", mps_s, mps_v, mps_v / mps_s, mps_u, mps_u / mps_s)
end

function perf_view(a::Array, i1, i2; rtimes::Int=100000)
    et_s, _ = time_view2d(sub(a, i1, i2), rtimes)
    et_v, _ = time_view2d(view(a, i1, i2), rtimes)
    et_u, _ = time_view2d(unsafe_view(a, i1, i2), rtimes)

    v = view(a, i1, i2)
    mps_s = mps(v, rtimes, et_s)
    mps_v = mps(v, rtimes, et_v)
    mps_u = mps(v, rtimes, et_u)

    @printf("%-24s:   %10.3f MPS   %10.3f MPS (%6.3fx)   %10.3f MPS (%6.3fx)\n",
        "[$(myrepr(i1)), $(myrepr(i2))]", mps_s, mps_v, mps_v / mps_s, mps_u, mps_u / mps_s)
end

function perf_view(a::Array, i1, i2, i3; rtimes::Int=100000)
    et_s, _ = time_view3d(sub(a, i1, i2, i3), rtimes)
    et_v, _ = time_view3d(view(a, i1, i2, i3), rtimes)
    et_u, _ = time_view3d(unsafe_view(a, i1, i2, i3), rtimes)

    v = view(a, i1, i2, i3)
    mps_s = mps(v, rtimes, et_s)
    mps_v = mps(v, rtimes, et_v)
    mps_u = mps(v, rtimes, et_u)

    @printf("%-24s:   %10.3f MPS   %10.3f MPS (%6.3fx)   %10.3f MPS (%6.3fx)\n",
        "[$(myrepr(i1)), $(myrepr(i2)), $(myrepr(i3))]", mps_s, mps_v, mps_v / mps_s, mps_u, mps_u / mps_s)
end

# benchmarks

println("Indexing                      sub              view                       unsafe_view")
println("--------------------------------------------------------------------------------------------------")

const a1 = rand(1024)

gc_disable()

println("1D views")
perf_view(a1, :)
perf_view(a1, 1:1024)
perf_view(a1, 1:2:1024)
println()

gc_enable()

const a2 = rand(32, 32)

gc_disable()

println("2D views")
perf_view(a2, :, :)
perf_view(a2, :, 1:32)
perf_view(a2, :, 1:2:32)

perf_view(a2, 1:30, :)
perf_view(a2, 1:30, 1:32)
perf_view(a2, 1:30, 1:2:32)

perf_view(a2, 1:2:30, :)
perf_view(a2, 1:2:30, 1:32)
perf_view(a2, 1:2:30, 1:2:32)
println()

gc_enable()

const a3 = rand(16, 8, 8)

gc_disable()

println("3D views")
perf_view(a3, :, :, :)
perf_view(a3, :, :, 1:8)
perf_view(a3, :, :, 1:2:8)

perf_view(a3, :, 1:8, :)
perf_view(a3, :, 1:8, 1:8)
perf_view(a3, :, 1:8, 1:2:8)

perf_view(a3, :, 1:2:8, :)
perf_view(a3, :, 1:2:8, 1:8)
perf_view(a3, :, 1:2:8, 1:2:8)

perf_view(a3, 1:16, :, :)
perf_view(a3, 1:16, :, 1:8)
perf_view(a3, 1:16, :, 1:2:8)

perf_view(a3, 1:16, 1:8, :)
perf_view(a3, 1:16, 1:8, 1:8)
perf_view(a3, 1:16, 1:8, 1:2:8)

perf_view(a3, 1:16, 1:2:8, :)
perf_view(a3, 1:16, 1:2:8, 1:8)
perf_view(a3, 1:16, 1:2:8, 1:2:8)

perf_view(a3, 1:2:16, :, :)
perf_view(a3, 1:2:16, :, 1:8)
perf_view(a3, 1:2:16, :, 1:2:8)

perf_view(a3, 1:2:16, 1:8, :)
perf_view(a3, 1:2:16, 1:8, 1:8)
perf_view(a3, 1:2:16, 1:8, 1:2:8)

perf_view(a3, 1:2:16, 1:2:8, :)
perf_view(a3, 1:2:16, 1:2:8, 1:8)
perf_view(a3, 1:2:16, 1:2:8, 1:2:8)
println()

gc_enable()


using CSV
using Dates
using DataFrames
using DelimitedFiles
using XLSX
using YAML
using InteractiveUtils
const IU = InteractiveUtils

using SLiDE  # see src/SLiDE.jl

READ_DIR = joinpath("data", "readfiles")

files_map = [
    XLSXInput("generate_yaml.xlsx", "map_parse",     "B1:Z150", "map_parse"),
    XLSXInput("generate_yaml.xlsx", "map_scale",     "B1:Z150", "map_scale"),
    XLSXInput("generate_yaml.xlsx", "map_bluenote",  "B1:Z150", "map_bluenote"),
    XLSXInput("generate_yaml.xlsx", "map_crosswalk", "B1:Z150", "map_crosswalk")
]

files_map = write_yaml(READ_DIR, files_map)
files_map = run_yaml(files_map)
include(joinpath(SLIDE_DIR, "dev", "datastream", "adjust_maps.jl"))


using VITA2D
using Test
using Random
Random.seed!(42)

@testset "VITA2D.jl" begin
    @test half(4) == 2
    @test third(30) == 10

    L = 6
    P = 2
    θ = rand(3P)
    h = (ZZ = 1.0,
         X = 1.0,
         Z = 1.0)

    M = Model(L, θ, h)
    Obs = Thermo(P)
    Grad = Gradient(Obs, P)
    Sto = StochasticMatrix(Obs, P)
    C = Config((L, L, 2P+1))

    warmup!(C, M, 1)
    mc(C, M, Obs, Grad, 1, Sto)

    sgd_params = VITA2D.SGDParams()

    E = sgd(L=L,
              P=P,
              hX = 1.0,
              θ = θ,
              SGD=SGDParams(),
              path=false)

    E = sgd(L=L,
            P=P,
            hX = 3.0,
            θ = θ,
            SGD=sgd_params,
            print=true)
end


using Catalyst
using Test
using DelaySSAToolkit
rn = @reaction_network begin
    ρ, S+I --> E+I
    1/I, I --> R
    r, I --> R
end ρ r

jumpsys = convert(JumpSystem, rn, combinatoric_ratelaws=false)

u0 = [999,1,0,0] # S, I, E, R
tf = 400.
tspan = (0,tf)
ps = [1e-4, 1e-2] # parameters for ρ, r
τ = 20.
dprob = DiscreteProblem(jumpsys,u0,tspan,ps)

delay_trigger_affect! = function (integrator, rng)
    append!(integrator.de_chan[1], τ)
end
delay_trigger = Dict(1=>delay_trigger_affect!)
delay_interrupt = Dict()
delay_complete = Dict(1=>[2=>1, 3=>-1])
delayjumpset = DelayJumpSet(delay_trigger, delay_complete, delay_interrupt)

de_chan0 = [[]]

meanlist=[]
medianlist=[]
minlist=[]
maxlist=[]
stdlist=[]

for algo in algo_list
    djprob = DelayJumpProblem(jumpsys,dprob,algo,delayjumpset, de_chan0, save_positions=(false,false))
    a=@benchmark solve(djprob, SSAStepper())
    push!(meanlist,copy(mean(a).time/1e9))
    push!(medianlist,copy(median(a).time/1e9))
    push!(minlist,copy(minimum(a).time/1e9))
    push!(maxlist,copy(maximum(a).time/1e9))
    push!(stdlist,copy(std(a).time/1e9))
    print(djprob.aggregator," OK","\n")
end

df=DataFrame(mean=meanlist,median=medianlist,min=minlist,max=maxlist,std=stdlist)
CSV.write("test/zxytest/results/SEIR_save_positions=f.csv",df)

df=CSV.read("test/zxytest/results/SEIR_save_positions=f.csv",DataFrame)
medianlist=df.median

medianvalue=[string(round(mt,digits=6),"s") for mt in medianlist]

algo_name = ["DelayDirect","DelayRejection","DelayMNRM","DelayDirectCR"]
using Plots
p1=bar(algo_name,medianlist,legend=:false,title="SEIR",ylabel="mediantime")
scatter!(algo_name, 0.0002 .+ medianlist , markeralpha=0, series_annotations=medianvalue)
savefig(p1,"test/zxytest/results/SEIR_median.png")

#=
function runbenchmark(algo)
    djprob = DelayJumpProblem(dprob, algo, jumpset, delayjumpset, de_chan0, save_positions = (true,true),save_delay_channel = true)
    a=@benchmark solve(djprob, SSAStepper())
    #a=@bprofile solve(jprob, SSAStepper(), saveat = timestamps)
    return a
end

function testalgo(algo_list,times)
    for algo in algo_list
        onealgotext=zeros(1,times)
        for i=1:times
            bench=runbenchmark(algo)
            onealgotext[i]=median(bench).time/1e3
        end
        push!(test_all_algo,onealgotext)
    end
end

times=5
algo_list = [DelayMNRM(), DelayRejection(), DelayDirectCR()]

test_all_algo=[]
@time testalgo(algo_list,times)

test_all_algo
length(test_all_algo[1])
#mean and std
#DelayMNRM()
mean(test_all_algo[1])
std(test_all_algo[1])
#DelayRejection()
mean(test_all_algo[2])
std(test_all_algo[2])
#DelayDirectCR()
mean(test_all_algo[3])
std(test_all_algo[3])

algo_name = ["MNRM", "Rejection", "DirectCR"]
meanlist=[mean(test_all_algo[i]) for i=1:length(test_all_algo)]
stdlist=[std(test_all_algo[i]) for i=1:length(test_all_algo)]

using Printf
strmean = [@sprintf("%.3fus", yi) for yi in meanlist]
strstd = [@sprintf("%.3f", yi) for yi in stdlist]

using Plots
p1=bar(algo_name,meanlist,ylims=(16,22),text=strmean,bar_width=0.1,
       label="mean of time",text_position="outside")
p2=bar(algo_name,stdlist,ylims=(0.15,0.5),text=strstd,bar_width=0.1,
       label="std of time")
plot(p1, p2, layout = (2,1))
=#


#=
    This file includes all the funtions needed to update the operation
    and investment informations
 =#

function update_operation_informations!(h::Int64, y::Int64, s::Int64, mg::Microgrid, ω::Scenarios)
    # Demands
    for (k, a) in enumerate(mg.demands)
        a.timestamp[h,y,s] = ω.demands[k].t[h,y,s]
        a.carrier.power[h,y,s] = ω.demands[k].power[h,y,s]
    end
    # Generations
    for (k, a) in enumerate(mg.generations)
        a.timestamp[h,y,s] = ω.generations[k].t[h,y,s]
        a.carrier.power[h,y,s] = a.powerMax[y,s] * ω.generations[k].power[h,y,s]
    end
    # Grids - We assume the price of electricity is known over the year
    for (k, a) in enumerate(mg.grids)
        if h == 1
            a.cost_in[1:end,y,s] = ω.grids[k].cost_in[1:end,y,s]
            a.cost_out[1:end,y,s] = ω.grids[k].cost_out[1:end,y,s]
        end
    end
end

function update_investment_informations!(y::Int64, s::Int64, mg::Microgrid, ω::Scenarios)
    # Generations
    for (k, a) in enumerate(mg.generations)
        a.cost[y,s] = ω.generations[k].cost[y,s]
    end
    # Storages
    for (k, a) in enumerate(mg.storages)
        a.cost[y,s] = ω.storages[k].cost[y,s]
    end
    # Converters
    for (k, a) in enumerate(mg.converters)
        a.cost[y,s] = ω.converters[k].cost[y,s]
    end
end


#!/usr/bin/env julia

using Pkg
Pkg.activate(".")
#Pkg.instantiate()
Pkg.status()

time_start = time()
println("start pre-compilation")

using Distributed
using SparseArrays

using JSON

using JuMP

using Ipopt

using InfrastructureModels
using PowerModels
using Memento

println("pre-compilation finished ($(time() - time_start))")


module Multigrid
	using LinearAlgebra
	using SparseArrays
	using Statistics
	using ParSpMatVec;
	using jInv.LinearSolvers
	using jInv.Utils

	const global hasParSpMatVec = ParSpMatVec.isBuilt();
	if hasParSpMatVec == false
		println("hasParSpMatVec==false. ParSpMatVec has failed to build!!!!!!!");
	end

	import jInv.LinearSolvers.JuliaSolver, jInv.LinearSolvers.getJuliaSolver
	import jInv.LinearSolvers.AbstractSolver
	import jInv.LinearSolvers.solveLinearSystem!,jInv.LinearSolvers.solveLinearSystem, jInv.LinearSolvers.setupSolver
	import jInv.Utils.clear!

	SparseCSCTypes = Union{SparseMatrixCSC{ComplexF64,Int64},SparseMatrixCSC{Float64,Int64},SparseMatrixCSC{ComplexF32,Int64},SparseMatrixCSC{Float32,Int64}}
	ArrayTypes = Union{Array{ComplexF64},Array{ComplexF32},Array{Float64},Array{Float32}}

	export SparseCSCTypes,ArrayTypes
	include("ParallelJuliaSolver/parallelJuliaSolver.jl");
	include("DomainDecomposition/DomainDecomposition.jl");
	include("Multigrid/MGdef.jl");
end

# check if MUMPS can be used
# const minMUMPSversion = VersionNumber(0,0,1)
# hasMUMPS=false
# vMUMPS = VersionNumber(0,0,0)
# try
	# vMUMPS = Pkg.installed("MUMPS")
	# hasMUMPS = vMUMPS >= minMUMPSversion
	# if hasMUMPS
		# using MUMPS;
	# end
# catch
# end

# end # module