include("../src/utils/model.jl")
include("../src/solvers/descent.jl")
include("../src/solvers/annealing.jl")
include("../src/solvers/lp_timing.jl")
include("../src/solvers/neighbour_operators.jl")

# Création d'une struct encapsulant la spécification d'un test LP
struct LpSolverSpec
    algo::Union{Symbol,Nothing}
    external_mip_solver::Union{Symbol,Nothing}
    lp_timing_solver::Union{DataType,Nothing}
    backend_model::Union{DataType,Nothing}
end

function test_one_lp_descent(spec)
    ln2("Création sol avec algo ")
    lg2(spec.algo, ":", spec.external_mip_solver)
    lg2(" (", spec.lp_timing_solver, "... ")
    Args.set(:external_mip_solver, spec.external_mip_solver)
    sol = Solution(inst, algo=spec.algo)
    model = sol.solver.model
    initial_sort!(sol, presort=:shuffle)

    @test isa(sol.solver, spec.lp_timing_solver)
    # On vérifie par exemple que le nom du solver externe "Clp" effectif
    # correspondant bien au symbole :clp du solveur demandé
    @test Symbol(lowercase(solver_name(model))) == spec.external_mip_solver
    lg2("ok (avec sol.cost=$(sol.cost))\n")

    # ===========
    lg2("Création DescentSolver nb_cons_reject_max=$(Args.get(:itermax))")
    lg2(" (cost=700<=850.0?)... ")
    sv = DescentSolver(inst)
    # println("\nconstruction faite")
    sv.do_save_bestsol = false
    sv.nb_cons_reject_max = Args.get(:itermax) # Devra être écrasé
    sv.durationmax = 1                         # Devra être écrasé
    # solve(sv, startsol=sol, nb_cons_reject_max=Args.get(:itermax))
    # Args.set(:level, 3)
    solve(sv, small_shift!, startsol=sol, nb_cons_reject_max=200, durationmax=3)
    # println("\nrésolution faite")
    @test sv.durationmax == 3
    @test sv.nb_cons_reject_max == 200
    @test isa(sv.bestsol.solver, spec.lp_timing_solver)
    # @test sv.bestsol.cost <= 850.0
    # @test sv.bestsol.cost <= 900.0
    # @test sv.bestsol.cost >= 700.0
    @test 700 <= sv.bestsol.cost <= 1_000.0
    lg1("ok ($(sv.bestsol.cost))\n")
end

function test_one_lp_annealing(spec)
    ln2("Création sol avec algo ")
    lg2(spec.algo, ":", spec.external_mip_solver)
    lg2(" (", spec.lp_timing_solver, "... ")
    Args.set(:external_mip_solver, spec.external_mip_solver)
    sol = Solution(inst, algo=spec.algo)
    model = sol.solver.model
    initial_sort!(sol, presort=:shuffle)

    @test isa(sol.solver, spec.lp_timing_solver)
    # On vérifie par exemple que le nom du solver externe "Clp" effectif
    # correspondant bien au symbole :clp du solveur demandé
    @test Symbol(lowercase(solver_name(model))) == spec.external_mip_solver
    lg2("ok (avec sol.cost=$(sol.cost))\n")

    # ===========
    lg2("Création AnnealingSolver nb_cons_reject_max=$(Args.get(:itermax))")
    lg2(" (cost=700<=850.0?)... ")
    sv = AnnealingSolver(inst, startsol=sol, nb_cons_reject_max=200,)
    solve(sv, small_shift!, durationmax=3)

    @test sv.durationmax == 3
    @test sv.nb_cons_reject_max == 200
    @test isa(sv.bestsol.solver, spec.lp_timing_solver)
    # @test sv.bestsol.cost <= 850.0
    # @test sv.bestsol.cost <= 900.0
    # @test sv.bestsol.cost >= 700.0
    @test 700 <= sv.bestsol.cost <= 1_000.0
    lg1("ok ($(sv.bestsol.cost))\n")
end


k = Kosmo()

mutable struct Health <: Component
    v::Int
end

ply = Entity!(k, Health(100))

system!(k) do x::Health
    x.v = x.v-3
    println(x.v)
end

t = Kosmos.run_systems(k)


type Parcel
  ID
  Index
  sDate
  Duration
  function Parcel()
    return new()
  end
end

"""
   Find gaps in time series (tsParm) structure and return duration of good data
     as well as an index into parcels of data which have contigously good data.

   Good data have values other than NaN.  Where there is a gap in the data
     larger than in.days4gap, a new parcel of data will be indicated.

   Any data which is NaN will be considered to form part of a gap in the time
     series such that where there are contiguous NaN's for a duration larger
     than days4gap will be marked as two separate parcels

 INPUT
  tsParm structure    -  from MetoceanTools package
  		      -  requires tsParm.sDate[1:nsDate] to be filled
		      -  requires tsParm.Data[1:nsDate,:] to be filled
  iDataColumn         -  column of tsParm.Data to use for data to be checked for gaps
  days4gap[1]         -  duration in days which is to be regarded as a gap in
                           the data

 OUTPUT:
   Parcel definitions as Parcel type
     ID 		- either 0 for a gap parcel or integer sequence number of parcel
     sDate		- start and end date of each parcel
     Index		- start and end index of each parcel where index is into tsParm.sDate
     Duration           - duration [dyas] of each parcel
   iParcel[1:nsDate]   -  either 0 for a gap larger than days4gap or a sequence number
                           giving the parcel number
   goodDurationDays    - the effective duration in days of the complete time series
                           (sum of the durations of each parcels)

 Created by Jason McConochie
 Revision 1 26Dec2016 by Jason McConochie
"""
function findGaps(in::tsParm,iDataColumn::Int64,days4gap::Float64)

   # A. Find the gaps greater than days4gap
   iParcel=zeros(Int64,size(in.sDate,1))
   ms2days=1/(1000 * 3600 * 24)
   inGap=false
   tParcel=1
   iSt=1
   iEd=1
   for i=1:length(in.sDate)
     isBad=isnan(in.Data[i,iDataColumn])
     if isBad & !inGap
       iSt=i
       inGap=true
     end
     if !isBad
       iEd=i-1
       dt=(Dates.value(in.sDate[iEd+1])-Dates.value(in.sDate[iSt]))*ms2days
       if inGap
         if 1.*dt>days4gap
           iParcel[iSt:iEd]=0
	   if iSt!=1
             tParcel+=1
	   end
	 else
           iParcel[iSt:iEd]=tParcel
	 end
       end
       iParcel[i]=tParcel
       inGap=false
     end
   end

   # B. Set duration of all parcels
   # B1. count number of parcels
   k=0
   lastParcel=-1
   for i=1:length(iParcel)-1
     if iParcel[i]!=lastParcel
        k=k+1
	lastParcel=iParcel[i]
     end
   end
   nParcels=k
   # B2. set start and end dates of parcels and parcel number associated
   p=Parcel()
   p.ID=Array{Int64,1}(nParcels)
   p.sDate=Array{DateTime,2}(nParcels,2)
   p.Index=Array{Int64,2}(nParcels,2)
   p.Duration=Array{Float64,1}(nParcels)
   lastParcel=-1
   k=0
   for i=1:length(iParcel)-1
     if iParcel[i]!=lastParcel
       k=k+1
       p.sDate[k,1]=in.sDate[i]
       p.Index[k,1]=i
       p.ID[k]=iParcel[i]
       lastParcel=iParcel[i]
     end
   end
   for i=1:nParcels-1
     p.sDate[i,2]=p.sDate[i+1,1]
     p.Index[i,2]=p.Index[i+1,1]
   end
   p.sDate[nParcels,2]=in.sDate[end]
   p.Index[nParcels,2]=length(in.sDate)

   goodDurationDays=0.
   for i=1:nParcels
     p.Duration[i]=ms2days*(Dates.value(p.sDate[i,2])-Dates.value(p.sDate[i,1]))
     if p.ID!=0
       goodDurationDays+=p.Duration[i]
     end
     #println("$(i) : $(p.ID[i]) : $(p.Index[i,1]),$(p.Index[i,2]) : $(p.sDate[i,1]),$(p.sDate[i,2]) : $(p.Duration[i])")
   end

   return p,goodDurationDays,iParcel

end

"""
   Function test_findGaps() to test findGaps function
"""
function test_findGaps()
   in=tsParm()
   in.sDate=[
      DateTime(1900,1,1),
      DateTime(1900,1,2),
      DateTime(1900,1,3),
      DateTime(1900,1,4),
      DateTime(1900,1,5),
      DateTime(1900,1,6),
      DateTime(1900,1,7),
      DateTime(1900,1,8),
      DateTime(1900,1,9),
      DateTime(1900,1,10),
      ]
   in.Data=zeros(10,1)
   in.Data[1:10,1]=[1,1,NaN,NaN,1,1,1,1,NaN,NaN]
   days4gap=1.5
   p,goodDurationDays,iParcel=findGaps(in,1,days4gap)

end


using SEAL
using Printf

function print_example_banner(title)
  if isempty(title)
    return nothing
  end

  title_length = length(title)
  banner_length = title_length + 2 * 10
  banner_border = "+" * "-"^(banner_length - 2) * "+"
  banner_center = "|" * " "^9 * title * " "^9 * "|"

  println()
  println(banner_border)
  println(banner_center)
  println(banner_border)
end

function print_parameters(context::SEALContext)
  context_data = key_context_data(context)
  encryption_parms = parms(context_data)
  scheme_type = scheme(encryption_parms)
  if scheme_type == SchemeType.bfv
    scheme_name = "BFV"
  elseif scheme_type == SchemeType.ckks
    scheme_name = "CKKS"
  else
    error("unsupported scheme")
  end

  println("/")
  println("| Encryption parameters :")
  println("|   scheme: ", scheme_name)
  println("|   poly_modulus_degree: ", poly_modulus_degree(encryption_parms))

  print("|   coeff_modulus size: ", total_coeff_modulus_bit_count(context_data), " (")
  bit_counts = [bit_count(modulus) for modulus in coeff_modulus(encryption_parms)]
  print(join(bit_counts, " + "))
  println(") bits")

  if scheme_type == SchemeType.bfv
    println("|   plain_modulus: ", value(plain_modulus(encryption_parms)))
  end

  println("\\")
end

function print_line(line_number)
  if line_number < 10
    padding = "  "
  elseif line_number < 100
    padding = " "
  else
    padding = ""
  end
  print("Line ", padding, line_number, " --> ")
end

function print_vector(vector, print_size=4, prec=3)
  slot_count = length(vector)
  println()
  if slot_count <= 2 * print_size
    print("    [ ")
    if prec == 7
      @printf "%.7f" first(vector)
    else
      @printf "%.3f" first(vector)
    end
    for i in 2:slot_count
      if prec == 7
        @printf ", %.7f" vector[i]
      else
        @printf ", %.3f" vector[i]
      end
    end
    println(" ]")
  else
    print("    [ ")
    if prec == 7
      @printf "%.7f" first(vector)
    else
      @printf "%.3f" first(vector)
    end
    for i in 2:print_size
      if prec == 7
        @printf ", %.7f" vector[i]
      else
        @printf ", %.3f" vector[i]
      end
    end
    if slot_count > 2 * print_size
      print(", ...")
    end
    for i in (slot_count - print_size + 1):slot_count
      if prec == 7
        @printf ", %.7f" vector[i]
      else
        @printf ", %.3f" vector[i]
      end
    end
    println(" ]")
  end
  println()
end

function print_matrix(matrix, row_size)
  print_size = 5

  println()

  # First row
  print("    [")
  for i in 1:print_size
    @printf("%3d,", matrix[i])
  end
  print(" ...,")
  for i in (row_size - print_size + 1):row_size
    if i != row_size
      @printf("%3d,", matrix[i])
    else
      @printf("%3d ]\n", matrix[i])
    end
  end

  # Second row
  print("    [")
  for i in (row_size + 1):(row_size + print_size)
    @printf("%3d,", matrix[i])
  end
  print(" ...,")
  for i in (2 * row_size - print_size + 1):(2 * row_size)
    if i != 2 * row_size
      @printf("%3d,", matrix[i])
    else
      @printf("%3d ]\n", matrix[i])
    end
  end

  println()
end

"""
    @elapsedus(ex)

Return integer number of elapsed microseconds required for executing `ex`. Modified from
`Base.@elapsed`, which returns the number of seconds as a floating point number.

See also: [`@elapsed`](@ref)
"""
macro elapsedus(ex)
  quote
    while false; end # compiler heuristic: compile this block (alter this if the heuristic changes)
    local t0 = time_ns()
    $(esc(ex))
    Int(div(time_ns() - t0, 1000))
  end
end


# MIT License
#
# Copyright (c) 2018 Martin Biel
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

struct SMPSStage{T <: AbstractFloat, M <: AbstractMatrix}
    id::Int
    model::LPData{T,M}
    uncertain::LPData{T,M}
    technology::UnitRange{Int}
    recourse::UnitRange{Int}

    function SMPSStage(model::LPData{T,M}) where {T <: AbstractFloat, M <: AbstractMatrix}
        return new{T, M}(1, model, LPData(T,M,0,0,0), 0:0, 0:0)
    end

    function SMPSStage(id::Integer, model::LPData{T,M}, uncertain::LPData{T,M}, technology::UnitRange{Int}, recourse::UnitRange{Int}) where {T <: AbstractFloat, M <: AbstractMatrix}
        return new{T, M}(id, model, uncertain, technology, recourse)
    end
end


# Read and write cgs files
using Dates
export export_compgraph, import_compgraph;

"""
    function export_compgraph(
            graph,
        fname;
        main_output = nothing,
        order = get_topo_order(graph)[1],
        fun = "",
        dom = "",
        err = "",
        genby = "",
        descr = "",
        user = splitdir(homedir())[end],
    )

Exports the graph to the file `fname` using the computation graph format (cgr).
These kwargs are strings or values that are stored as comments in the file

  - `descr` of the graph
  - `fun` function it approximates
  - `dom` domain it approximates
  - `err` error in this domain
  - `genby` script that generated this file
  - `user` a username that created the file

These kwarg influence how the graph is stored:

  - `order` specifies an order the graph nodes are stored
  - `main_output` is a `Symbol` specifying what is the output.

CGR format:

Every line corresponds to a node / operation. The syntax is matlab compatible,
although [`gen_code`](@ref) produces faster matlab code.

See also [`export_compgraph`](@ref).
"""
function export_compgraph(
    graph,
    fname;
    main_output = nothing,
    order = get_topo_order(graph)[1],
    fun = "",
    dom = "",
    err = "",
    genby = "",
    descr = "",
    user = splitdir(homedir())[end],
)
    # USER extraction for windows and linux:
    # https://stackoverflow.com/questions/27809812/get-current-username-in-julia-linux
    fname = abspath(fname)
    file = open(fname, "w+")

    nof_written_outputs = 0
    T = eltype(graph)

    println(file, "%# Representation of a computation graph")
    if !isempty(fun)
        println(file, "%# Function: ", fun)
    end
    if !isempty(descr)
        println(file, "%# ", descr)
    end
    if !isempty(dom)
        println(file, "%# Domain: ", dom)
    end
    if !isempty(err)
        println(file, "%# Error: ", err)
    end
    if !isempty(genby)
        println(file, "%# Generated by command: ", genby)
    end

    println(file, "%# Created: ", now(), " by user ", user)
    println(file, "")
    println(file, "graph_coeff_type=\"", string(T), "\";")
    println(file, "")
    for node in order
        op = graph.operations[node]
        if op == :mult
            println(
                file,
                String(node),
                "=",
                String(graph.parents[node][1]),
                "*",
                String(graph.parents[node][2]),
                ";",
            )
        elseif op == :ldiv
            println(
                file,
                String(node),
                "=",
                String(graph.parents[node][1]),
                "\\",
                String(graph.parents[node][2]),
                ";",
            )
        elseif op == :lincomb
            for i = 1:2
                print(file, "coeff$i=", real(graph.coeffs[node][i]))
                if T <: Complex
                    t = imag(graph.coeffs[node][i])
                    if t >= 0
                        print(file, " + ", t)
                    else
                        print(file, " - ", abs(t))
                    end
                    print(file, "i")
                end
                println(file, ";")
            end
            println(
                file,
                String(node),
                "=",
                "coeff1*",
                String(graph.parents[node][1]),
                "+",
                "coeff2*",
                String(graph.parents[node][2]),
                ";",
            )
        end

        if any(graph.outputs .== node) && !(node == main_output)
            println(file, "output$nof_written_outputs", "=", String(node))
            nof_written_outputs += 1
        end
    end

    if !isnothing(main_output)
        println(file, "output$nof_written_outputs", "=", String(main_output))
    end
    return close(file)
end

"""
    graph=import_compgraph(fname)

Reads a graph stored in the computation graph format (cgr) in the file `fname`.

See also [`import_compgraph`](@ref).
"""
function import_compgraph(fname)
    fname = abspath(fname)
    file = open(fname, "r")
    line = readline(file)
    while !eof(file) && !startswith(line, "graph_coeff_type=\"")
        line = readline(file)
    end
    num = split(line, r"^.*=")[2] #remove everything upto and including "="
    num = num[2:end-2] #remove trailing semicolon and quotation marks
    T = eval(Meta.parse(num))
    graph = Compgraph(T)

    while !eof(file)
        line = readline(file)

        if startswith(line, "coeff1=") #Start of three lines defining :lincomb
            α = Vector{T}(undef, 2)
            for i = 1:2
                # Remove everything upto "=" inclusive.
                num = replace(line, r".*=" => "")
                α[i] = parse(T, num[1:end-1])
                line = readline(file)
            end
            parts = split(line, "=") # Split string on "="
            target = Symbol(parts[1])
            subparts = split(parts[2], "+")
            if endswith(subparts[2], ";")
                subparts[2] = subparts[2][1:end-1]
            end
            p = Vector{Symbol}(undef, 2)
            for i = 1:2
                # Remove everything upto "*" inclusive
                p[i] = Symbol(replace(subparts[i], r".*\*" => ""))
            end
            add_lincomb!(graph, target, α[1], p[1], α[2], p[2])
        elseif occursin(r"\*", line) # Looks for "*". Found :mult
            parts = split(line, "=") # Split string on "="
            target = Symbol(parts[1])
            subparts = split(parts[2], "*")
            if endswith(subparts[2], ";")
                subparts[2] = subparts[2][1:end-1]
            end
            p = Vector{Symbol}(undef, 2)
            for i = 1:2
                p[i] = Symbol(subparts[i])
            end
            add_mult!(graph, target, p[1], p[2])

        elseif occursin(r"\\\\", line) #Looks for "\". Found :ldiv
            parts = split(line, "=") # Split string on "="
            target = Symbol(parts[1])
            subparts = split(parts[2], "\\")
            if endswith(subparts[2], ";")
                subparts[2] = subparts[2][1:end-1]
            end
            p = Vector{Symbol}(undef, 2)
            for i = 1:2
                p[i] = Symbol(subparts[i])
            end
            add_ldiv!(graph, target, p[1], p[2])

        elseif startswith(line, r"output[0-9][0-9]*=") #Define output
            parts = split(line, "=") # Split string on "="
            out = Symbol(parts[2])
            add_output!(graph, out)
        end
    end

    close(file)
    return graph
end


using SortMark

df = make_df()

skip = (df.Type .<: Union{UInt64,UInt128}) .&
    (df.source_key .== :small_positive)
compute!(df[(!).(skip), :])
stat!(df)
SortMark


@testset "009" begin
    @test tdpe.e009(3+4+5) == 3*4*5
end


using sem, Test, SafeTestsets

@time @safetestset "Examples" begin include("examples/example_tests.jl") end


struct ODEInterpolatingAdjointSensitivityFunction{rateType,uType,uType2,UF,PF,G,JC,GC,DG,TJ,PJT,PJC,CP,SType,INT,CV} <: SensitivityFunction
  uf::UF
  pf::PF
  g::G
  J::TJ
  pJ::PJT
  dg_val::uType
  jac_config::JC
  g_grad_config::GC
  paramjac_config::PJC
  sensealg::InterpolatingAdjoint
  f_cache::rateType
  discrete::Bool
  y::uType2
  sol::SType
  dg::DG
  checkpointing::Bool
  checkpoints::CP
  integrator::INT
  colorvec::CV
end

@noinline function ODEInterpolatingAdjointSensitivityFunction(g,u0,p,sensealg,discrete,sol,dg,checkpoints,tspan,colorvec)
  numparams = length(p)
  numindvar = length(u0)
  # if there is an analytical Jacobian provided, we are not going to do automatic `jac*vec`
  f = sol.prob.f
  checkpointing = sensealg.checkpointing isa Bool ? sensealg.checkpointing : !sol.dense
  isautojacvec = DiffEqBase.has_jac(f) ? false : get_jacvec(sensealg)
  J = isautojacvec ? nothing : similar(u0, numindvar, numindvar)

  if !discrete
    if dg != nothing || isautojacvec
      pg = nothing
      pg_config = nothing
    else
      pg = UGradientWrapper(g,tspan[2],p)
      pg_config = build_grad_config(sensealg,pg,u0,p)
    end
  else
    pg = nothing
    pg_config = nothing
  end

  if DiffEqBase.has_jac(f) || isautojacvec
    jac_config = nothing
    uf = nothing
  else
    uf = DiffEqDiffTools.UJacobianWrapper(f,tspan[2],p)
    jac_config = build_jac_config(sensealg,uf,u0)
  end

  y = copy(sol(tspan[1])) # TODO: Has to start at interpolation value!

  if DiffEqBase.has_paramjac(f) || isautojacvec
    paramjac_config = nothing
    pf = nothing
  else
    pf = DiffEqDiffTools.ParamJacobianWrapper(f,tspan[1],y)
    paramjac_config = build_param_jac_config(sensealg,pf,y,p)
  end

  pJ = isautojacvec ? nothing : similar(sol.prob.u0, numindvar, numparams)

  integrator = if checkpointing
    integ = init(sol.prob, sol.alg, save_on=false)
    integ.u = y
    integ
  else
    nothing
  end
  dg_val = similar(u0, numindvar) # number of funcs size
  f_cache = deepcopy(u0)

  return ODEInterpolatingAdjointSensitivityFunction(uf,pf,pg,J,pJ,dg_val,
                               jac_config,pg_config,paramjac_config,
                               sensealg,f_cache,
                               discrete,y,sol,dg,checkpointing,checkpoints,integrator,colorvec)
end

# u = λ'
function (S::ODEInterpolatingAdjointSensitivityFunction)(du,u,p,t)
  @unpack y, sol, J, uf, sensealg, checkpointing, f_cache, jac_config, discrete, dg, dg_val, g, g_grad_config = S
  idx = length(y)
  f = sol.prob.f
  isautojacvec = DiffEqBase.has_jac(f) ? false : get_jacvec(sensealg)

  if checkpointing
    @unpack integrator, checkpoints = S
    # assuming that in the forward direction `t0` < `t1`, and the
    # `checkpoints` vector is sorted with respect to the forward direction
    tidx = findlast(x->x <= t, checkpoints)
    t0 = checkpoints[tidx]
    dt = t-t0
    if abs(dt) > integrator.opts.dtmin
      sol(integrator.u, t0)
      copyto!(integrator.uprev, integrator.u)
      integrator.t = t0
      # set `iter` to some arbitrary integer so that there won't be max maxiters error
      integrator.iter=100
      u_modified!(integrator, true)
      step!(integrator, dt, true)
      # `integrator.u` is aliased to `y`
    else
      sol(y,t)
    end
  else
    sol(y,t)
  end

  λ     = @view u[1:idx]
  dλ    = @view du[1:idx]
  grad  = @view u[idx+1:end]
  dgrad = @view du[idx+1:end]

  if !isautojacvec
    if DiffEqBase.has_jac(f)
      f.jac(J,y,p,t) # Calculate the Jacobian into J
    else
      uf.t = t
      jacobian!(J, uf, y, f_cache, sensealg, jac_config)
    end
    mul!(dλ',λ',J)
  else
    _dy, back = Tracker.forward(y, sol.prob.p) do u, p
      if DiffEqBase.isinplace(sol.prob)
        out_ = map(zero, u)
        f(out_, u, p, t)
        Tracker.collect(out_)
      else
        vec(f(u, p, t))
      end
    end
    dλ[:], dgrad[:] = Tracker.data.(back(λ))
  end

  dλ .*= -one(eltype(λ))

  if !discrete
    if dg != nothing
      dg(dg_val,y,p,t)
    else
      g.t = t
      gradient!(dg_val, g, y, sensealg, g_grad_config)
    end
    dλ .+= dg_val
  end

  if !isautojacvec
    @unpack pJ, pf, paramjac_config = S
    if DiffEqBase.has_paramjac(f)
      f.paramjac(pJ,y,sol.prob.p,t) # Calculate the parameter Jacobian into pJ
    else
      jacobian!(pJ, pf, sol.prob.p, f_cache, sensealg, paramjac_config)
    end
    mul!(dgrad',λ',pJ)
  end
  nothing
end

# g is either g(t,u,p) or discrete g(t,u,i)
@noinline function ODEAdjointProblem(sol,sensealg::InterpolatingAdjoint,
                                     g,t=nothing,dg=nothing;
                                     checkpoints=sol.t,
                                     callback=CallbackSet())
  f = sol.prob.f
  tspan = (sol.prob.tspan[2],sol.prob.tspan[1])
  discrete = t != nothing

  p = sol.prob.p
  p === DiffEqBase.NullParameters() && error("Your model does not have parameters, and thus it is impossible to calculate the derivative of the solution with respect to the parameters. Your model must have parameters to use parameter sensitivity calculations!")

  u0 = zero(sol.prob.u0)

  len = length(u0)+length(p)
  λ = similar(u0, len)
  sense = ODEInterpolatingAdjointSensitivityFunction(g,u0,
                                                     p,sensealg,discrete,
                                                     sol,dg,checkpoints,tspan,
                                                     f.colorvec)

  init_cb = t !== nothing && sol.prob.tspan[2] == t[end]
  cb = generate_callbacks(sense, g, λ, t, callback, init_cb)
  z0 = vec(zero(λ))
  ODEProblem(sense,z0,tspan,p,callback=cb)
end


#=
types and methods to facilitate the interaction between orders, rules, and portfolios
=#

abstract type AbstractOrder end

struct MarketOrder <: AbstractOrder
    asset::String
    quantity::Number
end

struct LimitOrder <: AbstractOrder
    asset::String
    quantity::Number
    limit::Number
end

struct StopOrder <: AbstractOrder
    asset::String
    quantity::Number
    stop::Number
end

#TODO: complete this logic, enable interaction with strategy/portfolio objects
#TODO: diffierentiate between buying vs. going long and the like (the latter should reverse position if short)

#TODO: add logic whereby the order logic is altered by the type `T` of qty
# (if T<:Int, order that many *shares*, else if T<:Float64, interpret qty as a fraction of the portfolio at time t)

liquidate{T<:Real}(qty::T) = qty

long{T<:Real}(qty::T)::T = qty
buy{T<:Real}(qty::T)::T = qty

short{T<:Real}(qty::T)::T = qty
sell{T<:Real}(qty::T)::T = qty


"""
A Julia suite for chaos and nonlinear dynamics
"""
module DynamicalSystems

using StaticArrays

export SVector, SMatrix, @SVector, @SMatrix, Size

using Reexport

@reexport using DelayEmbeddings
@reexport using DynamicalSystemsBase
@reexport using ChaosTools
@reexport using RecurrenceAnalysis

display_update = true
update_name = "update_v1.1.0"

if display_update
if !isfile(joinpath(@__DIR__, update_name))
printstyled(stdout,
"""
\nUpdate message: DynamicalSystems v1.1

A new package has joined DynamicalSystems: RecurrenceAnalysis !

This new package offers tools to compute and analyze
recurrences in your timeseries!

Check out the documentation for more!\n
"""; color = :light_magenta)
touch(joinpath(@__DIR__, update_name))
end
end

end # module


using JSON
using Test
using ComparativeAutograder

const ARITHMETIC_SOLUTION = """
function h(x, y, z)
    return x*y, y*z, x*z, x+y+z
end
"""

@testset "Arithmetic" begin

    cases = [TestCase((randn(), randn(), randn())) for _ in 1:100]
    suite = TestSuite("h", cases)

    soln_path, soln_io = mktemp()
    sub_path, sub_io = mktemp()
    sub_stdout, sub_stderr = nothing, nothing

    try
        write(soln_io, ARITHMETIC_SOLUTION)
        close(soln_io)
        write(sub_io, """h(x, y, z) = x*y, y*z, x*z, x+y+z""")
        close(sub_io)
        sub_stdout, sub_stderr = @capture_stdstreams runtestsuite(
            suite, soln_path, sub_path
        )
    finally
        rm.([soln_path, sub_path])
    end

    @test sub_stdout != nothing
    sub_data = JSON.parse(sub_stdout)
    @test sub_data["passed"] == true
end


module PeeEmTee

using HDF5
using DocStringExtensions
using StatsBase
using Distributions
using Optim
using RecipesBase
using DelimitedFiles
import Base
export high_voltages, WaveSet, calculate_charges, ChargeDist, pmtresp,
       pre_fit, gauss, norm_gauss, pmtresp_fit, pmtresp_uap, plot, plot!,
       calculate_transit_times, bin_data, subtract_baseline, simulate_charges,
       baseline_max, PMTRespFit, PMTRespUapFit, make_qfunc, PreFit, mean_waveform

include("tools.jl")
include("fit.jl")
include("plot.jl")

end # module


module Images

import Base.Order: Ordering, ForwardOrdering, ReverseOrdering
import Base.Graphics: width, height
importall Base

using Color
using Cartesian

# We'll use Grid's Counter, but we don't need the rest of it
include(joinpath(Pkg.dir(),"Grid","src","counter.jl"))

include("core.jl")
include("config.jl")
include("iterator.jl")
include("io.jl")
include("scaling.jl")
include("labeledarrays.jl")
include("algorithms.jl")

export # types
    AbstractImage,
    AbstractImageDirect,
    AbstractImageIndexed,
    Image,
    ImageCmap,
    BitShift,
    ClipMin,
    ClipMax,
    ClipMinMax,
    LabeledArray,
    Overlay,
    OverlayImage,
    ScaleAutoMinMax,
    ScaleInfo,
    ScaleMinMax,
    ScaleNone,
    ScaleSigned,
    SliceData,

    # constants
    palette_fire,
    palette_gray32,
    palette_gray64,
    palette_rainbow,

    # core functions
    assert2d,
    assert_scalar_color,
    assert_timedim_last,
    assert_xfirst,
    assert_yfirst,
    colordim,
    colorspace,
    coords,
    coords_spatial,
    data,
    dimindex,
    dimindexes,
    getindexim,
    grayim,
    height,
    isdirect,
    isxfirst,
    isyfirst,
    limits,
    maxabsfinite,
    maxfinite,
    minfinite,
    nimages,
    pixelspacing,
    properties,
    refim,
    rerange!,
    reslice!,
    sdims,
    size_spatial,
    share,
    sliceim,
    spatialorder,
    spatialpermutation,
    spatialproperties,
    storageorder,
    subim,
    timedim,
    width,
    widthheight,

    # io functions
    add_image_file_format,
    imread,
    imwrite,
    loadformat,

    # iterator functions
    first_index,
    iterate_spatial,
    parent,

    # color-related functions
    alphaval,
    blueval,
    greenval,
    redval,
    red,
    green,
    blue,
    hsi2rgb,
    imadjustintensity,
    indexedcolor,
    lut,
    ntsc2rgb,
    rgb2gray,
    rgb2hsi,
    rgb2ntsc,
    rgb2ycbcr,
    uint32color,
    uint32color!,
    ycbcr2rgb,

    # Scaling of intensity
    climdefault,
    float32sc,
    float64sc,
    sc,
    scale,
    scaleinfo,
    scaleminmax,
    scalesigned,
    uint8sc,
    uint16sc,
    uint32sc,

    # algorithms
    backdiffx,
    backdiffy,
    dilate,
    erode,
    opening,
    closing,
    forwarddiffx,
    forwarddiffy,
    gaussian2d,
    imaverage,
    imcomplement,
    imdog,
    imedge,
    imfilter,
    imfilter_gaussian,
    imfilter_gaussian!,
    imgaussiannoise,
    imlaplacian,
    imlineardiffusion,
    imlog,
    imROF,
    imstretch,
#     imthresh,
    label_components,
    label_components!,
    ncc,
    padarray,
    prewitt,
    sad,
    sadn,
    sobel,
    ssd,
    ssdn,

    # phantoms
    shepp_logan

@deprecate cairoRGB     uint32color!
@deprecate refim        getindexim
@deprecate scaledefault climdefault

end


export AbstractLinearSolver, AbstractLinearSolverCache
export AbstractKrylovMethod
export LinearSolver
export settolerance!
export linearsolve!
export LSinitialize!, LSsolve!

"""
    AbstractLinearSolver

This is an abstract type representing a generic iterative
linear solver.

The available concrete implementations are:

- [`Solvent.GeneralizedMinimalResidualMethod`](@ref)
- [`Solvent.ConjugateGradientMethod`](@ref)
- [`Solvent.GeneralizedConjugateResidualMethod`](@ref)
"""
abstract type AbstractLinearSolver <: AbstractSystemSolver end

abstract type AbstractLinearSolverCache end
abstract type AbstractKrylovMethod end

"""
    LinearSolver(
        linearoperator!,
        krylov_alg::AbstractKrylovMethod,
        Q::AT;
        pc_alg = Identity(),
        rtol = √eps(eltype(AT)),
        atol = eps(eltype(AT)),
    ) where {AT}

Object for a linear system solver with operator `linearoperator!`
i.e,
```math
L(Q) = Q_{rhs}
```
with solver method specified by `krylov_alg`. Preconditioning may
be specified by `pc_alg`.
"""
mutable struct LinearSolver{
    OP,
    krylovType <: AbstractKrylovMethod,
    pcType <: AbstractPreconditioner,
    fType,
    lscType <: AbstractLinearSolverCache,
} <: AbstractLinearSolver
    linop!::OP
    krylov_alg::krylovType
    pc::pcType
    rtol::fType
    atol::fType
    cache::lscType

    function LinearSolver(
        linearoperator!,
        krylov_alg::AbstractKrylovMethod,
        Q::AT;
        pc_alg = Identity(),
        rtol = √eps(eltype(AT)),
        atol = eps(eltype(AT)),
    ) where {AT}
        lscache = cache(
            krylov_alg,
            Q,
        )
        pc = Preconditioner(pc_alg, linearoperator!, Q)

        OP = typeof(linearoperator!)
        krylovType = typeof(krylov_alg)
        pcType = typeof(pc)
        fType = typeof(rtol)
        lscType = typeof(lscache)

        return new{OP, krylovType, pcType, fType, lscType}(
            linearoperator!,
            krylov_alg,
            pc,
            rtol,
            atol,
            lscache,
        )
    end
end

"""
    linearsolve!(
        linearsolver::LinearSolver,
        Q,
        Qrhs,
        args...;
        cvg = Ref{Bool}(),
    )

Solves a linear problem defined by the `linearsolver` object and the state
`Qrhs`, i.e,
```math
L(Q) = Q_{rhs}
```
using the initial guess `Q`. After the call `Q` contains the solution.
"""
function linearsolve!(
    linearsolver::LinearSolver,
    Q,
    Qrhs,
    args...;
    cvg = Ref{Bool}(),
)
    converged = false
    iters = 0
    converged, threshold = LSinitialize!(
        linearsolver,
        Q,
        Qrhs,
        args...,
    )
    converged && return iters

    converged, iters, residual_norm = LSsolve!(
        linearsolver,
        threshold,
        Q,
        Qrhs,
        args...,
    )
    converged || @warn "Linear solver did not attain convergence after $iters iterations"
    cvg[] = converged
    iters
end

"""
    settolerance!(linearsolver::LinearSolver, tol; relative=false)

Set the relative or absolute tolerance of `linearsolver` to `tol`.

Set `rtol` if `relative = true` or `atol` if `relative = false`.
"""
function settolerance!(linearsolver::LinearSolver, tol; relative = false)
    if relative
        linearsolver.rtol = tol
    else
        linearsolver.atol = tol
    end
    nothing
end

"""
    LSinitialize!(
        linearsolver::LinearSolver,
        Q,
        Qrhs,
        args...,
    )

Initialize the preconditioner and linear solver.
"""
function LSinitialize!(
    linearsolver::LinearSolver,
    Q,
    Qrhs,
    args...,
)
    # Initializes the preconditioner
    PCinitialize!(linearsolver.pc, Q, Qrhs, args...)
    LSinitialize!(
        linearsolver.krylov_alg,
        linearsolver,
        Q,
        Qrhs,
        args...,
    )
end

"""
    LSsolve!(
        linearsolver::LinearSolver,
        threshold,
        Q,
        Qrhs,
        args...,
    )

Implement linear solver iterations as specified by the solver type.
"""
function LSsolve!(
    linearsolver::LinearSolver,
    threshold,
    Q,
    Qrhs,
    args...,
)
    LSsolve!(
        linearsolver.krylov_alg,
        linearsolver,
        threshold,
        Q,
        Qrhs,
        args...,
    )
end

@kernel function linearcombination!(Q, cs, Xs, increment::Bool)
    i = @index(Global, Linear)
    if !increment
        @inbounds Q[i] = -zero(eltype(Q))
    end
    @inbounds for j in 1:length(cs)
        Q[i] += cs[j] * Xs[j][i]
    end
end

include("gmres.jl")
include("gcr.jl")
include("cg.jl")


using CSV, DataFrames, HDF5

"""
This function read the orginal porto taxi csv file and save all trips into
hdf5 file, each trip will be a 2xn matrix where n is the length of the trip.
The first row is longitude and second row is latitude.

The output hdf5 file will be used to construct vocabulary by `makeVocab!()` in
`SpatialRegionTools.jl`.
"""
function porto2h5(csvfile::String)
    df = CSV.read(csvfile)
    #df[:TIMESTAMP] = convert(Array{String}, df[:TIMESTAMP])
    #df[:MISSING_DATA] = convert(Array{String}, df[:MISSING_DATA])
    #df[:POLYLINE] = convert(Array{String, 1}, df[:POLYLINE])

    df = df[df.MISSING_DATA .== "False", :]
    sort!(df, [:TIMESTAMP])

    println("Processing $(size(df, 1)) trips...")
    ## writing in pure text
    #CSV.write("porto.csv", df[:POLYLINE])
    #open("porto.csv", "w") do f
    #    for s in df[:POLYLINE]
    #        write(f, s * "\n")
    #    end
    #end
    ## writing in numeric matrix with hdf5
    h5open("../data/porto.h5", "w") do f
        num = 0
        for trip in df.POLYLINE
            trip = Meta.parse(trip) |> eval
            tripLength = length(trip)
            tripLength == 0 && continue
            trip = hcat(trip...)
            num += 1
            f["/trips/$num"] = trip
            f["/timestamps/$num"] = collect(0:tripLength-1) * 15.0
            num % 100_000 == 0 && println("$num")
        end
        attrs(f)["num"] = num
        println("Saved $num trips.")
    end
end

function porto2standardcsv(portocsv::String)
    df = CSV.read(portocsv)
    df = df[df.MISSING_DATA .== "False", :]
    #sort!(df, [:TIMESTAMP])
    trips = DataFrame[]
    for (i, row) in enumerate(eachrow(df))
        gps = Meta.parse(row.POLYLINE) |> eval
        tripLength = length(gps)
        tripLength == 0 && continue
        gps = hcat(gps...)
        trip = DataFrame(tripid=fill(row.TRIP_ID, tripLength),
                         timestamps=row.TIMESTAMP .+ collect(0:tripLength-1) * 15,
                         lon=gps[1, :],
                         lat=gps[2, :])
        push!(trips, trip)
        i % 100_000 == 0 && println("Processed $i rows.")
    end
    #CSV.write("../data/porto-trips.csv", trips, header=[:tripid, :timestamps, :lon, :lat])
    trips
end

"""
Distorting a trip using Gaussian noise
"""
function distort(trip::Matrix{Float64}, rate::Float64; radius=50.0)
    noisetrip = copy(trip)
    for i = 1:size(noisetrip, 2)
        if rand() <= rate
            x, y = lonlat2meters(noisetrip[:, i]...)
            xnoise, ynoise = 2rand()-1, 2rand()-1
            normz = hypot(xnoise, ynoise)
            xnoise, ynoise = xnoise*radius/normz, ynoise*radius/normz
            noisetrip[:, i] = meters2lonlat(x + xnoise, y + ynoise) |> collect
        end
    end
    noisetrip
end

"""
Accepting one trip and producing its 10 different noise rate distorted variants
"""
function distort(trip::Matrix{Float64}, nsplit::Int)
    noisetrips = Matrix{Float64}[]

    for rate in 0:0.1:0.9
        noisetrip = distort(trip, rate)
        push!(noisetrips, noisetrip)
    end
    noisetrips
end

"""
Downsampling one trip, rate is dropping rate
"""
function downsampling(trip::Matrix{Float64}, rate::Float64)
    keep_idx = Int[1]
    for i = 2:size(trip, 2)-1
        rand() > rate && push!(keep_idx, i)
    end
    push!(keep_idx, size(trip, 2))
    copy(trip[:, keep_idx])
end

"""
Accepting one trip and producing its 9 different lowsampling rate variants
"""
function downsampling(trip::Matrix{Float64}, nsplit::Int)
    noisetrips = Matrix{Float64}[]
    dropping_rates = 0:0.1:0.8
    for rate in dropping_rates
        noisetrip = downsampling(trip, rate)
        push!(noisetrips, noisetrip)
    end
    noisetrips
end

"""
First downsampling and then distorting the trip, producing its 20 different variants
"""
function downsamplingDistort(trip::Matrix, nsplit::Int)
    noisetrips = Matrix{Float64}[]
    dropping_rates = [0, 0.2, 0.4, 0.5, 0.6]
    distorting_rates = [0, 0.2, 0.4, 0.6]
    for dropping_rate in dropping_rates
        noisetrip1 = downsampling(trip, dropping_rate)
        for distorting_rate in distorting_rates
            noisetrip2 = distort(noisetrip1, distorting_rate)
            push!(noisetrips, noisetrip2)
        end
    end
    noisetrips
end

"""
longitude, latitude to Web Mercator coordinate
"""
function lonlat2meters(lon::Float64, lat::Float64)
    semimajoraxis = 6378137.0
    east = lon * 0.017453292519943295
    north = lat * 0.017453292519943295
    t = sin(north)
    semimajoraxis * east, 3189068.5 * log((1 + t) / (1 - t))
end

function meters2lonlat(x::Float64, y::Float64)
    semimajoraxis = 6378137.0
    lon = x / semimajoraxis / 0.017453292519943295
    t = exp(y / 3189068.5)
    lat = asin((t - 1) / (t + 1)) / 0.017453292519943295
    lon, lat
end

#x, y = lonlat2meters(-8.7351,40.95)
#meters2lonlat(x, y)

#h5open("/tmp/test.h5", "w") do f
#    f["/trips/1"] = rand(10,2)
#    f["/trips/2"] = rand(30,2)
#    attrs(f)["num"] = 2
#    attrs(f)["/trips/1"] = 10
#    attrs(f)["/trips/2"] = 11
#end
#
#h5open("/tmp/test.h5", "r") do f
#    read(f["/trips/1"])
#    attrs(f)["/trips/1"] |> read
#end

#porto2h5("/Users/fineday/Downloads/train.csv")

#tic()
#tripLengths = Int[]
#h5open("preprocessing/porto.h5", "r") do f
#    num = read(attrs(f)["num"])
#    for i = 1:num
#        trip = read(f["/trips/$i"])
#        push!(tripLengths, size(trip, 2))
#        #i % 100_000 == 0 && println(i)
#    end
#end
#toc()
#h5open("porto.h5", "r") do fread
#    num = attrs(fread)["num"] |> read
#    h5open("/tmp/porto.h5", "w") do fwrite
#        attrs(fwrite)["num"] = num
#        for i = 1:num
#            trip = read(fread["/trips/$i"])
#            fwrite["/trips/$i"] = trip
#            tripLength = size(trip, 2)
#            fwrite["/timestamps/$i"] = collect(0:tripLength-1) * 15.0
#            i % 100_000 == 0 && println(i)
#        end
#    end
#end
#toc()

#filter(x-> x >= 20 && x <= 100, tripLengths) |> length
#sum(tripLengths)
#
#sum(tripLengths[1:800_000])

#h5open("porto.h5", "r") do fread
#    h5open("/tmp/porto.h5", "w") do fwrite
#        num = attrs(fread)["num"] |> read
#        attrs(fwrite)["num"] = num
#        for i = 1:num
#            fwrite["/trips/$i"] = fread["/trips/$i"] |> read |> transpose
#            i % 100_000 == 0 && println(i)
#        end
#    end
#end

#tripLengths = Int[]
#h5open("../preprocessing/porto.h5", "r") do f
#    num = read(attrs(f)["num"])
#    for i = 1:num
#        trip = read(f["/trips/$i"])
#        tripLength = size(trip, 2)
#        push!(tripLengths, tripLength)
#        #tripLength >= 50 && push!(tripLengths, tripLength)
#        #i % 100_000 == 0 && println(i)
#    end
#end
#t = filter(x -> 30<=x, tripLengths)
#filter(x -> 60 <= x, t[800000:end])
#sum(tripLengths)
#mean(tripLengths)
#tripLengths
#maximum(tripLengths) #3881
#129_535_632 / 1527348
#trip = rand(40:50, 2, 40)*1.0
#distort(trip, 10)
#downsampling(trip, 10)
#sdownsamplingDistort(trip, 5)

#lonlat2meters(-8.735152,40.953673)


using .VectorizationBase: AbstractSIMD
import .ForwardDiff
import .ChainRulesCore

@generated function SLEEFPirates.tanh_fast(x::ForwardDiff.Dual{T,S,N}) where {T,S,N}
    quote
        $(Expr(:meta,:inline))
        t = tanh_fast(x.value)
        ∂t = vfnmadd_fast(t, t, one(S))
        p = x.partials
        ForwardDiff.Dual(t, ForwardDiff.Partials(Base.Cartesian.@ntuple $N n -> mul_fast(∂t, p[n])))
    end
end
function ChainRulesCore.rrule(::typeof(tanh_fast), x)
    t = tanh_fast(x)
    ∂ = let t = t
        y -> (ChainRulesCore.Zero(), mul_fast(vfnmadd_fast(t, t, one(t)), y))
    end
    t, ∂
end
@generated function SLEEFPirates.sigmoid_fast(x::ForwardDiff.Dual{T,S,N}) where {T,S,N}
    quote
        $(Expr(:meta,:inline))
        s = sigmoid_fast(x.value)
        ∂s = vfnmadd_fast(s,s,s)
        p = x.partials
        ForwardDiff.Dual(s, ForwardDiff.Partials(Base.Cartesian.@ntuple $N n -> mul_fast(∂s, p[n])))
    end
end
function ChainRulesCore.rrule(::typeof(sigmoid_fast), x)
    s = sigmoid_fast(x)
    ∂ = let s = s
        y -> (ChainRulesCore.Zero(), mul_fast(vfnmadd_fast(s, s, s), y))
    end
    s, ∂
end
@generated function VectorizationBase.relu(x::ForwardDiff.Dual{T,S,N}) where {T,S,N}
    quote
        $(Expr(:meta,:inline))
        v = x.value
        z = zero(v)
        cmp = v < z
        r = ifelse(cmp, z, v)
        p = x.partials
        ForwardDiff.Dual(r, ForwardDiff.Partials(Base.Cartesian.@ntuple $N n -> ifelse(cmp, z, p[n])))
    end
end
function ChainRulesCore.rrule(::typeof(relu), v)
    z = zero(v)
    cmp = v < z
    r = ifelse(cmp, z, v)
    ∂ = let cmp = cmp
        y -> (ChainRulesCore.Zero(), ifelse(cmp, zero(y), y))
    end
    r, ∂
end

@generated function init_dual(v::Tuple{Vararg{AbstractSIMD,A}}) where {A}
    res = Expr(:tuple)
    q = Expr(:block, Expr(:meta,:inline))
    for a ∈ 1:A
        v_a = Symbol(:v_,a)
        push!(q.args, Expr(:(=), v_a, Expr(:ref, :v, a)))
        partials = Expr(:tuple)
        for i ∈ 1:A
            push!(partials.args, Expr(:call, i == a ? :one : :zero, v_a))
        end
        push!(res.args, :(ForwardDiff.Dual($v_a, ForwardDiff.Partials($partials))))
    end
    push!(q.args, res)
    q
end

@generated function dual_store!(∂p::Tuple{Vararg{AbstractStridedPointer,A}}, p::AbstractStridedPointer, ∂v, im::Vararg{Any,N}) where {A,N}
    quote
        $(Expr(:meta,:inline))
        v = ∂v.value
        ∂ = ∂v.partials
        Base.Cartesian.@nextract $N im im
        Base.Cartesian.@ncall $N VectorizationBase.vnoaliasstore! p v im  # store
        Base.Cartesian.@nexprs $A a -> begin # for each of `A` partials
            ∂p_a = ∂p[a]
            ∂_a = ∂[a]
            Base.Cartesian.@ncall $N VectorizationBase.vnoaliasstore! ∂p_a ∂_a im # store
        end
        nothing
    end
end

function ∂vmap_singlethread!(
    f::F, ∂y::Tuple{Vararg{DenseArray{T},A}}, y::DenseArray{T},
    args::Vararg{<:DenseArray{<:Base.HWReal},A}
) where {F,T <: Base.HWReal, A}
    N = length(y)
    ptry = VectorizationBase.zero_offsets(stridedpointer(y))
    ptrargs = VectorizationBase.zero_offsets.(stridedpointer.(args))
    ptr∂y = VectorizationBase.zero_offsets.(stridedpointer.(∂y))

    i = 0
    V = VectorizationBase.pick_vector_width(T)
    W = Int(V)
    st = VectorizationBase.static_sizeof(T)
    zero_index = MM{W}(StaticInt(0), st)
    while i < N - ((W << 2) - 1)
        index = VectorizationBase.Unroll{1,W,4,1,W,0x0000000000000000}((i,))
        v = f(init_dual(vload.(ptrargs, index))...)
        dual_store!(ptr∂y, ptry, v, index)
        i = vadd_fast(i, 4W)
    end
    while i < N - (W - 1)
        vᵣ = f(init_dual(vload.(ptrargs, ((MM{W}(i),),)))...)
        dual_store!(ptr∂y, ptry, vᵣ, (MM{W}(i),))
        i = vadd_fast(i, W)
    end
    if i < N
        m = mask(T, N & (W - 1))
        dual_store!(ptr∂y, ptry, f(init_dual(vload.(ptrargs, ((MM{W}(i),),), m))...), (MM{W}(i,),), m)
    end
    nothing
end

struct SIMDMapBack{K,T<:Tuple{Vararg{Any,K}}}
    jacs::T
end
@generated function (b::SIMDMapBack{K,T})(Δ::A) where {K,T,A}
    preloop = Expr(:block, :(jacs = b.jacs))
    loop_body = Expr(:block, :(Δᵢ = Δ[i]))
    ret = Expr(:tuple, ChainRulesCore.Zero(), ChainRulesCore.Zero())
    for k ∈1:K
        jₖ = Symbol(:j_, k)
        push!(preloop.args, :($jₖ = jacs[$k]))
        push!(loop_body.args, :($jₖ[i] *= Δᵢ))
        push!(ret.args, jₖ)
    end
    quote
        $preloop
        @avx for i ∈ eachindex(Δ)
            $loop_body
        end
        $ret
    end
end

function ChainRulesCore.rrule(::typeof(vmap), f::F, args::Vararg{Any,K}) where {F,K}
    out = similar(first(args))
    jacs = map(similar, args)
    ∂vmap_singlethread!(f, jacs, out, args...)
    out, SIMDMapBack(jacs)
end


# Allows UMAP to be called on Experiment objects
function UMAP.umap(e::Experiment, n_components=2; kwargs...)
    umap(convert(Matrix, getdata(e))', n_components; kwargs...)
end


"""
An aggregator interface for SSA-like algorithms.

### Required Fields
- `next_jump`
- `next_jump_time`
- `end_time`
- `cur_rates`
- `sum_rate`
- `ma_jumps`
- `rates`
- `affects!`
- `save_positions`
- `rng`
"""
abstract type AbstractSSAJumpAggregator <: AbstractJumpAggregator end

DiscreteCallback(c::AbstractSSAJumpAggregator) = DiscreteCallback(c, c, initialize = c, save_positions = c.save_positions)

########### The following routines should be templates for all SSAs ###########

# # condition for jump to occur
# @inline function (p::SSAJumpAggregator)(u, t, integrator)
#   p.next_jump_time == t
# end

# # executing jump at the next jump time
# function (p::SSAJumpAggregator)(integrator)
#   execute_jumps!(p, integrator, integrator.u, integrator.p, integrator.t)
#   generate_jumps!(p, integrator, integrator.u, integrator.p, integrator.t)
#   register_next_jump_time!(integrator, p, integrator.t)
#   nothing
# end

# # setting up a new simulation
# function (p::SSAJumpAggregator)(dj, u, t, integrator) # initialize
#   initialize!(p, integrator, u, integrator.p, t)
#   register_next_jump_time!(integrator, p, integrator.t)
#   nothing
# end

############################## Generic Routines ###############################

@inline function register_next_jump_time!(integrator, p::AbstractSSAJumpAggregator, t)
  if p.next_jump_time < p.end_time
    add_tstop!(integrator, p.next_jump_time)
  end
  nothing
end

# helper routine for setting up standard fields of SSA jump aggregations
function build_jump_aggregation(jump_agg_type, u, p, t, end_time, ma_jumps, rates,
                                affects!, save_positions, rng; kwargs...)

  # mass action jumps
  majumps = ma_jumps
  if majumps === nothing
    majumps = MassActionJump(Vector{typeof(t)}(),
                             Vector{Vector{Pair{Int,eltype(u)}}}(),
                             Vector{Vector{Pair{Int,eltype(u)}}}())
  end

  # current jump rates, allows mass action rates and constant jumps
  cur_rates = Vector{typeof(t)}(undef,get_num_majumps(majumps) + length(rates))

  sum_rate = zero(typeof(t))
  next_jump = 0
  next_jump_time = typemax(typeof(t))
  jump_agg_type(next_jump, next_jump_time, end_time, cur_rates, sum_rate,
                majumps, rates, affects!, save_positions, rng; kwargs...)
end

# reevaluate all rates and total rate
function fill_rates_and_sum!(p::AbstractSSAJumpAggregator, u, params, t)
  sum_rate = zero(typeof(p.sum_rate))

  # mass action jumps
  majumps   = p.ma_jumps
  cur_rates = p.cur_rates
  @inbounds for i in 1:get_num_majumps(majumps)
      cur_rates[i] = evalrxrate(u, i, majumps)
      sum_rate    += cur_rates[i]
  end

  # constant rates
  rates = p.rates
  idx   = get_num_majumps(majumps) + 1
  @inbounds for rate in rates
      cur_rates[idx] = rate(u, params, t)
      sum_rate += cur_rates[idx]
      idx += 1
  end

  p.sum_rate = sum_rate
  nothing
end

# recalculate jump rates for jumps that depend on the just executed jump
# requires dependency graph
function update_dependent_rates!(p::AbstractSSAJumpAggregator, u, params, t)
  @inbounds dep_rxs = p.dep_gr[p.next_jump]
  num_majumps = get_num_majumps(p.ma_jumps)
  cur_rates   = p.cur_rates
  sum_rate    = p.sum_rate
  majumps     = p.ma_jumps
  @inbounds for rx in dep_rxs
      sum_rate -= cur_rates[rx]
      if rx <= num_majumps
          @inbounds cur_rates[rx] = evalrxrate(u, rx, majumps)
      else
          @inbounds cur_rates[rx] = p.rates[rx-num_majumps](u, params, t)
      end
      sum_rate += cur_rates[rx]
  end

  p.sum_rate = sum_rate
  nothing
end


function AbstractLambdaMethod()
    return AbstractLambdaMethod(())
end

function solve_ils(obj::AbstractLambdaMethod, arg0::jint, arg1::Vector{jdouble}, arg2::Vector{jint}, arg3::RealMatrix)
    return jcall(obj, "solveILS", Vector{IntegerLeastSquareSolution}, (jint, Vector{jdouble}, Vector{jint}, RealMatrix), arg0, arg1, arg2, arg3)
end


using BusinessDays
using Test
import Dates

println("Perftests")

d0 = Dates.Date(2015, 06, 29) ; d1 = Dates.Date(2100, 12, 20)

cal_type = BusinessDays.Brazil()
cal_sym = :Brazil
cal_str = "Brazil"

@time BusinessDays.initcache(cal_type)
bdays(cal_type, d0, d1) # force JIT compilation
@time bdays(cal_type, d0, d1)
@time for i in 1:1000 bdays(cal_type, d0, d1) end
bdays(cal_sym, d0, d1) # force JIT compilation
bdays(cal_str, d0, d1) # force JIT compilation
@time for i in 1:1000 bdays(cal_sym, d0, d1) end
@time for i in 1:1000 bdays(cal_str, d0, d1) end

N = 1000
d0vec = fill(d0, N)
d1vec = fill(d1, N)
cal_type_vec = fill(cal_type, N)
cal_sym_vec = fill(cal_sym, N)
cal_str_vec = fill(cal_str, N)

# Warmup
BusinessDays.initcache(BusinessDays.Brazil())
BusinessDays.initcache("USSettlement")
BusinessDays.initcache(:UKSettlement)
BusinessDays.bdays(cal_type, d0, d1)
BusinessDays.bdays(cal_sym, d0, d1)
BusinessDays.bdays(cal_str, d0, d1)
BusinessDays.bdays(cal_type, d0vec, d1vec)
BusinessDays.bdays(cal_sym, d0vec, d1vec)
BusinessDays.bdays(cal_str, d0vec, d1vec)
BusinessDays.bdays(cal_type_vec, d0vec, d1vec)
BusinessDays.bdays(cal_sym_vec, d0vec, d1vec)
BusinessDays.bdays(cal_str_vec, d0vec, d1vec)

println("type")
@time for i in 1:1000 BusinessDays.bdays(cal_type, d0, d1) end
@time BusinessDays.bdays(cal_type, d0vec, d1vec)
@time BusinessDays.bdays(cal_type_vec, d0vec, d1vec)
println("sym")
@time for i in 1:1000 BusinessDays.bdays(cal_sym, d0, d1) end
@time BusinessDays.bdays(cal_sym, d0vec, d1vec)
@time BusinessDays.bdays(cal_sym_vec, d0vec, d1vec)
println("str")
@time for i in 1:1000 BusinessDays.bdays(cal_str, d0, d1) end
@time BusinessDays.bdays(cal_str, d0vec, d1vec)
@time BusinessDays.bdays(cal_str_vec, d0vec, d1vec)

BusinessDays.cleancache()

println("initcache type")
@time BusinessDays.initcache(cal_type)
BusinessDays.cleancache()

println("initcache sym")
@time BusinessDays.initcache(cal_sym)
BusinessDays.cleancache()

println("initcache str")
@time BusinessDays.initcache(cal_str)
BusinessDays.cleancache()

let
    cal = BusinessDays.WeekendsOnly()
    println("WeekendsOnly no cache")
    @time for i in 1:1000 BusinessDays.bdays(cal, d0, d1) end
    BusinessDays.initcache(cal)
    println("WeekendsOnly cache enabled")
    @time for i in 1:1000 BusinessDays.bdays(cal, d0, d1) end
end


using MaxMinFilters
using Test
using Random

len = 1000000
Random.seed!(0)
T = Float64
vec = rand(T, len)
window1 = 5
window2 = 200

mx_ref = similar(vec)
mn_ref = similar(vec)

mx = similar(vec)
mn = similar(vec)
using BenchmarkTools
deck1 = @benchmark movmaxmin!($mx, $mn, $vec, $window1)
cbuf1 = @benchmark movmaxmin_!($mx, $mn, $vec, $window1)
inli1 = @benchmark _movmaxmin!($mx,$mn, $vec, $window1)
deck2 = @benchmark movmaxmin!($mx, $mn, $vec, $window1)
cbuf2 = @benchmark movmaxmin_!($mx,$mn, $vec, $window1)
inli2 = @benchmark _movmaxmin!($mx,$mn, $vec, $window1)

t21 = @benchmark    _movmaxmin!($mx,$mn, $vec, $window2)
t22 = @benchmark  ie_movmaxmin!($mx,$mn, $vec, $window2)
t22 = @benchmark ien_movmaxmin!($mx,$mn, $vec, $window2)
t21 = @benchmark    _movmaxmin!($mx,$mn, $vec, $window2)
t22 = @benchmark  ie_movmaxmin!($mx,$mn, $vec, $window2)
t22 = @benchmark ien_movmaxmin!($mx,$mn, $vec, $window2)

t21 = @benchmark    _movmaxmin!($mx,$mn, $vec, $window2)
t22 = @benchmark  ie_movmaxmin!($mx,$mn, $vec, $window2)
t22 = @benchmark ien_movmaxmin!($mx,$mn, $vec, $window2)
t21 = @benchmark    _movmaxmin!($mx,$mn, $vec, $window2)
t22 = @benchmark  ie_movmaxmin!($mx,$mn, $vec, $window2)
t22 = @benchmark ien_movmaxmin!($mx,$mn, $vec, $window2)

e = 1


export extract_amp, extract_vis, extract_lcamp, extract_cphase

function getvisfield(obs)
    obsamps = obs.data::PyObject
    u = deepcopy((get(obsamps, Vector{Float64}, "u")))
    v = deepcopy((get(obsamps, Vector{Float64}, "v")))
    err = deepcopy((get(obsamps, Vector{Float64}, "sigma")))
    vis = deepcopy((get(obsamps, Vector{Complex{Float64}}, "vis")))
    t1 = Symbol.(deepcopy((get(obsamps, Vector{String}, "t1"))))
    t2 = Symbol.(deepcopy((get(obsamps, Vector{String}, "t2"))))
    baseline = tuple.(t1, t2)
    time = deepcopy((get(obsamps, Vector{Float64}, "time")))
    freq = zeros(length(time))
    bw = zeros(length(time))

    return  StructArray{Comrade.EHTVisibilityDatum{Float64}}(
        visr = real.(vis),
        visi = imag.(vis),
        u = u,
        v = v,
        error = err,
        time = time,
        frequency = freq,
        bandwidth = bw,
        baseline = baseline
    )
end

function getampfield(obs)
    obsamps = obs.amp::PyObject
    uamp = deepcopy(get(obsamps, Vector{Float64}, "u"))
    vamp = deepcopy(get(obsamps, Vector{Float64}, "v"))
    erramp = deepcopy(get(obsamps, Vector{Float64}, "sigma"))
    amps = deepcopy(get(obsamps, Vector{Float64}, "amp"))
    t1 = Symbol.(deepcopy(get(obsamps, Vector{String}, "t1")))
    t2 = Symbol.(deepcopy(get(obsamps, Vector{String}, "t2")))
    baseline = tuple.(t1, t2)
    time = deepcopy(get(obsamps, Vector{Float64}, "time"))
    freq = zeros(length(time))
    bw = zeros(length(time))

    return  StructArray{Comrade.EHTVisibilityAmplitudeDatum{Float64}}(
        amp = amps,
        u = uamp,
        v = vamp,
        error = erramp,
        time = time,
        frequency = freq,
        bandwidth = bw,
        baseline = baseline
    )
end

function getcpfield(obs)
    obscp = obs.cphase::PyObject
    u1 = deepcopy((get(obscp, Vector{Float64}, "u1")))
    v1 = deepcopy((get(obscp, Vector{Float64}, "v1")))
    u2 = deepcopy((get(obscp, Vector{Float64}, "u2")))
    v2 = deepcopy((get(obscp, Vector{Float64}, "v2")))
    u3 = deepcopy((get(obscp, Vector{Float64}, "u3")))
    v3 = deepcopy((get(obscp, Vector{Float64}, "v3")))
    cp = deg2rad.(deepcopy((get(obscp, Vector{Float64}, "cphase"))))
    errcp = deg2rad.(deepcopy((get(obscp, Vector{Float64}, "sigmacp"))))

    t1 = Symbol.(deepcopy((get(obscp, Vector{String}, "t1"))))
    t2 = Symbol.(deepcopy((get(obscp, Vector{String}, "t2"))))
    t3 = Symbol.(deepcopy((get(obscp, Vector{String}, "t3"))))
    baseline = tuple.(t1, t2, t3)
    time = deepcopy(get(obscp, Vector{Float64}, "time"))
    freq = zeros(length(time))
    bw = zeros(length(time))

    return StructArray{Comrade.EHTClosurePhaseDatum{Float64}}(
        phase = cp,
        u1 = u1,
        v1 = v1,
        u2 = u2,
        v2 = v2,
        u3 = u3,
        v3 = v3,
        error = errcp,
        time = time,
        frequency = freq,
        bandwidth = bw,
        triangle = baseline
    )

end

function getlcampfield(obs)
    obslcamp = obs.logcamp::PyObject
    u1 = deepcopy((get(obslcamp, Vector{Float64}, "u1")))
    v1 = deepcopy((get(obslcamp, Vector{Float64}, "v1")))
    u2 = deepcopy((get(obslcamp, Vector{Float64}, "u2")))
    v2 = deepcopy((get(obslcamp, Vector{Float64}, "v2")))
    u3 = deepcopy((get(obslcamp, Vector{Float64}, "u3")))
    v3 = deepcopy((get(obslcamp, Vector{Float64}, "v3")))
    u4 = deepcopy((get(obslcamp, Vector{Float64}, "u4")))
    v4 = deepcopy((get(obslcamp, Vector{Float64}, "v4")))
    camp = deepcopy((get(obslcamp, Vector{Float64}, "camp")))
    errcamp = deepcopy((get(obslcamp, Vector{Float64}, "sigmaca")))

    t1 = Symbol.(deepcopy((get(obslcamp, Vector{String}, "t1"))))
    t2 = Symbol.(deepcopy((get(obslcamp, Vector{String}, "t2"))))
    t3 = Symbol.(deepcopy((get(obslcamp, Vector{String}, "t3"))))
    t4 = Symbol.(deepcopy((get(obslcamp, Vector{String}, "t4"))))
    baseline = tuple.(t1, t2, t3, t4)
    time = deepcopy(get(obslcamp, Vector{Float64}, "time"))
    freq = zeros(length(time))
    bw = zeros(length(time))

    return StructArray{Comrade.EHTLogClosureAmplitudeDatum{Float64}}(
        amp = camp,
        u1 = u1,
        v1 = v1,
        u2 = u2,
        v2 = v2,
        u3 = u3,
        v3 = v3,
        u4 = u4,
        v4 = v4,
        error = errcamp,
        time = time,
        frequency = freq,
        bandwidth = bw,
        quadrangle = baseline
    )

end

function getradec(obs)::Tuple{Float64, Float64}
    return (float(obs.ra), float(obs.dec))
end

function getmjd(obs)::Int
    return Int(obs.mjd)
end

function getsource(obs)::Symbol
    return Symbol(obs.source)
end

"""
    extract_amp(obs)
Extracts the visibility amplitudes from an ehtim observation object

Returns an EHTObservation with visibility amplitude data
"""

function extract_amp(obs; kwargs...)
    obs.add_amp(;kwargs...)
    data = getampfield(obs)
    ra, dec = getradec(obs)
    mjd = getmjd(obs)
    source = getsource(obs)
    bw = obs.bw
    rf = obs.rf

    return Comrade.EHTObservation(data = data, mjd = mjd,
                   ra = ra, dec= dec,
                   bandwidth=bw, frequency=rf,
                   source = source,
    )
end

"""
    extract_vis(obs)
Extracts the complex visibilities from an ehtim observation object

Returns an EHTObservation with complex visibility data
"""
function extract_vis(obs)
    data = getvisfield(obs)
    ra, dec = getradec(obs)
    mjd = getmjd(obs)
    source = getsource(obs)
    bw = obs.bw
    rf = obs.rf
    return Comrade.EHTObservation(
                   data = data, mjd = mjd,
                   ra = ra, dec= dec,
                   bandwidth=bw, frequency=rf,
                   source = source,
    )
end

"""
    extract_cphase(obs)
Extracts the closure phases from an ehtim observation object

Returns an EHTObservation with closure phases datums
"""
function extract_cphase(obs; kwargs...)
    obs.add_cphase(;kwargs...)
    data = getcpfield(obs)
    ra, dec = getradec(obs)
    mjd = getmjd(obs)
    source = getsource(obs)
    bw = obs.bw
    rf = obs.rf

    return Comrade.EHTObservation(data = data, mjd = mjd,
                   ra = ra, dec= dec,
                   bandwidth=bw, frequency=rf,
                   source = source,
    )

end

"""
    extract_lcamp(obs)
Extracts the log-closure amp. from an ehtim observation object

Returns an EHTObservation with closure amp. datums
"""
function extract_lcamp(obs; kwargs...)
    obs.add_logcamp(;kwargs...)
    data = getlcampfield(obs)
    ra, dec = getradec(obs)
    mjd = getmjd(obs)
    source = getsource(obs)
    bw = obs.bw
    rf = obs.rf

    return Comrade.EHTObservation(data = data, mjd = mjd,
                   ra = ra, dec= dec,
                   bandwidth=bw, frequency=rf,
                   source = source,
    )

end


# Recreates the analysis from the original data files with different parameters

datapath = "dataset"
experimentdirnames = ["Mdm2-p53", "MEG3-p53"]

samplenames = ["A", "B", "C", "D"]

nreplicates = 3
nsamples = 4
ncells = 10

outputdir = "output/optimize"
mkpath(outputdir)
outputdatapath = joinpath(outputdir, "results_optimize.jld2")

using Distributed
currentworkers = addprocs(exeflags = "--project")
@everywhere using SMLMAssociationAnalysis_NCB
using Printf
using LocalizationMicroscopy
using Statistics
using FileIO

experimentresults = Vector{Vector{Vector{ResultOptimizing}}}[]
for experimentdirname ∈ experimentdirnames
    println("Starting experiment $experimentdirname.")
    experimentpath = joinpath(datapath, experimentdirname)
    replicateresults = Vector{Vector{ResultOptimizing}}[]
    for i ∈ 1:nreplicates
        sampleresults = Vector{ResultOptimizing}[]
        println("    Starting replicate $i.")
        replicatepath = joinpath(experimentpath, "Replicate $i")
        for samplename ∈ samplenames
            results = ResultOptimizing[]
            println("        Starting sample $samplename.")
            for j ∈ 1:ncells
                println("            Starting cell $j.")
                cellpath = joinpath(replicatepath, "$samplename $(Printf.@sprintf("%03i", j)).bin.txt")
                localizations = loadlocalizations(cellpath, LocalizationMicroscopy.nikonelementstext)
                # account for variances in data collection
                if experimentdirname == experimentdirnames[2] && samplename ∈ samplenames[3:4]
                    ch1_name = "561"
                else
                    ch1_name = "647"
                end
                ch2_name = "488"
                if experimentdirname == experimentdirnames[2] &&
                   ((i == 3 && samplename ∈ samplenames[1:3]) || (i == 2 && samplename == samplenames[2]))
                    ch1_startframe = 1
                    ch2_startframe = 15001
                else
                    ch1_startframe = 1
                    ch2_startframe = 11001
                end
                ch1 = getmolecules.(
                    Ref(localizations),
                    Ref(ch1_name),
                    Ref(ch1_startframe),
                    11000,
                    100,
                    10,
                    34.2,
                    [0, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000],
                    0
                )
                ch2 = getmolecules.(
                    Ref(localizations),
                    Ref(ch2_name),
                    Ref(ch2_startframe),
                    11000,
                    100,
                    10,
                    34.2,
                    [0, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000],
                    0
                )

                ch1_molecules, ch1_localizations = getmolecules(
                    localizations,
                    ch1_name,
                    ch1_startframe,
                    11000,
                    100,
                    10,
                    34.2,
                    500,
                    200,
                )
                ch2_molecules, ch2_localizations = getmolecules(
                    localizations,
                    ch2_name,
                    ch2_startframe,
                    11000,
                    100,
                    10,
                    34.2,
                    500,
                    200,
                )

                ch1_neighbors, ch2_neighbors, distances = exclusivenearestneighbors(ch1_molecules, ch2_molecules)

                percentileranks_by_localdensity = montecarloaffinity.(
                    Ref(ch1_molecules),
                    Ref(ch2_molecules),
                    Ref(ch1_neighbors),
                    Ref(ch2_neighbors),
                    Ref(distances),
                    200:200:2000,
                    10000
                )

                if length(distances) == 0
                    mediandistance = NaN
                else
                    mediandistance = median(distances)
                end
                println("                Done:$(length(distances)) neighbors from $(length(ch1_molecules)) and $(length(ch2_molecules)) molecules, $(length(ch1_localizations)) and $(length(ch2_localizations)) localizations; median distance $mediandistance")
                ch1_data = ChannelDataOptimizing(ch1_name, first.(ch1), ch1_neighbors)
                ch2_data = ChannelDataOptimizing(ch2_name, first.(ch2), ch2_neighbors)
                result = ResultOptimizing(
                    "",
                    experimentdirname,
                    i,
                    samplename,
                    j,
                    [ch1_data, ch2_data],
                    distances,
                    mediandistance,
                    percentileranks_by_localdensity
                )
                push!(results, result)
            end
            push!(sampleresults, results)
        end
        push!(replicateresults, sampleresults)
    end
    push!(experimentresults, replicateresults)
end

save(outputdatapath, "experimentresults", experimentresults)

rmprocs(currentworkers)


export dimension, transform, transform_and_logjac, transform_logdensity, inverse, inverse!,
    inverse_eltype, as, random_arg, random_value

###
### log absolute Jacobian determinant
###

"""
$(TYPEDEF)

Flag used internally by the implementation of transformations, as explained below.

When calculating the log jacobian determinant for a matrix, initialize with
```julia
logjac_zero(flag, x)
```
and then accumulate with log jacobians as needed with `+`.

When `flag` is `LogJac`, methods should return the log Jacobian as the second
argument, otherwise `NoLogJac`, which simply combines to itself with `+`,
serving as an empty placeholder. This allows methods to share code of the two
implementations.
"""
abstract type LogJacFlag end

"""
Calculate log Jacobian as the second value.
"""
struct LogJac <: LogJacFlag end

const LOGJAC = LogJac()

"""
Don't calculate log Jacobian, return `NOLOGJAC` as the second value.
"""
struct NoLogJac <: LogJacFlag end

const NOLOGJAC = NoLogJac()

Base.:+(::NoLogJac, ::NoLogJac) = NOLOGJAC

"""
$(SIGNATURES)

Initial value for log Jacobian calculations.
"""
logjac_zero(::LogJac, T::Type{<:Real}) = log(one(T))

logjac_zero(::NoLogJac, _) = NOLOGJAC

###
### internal methods that implement transformations
###

"""
    transform_with(flag::LogJacFlag, transformation, x::AbstractVector, index)

Transform elements of `x` from `index`, using `transformation`.

Return `(y, logjac), index′`, where

- `y` is the result of the transformation,

- `logjac` is the the log Jacobian determinant or a placeholder, depending on `flag`,

- `index′` is the next index in `x` after the elements used for the transformation

**Internal function**. Implementations

1. can assume that `x` has enough elements for `transformation` (ie `@inbounds` can be
used),

2. should work with generalized indexing on `x`.
"""
function transform_with end

"""
    inverse_at!(x, index, transformation, y)

Invert transformation at `y` and put the result in `x` starting at `index`.

**Internal function**. Implementations

1. can assume that `x` has enough elements for the result (ie `@inbounds` can be used),

2. should work with generalized indexing on `x`.
"""
function inverse_at! end

####
#### API
####

"""
$(TYPEDEF)

Supertype for all transformations in this package.

# Interface

The user interface consists of

- [`dimension`](@ref)
- [`transform`](@ref)
- [`transform_and_logjac`](@ref)
- [`inverse`]@(ref), [`inverse!`](@ref)
- [`inverse_eltype`](@ref).
"""
abstract type AbstractTransform end

Base.broadcastable(t::AbstractTransform) = Ref(t)

"""
$(TYPEDEF)

Inverse of the wrapped transform. Use the 1-argument version of
[`inverse`](@ref) to construct.
"""
struct InverseTransform{T}
    transform::T
end

"""
    inverse(t::AbstractTransform, y)

Return `x` so that `transform(t, x) ≈ y`.

    $SIGNATURES

Return a callable equivalen to `y -> inverse(t, y)`.
"""
inverse(t::AbstractTransform) = InverseTransform(t)

(ι::InverseTransform)(y) = inverse(ι.transform, y)

"""
    inverse_eltype(t::AbstractTransform, y)

The element type for vector `x` so that `inverse!(x, t, y)` works.
"""
function inverse_eltype end

"""
$(SIGNATURES)

Put `inverse(t, y)` into a preallocated vector `x`, returning `x`.

Generalized indexing should be assumed on `x`.

See [`inverse_eltype`](@ref) for determining the type of `x`.
"""
function inverse!(x::AbstractVector, transformation::AbstractTransform, y)
    @argcheck dimension(transformation) == length(x)
    inverse_at!(x, firstindex(x), transformation, y)
    x
end

"""
$(SIGNATURES)

Let ``y = t(x)``, and ``f(y)`` a log density at `y`. This function evaluates `f
∘ t` as a log density, taking care of the log Jacobian correction.
"""
function transform_logdensity(t::AbstractTransform, f, x)
    y, ℓ = transform_and_logjac(t, x)
    ℓ + f(y)
end

"""
    dimension(t::AbstractTransform)

The dimension (number of elements) that `t` transforms.

Types should implement this method.
"""
function dimension end

"""
    as(T, args...)

Shorthand for constructing transformations with image in `T`. `args` determines
or modifies behavior, details depend on `T`.

Not all transformations have an `as` method, some just have direct constructors.
See `methods(as)` for a list.

# Examples

```julia
as(Real, -∞, 1)          # transform a real number to (-∞, 1)
as(Array, 10, 2)         # reshape 20 real numbers to a 10x2 matrix
as(Array, as𝕀, 10)       # transform 10 real numbers to (0, 1)
as((a = asℝ₊, b = as𝕀)) # transform 2 real numbers a NamedTuple, with a > 0, 0 < b < 1
```
"""
function as end

####
#### vector transformations
####

"""
$(TYPEDEF)

Transformation that transforms `<: AbstractVector`s to other values.

# Implementation

Implements [`transform`](@ref) and [`transform_and_logjac`](@ref) via
[`transform_with`](@ref), and [`inverse`](@ref) via [`inverse!`](@ref).
"""
abstract type VectorTransform <: AbstractTransform end

"""
$(SIGNATURES)

Transform `x` using `t`.
"""
function transform(t::VectorTransform, x::AbstractVector)
    @argcheck dimension(t) == length(x)
    first(transform_with(NOLOGJAC, t, x, firstindex(x)))
end

"""
$(SIGNATURES)

Transform `x` using `t`; calculating the log Jacobian determinant, returned as
the second value.
"""
function transform_and_logjac(t::VectorTransform, x::AbstractVector)
    @argcheck dimension(t) == length(x)
    y, ℓ, _ = transform_with(LOGJAC, t, x, firstindex(x))
    y, ℓ
end

function inverse(t::VectorTransform, y)
    inverse!(Vector{inverse_eltype(t, y)}(undef, dimension(t)), t, y)
end

"""
$(SIGNATURES)

A random argument for a transformation.

# Keyword arguments

$(_RANDOM_REALS_KWARGS_DOC)
"""
random_arg(x::VectorTransform; kwargs...) = random_reals(dimension(x); kwargs...)

"""
$(SIGNATURES)

Random value from a transformation.

# Keyword arguments

$(_RANDOM_REALS_KWARGS_DOC)
"""
random_value(t::AbstractTransform; kwargs...) = transform(t, random_arg(t; kwargs...))


# !! auto-generated code, do not edit !!
module shape_msgs
using RobotOSData.Messages
using RobotOSData.StdMsgs
using RobotOSData.CommonMsgs
struct MeshTriangle <: Readable
    vertex_indices::SVector{3, UInt32}
end
struct SolidPrimitive <: Readable
    type_::UInt8
    dimensions::Vector{Float64}
end
struct Plane <: Readable
    coef::SVector{4, Float64}
end
struct Mesh <: Readable
    triangles::Vector{MeshTriangle}
    vertices::Vector{geometry_msgs.Point}
end
end


module QJuliaIntrinsics

import QJuliaRegisters

#create function/type alias
#SSE
m128d   = QJuliaRegisters.double2
m128    = QJuliaRegisters.float4
#AVX/AVX2
m256d   = QJuliaRegisters.double4
m256    = QJuliaRegisters.float8
#AVX3
m512d   = QJuliaRegisters.double8
m512    = QJuliaRegisters.float16

m256dfma(a,b,c) = ccall("llvm.fma.v4f64", llvmcall, m256d, (m256d, m256d, m256d), a, b, c)
m256fma(a,b,c ) = ccall("llvm.fma.v8f32", llvmcall, m256, (m256, m256, m256), a, b, c)

m512dfma(a,b,c) = ccall("llvm.fma.v8f64", llvmcall, m512d, (m512d, m512d, m512d), a, b, c)
m512fma(a,b,c ) = ccall("llvm.fma.v16f32", llvmcall, m512, (m512, m512, m512), a, b, c)

prefetch(a, i1, i2, i3) = ccall("llvm.prefetch", llvmcall, Cvoid, (Ptr{Cchar}, Int32, Int32, Int32), a, i1, i2, i3)

prefetchT0(a) = mmprefetch(a, Int32(0), Int32(3), Int(1))
prefetchT1(a) = mmprefetch(a, Int32(0), Int32(2), Int(1))
prefetchT2(a) = mmprefetch(a, Int32(0), Int32(1), Int(1))

@inline function mm_mul(x::T, y::T) where T <:AbstractFloat
   return x * y
end

@inline function mm_mul(xmm::m128, ymm::m128)
  (VecElement(xmm[1].value * ymm[1].value),
   VecElement(xmm[2].value * ymm[2].value),
   VecElement(xmm[3].value * ymm[3].value),
   VecElement(xmm[4].value * ymm[4].value))
end

@inline function mm_mul(xmm::m128d, ymm::m128d)
  (VecElement(xmm[1].value * ymm[1].value),
   VecElement(xmm[2].value * ymm[2].value))
end

@inline function mm_mul(xmm::m256, ymm::m256)
  (VecElement(xmm[1].value * ymm[1].value),
   VecElement(xmm[2].value * ymm[2].value),
   VecElement(xmm[3].value * ymm[3].value),
   VecElement(xmm[4].value * ymm[4].value),
   VecElement(xmm[5].value * ymm[5].value),
   VecElement(xmm[6].value * ymm[6].value),
   VecElement(xmm[7].value * ymm[7].value),
   VecElement(xmm[8].value * ymm[8].value))
end

@inline function mm_mul(xmm::m256d, ymm::m256d)
  (VecElement(xmm[1].value * ymm[1].value),
   VecElement(xmm[2].value * ymm[2].value),
   VecElement(xmm[3].value * ymm[3].value),
   VecElement(xmm[4].value * ymm[4].value))
end

@inline function mm_add(x::T, y::T) where T <:AbstractFloat
   return x + y
end

@inline function mm_add(xmm::m128, ymm::m128)
  (VecElement(xmm[1].value + ymm[1].value),
   VecElement(xmm[2].value + ymm[2].value),
   VecElement(xmm[3].value + ymm[3].value),
   VecElement(xmm[4].value + ymm[4].value))
end

@inline function mm_add(xmm::m128d, ymm::m128d)
  (VecElement(xmm[1].value + ymm[1].value),
   VecElement(xmm[2].value + ymm[2].value))
end

@inline function mm_add(xmm::m256, ymm::m256)
  (VecElement(xmm[1].value + ymm[1].value),
   VecElement(xmm[2].value + ymm[2].value),
   VecElement(xmm[3].value + ymm[3].value),
   VecElement(xmm[4].value + ymm[4].value),
   VecElement(xmm[5].value + ymm[5].value),
   VecElement(xmm[6].value + ymm[6].value),
   VecElement(xmm[7].value + ymm[7].value),
   VecElement(xmm[8].value + ymm[8].value))
end

@inline function mm_add(xmm::m256d, ymm::m256d)
  (VecElement(xmm[1].value + ymm[1].value),
   VecElement(xmm[2].value + ymm[2].value),
   VecElement(xmm[3].value + ymm[3].value),
   VecElement(xmm[4].value + ymm[4].value))
end

@inline function mm_sub(x::T, y::T) where T <:AbstractFloat
   return x - y
end

@inline function mm_sub(xmm::m128, ymm::m128)
  (VecElement(xmm[1].value - ymm[1].value),
   VecElement(xmm[2].value - ymm[2].value),
   VecElement(xmm[3].value - ymm[3].value),
   VecElement(xmm[4].value - ymm[4].value))
end

@inline function mm_sub(xmm::m128d, ymm::m128d)
  (VecElement(xmm[1].value - ymm[1].value),
   VecElement(xmm[2].value - ymm[2].value))
end

@inline function mm_sub(xmm::m256, ymm::m256)
  (VecElement(xmm[1].value - ymm[1].value),
   VecElement(xmm[2].value - ymm[2].value),
   VecElement(xmm[3].value - ymm[3].value),
   VecElement(xmm[4].value - ymm[4].value),
   VecElement(xmm[5].value - ymm[5].value),
   VecElement(xmm[6].value - ymm[6].value),
   VecElement(xmm[7].value - ymm[7].value),
   VecElement(xmm[8].value - ymm[8].value))
end

@inline function mm_sub(xmm::m256d, ymm::m256d)
  (VecElement(xmm[1].value - ymm[1].value),
   VecElement(xmm[2].value - ymm[2].value),
   VecElement(xmm[3].value - ymm[3].value),
   VecElement(xmm[4].value - ymm[4].value))
end

@inline function mm_mad(a::T, x::T, y::T) where T <:AbstractFloat
   return a * x + y
end

@inline function mm_mad(a::T, xmm::m128, ymm::m128) where T <: AbstractFloat
  (VecElement(a * xmm[1].value + ymm[1].value),
   VecElement(a * xmm[2].value + ymm[2].value),
   VecElement(a * xmm[3].value + ymm[3].value),
   VecElement(a * xmm[4].value + ymm[4].value))
end

@inline function mm_mad(a::T, xmm::m128d, ymm::m128d) where T <: AbstractFloat
  (VecElement(a * xmm[1].value + ymm[1].value),
   VecElement(a * xmm[2].value + ymm[2].value))
end

@inline function mm_mad(a::T, xmm::m256, ymm::m256) where T <: AbstractFloat
   zmm = m256((a, a, a, a, a, a, a, a))
   return m256fma(zmm,xmm,ymm)
end

@inline function mm_mad(a::T, xmm::m256d, ymm::m256d) where T <: AbstractFloat
   zmm = m256d((a, a, a, a))
   return m256dfma(zmm,xmm,ymm)
end

end #QJuliaIntrinsics


"""
`module JuLIP.Solve`

Contains a few geometry optimisation routines, for now see
the help for these:

* `minimise!`
"""
module Solve

import Optim
import LineSearches

using Optim: OnceDifferentiable, optimize, ConjugateGradient, LBFGS
using LineSearches: BackTracking

using JuLIP: AbstractAtoms, Preconditioner, update!, Identity,
            dofs, energy, gradient, set_dofs!, set_constraint!, site_energies,
            Dofs, calculator, constraint, AbstractCalculator

using JuLIP.Potentials: SitePotential
using JuLIP.Preconditioners: Exp
using JuLIP.Constraints: FixedCell

# TODO: move to Optim 0.11, and exploit the new alphaguess functionality of
#       Linesearches. This gives awful error messages.

export minimise!

Ediff(V::AbstractCalculator, at::AbstractAtoms, Es0::Vector{Float64}) =
   sum_kbn(site_energies(V, at) - Es0)

Ediff(at::AbstractAtoms, Es0::Vector{Float64}, x::Dofs) =
   Ediff(calculator(at), set_dofs!(at, x), Es0)

"""
`minimise!(at::AbstractAtoms)`: geometry optimisation

`at` must have a calculator and a constraint attached.

## Keyword arguments:
* `precond = :auto` : preconditioner; more below
* `gtol = 1e-6` : gradient tolerance (max-norm)
* `ftol = 1e-32` : objective tolerance
* `Optimiser = :auto`, `:auto` should always work, at least on the master
   branch of `Optim`; `:lbfgs` needs the `extraplbfgs2` branch, which is not
   yet merged. Other options might be introduced in the future.
* `verbose = 1`: 0 : no output, 1 : final, 2 : iteration and final
* `robust_energy_difference = false` : if true use Kahan summation of site energies
* `store_trace = false` : store history of energy and norm of forces
* `extended_trace = false`: also store full history of postions and forces
* `maxstep = Inf`: maximum step size, useful if initial gradient is very large
* `callback`: callback function to pass to `optimize()`, e.g. to use alternate convergence criteria

## Preconditioner

`precond` may be a valid preconditioner, e.g.,
`Identity()` or `Exp(at)`, or one of the following symbols

* `:auto` : the code will make the best choice it can with the avilable
   information
* `:exp` : will use `Exp(at)`
* `:id` : will use `Identity()`
"""
function minimise!(at::AbstractAtoms;
                  precond = :auto,
                  method = :auto,
                  gtol=1e-5, ftol=1e-32,
                  verbose = 1,
                  robust_energy_difference = false,
                  store_trace = false,
		  extended_trace = false,
                  maxstep = Inf,
                  callback = nothing)

   # create an objective function
   if robust_energy_difference
      Es0 = site_energies(at)
      obj_f = x -> Ediff(at, Es0, x)
   else
      obj_f = x->energy(at, x)
   end
   obj_g! = (g, x) -> copy!(g, gradient(at, x))

   # create a preconditioner
   if isa(precond, Symbol)
      if precond == :auto
         if isa(constraint(at), FixedCell)
            precond = :exp
         else
            precond = :id
         end
      end
      if precond == :exp
         if method == :lbfgs
            precond = Exp(at, energyscale = :auto)
         else
            precond = Exp(at)
         end
      elseif precond == :id
         precond = Identity()
      else
         error("unknown symbol for precond")
      end
   end

   # choose the optimisation method Optim.jl
   if method == :auto || method == :cg
      if isa(precond, Identity)
         optimiser = ConjugateGradient(linesearch = BackTracking(order=2, maxstep=maxstep))
      else
         optimiser = ConjugateGradient( P = precond,
                           precondprep = (P, x) -> update!(P, at, x),
                           linesearch = BackTracking(order=2, maxstep=maxstep) )
      end
   elseif method == :lbfgs
      optimiser = LBFGS( P = precond,
                        precondprep = (P, x) -> update!(P, at, x),
                        alphaguess = LineSearches.InitialHagerZhang(),
                        linesearch = BackTracking(order=2, maxstep=maxstep) )
   else
      error("JuLIP.Solve.minimise!: unknown `method` option")
   end

   results = optimize( obj_f, obj_g!, dofs(at), optimiser,
                        Optim.Options( f_tol = ftol, g_tol = gtol,
                                       store_trace = store_trace,
                                       extended_trace = extended_trace,
                                       callback = callback,
                                       show_trace = (verbose > 1)) )
   set_dofs!(at, Optim.minimizer(results))
   # analyse the results
   if verbose > 0
      println(results)
   end
   return results
end

end


module BilevelOptimization

import JuMP
using JuMP: @variable, @constraint, @objective

export BilevelLP, build_blp_model, BilevelFlowProblems, VariableType,
       SOS1Complementarity, BoundComplementarity

include("types.jl")
include("complementarity.jl")
include("build_model.jl")
include("bilevel_flows.jl")

end # module


module CompressedIndices

import Base: eltype, length, endof, size, checkbounds, eachindex, getindex, start, next, done, push!, append!, intersect, intersect!, +, -
export Indices

AbstractIndices = Union{Int, UnitRange{Int}, Vector{Int}}

immutable Indices <: AbstractVector{Int}
    vectors::Vector{Int}
    ranges::Vector{Int}
    length::Vector{Int}
    isrange::BitVector
end

eltype(I::Indices) = Int
length(I::Indices) = sum(I.length)
endof(I::Indices) = length(I)
size(I::Indices) = (length(I),)
eachindex(I::Indices) = 1:length(I)

start(I::Indices) = (1, 0, 0, 1)
function next(I::Indices, state::NTuple{4, Int})
    @inbounds l = state[1]
    @inbounds vi = state[2]
    @inbounds ri = state[3]
    @inbounds li = state[4]
    @inbounds isrange = I.isrange[li]

    item = 0
    if isrange
        @inbounds item = I.ranges[ri + 1] + l - 1
    else
        @inbounds item = I.vectors[vi + l]
    end

    l += 1
    @inbounds l_li = I.length[li]
    if l > l_li
        if isrange
            ri += 1
        else
            vi += l - 1
        end

        l = 1
        li += 1
    end

    return item, (l, vi, ri, li)
end

done(I::Indices, state::NTuple{4, Int}) = (@inbounds s = state[4] ; s > length(I.length))

function push!(I::Indices, index::Int)
    if length(I.length) == 0
        push!(I.ranges, index)
        push!(I.length, 1)
        push!(I.isrange, true)

        return I
    end

    li = endof(I.length)
    @inbounds isrange = I.isrange[li]

    if isrange
        @inbounds rs = I.ranges[end]
        @inbounds rl = I.length[li]
        if index == rs + rl
            @inbounds I.length[li] += 1
        else
            push!(I.vectors, index)
            push!(I.length, 1)
            push!(I.isrange, false)
        end
    else
        @inbounds vl = I.vectors[end]
        if index == vl + 1
            pop!(I.vectors)
            @inbounds l_li = I.length[li]
            if l_li > 1
                @inbounds I.length[li] -= 1
                push!(I.length, 2)
                push!(I.isrange, true)
            else
                @inbounds I.length[li] += 1
                @inbounds I.isrange[li] = true
            end
            push!(I.ranges, vl)
        else
            push!(I.vectors, index)
            @inbounds I.length[li] += 1
        end
    end

    return I
end

function getindex(I::Indices, index::Int)
    n = 0
    ri = 0
    vi = 0

    for (i, l) in enumerate(I.length)
        @inbounds isrange = I.isrange[i]
        if n + l >= index
            if isrange
                return I.ranges[ri + 1] + (index - n - 1)
            else
                return I.vectors[vi + index - n]
            end
        end

        n += l
        if isrange
            ri += 1
        else
            vi += l
        end
    end

    return 0
end

@generated function Indices(ind::AbstractIndices...)
    length(ind) == 0 && return :(Indices(Vector{Int}(), Vector{Int}(), Vector{Int}(), BitVector()))
    num_ranges = count(x -> x == UnitRange{Int}, ind)

    isrange = BitVector([ind[1] == UnitRange{Int}])

    for i in 2:endof(ind)
        b = ind[i] == UnitRange{Int}
        if last(isrange) || b
            push!(isrange, b)
        end
    end

    ex = Expr[
        :(vectors = Vector{Int}())
        :(ranges = Vector{Int}($num_ranges))
        :(len = Vector{Int}($(length(isrange))))
        :(isrange = BitVector($isrange))
    ]

    ri = 0
    oi = 0
    last_isrange = true
    for (i, i_type) in enumerate(ind)
        if i_type == UnitRange{Int}
            ri += 1
            push!(ex, :(ranges[$ri] = first(ind[$i])))

            oi += 1
            push!(ex, :(len[$oi] = length(ind[$i])))

            last_isrange = true
        elseif i_type == Int
            if last_isrange
                oi += 1
                push!(ex, :(len[$oi] = 1))
                last_isrange = false
            else
                push!(ex, :(len[$oi] += 1))
            end

            push!(ex, :(push!(vectors, ind[$i])))
        elseif i_type == Vector{Int}
            if last_isrange
                oi += 1
                push!(ex, :(len[$oi] = length(ind[$i])))
                last_isrange = false
            else
                push!(ex, :(len[$oi] += length(ind[$i])))
            end

            push!(ex, :(append!(vectors, ind[$i])))
        else
            throw(ArgumentError("Invalid types in constructor Indices()"))
        end
    end

    push!(ex, :(Indices(vectors, ranges, len, isrange)))

    return Expr(:block, ex...)
end

end


using LadeGCM

@static if VERSION < v"0.7.0-DEV.2005"
    using Base.Test
else
    using Test
end

c = constants();
@testset "Configuration" begin
    @testset "Defaults" begin
        @test c.cₐ₀ ≈ 589.0
        @test c.cₜ₀ ≈ 1875.0
        @test c.cₘ₀ ≈ 900.0
        @test c.λ ≈ 1.8
        @test c.τ ≈ 4.0
        @test c.D ≈ 1.0
        @test c.r ≈ 12.5
        @test c.DT ≈ 0.0423
        @test c.B₀ ≈ 13.0
        @test c.BT ≈ 0.032
        @test c.w₀ ≈ 0.1
        @test c.wT ≈ 0.1
        @test c.QR ≈ 1.72
        @test c.NPP₀ ≈ 55.0
        @test c.KC ≈ 0.3
    end
    @testset "Overrides" begin
        co = constants(cₐ₀=2.0, cₜ₀=3.0, QR=4.0, DT=5.0);
        @test co.cₐ₀ ≈ 2.0 #Altered
        @test co.cₜ₀ ≈ 3.0 #Altered
        @test co.cₘ₀ ≈ 900.0
        @test co.λ ≈ 1.8
        @test co.τ ≈ 4.0
        @test co.D ≈ 1.0
        @test co.r ≈ 12.5
        @test co.DT ≈ 5.0 #Altered
        @test co.B₀ ≈ 13.0
        @test co.BT ≈ 0.032
        @test co.w₀ ≈ 0.1
        @test co.wT ≈ 0.1
        @test co.QR ≈ 4.0 #Altered
        @test co.NPP₀ ≈ 55.0
        @test co.KC ≈ 0.3
    end
end

@testset "Pathways" begin
    rcp26 = RCP3PD;
    rcp45 = RCP45;
    rcp6 = RCP6;
    rcp85 = RCP85;
    @testset "Types" begin
        @test typeof(rcp26) <: LadeGCM.Pathway
        @test isa(rcp26, LadeGCM.RCP3PDPathway)
        @test typeof(rcp45) <: LadeGCM.Pathway
        @test isa(rcp45, LadeGCM.RCP45Pathway)
        @test typeof(rcp6) <: LadeGCM.Pathway
        @test isa(rcp6, LadeGCM.RCP6Pathway)
        @test typeof(rcp85) <: LadeGCM.Pathway
        @test isa(rcp85, LadeGCM.RCP85Pathway)
    end
    @testset "Show" begin
        @test sprint(show, rcp26) == "+2.6 W/m² (Peak & Decline) Representative Concentration Pathway"
        @test sprint(show, rcp45) == "+4.5 W/m² Representative Concentration Pathway"
        @test sprint(show, rcp6) == "+6.0 W/m² Representative Concentration Pathway"
        @test sprint(show, rcp85) == "+8.5 W/m² Representative Concentration Pathway"
    end
    @testset "Summary" begin
        @test summary(rcp26) == "RCP3PD"
        @test summary(rcp45) == "RCP45"
        @test summary(rcp6) == "RCP6"
        @test summary(rcp85) == "RCP85"
    end
end

@testset "Emission Parameters" begin
    @testset "Loading data" begin
        data_frame = LadeGCM.load_pathway_data(RCP45);
        @test data_frame[:Year][1] == 1765
        @test data_frame[:FossilCO2][345] ≈ 3.19374
        @test data_frame[:OtherCO2][end] ≈ 0.0
    end
    @testset "Generate Continuous Data" begin
        (E, LUC) = LadeGCM.generate_emission_parameters(RCP45);
        @test E(1768.8) ≈ 0.003
        @test E(2100) ≈ 4.203
        @test LUC(1982.77) ≈ 1.217443742
        @test LUC(2019) ≈ 0.39778
    end
end

@testset "Solutions" begin
    # NOTE: These are going to have some numerical error in them,
    # so we round everything for stability.
    r6 = calculate(RCP6, saveat=1765:10:2100);
    r85 = calculate(RCP85, saveat=1765:10:2100);
@static if VERSION < v"0.7.0-DEV.2005"
    @test isapprox(round(r6.cₜ[13],0), 1863.0; atol = 2)
    @test isapprox(round(r6.cₘ[28],0), 930.0; atol = 2)
    @test isapprox(round(r6.cₐ[3],0), 590.0; atol = 2)
    @test isapprox(round(r6.ΔT[9],0), 0.0; atol = 2)
    @test isapprox(round(r6.Δcₜ[27],0), 2.0; atol = 2)
    @test isapprox(round(r6.Δcₘ[14],0), 0.0; atol = 2)
    @test isapprox(round(r6.ΔcM[23],0), 1.0; atol = 2)
    @test isapprox(round(r6.Δcₛ[16],0), 1.0; atol = 2)
    @test isapprox(round(r6.Δcₐ[31],0), 10.0; atol = 2)
    @test isapprox(round(r6.year[24],0), 1995.0; atol = 2)
    @test isapprox(round(r85.cₜ[30],0), 1991.0; atol = 2)
    @test isapprox(round(r85.cₘ[8],0), 900.0; atol = 2)
    @test isapprox(round(r85.cₐ[16],0), 614.0; atol = 2)
    @test isapprox(round(r85.ΔT[29],0), 2.0; atol = 2)
    @test isapprox(round(r85.Δcₜ[21],0), 0.0; atol = 2)
    @test isapprox(round(r85.Δcₘ[18],0), 0.0; atol = 2)
    @test isapprox(round(r85.ΔcM[4],0), 0.0; atol = 2)
    @test isapprox(round(r85.Δcₛ[26],0), 7.0; atol = 2)
    @test isapprox(round(r85.Δcₐ[10],0), 0.0; atol = 2)
    @test isapprox(round(r85.year[28],0), 2035.0; atol = 2)
else
    @test isapprox(round(r6.cₜ[13];digits=0), 1863.0; atol = 2)
    @test isapprox(round(r6.cₘ[28];digits=0), 930.0; atol = 2)
    @test isapprox(round(r6.cₐ[3];digits=0), 590.0; atol = 2)
    @test isapprox(round(r6.ΔT[9];digits=0), 0.0; atol = 2)
    @test isapprox(round(r6.Δcₜ[27];digits=0), 2.0; atol = 2)
    @test isapprox(round(r6.Δcₘ[14];digits=0), 0.0; atol = 2)
    @test isapprox(round(r6.ΔcM[23];digits=0), 1.0; atol = 2)
    @test isapprox(round(r6.Δcₛ[16];digits=0), 1.0; atol = 2)
    @test isapprox(round(r6.Δcₐ[31];digits=0), 10.0; atol = 2)
    @test isapprox(round(r6.year[24];digits=0), 1995.0; atol = 2)
    @test isapprox(round(r85.cₜ[30];digits=0), 1991.0; atol = 2)
    @test isapprox(round(r85.cₘ[8];digits=0), 900.0; atol = 2)
    @test isapprox(round(r85.cₐ[16];digits=0), 614.0; atol = 2)
    @test isapprox(round(r85.ΔT[29];digits=0), 2.0; atol = 2)
    @test isapprox(round(r85.Δcₜ[21];digits=0), 0.0; atol = 2)
    @test isapprox(round(r85.Δcₘ[18];digits=0), 0.0; atol = 2)
    @test isapprox(round(r85.ΔcM[4];digits=0), 0.0; atol = 2)
    @test isapprox(round(r85.Δcₛ[26];digits=0), 7.0; atol = 2)
    @test isapprox(round(r85.Δcₐ[10];digits=0), 0.0; atol = 2)
    @test isapprox(round(r85.year[28];digits=0), 2035.0; atol = 2)
end
end


module Models

using NearestNeighbors
using StatsBase
using Statistics
using LinearAlgebra

using Random
using Flux
using ImageTransformations
using MLDataPattern
using ProgressMeter: Progress, next!

include("utils/utils.jl")
include("utils/two-stage.jl")
include("utils/losses.jl")
include("utils/nn_builders.jl")
include("utils/lkl_quantile.jl")

include("knn.jl")
include("pidforest.jl")
include("GANomaly.jl")
include("SkipGANomaly.jl")
include("skmodels.jl")
include("pyodmodels.jl")
include("torchmodels.jl")
include("tabular_flows.jl")
include("vae.jl")
include("aae.jl")
include("adVAE.jl")
include("DeepSVDD.jl")
include("fanogan.jl")
include("gan.jl")
include("sptn.jl")

# this contains dependencies from vae and aae
include("utils/vae_utils.jl")

end