using Pkg
Pkg.add("Agents")
using Agents

using Pkg
Pkg.add("CSV")
using CSV

global dims_glob = (100,150);
global maxSize = 23; # after this size we no longer grow our crab
global minTemp = 9; # dormancy temp
global lowTemp = 10.8; # from Brylawski & Miller 2007 - the minimum physiological temp
global lowSalt = 10;
global step = 1;
global nurseryx = 40 # the horizontal limiter on the space for spawn re-entry
global nurseryy = 40 # the vertical limiter on the space for spawn re-entry
println("Initialized.")

global temps_all = CSV.read("model_temps.csv") # the observed temp at each grid cell
global salt_all = CSV.read("model_salt.csv") # the observed salt at each grid cell

println("Read data.")

function getSalt(tempTup,simTime)
    saltstring = salt[convert(Int,tempTup[1]),convert(Int, convert(Int,tempTup[2])+150*simTime)]
    if saltstring == "NA"
        return "NA"
    else
        return parse(Float64, saltstring)
    end
end

function getTemp(tempTup,simTime)
    tempstring = temps[convert(Int,tempTup[1]),convert(Int, convert(Int,tempTup[2])+150*simTime)]
    if tempstring == "NA"
        return "NA"
    else
        return parse(Float64, tempstring)
    end
end

# each agent is a crab that contains its own position, its own ID, its own other physiological params
mutable struct ChesapeakeBayCrab <: AbstractAgent
    id::Int
    pos::NTuple{2, Int}
    temp::Float64 # the temperature where the crab is placed
    salt::Float64 # the salinity where thecrab is placed
    simTime::Int # time since the simulation began
    age::Int
    sex::Int
    size::Float64
    mated::Int
    numSperm::Int
    eggs::Int
    degreeDays::Int
    tOpt::Int
    lifespan::Int
end

# really we have 120 million crabs
function crab_model_2D(;dims = dims_glob, M = 120)
  global step = 1;
  space = Space(dims, periodic = true)
  age = rand(0:10)
  model = ABM(ChesapeakeBayCrab, space; properties = step, scheduler = random_activation)
  for i in 1:M # add agents in random nodes
      if rand(0:10) > 7
          age = rand(200:1100)
      else
          age = rand(0:200)
      end
      sex = rand(0:1)
      mated = 0
      if rand(1:10) == 1 # fewer need to start out mated. ~10% mated at any given time.
          mated = 1
      end
      tOpt = rand(20:26)
      degreeDays = rand(0:24)
      simTime = 0
      pos = (rand(1:100), rand(1:150))
      temp = getTemp(pos,simTime)
      while temp == "NA"
          pos = (rand(1:100), rand(1:150))
          temp = getTemp(pos,simTime)
      end
      salt = getSalt(pos,simTime)
      size = rand(0:23)
      numSperm = size * 2
      eggs = round(rand(0:46), digits=0)
      lifespan = rand(200:1100)
      node_to_add = coord2vertex(pos, model)
      add_agent!(node_to_add, model, temp, salt, simTime, age, sex, size, mated, numSperm, eggs, degreeDays, tOpt, lifespan)
  end
  numSpawns = round(0,digits=0)#M/75, digits = 0)
  simTime = 0
  for i in 1:numSpawns
    age = 0
    sex = rand(0:1)
    numSperm = 0
    size = rand(0:5)
    mated = 0
    tOpt = rand(20:26)
    degreeDays = 0
    lifespan = rand(200:1100)
    pos = (rand(nurseryx:100), rand(1:nurseryy)) # spawns can't start just anywhere
    temp = getTemp(pos,simTime)
    numSperm = 0
    eggs = 0
    while temp == "NA"
        pos = (rand(nurseryx:100), rand(1:nurseryy))
        temp = getTemp(pos,simTime)
    end
    salt = getSalt(pos,simTime)
    node_to_add = coord2vertex(pos, model)
    add_agent!(node_to_add, model, temp, salt, simTime, age, sex, size, mated, numSperm, eggs, degreeDays, tOpt, lifespan)
  end
  return model
end

function bay_step!(bay)
    if nagents(bay) > 1
        for node in nodes(bay, by = :random)
            if nagents(bay) == 0
                return
            end
            nc = get_node_contents(node, bay)
            for curr_agent in nc
                crab = id2agent(curr_agent, bay)
                randnumber = rand();
                if crab.age >= crab.lifespan
                    delete!(bay.agents, crab.id)

                    twoDpos = node
                    splice!(bay.space.agent_positions[twoDpos],
                      findfirst(a->a==crab.id, bay.space.agent_positions[twoDpos]))
                elseif (crab.salt < lowSalt) & (crab.temp < lowTemp) & (randnumber <= (0.25 / 365))
                    delete!(bay.agents, crab.id)
                    twoDpos = node
                    splice!(bay.space.agent_positions[twoDpos],
                      findfirst(a->a==crab.id, bay.space.agent_positions[twoDpos]))
                elseif (crab.temp < lowTemp) & (randnumber <= (0.20 / 365))
                    delete!(bay.agents, crab.id)
                    twoDpos = node
                    splice!(bay.space.agent_positions[twoDpos],
                      findfirst(a->a==crab.id, bay.space.agent_positions[twoDpos]))
                elseif (randnumber <= (0.17 / 365))
                    delete!(bay.agents, crab.id)
                    twoDpos = node
                    splice!(bay.space.agent_positions[twoDpos],
                      findfirst(a->a==crab.id, bay.space.agent_positions[twoDpos]))
                end
            end
        end
    end
end

function agent_step_2d!(agent, model)
    temp = getTemp(agent.pos,agent.simTime)
    salt = getSalt(agent.pos,agent.simTime)
    agent.age += 1
    if (agent.age > 800)
        agent.mated = 2 # don't care about mating anymore
    end
    agent.temp = temp
    agent.salt = salt
    agent.simTime = agent.simTime + 1
    if temp > minTemp
        spawnx = 1 # the horizontal limiter on the space for spawning - any part of lower bay
        spawny = 23 # the vertical limiter on the space for spawning - lower part of bay
        agent_node = coord2vertex(agent.pos, model)
        agent.degreeDays += round(agent.temp - minTemp,digits=0)
        probMolt = 0.669 * log(agent.degreeDays) - 3.71 # from Brylawski & Miller 2007
        growthAmount = real(rand(1044:1598)) / 1000
        if (probMolt > rand()) & ((agent.size + growthAmount) < maxSize)
            agent.size += growthAmount
        end
        if agent.pos[2] > 80
            if agent.sex == 0 # if a female
                agent_node = coord2vertex(agent.pos, model)
                neighboring_nodes = node_neighbors(agent_node, model)
                push!(neighboring_nodes, agent_node) # also consider current node
                available_agents = []
                for curr in neighboring_nodes
                    append!(available_agents, get_node_contents(curr, model))
                end
                done_ = 0
                while length(available_agents) > 0 & done_ == 0
                    tester = rand(available_agents)
                    random_neighbor_agent = id2agent(tester, model)
                    if (random_neighbor_agent.numSperm > 0) & (random_neighbor_agent.sex == 1)
                        agent.mated = 1
                        agent.eggs = random_neighbor_agent.numSperm
                        if (random_neighbor_agent.mated == 1)
                            random_neighbor_agent.mated = 2 # now this sperm donor doesn't care about mating
                            # this will cause there to be a better global distribution of crabs.
                            # and possibly fewer overall crabs.
                        else
                            random_neighbor_agent.mated = 1
                        end
                        done_ = 1
                    end

                    deleteat!(available_agents, findfirst(x->x == tester,available_agents))
                end

                if (rand(1:1000) == 1) & (agent.mated == 0)
                    agent.eggs = rand(0:23) * 2
                    agent.mated = 1
                end
            else
                agent.numSperm = convert(Int,round(agent.size,digits=0))
            end
        end

        # Randomly decide whether the agent will move position. Each coord can change by up to 1
        newx = agent.pos[1] + rand(-5:5);
        newy = agent.pos[2] + rand(-5:5);

        # Take into account that females that have mated are trying to go southeast (higher x & lower y)
        if (agent.mated == 1) & (agent.sex == 0 | agent.pos[2] > 125)
           newx = agent.pos[1] + rand(-5:5); # + 1 + rand(-5:5);
           newy = agent.pos[2] - 8 + rand(-2:2);
        elseif (agent.pos[2] > 125)
           newy = agent.pos[2] - 4 + rand(-2:2);
        end

        # Take into account that individuals that have not mated are trying to go north & stay west
        if (agent.mated == 0) & (agent.pos[2] < 80) & (agent.pos[1] > 40) & (agent.pos[1] < 65)
            newx = agent.pos[1] + rand(-5:5);
            newy = agent.pos[2] + 8 + rand(-2:2);
        end

        # Take into account that individuals that are in branches need to get to the main shaft eventually
        if (agent.mated < 2) & (agent.pos[2] < 80)
            if agent.pos[1] < 40
                newx = agent.pos[1] + rand(-2:4);
            elseif agent.pos[1] > 55
                newx = agent.pos[1] + rand(-4:2);
            end
        end

        if (agent.pos[1] > 55) & (agent.pos[2] < 40)
            newx = agent.pos[1] - 1 + rand(-4:2);
        end

        # distribute crabs that don't care about mating to low-density areas.
        if (agent.mated == 2)
            agent_node = coord2vertex(agent.pos, model)
            currmin = 5
            for cell in node_neighbors(agent_node, model)
                neighboringcrabs = get_node_contents(cell, model)
                if (length(neighboringcrabs) < currmin)
                    (newx,newy) = vertex2coord(cell, model)
                    currmin = length(neighboringcrabs)
                end
            end
        end

        # Make sure we don't go off the grid
        if newx > dims_glob[1]
            newx = agent.pos[1]
        elseif newx < 1
            newx = 1
        end

        if newy > dims_glob[2]
            newy = agent.pos[2]
        elseif newy < 1
            newy = 1
        end

        temp = getTemp((newx,newy),agent.simTime)
        numNAs = 0
        while temp == "NA"
            changex = rand(-2:2);
            changey = rand(-2:2);
            if agent.mated == 0
                changey = rand(-2:5);
            else
                changey = rand(-8:0);
            end

            if (agent.mated < 2)
                if agent.pos[1] < 40
                    changex = rand(-1:2);
                elseif agent.pos[1] > 55
                    changex = rand(-2:1); #+ rand(-5:5);
                end
            end

            if changex == 0
                changex = rand(-1:1)
            end
            if changey == 0
                changey = rand(-1:1)
            end
            newx = agent.pos[1] + changex
            newy = agent.pos[2] + changey

            # Make sure we don't go off the grid
            if newx > dims_glob[1]
                newx = dims_glob[1]
            elseif newx < 1
                newx = 1
            end

            if newy > dims_glob[2]
                newy = dims_glob[2]
            elseif newy < 1
                newy = 1
            end

            temp = getTemp((newx,newy),agent.simTime)
            numNAs += 1

            if (temp == "NA") & (numNAs >= 3)
                newx = agent.pos[1]
                newy = agent.pos[2]
                temp = agent.temp
            end

        end

        agent.pos = (newx,newy)
        agent.temp = temp
        agent.salt = getSalt(agent.pos,agent.simTime)
        if (agent.pos[1] > spawnx) & (agent.pos[2] <= spawny) & (agent.mated == 1) & (rand(1:100) == 1)
            numSpawns = round(agent.eggs * 0.3, digits=0) #0.35#rand(20:60); #220);
            simTime = agent.simTime
            for i in 1:numSpawns
                age = 0
                sex = rand(0:1)
                numSperm = 0
                size = rand(0:5)
                mated = 0
                tOpt = rand(20:26)
                degreeDays = 0
                lifespan = rand(200:1100)
                pos = (rand(nurseryx:100), rand(1:nurseryy)) # spawns can't start just anywhere
                temp = getTemp(pos,simTime)
                numSperm = 0
                eggs = 0
                while temp == "NA"
                    pos = (rand(nurseryx:100), rand(1:nurseryy))
                    temp = getTemp(pos,simTime)
                end
                salt = getSalt(pos,simTime)
                node_to_add = coord2vertex(pos, model)
                add_agent!(node_to_add, model, temp, salt, simTime, age, sex, size, mated, numSperm, eggs, degreeDays, tOpt, lifespan)
            end
            agent.mated = 0
        end
    end
end

using Random
global temps_trunc = CSV.read("model_temps_truncated.csv")[:,1:(150*365)]
global salts_trunc = CSV.read("model_salts_truncated.csv")[:,1:(150*365)]
global temps = hcat(temps_trunc, CSV.read("model_temps_climatechange.csv")[:,1:(150*1460)],makeunique=true)
# without salt being changed ->
global salt = salt_all[:,1:length(temps[1,:])]
model = crab_model_2D(;M = 2000)
agent_properties = [:id, :pos, :temp, :simTime, :age, :sex, :mated, :eggs, :size, :numSperm, :degreeDays, :tOpt, :lifespan]
numdays = Int(round(length(temps[1,:]) / 150, digits=0)-1)
data = step!(model, agent_step_2d!, bay_step!, numdays, agent_properties, when =[50,100,150,160,170,180,190,200,300,400,500,600,700,800,900,1000,1100,1400,1700,1800,numdays],replicates=3,parallel=true,step0=true)

CSV.write(joinpath("output","climatechangetemps_29August2020.csv",data)

global temps = hcat(temps_trunc, CSV.read("model_temps_truncated.csv")[:,1:(150*1460)], makeunique=true)
model2 = crab_model_2D(;M = 2000)
data2 = step!(model2, agent_step_2d!, bay_step!, numdays, agent_properties, when =[50,100,150,160,170,180,190,200,300,400,500,600,700,800,900,1000,1100,1400,1700,1800,numdays],replicates=3,parallel=true,step0=true)

Pkg.add("DataFrames")
using DataFrames
CSV.write(joinpath("output","normaltruncatedtemps_29August2020.csv"),data2)


## CrystalNets main API

"""
    topological_genome(net::CrystalNet{D,T})::String where {D,T}

Compute the topological genome of a net. The topological genome is an invariant
if the net, meaning that it does not depend on its representation. It is also
the string representation of a D-periodic graph such that
`PeriodicGraph{D}(topological_genome(net))` is isomorphic to `net.pge.g` (except
possibly if the `ignore_types` option is unset).

!!! info
    Options must be passed directly within `net`.
"""
function topological_genome(net::CrystalNet{D,T})::TopologicalGenome where {D,T}
    isempty(net.types) && return TopologicalGenome(net.options.error)
    if net.options.ignore_types
        net = CrystalNet{D,T}(net.pge, fill(Symbol(""), length(net.types)), net.options)
    end
    shrunk_net, _collisions = collision_nodes(net)
    _collisions isa Nothing && return TopologicalGenome(net.pge.g, nothing, true)
    collisions::Vector{CollisionNode} = _collisions

    if !net.options.skip_minimize
        flag = true
        try
            shrunk_net, _collisions = minimize(shrunk_net, collisions)
            flag = false
        catch e
            isinterrupt(e) && rethrow()
            if T == Rational{BigInt} || !isoverfloworinexact(e)
                net.options.throw_error && rethrow()
                return TopologicalGenome(string(e)::String)
            end
        end
        if flag # not in the catch to avoid a StackOverflow of errors in case something goes wrong
            newnet = CrystalNet{D,widen(T)}(net; ignore_types=false)
            return topological_genome(newnet)
        end
        _collisions isa Nothing && return TopologicalGenome(shrunk_net.pge.g, nothing, true)
        collisions = _collisions
    end

    export_net = isempty(net.options.export_net) ? isempty(net.options._pos) ?
                 net.options.export_subnets : "" : net.options.export_net
    export_default(net, "net", net.options.name, export_net)

    return topological_genome(shrunk_net, collisions)
end

topological_genome(net::CrystalNet{0,T}) where {T} = TopologicalGenome(net.options.error)

function topological_genome(net::CrystalNet{D,T}, collisions::Vector{CollisionNode})::TopologicalGenome where {D,T}
    try
        g::PeriodicGraph{D} = topological_key(net, collisions)
        unstable = g.width[] == -2
        unstable && (g.width[] = -1)
        ne(g) == 0 && return TopologicalGenome(net.options.error)
        return TopologicalGenome(g, recognize_topology(g), unstable)
    catch e
        isinterrupt(e) && rethrow()
        if T == Rational{BigInt} || !isoverfloworinexact(e)
            net.options.throw_error && rethrow()
            return TopologicalGenome(string(e)::String)
        end
    end
    return topological_genome(CrystalNet{D,widen(T)}(net), collisions)
end

"""
    topological_genome(g::Union{String,PeriodicGraph}, options::Options=Options())
    topological_genome(g::Union{String,PeriodicGraph}; kwargs...)

Compute the topological genome of a periodic graph.
If given a topological key (as a string), it is converted to a `PeriodicGraph` first.
"""
function topological_genome(g::PeriodicGraph, options::Options)
    nets = UnderlyingNets(g, options)
    return topological_genome(nets)
end
topological_genome(s::String, options::Options) = topological_genome(PeriodicGraph(s), options)
topological_genome(g::Union{String,PeriodicGraph}; kwargs...) = topological_genome(g, Options(; throw_error=true, kwargs...))

function _loop_group!(ex::Expr, id::Symbol, net::Symbol, group::Expr)
    for j in 1:length(ex.args)
        arg = ex.args[j]
        if arg isa Symbol
            if arg === :id
                ex.args[j] = id
            elseif arg === :net
                ex.args[j] = net
            elseif arg === :group
                ex.args[j] = group
            end
        elseif arg isa Expr
            _loop_group!(arg, id, net, group)
        end
    end
    nothing
end
macro loop_group(ex)
    @toggleassert ex.head === :for
    ret = Expr(:block)
    for i in 1:3
        id = Symbol("id", i)
        net = Symbol("net", i)
        D = Symbol("D", i)
        group = :(group.$D)
        newex = quote
            if !isempty(group)
                currallowed = first(group)[2][1].options.dimensions
                if isempty(currallowed) || $i in currallowed
                    $(deepcopy(ex))
                end
            end
        end
        _loop_group!(newex, id, net, group)
        push!(ret.args, esc(newex))
    end
    return ret
end

"""
    topological_genome(group::UnderlyingNets)::Vector{Tuple{Vector{Int},String}}

Compute the topological genome of each subnet stored in `group`.

!!! info
    Options must be passed directly within the subnets.
"""
function topological_genome(group::UnderlyingNets)
    ret = Tuple{Vector{Int},TopologyResult}[]
    @loop_group for (id, net) in group
        encountered = Dict{PeriodicGraph,_Clustering}()
        subret = Vector{Tuple{_Clustering,Union{_Clustering,TopologicalGenome}}}(undef, length(net))
        for (j, subnet) in enumerate(net)
            clust = only(subnet.options.clusterings)
            refclust = get!(encountered, subnet.pge.g, clust)
            subret[j] = (clust, refclust == clust ? topological_genome(subnet) : refclust)
        end
        push!(ret, (id, TopologyResult(subret)))
    end
    return ret
end

"""
    recognize_topology(g::PeriodicGraph, arc=CRYSTAL_NETS_ARCHIVE)
    recognize_topology(genome::AbstractString, arc=CRYSTAL_NETS_ARCHIVE)

Attempt to recognize a topological genome from an archive of known genomes.

!!! warning
    This function does a simple lookup, not any kind of topology computation.
    To identify the topology of a `PeriodicGraph` or a `CrystalNet` `x`, query
    `topological_genome(x)` instead.
"""
function recognize_topology(genome::AbstractString, arc=CRYSTAL_NETS_ARCHIVE)
    get(arc, genome, nothing)
end
function recognize_topology(genome::PeriodicGraph, arc=CRYSTAL_NETS_ARCHIVE)
    recognize_topology(string(genome), arc)
end

"""
    determine_topology(path, options::Options)
    determine_topology(path; kwargs...)

Compute the topology of the structure described in the file located at `path`.
This is essentially equivalent to calling
`topological_genome(UnderlyingNets(parse_chemfile(path, options)))`.

In the case where the structure is not made of interpenetrating nets, return the topology
of the only net.
"""
function determine_topology(path, options::Options)
    genomes::Vector{Tuple{Vector{Int},TopologyResult}} =
        topological_genome(UnderlyingNets(parse_chemfile(path, options)))
    if length(genomes) == 1
        return genomes[1][2]
    end
    length(genomes) == 0 && return TopologyResult("")
    return genomes
end
determine_topology(path; kwargs...) = determine_topology(path, Options(; kwargs...))

macro ifvalidgenomereturn(opts, msg, skipcrystal=false)
    crystaldef = skipcrystal ? nothing : :(crystal = parse_chemfile(path, $opts))
    msg = lazy"(found by $msg)"
    ifprintinfo = isempty(msg) ? nothing : :(CrystalNets.@ifwarn @info $msg)

    return esc(quote
    $crystaldef
    group = UnderlyingNets(crystal)
    dim, subnets = isempty(group.D3) ? isempty(group.D2) ? (1, group.D1) : (2, group.D2) : (3, group.D3)
    for (_, nets) in subnets
        for net in nets
            net isa CrystalNet || continue
            sig = get(encountered_graphs, net.pge.g, net.pge.g)
            if haskey(encountered_genomes, net.pge.g)
                unstable, counter = encountered_genomes[sig]
                encountered_genomes[sig] = (unstable, counter + 1)
            else
                genome = topological_genome(net)::TopologicalGenome
                if dim == maxdim && genome.name !== nothing
                    $ifprintinfo
                    return genome
                end
                encountered_graphs[sig] = genome.genome
                encountered_genomes[genome.genome] = (genome.unstable, 1)
            end
        end
    end
    end)
end

"""
    guess_topology(path, options::Options)
    guess_topology(path; kwargs...)

Tries to determine the topology of the file at `path` by passing various options
(starting from the provided options if any) until finding a known topology.
If none is found, return the topological genome encountered most often through
the variation of options.
"""
function guess_topology(path, defopts::Options)
    maxdim = maximum(defopts.dimensions; init=0)
    encountered_graphs = Dict{PeriodicGraph,PeriodicGraph}()
    encountered_genomes = Dict{PeriodicGraph,Tuple{Bool,Int}}()
    if defopts.name == "unnamed"
        defopts = Options(defopts; name=splitext(splitdir(path)[2])[1])
    end

    @ifvalidgenomereturn defopts "using structure=StructureType.Guess" false

    # dryrun = Dict{Symbol,Union{Nothing,Set{Symbol}}}()
    # crystal = parse_chemfile(path, Options(defopts; dryrun))
    atoms = Set{Symbol}(crystal.types)

    if any(x -> (at = atomic_numbers[x]; (ismetal[at] | ismetalloid[at])), atoms)
        @ifvalidgenomereturn Options(defopts; structure=StructureType.MOF) "using MOF clusters"
    end

    defopts = Options(; structure=StructureType.Auto)
    # if defopts.cutoff_coeff == 0.75 # if default cutoff was used, try larger
    #     @ifvalidgenomereturn Options(defopts; cutoff_coeff=0.85) "using longer cutoff"
    # end
    # invalidatoms = union(get(dryrun, :invalidatoms, Set{Symbol}()),
    #                      get(dryrun, :suspect_smallbonds, Set{Symbol}()))
    # hashydrogens = :H ∈ atoms || :D ∈ atoms
    # if hashydrogens
    #     @ifvalidgenomereturn Options(defopts; ignore_atoms=(:H,:D)) "ignoring H"
    #     # delete!(invalidatoms, :H)
    #     # delete!(invalidatoms, :D)
    # end
    # for a in invalidatoms
    #     @ifvalidgenomereturn Options(defopts; ignore_atoms=tuple(a)) "ignoring $a"
    #     if hashydrogens
    #         @ifvalidgenomereturn Options(defopts; ignore_atoms=(a, :H, :D)) "ignoring H and $a"
    #     end
    # end
    # if any(==(:C), atoms)# && :C ∉ invalidatoms # organic solvent
    #     @ifvalidgenomereturn Options(defopts; ignore_atoms=(:C,)) "ignoring C"
    # end
    # if haskey(dryrun, :try_Input_bonds)
    #     @ifvalidgenomereturn Options(defopts; bonding=Bonding.Input) "using input bonds"
    # end
    # @ifvalidgenomereturn Options(defopts; ignore_low_occupancy=true) "removing atoms with occupancy < 0.5"
    if :Al ∈ atoms && (:P ∈ atoms || :Pc ∈ atoms) # ALPO datastructure
        # flag = false
        # for i in vertices(crystal.pge.g)
        #     crystal.types[i] === :O || continue
        #     currt = :O
        #     for x in neighbors(crystal.pge.g, i)
        #         t = crystal.types[x.v]
        #         if (t === :P) | (t === :Al)
        #             if currt === :O
        #                 currt = t
        #             elseif currt === t
        #                 flag = true
        #                 break
        #             end
        #         end
        #     end
        # end
        # if flag
            @ifvalidgenomereturn Options(defopts; ignore_homoatomic_bonds=(:Al,:P)) "removing Al-O-Al and P-O-P bonds"
        # end
        # if :Sn ∈ atoms # observed in some cases for ALPO databases
        #     @ifvalidgenomereturn Options(defopts; ignore_atoms=(:Sn,)) "ignoring Sn"
        # end
    end
    # if haskey(dryrun, :try_no_Auto_bonds)
    #     @ifvalidgenomereturn Options(defopts; bonding=Bonding.Guess) "discarding input bonds and guessing them"
    # end
    # if haskey(dryrun, :collisions)
    #     @ifvalidgenomereturn Options(defopts; authorize_pruning=false) "retaining all colliding atoms"
    # end

    # Finally, if everything fails, return the one encountered most
    maxencounter = 0
    most_plausible_genome::PeriodicGraph = PeriodicGraph{0}()
    final_unstable = false
    for (genome, (unstable, i)) in encountered_genomes
        if i > maxencounter
            maxencounter = i
            most_plausible_genome = genome
            final_unstable = unstable
        end
    end
    return most_plausible_genome == PeriodicGraph{0}() ? begin
            issetequal(defopts.dimensions, [1,2,3]) ? TopologicalGenome() :
                TopologicalGenome("could not guess (perhaps because no net with suitable dimension in $(defopts.dimensions) was found?)")
            end : TopologicalGenome(most_plausible_genome, nothing, final_unstable)
end
guess_topology(path; kwargs...) = guess_topology(path, Options(structure=StructureType.Guess; kwargs...))

"""
    determine_topology_dataset(path, save, autoclean, options::Options)
    determine_topology_dataset(path, save=true, autoclean=true; kwargs...)

Given a path to a directory containing structure input files, compute the
topology of each structure within the directory.
Return a dictionary linking each file name to the result.
The result is a [`TopologyResult`](@ref), containing the topological genome, the name if
known and the stability of the net. In case of error, the exception is reported.

It is strongly recommended to toggle warnings off (through [`toggle_warning`](@ref)) and
not to export any file since those actions may critically reduce performance,
especially for numerous files.

If `save` is set, the result is also stored in a julia serialized file located at
"\\\$path/../results_\\\$i" where `i` is the lowest integer such that this path does
not already exist at the start of the computation.
While processing, this path will be used to create a directory storing the
current state of the computation: to continue an interrupted computation, simply
pass this temporary directory as the path. If `autoclean` is set, this directory
is removed at the end if the computation was successful.

If `save` is set and `autoclean` is unset, the directory of temporary files will
be renamed into "\\\$path/../results_\\\$i.OLD\\\$j".
"""
function determine_topology_dataset(path, save, autoclean, options::Options)
    if isdirpath(path)
        path = dirname(path)
    end
    if startswith(basename(path), "results_") && isfile(joinpath(path), "data")
        resultdir = path
        path = only(readlines(joinpath(path, "data")))
        alreadydone = String[]
        for _f in readdir(resultdir; join=true)
            basename(_f) == "data" && continue
            io = open(_f, "r+")
            pos = max(filesize(io) - 2, 0)
            seek(io, pos)
            while pos > 0 && read(io, Char) != '\n'
                pos -= 1
                seek(io, pos)
            end
            truncate(io, pos+(pos>0)) # remove the last line if incomplete
            close(io)
            for l in eachline(_f)
                _splits = split(l, '/')
                isempty(last(_splits)) && continue
                push!(alreadydone, join(_splits[1:end-2], '/'))
            end
        end
        files = recursive_readdir(path)
        setdiff!(files, alreadydone)
    else
        resultdir = tmpexportname(dirname(path), "", "results", "")
        mkdir(resultdir)
        open(joinpath(resultdir, "data"), "w") do f
            println(f, path)
        end
        files = recursive_readdir(path)
    end
    resultdir::String
    files::Vector{String}

    @threads for file in files
        f = joinpath(path, file)
        # threadid() == 1 && @show f # to find infinite loops: the last one printed is probably running

        genomes::Vector{Tuple{Vector{Int},TopologyResult}} = try
            topological_genome(UnderlyingNets(parse_chemfile(f, options)))
        catch e
            (options.throw_error || isinterrupt(e)) && rethrow()
            [(Int[], TopologyResult(string(e)))]
        end
        if isempty(genomes)
            push!(genomes, (Int[], TopologyResult("")))
        end
        for (j, (_, genome)) in enumerate(genomes)
            newname = length(genomes) == 1 ? file * '/' : file * '/' * string(j)
            open(joinpath(resultdir, string(threadid())), "a") do results
                io = IOContext(results, :compact => true)
                println(io, newname, '/', genome)
            end
        end
    end

    ret = Pair{String,TopologyResult}[]
    for _f in readdir(resultdir; join=true)
        basename(_f) == "data" && continue
        for l in eachline(_f)
            isempty(l) && continue
            splits = split(l, '/')
            _genome = pop!(splits)
            isempty(splits[end]) && pop!(splits)
            push!(ret, Pair(join(splits, '/'), parse(TopologyResult, _genome)))
        end
    end
    result::Dict{String,TopologyResult} = Dict(ret)
    if save
        i = 0
        tmpresultdir = resultdir*".OLD"*string(i)
        while ispath(tmpresultdir)
            i += 1
            tmpresultdir = resultdir*".OLD"*string(i)
        end
        mv(resultdir, tmpresultdir)
        Serialization.serialize(resultdir, result)
        println("Topologies of ", path, " saved at ", resultdir)
        if autoclean
            rm(tmpresultdir; recursive=true)
        else
            println("Temporary files kept at ", tmpresultdir)
        end
    elseif autoclean
        rm(resultdir; recursive=true)
    else
        println("Temporary files kept at ", resultdir)
    end
    return result
end
function determine_topology_dataset(path, save=true, autoclean=true; kwargs...)
    opts, restore_warns = db_options(; kwargs...)
    ret = determine_topology_dataset(path, save, autoclean, opts)
    restore_warns && (DOWARN[] = true)
    ret
end

"""
    guess_topology_dataset(path, save, autoclean, options::Options)
    guess_topology_dataset(path, save=true, autoclean=true; kwargs...)

Given a path to a directory containing structure input files, guess the topology of each
structure within the directory using [`guess_topology`](@ref).
Return a dictionary linking each file name to the result.
The result is the corresponding topology name, if known, or the topological
genome preceded by an "UNKNOWN" mention otherwise. In case of error, the result
is the exception preceded by a "FAILED with" mention. Finally, if the input does
not represent a periodic structure, the result is "non-periodic".

It is strongly recommended to toggle warnings off (through [`toggle_warning`](@ref)) and
not to export any file since those actions may critically reduce performance,
especially for numerous files.

The `save` and `autoclean` arguments work identically to their counterpart for
[`determine_topology_dataset`](@ref).
"""
function guess_topology_dataset(path, save, autoclean, options::Options)
    if isdirpath(path)
        path = dirname(path)
    end
    if startswith(basename(path), "guessed_") && isfile(joinpath(path), "data")
        resultdir = path
        path = only(readlines(joinpath(path, "data")))
        alreadydone = String[]
        for _f in readdir(resultdir; join=true)
            basename(_f) == "data" && continue
            io = open(_f, "r+")
            pos = max(filesize(io) - 2, 0)
            seek(io, pos)
            while pos > 0 && read(io, Char) != '\n'
                pos -= 1
                seek(io, pos)
            end
            truncate(io, pos+(pos>0)) # remove the last line if incomplete
            close(io)
            for l in eachline(_f)
                isempty(l) && continue
                _splits = split(l, '/')
                push!(alreadydone, join(_splits[1:end-1], '/'))
            end
        end
        files = recursive_readdir(path)
        setdiff!(files, alreadydone)
    else
        resultdir = tmpexportname(dirname(path), "", "guessed", "")
        mkdir(resultdir)
        open(joinpath(resultdir, "data"), "w") do f
            println(f, path)
        end
        files = recursive_readdir(path)
    end
    resultdir::String
    files::Vector{String}

    @threads for file in files
        f = joinpath(path, file)

        genome::TopologicalGenome = try
            guess_topology(f, options)
        catch e
            (options.throw_error || isinterrupt(e)) && rethrow()
            TopologicalGenome(escape_string(string(e)::String))
        end
        open(joinpath(resultdir, string(threadid())), "a") do results
            println(results, file, '/', genome)
        end
    end

    ret = Pair{String,TopologicalGenome}[]
    for _f in readdir(resultdir; join=true)
        basename(_f) == "data" && continue
        for l in eachline(_f)
            splits = split(l, '/')
            isempty(last(splits)) && @show _f, l
            _genome = pop!(splits)
            push!(ret, Pair(join(splits, '/'), parse(TopologicalGenome, _genome)))
        end
    end
    result::Dict{String,TopologicalGenome} = Dict(ret)
    if save
        i = 0
        tmpresultdir = resultdir*".OLD"*string(i)
        while ispath(tmpresultdir)
            i += 1
            tmpresultdir = resultdir*".OLD"*string(i)
        end
        mv(resultdir, tmpresultdir)
        Serialization.serialize(resultdir, result)
        println("Guessed topologies of ", path, " saved at ", resultdir)
        if autoclean
            rm(tmpresultdir; recursive=true)
        else
            println("Temporary files kept at ", tmpresultdir)
        end
    elseif autoclean
        rm(resultdir; recursive=true)
    else
        println("Temporary files kept at ", resultdir)
    end
    return result
end
function guess_topology_dataset(path, save=true, autoclean=true; kwargs...)
    opts, restore_warns = db_options(; structure=StructureType.Guess, kwargs...)
    ret = guess_topology_dataset(path, save, autoclean, opts)
    restore_warns && (DOWARN[] = true)
    ret
end


# This needs actual performance testing.
@generated function Chunk(::Val{N}) where N
    if 4N <= VectorizationBase.REGISTER_SIZE
        P = N
    else
        p1 = round(Int, 4N / VectorizationBase.REGISTER_SIZE, RoundUp)
        P = round(Int, N / p1, RoundUp)
    end
    ForwardDiff.Chunk{P}()
end
@generated ValP1(::Val{N}) where N = Val{N+1}()
@generated ValM1(::Val{N}) where N = Val{N-1}()

mutable struct GradientDiffResult{V,P,R} <: DiffResults.DiffResult{1,V,Tuple{ConstantFixedSizeVector{P,V,R,R}}}
    grad::ConstantFixedSizeVector{P,V,R,R}
    value::V
    function GradientDiffResult{V,P,R}(::UndefInitializer) where {V, P, R}
        new{V,P,R}()
    end
    @generated function GradientDiffResult{V,P}(::UndefInitializer) where {V, P}
        R = PaddedMatrices.calc_padding(P, V)
        :(GradientDiffResult{$V,$P,$R}(undef))
    end
    function GradientDiffResult(g::ConstantFixedSizeVector{P,V,R,R}, v::V = zero(V)) where {P,V,R}
        GradientDiffResult{P,V,R}(g, v)
    end
end
mutable struct HessianDiffResult{V,P,R,L} <: DiffResults.DiffResult{2,V,Tuple{ConstantFixedSizeVector{P,V,R,R},ConstantFixedSizeMatrix{P,P,V,R,L}}}
    grad::ConstantFixedSizeVector{P,V,R,R}
    hess::ConstantFixedSizeMatrix{P,P,V,R,L}
    value::V
    function HessianDiffResult{V,P,R,L}(::UndefInitializer) where {V,P,R,L}
        new{V,P,R,L}()
    end
    @generated function HessianDiffResult{V,P}(::UndefInitializer) where {V, P}
        N,R,L = PaddedMatrices.calc_NPL((P,P), V)
        :(HessianDiffResult{$V,$P,$R,$L}(undef))
    end
    function GradientDiffResult(g::ConstantFixedSizeVector{P,V,R,R}, H::ConstantFixedSizeMatrix{P,P,V,R,L}, v::V = zero(V)) where {P,V,R,L}
        GradientDiffResult{P,V,R,R}(g, H, v)
    end
end
const GradResult{V,P,R} = Union{GradientDiffResult{V,P,R},HessianDiffResult{V,P,R}}
@inline Base.pointer(result::GradResult{V}) where {V} = Base.unsafe_convert(Ptr{V}, pointer_from_objref(result))
@inline DiffResults.derivative(result::GradResult{V,P,R}) where {V,P,R} = PtrVector{P,V,R}(pointer(result))
@inline DiffResults.derivative(result::GradResult{V,P,R}, ::Type{Val{1}}) where {V,P,R} = PtrVector{P,V,R}(pointer(result))
@inline DiffResults.derivative(result::HessianDiffResult{V,P,R}, ::Type{Val{2}}) where {V,P,R} = PtrVector{P,V,R}(pointer(result) + sizeof(V)*R )
@inline DiffResults.gradient(result::GradResult{V,P,R}) where {V,P,R} = PtrVector{P,V,R}(pointer(result))
@inline DiffResults.hessian(result::HessianDiffResult{V,P,R})  where {V,P,R}= PtrMatrix{P,P,V,R}(pointer(result) + sizeof(V)*R )

@inline DiffResults.value!(r::GradResult, x::Number) = (r.value = x; return r)
# @inline DiffResults.value!(r::GradResult, x::AbstractArray) = (copyto!(value(r), x); return r)
@inline DiffResults.value!(f::F, r::GradResult, x::Number) where {F} = (r.value = f(x); return r)
# @inline DiffResults.value!(f, r::GradResult, x::AbstractArray) = (map!(f, value(r), x); return r)

function DiffResults.derivative!(r::GradResult, x::AbstractArray)
    copyto!(DiffResults.gradient(r), x)
    return r
end
function DiffResults.derivative!(r::GradResult, x::AbstractArray, ::Type{Val{1}})
    copyto!(DiffResults.gradient(r), x)
    return r
end
function DiffResults.derivative!(r::HessianDiffResult, x::AbstractArray, ::Type{Val{2}})
    copyto!(DiffResults.hessian(r), x)
    return r
end
function DiffResults.derivative!(r::GradResult{P,V,R}, x::ConstantFixedSizeVector{P,V,R}) where {P,V,R}
    r.grad = x
    return r
end
function DiffResults.derivative!(r::GradResult{P,V,R}, x::ConstantFixedSizeVector{P,V,R}, ::Type{Val{1}}) where {P,V,R}
    r.grad = x
    return r
end
function DiffResults.derivative!(r::HessianDiffResult{P,V,R,L}, x::ConstantFixedSizeMatrix{P,P,V,R,L}, ::Type{Val{2}}) where {P,V,R,L}
    r.hess = x
    return r
end
# function DiffResults.derivative!(f, r::GradResult, x::Number)
#     r.grad = f(x)
#     return r
# end
# function DiffResults.derivative!(f, r::GradResult, x::AbstractArray)
#     map!(f, r.grad, x)
#     return r
# end

function GradientDiffResult(x::SizedVector{P,T}) where {T,P}
    GradientDiffResult{T,P}(undef)
end
function HessianDiffResult(x::SizedVector{P,T}) where {T,P}
    HessianDiffResult{T,P}(undef)
    # HessianDiffResult(zero(T), similar(x), FixedSizeMatrix{P,P,T}(undef))
end
# function HessianDiffResult(x::MVector{P,T}) where {T,P}
#     HessianDiffResult{T,P}(undef)
#     # HessianDiffResult(zero(T), similar(x), MMatrix{P,P,T}(undef))
# end

abstract type AbstractAutoDiffDifferentiable{P,T} <: AbstractDifferentiableObject{P,T} end
abstract type AbstractOnceDifferentiableObject{P,T} <: AbstractAutoDiffDifferentiable{P,T} end
abstract type AbstractTwiceDifferentiableObject{P,T} <: AbstractAutoDiffDifferentiable{P,T} end
abstract type Configuration{P,V,F} end

struct GradientConfiguration{P,V,F,T,ND,DG,R} <: Configuration{P,V,F}
    f::F
    result::GradientDiffResult{V,P,R}
    gconfig::ForwardDiff.GradientConfig{T,V,ND,DG}
end

struct HessianConfiguration{P,V,F,T,T2,ND,DJ,DG,DG2, R_V, R_D, LSQM} <: Configuration{P,V,F}
    f::F
    result::HessianDiffResult{V,P,R_V,LSQM}
    inner_result::GradientDiffResult{ForwardDiff.Dual{T,V,ND},P,R_D}
    jacobian_config::ForwardDiff.JacobianConfig{T,V,ND,DJ}
    gradient_config::ForwardDiff.GradientConfig{T,ForwardDiff.Dual{T,V,ND},ND,DG}
    gconfig::ForwardDiff.GradientConfig{T2,V,ND,DG2}
end

struct TwiceDifferentiable{P,T,F,C<:Configuration{P,T,F}, SV_V <: SizedVector{P,T}} <: AbstractTwiceDifferentiableObject{P,T}
    x_f::SV_V # x used to evaluate f (stored in F)
    x_df::SV_V # x used to evaluate df (stored in DF)
    x_h::SV_V #??
    config::C
end
struct OnceDifferentiable{P,T,F,C<:GradientConfiguration{P,T,F}, SV_V <: SizedVector{P,T}} <: AbstractOnceDifferentiableObject{P,T}
    x_f::SV_V# x used to evaluate f (stored in F)
    x_df::SV_V # x used to evaluate df (stored in DF)
    config::C
end

function GradientConfiguration(f::F, x::SizedVector{P,T}) where {F,T,P}
    result = GradientDiffResult(x)
    chunk = Chunk(Val{P}())
    tag = ForwardDiff.Tag(f, T)
    gconfig = ForwardDiff.GradientConfig(f, x, chunk, tag)

    GradientConfiguration(f, result, gconfig)
end
extract_tag(::T) where {T} = T
function HessianConfiguration(f::F, x::SizedVector{P,T}) where {F,T,P}
    result = HessianDiffResult(x)
    chunk = Chunk(Val{P}())
    # tag = ForwardDiff.Tag(f, T)

    # here we construct a JacobianConfig so that yduals is a PtrVector to DiffResults.gradient(inner_result)
    tag_instance = ForwardDiff.Tag(f, T)
    tag_type = extract_tag(tag_instance)
    # @show tag
    seeds = ForwardDiff.construct_seeds(ForwardDiff.Partials{P,T})
    # inner_result = GradientDiffResult(jacobian_config.duals[2])
    inner_result = GradientDiffResult{ForwardDiff.Dual{tag_type,T,P},P}(undef)
    duals = (DiffResults.gradient(inner_result), similar(x, ForwardDiff.Dual{tag_type,T,P}))
    jacobian_config = ForwardDiff.JacobianConfig{tag_type,T,P,typeof(duals)}(seeds, duals)

    # jacobian_config = ForwardDiff.JacobianConfig((f,ForwardDiff.gradient), DiffResults.gradient(result), x, chunk, tag)
    # jacobian_config = ForwardDiff.JacobianConfig((f,ForwardDiff.gradient), DiffResults.gradient(result), x, chunk, tag)
    gradient_config = ForwardDiff.GradientConfig(f, jacobian_config.duals[2], chunk, tag_instance)

    gconfig = ForwardDiff.GradientConfig(f, x, chunk, tag_instance)

    HessianConfiguration(f, result, inner_result, jacobian_config, gradient_config, gconfig)
end

TwiceDifferentiable(f::F, ::Val{P}) where {F,P} = TwiceDifferentiable(f, FixedSizeVector{P,Float64}(undef))
function TwiceDifferentiable(f::F, x::SV_V) where {F,T,P,SV_V <: SizedVector{P,T}}
    TwiceDifferentiable(x, similar(x), similar(x), HessianConfiguration(f, x))
end
function TwiceDifferentiable(x_f::SV_V,x_df::SV_V,x_h::SV_V,config::C) where {T,F,C<:Configuration{T,F},P,SV_V <: SizedVector{P,T}}
    TwiceDifferentiable{P,T,F,C}(x_f, x_df, x_h, config)
end

function OnceDifferentiable(f, x_f::SizedVector{P,T}) where {T,P}
    OnceDifferentiable(x_f, similar(x_f), GradientConfiguration(f, x_f))
end
function OnceDifferentiable(x_f::SV_V, config::C) where {T,F,C<:GradientConfiguration{T,F},P,SV_V <: SizedVector{P,T}}
    OnceDifferentiable{P,T,F,C}(x_f, similar(x_f), config)
end

# ProfileDifferentiable(f::F, ::Val{N}) where {F,N} = ProfileDifferentiable(f, Vector{Float64}(undef, N), Val{N}())
# function ProfileDifferentiable(f::F, x::AbstractArray{T}, ::Val{N}) where {F,T,N}
#     x_f = Vector{T}(undef, N-1)
#     ProfileDifferentiable(x_f, similar(x_f), similar(x_f), Configuration(f, x_f, ValM1(Val{N}())), x, Ref(N), Ref{T}(),::Val{N})
# end
# function ProfileDifferentiable(x_f::Vector{T},x_df::Vector{T},x_h::Vector{T}, config::C, x,::A i::RefValue{Int}, v::RefValue{T},::Val{N}) where {T,A<:AbstractArray{T},C,N}
#     ProfileDifferentiable{N,T,A,C}(x_f, x_df, x_h, config, x, i, v)
# end

@inline DiffResults.value!(obj::AbstractAutoDiffDifferentiable, x::Real) = DiffResults.value!(obj.config.result, x)

@inline value(obj::AbstractAutoDiffDifferentiable) = obj.config.result.value
@inline gradient(obj::AbstractAutoDiffDifferentiable) = DiffResults.gradient(obj.config.result)
# @inline gradient(obj::AbstractAutoDiffDifferentiable, i::Integer) = DiffResults.gradient(obj.config.result)[i]
@inline hessian(obj::TwiceDifferentiable) = DiffResults.hessian(obj.config.result)

@inline f(obj::AbstractAutoDiffDifferentiable, x) = obj.config.f(x)

# """
# For DynamicHMC support. Copies data.
# """
# @inline function (obj::OnceDifferentiable)(x)
#     fdf(obj, x)
#     DiffResults.ImmutableDiffResult(-obj.config.result.value, (-1 .* obj.config.result.grad,))
# end
# @inline function (obj::TwiceDifferentiable)(x)
#     fdf(obj, x)
#     DiffResults.ImmutableDiffResult(-obj.config.result.value, (-1 .* obj.config.result.grad,))
# end

@inline function df(obj::AbstractAutoDiffDifferentiable, x)
    ForwardDiff.gradient!(gradient(obj), obj.config.f, x, obj.config.gconfig, Val{false}())
end

@inline function fdf(obj::AbstractAutoDiffDifferentiable, x)
    # obj.config.result.grad = gradient(obj)
    ForwardDiff.gradient!(obj.config.result, obj.config.f, x, obj.config.gconfig, Val{false}())
    # DiffResults.value(obj.config.result)
    obj.config.result.value
end
@inline function scale_fdf(obj::AbstractAutoDiffDifferentiable, x, scale_target)
    # obj.config.result.grad = gradient(obj)
    scale = scale_gradient!(obj.config.result, obj.config.f, x, scale_target, obj.config.gconfig, Val{false}())
    # DiffResults.value(obj.config.result)
    obj.config.result.value, scale
end
@inline function scaled_fdf(obj::AbstractAutoDiffDifferentiable, x, scale)
    # obj.config.result.grad = gradient(obj)
    scaled_gradient!(obj.config.result, obj.config.f, x, scale, obj.config.gconfig, Val{false}())
    # DiffResults.value(obj.config.result)
    obj.config.result.value
end

# function Configuration(f::F, x::AbstractArray{V}, chunk::Chunk = Chunk(x), tag = Tag(f, V)) where {F,V}
#     result = DiffResults.HessianResult(x)
#     jacobian_config = ForwardDiff.JacobianConfig((f,gradient), DiffResults.gradient(result), x, chunk, tag)
#     gradient_config = ForwardDiff.GradientConfig(f, jacobian_config.duals[2], chunk, tag)
#     inner_result = DiffResults.DiffResult(zero(eltype(jacobian_config.duals[2])), jacobian_config.duals[2])
#     gconfig = ForwardDiff.GradientConfig(f, x, chunk, tag)
#     Configuration(f, result, inner_result, jacobian_config, gradient_config, gconfig)
# end

@inline function (c::HessianConfiguration)(y, z)
    # c.inner_result.grad = y #Already true?
    # @show typeof(y), typeof(z)
    ForwardDiff.gradient!(c.inner_result, c.f, z, c.gradient_config, Val{false}())
    DiffResults.value!(c.result, ForwardDiff.value(DiffResults.value(c.inner_result)))
    # copyto!(y, DiffResults.gradient(c.inner_result))
    y
end
@inline function hessian!(c::HessianConfiguration, x::AbstractArray)
    ForwardDiff.jacobian!(DiffResults.hessian(c.result), c, DiffResults.gradient(c.result), x, c.jacobian_config, Val{false}())
    # DiffResults.hessian(c.result)
    DiffResults.hessian(c.result)
end

"""
Force (re-)evaluation of the objective value at `x`.

Returns `f(x)` and stores the value in `obj.F`
"""
@inline function value!!(obj::AbstractAutoDiffDifferentiable, x)
    # obj.f_calls .+= 1
    # copyto!(obj.x_f, x)
    DiffResults.value!(obj, f(obj, x) )
    value(obj)
end
# """
# Evaluates the objective value at `x`.
#
# Returns `f(x)`, but does *not* store the value in `obj.F`
# """
# @inline function value(obj::AbstractAutoDiffDifferentiable, x)
#     if x != obj.x_f
#         # obj.f_calls .+= 1
#         value!!(obj, x)
#     end
#     value(obj)
# end
"""
Evaluates the objective value at `x`.

Returns `f(x)` and stores the value in `obj.F`
"""
@inline function value!(obj::AbstractAutoDiffDifferentiable, x)
    # if x != obj.x_f
        value!!(obj, x)
    # end
    value(obj)
end

"""
Evaluates the gradient value at `x`

This does *not* update `obj.DF`.
"""
@inline function gradient(obj::AbstractAutoDiffDifferentiable, x)
    DF = gradient(obj)
    gradient!!(obj, x)
    DF
end
"""
Evaluates the gradient value at `x`.

Stores the value in `obj.DF`.
"""
@inline function gradient!(obj::AbstractAutoDiffDifferentiable, x)
    # if x != obj.x_df
        gradient!!(obj, x)
    # end
    gradient(obj)
end
"""
Force (re-)evaluation of the gradient value at `x`.

Stores the value in `obj.DF`.
"""
@inline function gradient!!(obj::AbstractAutoDiffDifferentiable, x)
    # obj.df_calls .+= 1
    copyto!(obj.x_df, x)
    df(obj, x)
end

@inline function value_gradient!(obj::AbstractAutoDiffDifferentiable, x)
    # if x != obj.x_f && x != obj.x_df
        value_gradient!!(obj, x)
    # elseif x != obj.x_f
    #     value!!(obj, x)
    # elseif x != obj.x_df
    #     gradient!!(obj, x)
    # end
    value(obj)
end
@inline function value_gradient!!(obj::AbstractAutoDiffDifferentiable, x)
    # obj.f_calls .+= 1
    # obj.df_calls .+= 1
    # copyto!(obj.x_f, x)
    # copyto!(obj.x_df, x)
    DiffResults.value!(obj, fdf(obj, x))
end

@inline function hessian!(obj::AbstractAutoDiffDifferentiable, x)
    # if x != obj.x_h
        hessian!!(obj, x)
    # end
    nothing
end
@inline function hessian!!(obj::AbstractAutoDiffDifferentiable, x)
    # obj.h_calls .+= 1
    # copyto!(obj.x_h, x)
    hessian!(obj.config, x)
end

# function ForwardDiff.vector_mode_dual_eval(f::F, x::FixedSizeVector, cfg::Union{JacobianConfig,GradientConfig}) where F
#     xdual = cfg.duals
#     seed!(xdual, x, cfg.seeds)
#     return f(xdual)
# end

@inline function ForwardDiff.seed!(duals::AbstractMutableFixedSizeVector{P,ForwardDiff.Dual{T,V,N}}, x,
                                    seed::ForwardDiff.Partials{N,V}) where {T,V,N,P}
    @inbounds for i in 1:N
        duals[i] = ForwardDiff.Dual{T,V,N}(x[i], seed)
    end
    return duals
end

@inline function ForwardDiff.seed!(duals::AbstractMutableFixedSizeVector{P,ForwardDiff.Dual{T,V,N}}, x,
                                    seeds::NTuple{N,ForwardDiff.Partials{N,V}}) where {T,V,N,P}
    @inbounds for i in 1:N
        duals[i] = ForwardDiff.Dual{T,V,N}(x[i], seeds[i])
    end
    return duals
end

@inline function ForwardDiff.seed!(duals::AbstractMutableFixedSizeVector{P,ForwardDiff.Dual{T,V,N}}, x, index,
                                    seed::ForwardDiff.Partials{N,V} = zero(ForwardDiff.Partials{N,V})) where {T,V,N,P}
    offset = index - 1
    @inbounds for i in 1:N
        j = i + offset
        duals[j] = ForwardDiff.Dual{T,V,N}(x[j], seed)
    end
    return duals
end

@inline function ForwardDiff.seed!(duals::AbstractMutableFixedSizeVector{P,ForwardDiff.Dual{T,V,N}}, x, index,
                                    seeds::NTuple{N,ForwardDiff.Partials{N,V}}, chunksize = N) where {T,V,N,P}
    offset = index - 1
    @inbounds for i in 1:chunksize
        j = i + offset
        duals[j] = ForwardDiff.Dual{T,V,N}(x[j], seeds[i])
    end
    return duals
end
function ForwardDiff.extract_jacobian!(::Type{T}, result::AbstractMutableFixedSizeMatrix{M,N}, ydual::AbstractArray, n) where {M,N,T}
    # out_reshaped = reshape(result, length(ydual), n)
    # @inbounds for col in 1:size(out_reshaped, 2), row in 1:size(out_reshaped, 1)
    #     out_reshaped[row, col] = partials(T, ydual[row], col)
    # end
    @inbounds for col in 1:N, row in 1:M
        # result[row, col] = ForwardDiff.partials(T, ydual[row], col)
        result[row, col] = ForwardDiff.partials(T, ydual[col], row)
    end
    return result
end


module EvoTrees

export grow_tree!, grow_gbtree, grow_gbtree!, Tree, Node, Params, predict, EvoTreeRegressor, EvoTreeRegressorR

using DataFrames
using Statistics
using CSV
using Base.Threads: @threads
using StatsBase: sample, quantile
using Random: seed!
using StaticArrays
using Flux: onehot
# import MLJBase
# import MLJ

include("struct.jl")
include("loss.jl")
include("eval.jl")
include("predict.jl")
include("find_split.jl")
include("trees.jl")
# include("MLJ.jl")

end # module


abstract type GOCInput <: AbstractInput end

##TODO
function get_baseMVA(bus::String)
    return 100
end

"""
    read_input(input_type::T, instance_path::String) where T<:Type{GOCInput}

Read instance in `instance_path` depending on `input_type`.\n
Return a structure OPFProblems.\n
For `GOCInput`, read files pscopf_data.mat, contingency.csv and generator.csv, store them in arrays and use information to fulfill a DataSource structure for the basecase scenario. Finally take into account contingencies.

"""
function read_input(input_type::T, instance_path::String) where T<:Type{GOCInput}
    #read
    power_data,generator_data,contingency_data = getdataGOC(instance_path)
    ##info in bus
    bus,index = read_data_bus(power_data)
    ##info in generator
    gendata_dict = generator_data_to_dict(generator_data,index)
    ##info in generator
    bus = add_generator_data!(power_data,gendata_dict,bus,index)
    ##info in branch
    link = read_branch_data(power_data,index)
    #info basecase
    ds = DataSource(bus,link)
    node_linksin, node_linksout = SortedDict{String, SortedSet{Link}}(), SortedDict{String, SortedSet{Link}}()
    node_vars = SortedDict{String, SortedDict{String, Variable}}()
    link_vars = SortedDict{Link, SortedDict{String, Variable}}()
    gs = GridStructure("BaseCase", node_linksin, node_linksout)
    node_formulations = SortedDict{String, SortedDict{String, Symbol}}()
    link_formulations = SortedDict{Link, SortedDict{String, Symbol}}()
    mp = MathematicalProgramming(node_formulations, link_formulations, node_vars,link_vars)
    ##read scenarios
    OPFproblems = scenarios_data(ds, gs, mp, contingency_data,index)
    return OPFproblems
end

#########################
## Utils functions
#########################
"""
    getdataGOC(instance_path)

Read 3 files (pscopf_data.mat,generator.csv,contingency.csv) in `instance_path` and returns 3 arrays of data (power_data,generator_data,contingency_data)
Return an error if one the three files does not exist in path_folder.
"""
function getdataGOC(instance_path) ##String instance+scenario
    power_data = getpowerdata(instance_path)
    generator_data = getgeneratordata(instance_path)
    contingency_data = getcontingencydata(instance_path)
    return power_data,generator_data,contingency_data
end

function getpowerdata(instance_path)
    path_folder = instance_path
    if !("pscopf_data.mat" ∈ readdir(path_folder))
        error("missing file : pscopf_data.mat must be in $path_folder")
    end
    try
        return read(matopen(joinpath(path_folder,"pscopf_data.mat")))
    catch
        warn("Not possible to read matopen(pscopf_data.mat)")
        return
    end
end

function getgeneratordata(instance_path)
    path_folder = instance_path
    if !("generator.csv" ∈ readdir(path_folder))
        error("missing file : generator.csv must be in $path_folder")
    end
    try
        return readdlm(joinpath(path_folder,"generator.csv"),',',skipstart=1)
    catch
        warn("Not possible to readdlm generator.csv")
        return
    end

end

function getcontingencydata(instance_path)
    path_folder = instance_path
    if !("contingency.csv" ∈ readdir(path_folder))
        error("missing file : contingency.csv must be in $path_folder")
    end
    try
        return readdlm(joinpath(path_folder,"contingency.csv"),',',skipstart=1)
    catch
        warn("Not possible to readdlm contingency.csv")
    end
end

#########################
function get_bus_data(bus_data, listofkeys, id)
    listofvalues = []
    for key in listofkeys
        if !haskey(bus_data, key)
            error("bus_data dictionary does not have a key $key")
        else
            append!(listofvalues, bus_data[key][id])
        end
    end
    return listofvalues
end

function get_bus_index(power_data)
    nb_bus = Int(power_data["totalbus"])
    all_busdata_key = filter(x->ismatch(r"all_busdata", x), collect(keys(power_data)))[1]
    raw_bus_data = power_data[all_busdata_key]
    nb_lines = size(raw_bus_data,1)
    if nb_lines!=nb_bus
        error("number of lines of all_busdata not equal to number of bus in power_data at key totalbus")
    end
    index = SortedDict{Int64,Int64}()
    for line in 1:nb_lines
        id_bus = Int64(raw_bus_data[line,1])
        index[id_bus] = id_bus
    end
    return index
end

"""
    read_data_bus(power_data)

Reads data in `power_data` to construct the DataSource field bus without generator data (only VOLTM, load and shunt data)
"""
function read_data_bus(power_data)
    nb_bus = Int(power_data["totalbus"])
    index = get_bus_index(power_data)
    bus_data = power_data["bus"]
    # all_busName_key = filter(x->ismatch(r"all_busName", x), collect(keys(power_data)))[1]
    # busname_data = power_data[all_busName_key]
    all_busdata_key = filter(x->ismatch(r"all_busdata", x), collect(keys(power_data)))[1]
    raw_bus_data = power_data[all_busdata_key]
    nb_lines = size(raw_bus_data,1)
    if nb_lines!=length(bus_data["BaseKV"])
        error("sizes of all_busName and bus not equal")
    end
    bus = SortedDict{String, SortedDict{String,Any}}()
    # bus = SortedDict(bus_name(line) => SortedDict{String,Any}() for line in 1:nb_bus)
    for line in 1:nb_lines
        id_bus = Int(raw_bus_data[line,1])
        # index[id_bus] = line
        busname = bus_name(index[id_bus])
        baseKV = bus_data["BaseKV"][line]
        baseMVA = 1.0
        voltage_magnitude_min , voltage_magnitude_max  = get_bus_data(bus_data, ["VoltageMagnitudeMin", "VoltageMagnitudeMax"], line)
        if !haskey(bus, busname)
            bus[busname] = SortedDict{String,Any}()
        end
        bus[busname][volt_name()] = GOCVolt(id_bus, baseKV, baseMVA, voltage_magnitude_min, voltage_magnitude_max)
    end

    all_loadID_key = filter(x->ismatch(r"all_loadID", x), collect(keys(power_data)))[1]
    load_id = power_data[all_loadID_key]
    all_loaddata_key = filter(x->ismatch(r"all_loaddata", x), collect(keys(power_data)))[1]
    load_data = power_data[all_loaddata_key]

    for line in 1:size(load_data,1)
        id_bus = Int(load_data[line,1])
        busname = bus_name(index[id_bus])
        load = load_data[line,5] + im * load_data[line,6]
        id_load = id_elem(load_id,line)
        loadname = load_name(id_load)
        bus[busname][loadname] = GOCLoad(id_bus,loadname,load)
    end

    all_fixedshuntID_key = filter(x->ismatch(r"all_fixedshuntID", x), collect(keys(power_data)))[1]
    shunt_id = power_data[all_fixedshuntID_key]
    all_fixedshuntdata_key = filter(x->ismatch(r"all_fixedshuntdata", x), collect(keys(power_data)))[1]
    shunt_data = power_data[all_fixedshuntdata_key]
    for line in 1:size(shunt_data,1)
        id_bus = Int(shunt_data[line,1])
        busname = bus_name(index[id_bus])
        shunt = shunt_data[line,3] + im * shunt_data[line,4]
        id_shunt = id_elem(shunt_id,line)
        shuntname = shunt_name(id_shunt)
        bus[busname][shuntname] = GOCShunt(id_bus,shuntname,shunt)
    end

    return bus,index
end

function id_elem(array_id,line)
    if typeof(array_id) == String
        temp_string = array_id
    else
        temp_string = array_id[line]
        if typeof(temp_string)==Float64
            temp_string = Int(temp_string)
        end
    end
    return temp_string
    #return matchall(r"\S+" ,temp_string)[1]
end

#########################
function get_generator_data(generator_data, listofkeys, id)
    listofvalues = []
    for key in listofkeys
        if !haskey(generator_data, key)
            error("generator_data dictionary does not have a key $key")
        else
            append!(listofvalues, generator_data[key][id])
        end
    end
    return listofvalues
end

function generator_data_to_dict(generator_data,index) ##conversion array to dict
    generator_data_dict = SortedDict{String, SortedDict{String,SortedDict{Int64,Float64}}}()
    for line in 1:size(generator_data,1)
        bus_id = Int(generator_data[line,1])
        busname = bus_name(index[bus_id])
        gen_id = generator_data[line,2]
        gen_id = remove_simple_quotes_if_present(gen_id)
        if typeof(gen_id)==Float64
            gen_id = Int(gen_id)
        end
        genname = generator_name(gen_id)
        if !haskey(generator_data_dict, busname)
            generator_data_dict[busname] = SortedDict{String,SortedDict{Int64,Float64}}()
        end
        if !haskey(generator_data_dict[busname],genname)
            generator_data_dict[busname][genname] = SortedDict{Int64,Float64}()
        end
        term_id = Int(generator_data[line,3])
        value = generator_data[line,4]
        if term_id!=9
            generator_data_dict[busname][genname][term_id] = value
        end
    end
    return generator_data_dict
end

function remove_simple_quotes_if_present(id)
    if typeof(id)==SubString{String}
        if id[end-1]==' '
            return id[2:(end-2)]
        else
            return id[2:(end-1)]
        end
    else
        if id[end]==' '
            return id[1:(end-1)]
        else
            return id
        end
    end
end

"""
    add_generator_data!(power_data,generator_data_dict,bus,index)

Add generator data contained in `power_data` in DataSource field `bus` depending on `generator_data_dict` and `index`.\n
Return `bus`.

"""
function add_generator_data!(power_data,generator_data_dict,bus,index)
    nb_bus = Int(power_data["totalbus"])
    generator_data = power_data["generator"]
    for gen in 1:length(generator_data["BusNum"])
        bus_id = Int(get_generator_data(generator_data,["BusNum"],gen)[1])
        # all_busName_key = filter(x->ismatch(r"all_busName", x), collect(keys(power_data)))[1]
        # busname_data = power_data[all_busName_key]
        busname = bus_name(index[bus_id])
        gen_id, Pmin, Qmin, Pmax, Qmax, participation_factor =  get_generator_data(generator_data, ["UnitID","RealPowerMin","ReactivePowerMin","RealPowerMax","ReactivePowerMax", "ParticipationFactor"], gen)
        power_min = Pmin + im*Qmin
        power_max = Pmax + im*Qmax
        # cost_degrees = generator_data["RealPowerCostExponent"][gen,:]
        # cost_coeffs = generator_data["RealPowerCostCoefficient"][gen,:]
        gen_id2 = gen_id
        gen_id2 = remove_simple_quotes_if_present(gen_id2)
        if typeof(gen_id2)==Float64
            gen_id2 = Int(gen_id2)
        end
        generatorname = generator_name(gen_id2)
        dict_obj_coeffs = SortedDict{Int64,Float64}()
        for (degree, value) in generator_data_dict[busname][generatorname]
            if (degree ∈ [0,1,2])
                dict_obj_coeffs[degree] = value
            end
        end
        if length(dict_obj_coeffs)<3
            #warn("A 2-order polynom cost with less than 3 values for generator $generatorname at bus $busname, default value 0")
            if !haskey(dict_obj_coeffs,0) dict_obj_coeffs[0] = 0 end
            if !haskey(dict_obj_coeffs,1) dict_obj_coeffs[1] = 0 end
            if !haskey(dict_obj_coeffs,2) dict_obj_coeffs[2] = 0 end
       end
        bus[busname][generatorname] = GOCGenerator(bus_id,gen_id,power_min,power_max,participation_factor,dict_obj_coeffs)
    end
    return bus
end

#########################
function get_branch_data(branch_data, listofkeys, id)
    listofvalues = []
    for key in listofkeys
        if !haskey(branch_data, key)
            error("branch_data dictionary does not have a key $key")
        else
            append!(listofvalues, branch_data[key][id])
        end
    end
    return listofvalues
end

"""
    read_branch_data(power_data, index)

Read branch data contained in `power_data` to fulfill DataSource field `link` depending on `index`. \n
Return dictionary `link`.
"""
function read_branch_data(power_data, index)
    nb_branch = Int(power_data["totalbranch"])
    nb_bus = Int(power_data["totalbus"])
    # all_busName_key = filter(x->ismatch(r"all_busName", x), collect(keys(power_data)))[1]
    # busname_data = power_data[all_busName_key]
    all_lineCKT_key = filter(x->ismatch(r"all_lineCKT", x), collect(keys(power_data)))[1]
    lines_data = power_data[all_lineCKT_key]
    nb_lines = length(lines_data)
    link = SortedDict{Link, SortedDict{String,Any}}()
    branch_data = power_data["branch"]

    #lines without transformer
    for branch in 1:nb_lines
        origin = Int(get_branch_data(branch_data, ["Origin"], branch)[1])
        destination = Int(get_branch_data(branch_data, ["Destination"], branch)[1])
        linkname = Link(bus_name(index[origin]),bus_name(index[destination]))
        br_id = lines_data[branch]
        br_id2 = br_id
        branch_id = remove_simple_quotes_if_present(br_id2)
        if typeof(branch_id)==Float64
            branch_id = Int(branch_id)
        end
        resistance, reactance, susceptance, power_magnitude_max = get_branch_data(branch_data, ["SeriesResistance","SeriesReactance","ChargingSusceptance","PowerMagnitudeMax"], branch)
        if !haskey(link, linkname)
            link[linkname] = SortedDict{String,Any}()
        end
        if resistance == reactance == 0
            println(linkname, " nullimpedance line without transformer")
            name_line = nullimpedance_notransformer_name(branch_id)
            link[linkname][name_line] = GOCNullImpedance_notransformer(origin,destination,br_id,susceptance, power_magnitude_max)
        else
            name_line = linepi_notransformer_name(branch_id)
            link[linkname][name_line] = GOCLineπ_notransformer(origin,destination,br_id,resistance,reactance,susceptance, power_magnitude_max)
        end
    end

    all_transformerCKT_key = filter(x->ismatch(r"all_transformerCKT", x), collect(keys(power_data)))[1]
    transformername_data = power_data[all_transformerCKT_key]

    #lines with transformers
    for branch in (nb_lines+1):nb_branch
        origin = Int(get_branch_data(branch_data, ["Origin"], branch)[1])
        destination = Int(get_branch_data(branch_data, ["Destination"], branch)[1])
        linkname = Link(bus_name(index[origin]),bus_name(index[destination]))
        br_id = transformername_data[branch-nb_lines]
        br_id2 = br_id
        branch_id = remove_simple_quotes_if_present(br_id2)
        if typeof(branch_id)==Float64
            branch_id = Int(branch_id)
        end
        resistance, reactance, susceptance, power_magnitude_max = get_branch_data(branch_data, ["SeriesResistance","SeriesReactance","ChargingSusceptance","PowerMagnitudeMax"], branch)
        transfo_ratio, transfo_phase = get_branch_data(branch_data, ["TapRatio", "PhaseShift"], branch)
        if !haskey(link, linkname)
            link[linkname] = SortedDict{String,Any}()
        end
        if resistance == reactance == 0
            println(linkname, " nullimpedance line with transformer")
            name_line = nullimpedance_withtransformer_name(branch_id)
            link[linkname][name_line] = GOCNullImpedance_withtransformer(origin,destination,br_id,susceptance, transfo_ratio,transfo_phase, power_magnitude_max)
        else
            name_line = linepi_withtransformer_name(branch_id)
            link[linkname][name_line] = GOCLineπ_withtransformer(origin,destination,br_id,resistance,reactance,susceptance, transfo_ratio,transfo_phase, power_magnitude_max)
        end
    end

    return link
end

#########################
"""
    scenarios_data(ds,gs,mp,power_data, contingency_data,index)

Use dictionaries `ds`, `gs` and `mp` to create `Scenario` structures for each contingency contained in `contingency_data`.\n
Return an `OPFproblems` structure : "scenario" => Scenario

"""
function scenarios_data(ds,gs,mp,contingency_data,index)
    output = SortedDict("BaseCase" => Scenario(ds,gs,mp))
    ds = output["BaseCase"].ds
    nb_contingencies = size(contingency_data,1)
    for ct in 1:nb_contingencies
        id_contingency = contingency_data[ct,1]
        scenario_name = scenarioname(id_contingency)
        type_contingency = contingency_data[ct,2]
        #B, T or G (Branch, Transformer, Generator)
        gs_scenario = copy(gs)
        gs_scenario.scenario = scenario_name
        ds_scenario_bus = SortedDict{String, SortedDict{String,Any}}()
        for busname in keys(ds.bus)
            ds_scenario_bus[busname] = SortedDict{String,Any}()
        end
        ds_scenario_link = SortedDict{Link, SortedDict{String,Any}}()
        for linkname in keys(ds.link)
            ds_scenario_link[linkname] = SortedDict{String,Any}()
        end
        ds_scenario = DataSource(ds_scenario_bus,ds_scenario_link)
        if type_contingency == "G"
            println(type_contingency)
            bus_id = contingency_data[ct,3]
            busname = bus_name(index[bus_id])
            gen_num = contingency_data[ct,4]
            CID = contingency_data[ct,5]
            if typeof(CID)==SubString{String}
                gen_ID = CID[2:(end-1)]
            else
                gen_ID = string(CID)
            end
            for (bus,dict) in ds.bus
                if bus==busname
                    for (buslabel,data) in dict
                        if buslabel != generator_name(gen_ID)
                            ds_scenario.bus[bus][buslabel] = data
                        end
                    end
                else
                    for (buslabel,data) in dict
                        ds_scenario.bus[bus][buslabel] = data
                    end
                end
            end
            for (linkname,dict) in ds_scenario.link
                for (linklabel,data) in ds.link[linkname]
                    ds_scenario.link[linkname][linklabel] = ds.link[linkname][linklabel]
                end
            end
        elseif type_contingency=="B" || type_contingency=="T"
            #println("Type of contingency $ct is $type_contingency")
            origin = Int(contingency_data[ct,3])
            destination = Int(contingency_data[ct,4])
            link_to_remove = Link(bus_name(index[origin]),bus_name(index[destination]))
            # println("link_to_remove : ", link_to_remove)
            CID = contingency_data[ct,5]
            if typeof(CID)==SubString{String}
                id_line = CID[2:(end-1)]
            else
                id_line = string(CID)
            end
            for (busname,dict) in ds.bus
                for (buslabel,data) in dict
                    ds_scenario.bus[busname][buslabel] = data
                end
            end
            for (linkname,dict) in ds.link
                if linkname.orig == link_to_remove.orig && linkname.dest == link_to_remove.dest
                    # println("link to remove : ", link_to_remove)
                    for (linklabel,data) in dict
                        if !contains(linklabel,id_line)
                            ds_scenario.link[linkname][linklabel] = data
                        end
                    end
                    if isempty(ds_scenario.link[linkname])
                        delete!(ds_scenario.link,linkname)
                    end
                else
                    for (linklabel,data) in dict
                    ds_scenario.link[linkname][linklabel] = data
                    end
                end
            end
        end
        output[scenario_name] = Scenario(ds_scenario,gs_scenario,mp)
    end

    return output
end


## info command format:
## info time 203 nodes 11111111 score cp 11168 pv 5e9i+ ....

const PV = 1
const NonPV = 2
const Root = 3

function rememberPV(gs::GameStatus)
    # remember the last PV, and also the 5 previous ones because
    # they usually contain good moves to try
    i::Int = 0

    gs.lastPVLength = gs.triangularLength[1]
    #println("last length = ", gs.triangularLength[1])
    for i = 1:gs.triangularLength[1]
	gs.lastPV[i] = gs.triangularArray[1,i];
    end
end

# alpha-beta search
function AlphaBeta( gs::GameStatus, ply::Int, depth::Float64, alpha::Int, beta::Int)
    gs.triangularLength[ply+1] = ply
    ev_sign = ((ply & 0x01) == 0)?-1:1
    teban::Int = ((ply & 0x1) == 0)?gs.side:((gs.side==SENTE)?GOTE:SENTE)

    if depth <= 0.0
        return Eval( gs.board.nextMove, gs.board)
    end

    if gs.inodes > 100000
        gs.timedout = true
        return 0
    end

    gs.moveBufLen[ply+1+1] = generateMoves(gs.board, gs.moveBuf, teban, gs.moveBufLen[ply+1], gs)

    for i = gs.moveBufLen[ply+1]+1:gs.moveBufLen[ply+1+1] # 1 origin
        makeMove( gs.board, i, gs.moveBuf, teban)
        if in_check( teban, gs.board)
            #println("check!")
            takeBack( gs.board, i, gs.moveBuf, teban)
        else
            gs.inodes += 1
            val::Int = -AlphaBeta( gs, ply+1, depth-1.0, -beta, -alpha)
            takeBack( gs.board, i, gs.moveBuf, teban)
            #if val >= beta
            #    return beta
            #end
            if val > alpha
                alpha = val
		# both sides want to maximize from *their* perspective

                gs.triangularArray[ply+1,ply+1] = gs.moveBuf[i]
		# save this move
                for j = (ply +1+1):gs.triangularLength[ply+1+1]
                    gs.triangularArray[ply+1,j] = gs.triangularArray[ply+1+1,j]
                    # and append the latest best PV from deeper plies
                end
                gs.triangularLength[ply+1] = gs.triangularLength[ply + 1+1]
                rememberPV(gs)
            end
        end
    end
    alpha
end

# search driver
function think( sengo::Int, gs::GameStatus)
    move::Move = Move(0,0,0,0,0,0) # null move
    gs.lastPV = [Move(0,0,0,0,0,0) for x = 1:MaxPly]::Array{Move,1}
    gs.lastPVLength = 0
    gs.whiteHeuristics = [0 for x = 1:0xff, y = 1:0xff]::Array{Int,2}
    gs.blackHeuristics = [0 for x = 1:0xff, y = 1:0xff]::Array{Int,2}
    gs.nsStart = time_ns()
    gs.inodes = 0
    gs.timedout = false
    gs.side = sengo
    gs.board.nextMove = sengo

    for IDdepth = 1.0:1.0:15.0 #MaxPly
        gs.moveBufLen = [0 for x = 1:MaxPly]::Array{Int,1}
        gs.moveBuf = [Move(0,0,0,0,0,0) for x = 1:MaxMoves]
        gs.triangularLength = [0 for x = 1:MaxPly]::Array{Int,1}
        gs.triangularArray  = [Move(0,0,0,0,0,0) for x = 1:MaxPly, y = 1:MaxPly]::Array{Move,2}
        gs.followpv = true
        gs.allownull = true
        gs.depth = IDdepth
        # score::Int = AlphaBeta(gs, 0, IDdepth,-Infinity,Infinity) # ply=0
        score::Int = PVS(gs, 0, IDdepth,-Infinity,Infinity) # ply=0
        # println("Depth = ", IDdepth, ", Score = ", score)
        if gs.timedout
            return gs.lastPV[1]
        end
        rememberPV(gs)
        timeInMSecs::Int = int((time_ns() - gs.nsStart)/1000000)
        NPS::Int = int(gs.inodes / ((time_ns() - gs.nsStart)/1000000000))
        ## info time 203 nodes 11111111 score cp 11168 pv 5e9i+ ....
        print("info time ",timeInMSecs," depth ", int(IDdepth), " nodes ", gs.inodes, " score cp ", score, " nps ",NPS," pv")
        for i = 1:gs.triangularLength[1]
	    print(" ",move2USIString(gs.lastPV[i]))
        end
        println()
    end
    move = gs.lastPV[1]
    return move
end

function showPVforUSI(IDdepth::Float64, score::Int64, gs::GameStatus)
    i = 0::Int
    timeInMSecs = int((time_ns() - gs.nsStart)/1000000)::Int
    NPS = int(gs.inodes / ((time_ns() - gs.nsStart)/1000000000))::Int
    ## info time 203 nodes 11111111 score cp 11168 pv 5e9i+ ....
    print("info time ",timeInMSecs," depth ", int(IDdepth), " nodes ", gs.inodes, " score cp ", score, " nps ",NPS," pv")
    for i = 1:gs.triangularLength[1]
	print(" ",move2USIString(gs.lastPV[i]))
    end
    println()
end

# search driver
function thinkASP( sengo::Int, gs::GameStatus)
    move::Move = Move(0,0,0,0,0,0) # null move
    bestMove::Move = Move(0,0,0,0,0,0) # null move # added recently
    gs.lastPV = [Move(0,0,0,0,0,0) for x = 1:MaxPly]::Array{Move,1}
    gs.lastPVLength = 0
    gs.whiteHeuristics = [0 for x = 1:0xff, y = 1:0xff]::Array{Int,2}
    gs.blackHeuristics = [0 for x = 1:0xff, y = 1:0xff]::Array{Int,2}
    gs.nsStart = time_ns()
    gs.inodes = 0
    gs.timedout = false
    gs.side = sengo
    gs.board.nextMove = sengo
    gs.tt = Dict{Uint64,TransP}()
    delta::Int = 48
    score::Int = 0
    middle::Int = 0
    smallAlpha::Int = 0
    smallBeta::Int  = 0

    ev_sign = gs.side == SENTE? 1: -1

    for IDdepth = 1.0:1.0:60.0 #MaxPly
        hitScore::Bool = false

        while hitScore == false
            gs.moveBufLen = [0 for x = 1:MaxPly]::Array{Int,1}
            gs.moveBuf = [Move(0,0,0,0,0,0) for x = 1:MaxMoves]
            gs.triangularLength = [0 for x = 1:MaxPly]::Array{Int,1}
            gs.triangularArray  = [Move(0,0,0,0,0,0) for x = 1:MaxPly, y = 1:MaxPly]::Array{Move,2}
            gs.followpv = true
            gs.allownull = true
            gs.MoveBeginIndex = 0

            # score::Int = AlphaBeta(gs, 0, IDdepth,-Infinity,Infinity) # ply=0
            if IDdepth == 1.0
                score = ev_sign * SimplePVS(gs, 0, IDdepth,-Infinity,Infinity,Root,false) # ply=0
                middle = score
                hitScore = true
                smallAlpha = middle - delta
                smallBeta  = middle + delta
                bestMove = gs.lastPV[1]
            else
                score = ev_sign * SimplePVS(gs, 0, IDdepth,smallAlpha,smallBeta,Root,false) # ply=0
                middle = score

                if smallAlpha == -Infinity
                    hitScore = true
                end
                if smallBeta  == Infinity
                    hitScore = true
                end

                if smallAlpha < score < smallBeta
                    hitScore = true
                elseif score <= smallAlpha
                    delta *= 2
                    smallAlpha = middle - delta
                    if delta >= 2000
                        smallAlpha = -Infinity
                    end
                    showPVforUSI(IDdepth,score,gs)
                elseif score >= smallBeta
                    delta *= 2
                    smallBeta = middle + delta
                    if delta >= 2000
                        smallBeta = Infinity
                    end
                    showPVforUSI(IDdepth,score,gs)
                end
            end
        end
        # println("Depth = ", IDdepth, ", Score = ", score)
        if gs.timedout
            return bestMove
        end
        rememberPV(gs)
        showPVforUSI(IDdepth,score,gs)
        bestMove = gs.lastPV[1]
    end
    move = bestMove
    return move
end

function ThinkTest(gs::GameStatus)
    m::Move = think(SENTE,gs)
end


function show_args(args)
  @show args
end

# Example of running this routine:

include("run_hmc_background_student_rescale.jl")

# File containing the transit times:
#fname = "../../data/T1_timings_20191203.txt"
fname = "timing_simon.txt"

felements = "../../data/elements_noprior_students.txt"

# Name an output file:
#foutput = string("T1_run_hmc_student_ecc_lgndof_V1exp2nuinv_nstep2000_eps0.1_nleap20_",show_args(ARGS)[1],".jld2")
foutput = string("T1_run_hmc_student_ecc_lgndof_V1exp2nuinv_nstep10_eps0.1_nleap20_",show_args(ARGS)[1],".jld2")

# Call the likelihood profile function:
#run_hmc(fname,felements,foutput,2000,0.1,20,false)
run_hmc(fname,felements,foutput,10,0.1,20,false)


module Rdatatable

using StatsBase, RCall, Query, DataFrames, DataFramesMeta, DataTables
using IndexedTables
using IndexedTables.Table

function R_datatable(N,K)

    R"""
    library(data.table)
    N <- $N
    K <- $K
    # copied from
    # https://github.com/Rdatatable/data.table/wiki/Benchmarks-%3A-Grouping
    set.seed(1)
    DT <- data.table(
      id1 = sample(sprintf("id%03d",1:K), N, TRUE),      # large groups (char)
      id2 = sample(sprintf("id%03d",1:K), N, TRUE),      # large groups (char)
      id3 = sample(sprintf("id%010d",1:(N/K)), N, TRUE), # small groups (char)
      id4 = sample(K, N, TRUE),                          # large groups (int)
      id5 = sample(K, N, TRUE),                          # large groups (int)
      id6 = sample(N/K, N, TRUE),                        # small groups (int)
      v1 =  sample(5, N, TRUE),                          # int in range [1,5]
      v2 =  sample(5, N, TRUE),                          # int in range [1,5]
      v3 =  sample(round(runif(100,max=100),4), N, TRUE) # numeric e.g. 23.5749
    )

    timings <- list()

    timings$sum1 <- system.time( DT[, sum(v1), keyby=id1] )[3]
    timings$sum2 <- system.time( DT[, sum(v1), keyby=id1] )[3]
    timings$sum3 <- system.time( DT[, sum(v1), keyby="id1,id2"] )[3]
    timings$sum4 <- system.time( DT[, sum(v1), keyby="id1,id2"] )[3]
    timings$sum_mean1 <- system.time( DT[, list(sum(v1),mean(v3)), keyby=id3] )[3]
    timings$sum_mean2 <- system.time( DT[, list(sum(v1),mean(v3)), keyby=id3] )[3]
    timings$mean7_9_by_id4_1 <- system.time( DT[, lapply(.SD, mean), keyby=id4, .SDcols=7:9] )[3]
    timings$mean7_9_by_id4_2 <- system.time( DT[, lapply(.SD, mean), keyby=id4, .SDcols=7:9] )[3]
    timings$sum7_9_by_id6_1 <- system.time( DT[, lapply(.SD, sum), keyby=id6, .SDcols=7:9] )[3]
    timings$sum7_9_by_id6_2 <- system.time( DT[, lapply(.SD, sum), keyby=id6, .SDcols=7:9] )[3]
    """
    @rget timings
    return timings
end

function R_dplyr(N,K)

    R"""
    library(dplyr)
    N <- $N
    K <- $K
    # copied from
    # https://github.com/Rdatatable/data.table/wiki/Benchmarks-%3A-Grouping
    set.seed(1)
    DF <- data.frame(
      id1 = sample(sprintf("id%03d",1:K), N, TRUE),      # large groups (char)
      id2 = sample(sprintf("id%03d",1:K), N, TRUE),      # large groups (char)
      id3 = sample(sprintf("id%010d",1:(N/K)), N, TRUE), # small groups (char)
      id4 = sample(K, N, TRUE),                          # large groups (int)
      id5 = sample(K, N, TRUE),                          # large groups (int)
      id6 = sample(N/K, N, TRUE),                        # small groups (int)
      v1 =  sample(5, N, TRUE),                          # int in range [1,5]
      v2 =  sample(5, N, TRUE),                          # int in range [1,5]
      v3 =  sample(round(runif(100,max=100),4), N, TRUE) # numeric e.g. 23.5749
    )

    timings <- list()

    timings$sum1             <- system.time( DF %>% group_by(id1) %>% summarise(sum(v1)) )[3]
    timings$sum2             <- system.time( DF %>% group_by(id1) %>% summarise(sum(v1)) )[3]
    timings$sum3             <- system.time( DF %>% group_by(id1,id2) %>% summarise(sum(v1)) )[3]
    timings$sum4             <- system.time( DF %>% group_by(id1,id2) %>% summarise(sum(v1)) )[3]
    timings$sum_mean1        <- system.time( DF %>% group_by(id3) %>% summarise(sum(v1),mean(v3)) )[3]
    timings$sum_mean2        <- system.time( DF %>% group_by(id3) %>% summarise(sum(v1),mean(v3)) )[3]
    timings$mean7_9_by_id4_1 <- system.time( DF %>% group_by(id4) %>% summarise_at(c("v1","v2","v3"),mean) )[3]
    timings$mean7_9_by_id4_2 <- system.time( DF %>% group_by(id4) %>% summarise_at(c("v1","v2","v3"),mean) )[3]
    timings$sum7_9_by_id6_1  <- system.time( DF %>% group_by(id6) %>% summarise_at(c("v1","v2","v3"),sum) )[3]
    timings$sum7_9_by_id6_2  <- system.time( DF %>% group_by(id6) %>% summarise_at(c("v1","v2","v3"),sum) )[3]
    """
    @rget timings
    return timings
end

function createDataFrame(N::Int,K::Int)

    df = DataFrame(id1 = sample(["id$x" for x in 1:K],N),
                   id2 = sample(["id$x" for x in 1:K],N),
                   id3 = sample(["id$x" for x in 1:(N/K)],N),
                   id4 = sample(1:K,N),
                   id5 = sample(1:K,N),
                   id6 = sample(1:(N/K),N),
                   v1 = sample(1:5,N),
                   v2 = sample(1:5,N),
                   v3 = sample(round.(rand(100),4),N))

    return df
end

function createDataTable(N::Int,K::Int)

    df = DataTable(id1 = sample(["id$x" for x in 1:K],N),
                   id2 = sample(["id$x" for x in 1:K],N),
                   id3 = sample(["id$x" for x in 1:(N/K)],N),
                   id4 = sample(1:K,N),
                   id5 = sample(1:K,N),
                   id6 = sample(1:(N/K),N),
                   v1 = sample(1:5,N),
                   v2 = sample(1:5,N),
                   v3 = sample(round.(rand(100),4),N))

    return df
end
function createIndexedTable(N::Int,K::Int)

    df = Table(Columns(id1 = sample(["id$x" for x in 1:K],N),
                   id2 = sample(["id$x" for x in 1:K],N),
                   id3 = sample(["id$x" for x in 1:(N/K)],N),
                   id4 = sample(1:K,N),
                   id5 = sample(1:K,N),
                   id6 = sample(1:(N/K),N),
                   v1 = sample(1:5,N),
                   v2 = sample(1:5,N),
                   v3 = sample(round.(rand(100),4),N)))

    return df
end

function benches(df::DataFrame)

    # timings
    ti = Dict()

    ti[:sum1] = @elapsed @from i in df begin
                             @group i by i.id1 into g
                             @select {r=sum(g.v1)}
                             @collect DataFrame
                         end
    ti[:sum2] = @elapsed @from i in df begin
                             @group i by i.id1 into g
                             @select {r=sum(g.v1)}
                             @collect DataFrame
                         end
    ti[:sum3] = @elapsed @from i in df begin
                             @group i by (i.id1,i.id2) into g
                             @select {r=sum(g.v1)}
                             @collect DataFrame
                         end
    ti[:sum4] = @elapsed @from i in df begin
                             @group i by (i.id1,i.id2) into g
                             @select {r=sum(g.v1)}
                             @collect DataFrame
                         end
    ti[:sum_mean1] = @elapsed @from i in df begin
                             @group i by i.id3 into g
                             @select {s=sum(g.v1),m=mean(g.v3)}
                             @collect DataFrame
                         end
    ti[:sum_mean2] = @elapsed @from i in df begin
                             @group i by i.id3 into g
                             @select {s=sum(g.v1),m=mean(g.v3)}
                             @collect DataFrame
                         end
    ti[:mean7_9_by_id4_1] = @elapsed @from i in df begin
                             @group i by i.id4 into g
                             @select {m7=mean(g.v1),m8=mean(g.v2),m9=mean(g.v3)}
                             @collect DataFrame
                         end
    ti[:mean7_9_by_id4_2] = @elapsed @from i in df begin
                             @group i by i.id4 into g
                             @select {m7=mean(g.v1),m8=mean(g.v2),m9=mean(g.v3)}
                             @collect DataFrame
                         end
    ti[:sum7_9_by_id6_1] = @elapsed @from i in df begin
                             @group i by i.id6 into g
                             @select {m7=mean(g.v1),m8=mean(g.v2),m9=mean(g.v3)}
                             @collect DataFrame
                         end
    ti[:sum7_9_by_id6_2] = @elapsed @from i in df begin
                             @group i by i.id6 into g
                             @select {m7=mean(g.v1),m8=mean(g.v2),m9=mean(g.v3)}
                             @collect DataFrame
                         end
    return ti
end
function benches(df::DataTable)

    # timings
    ti = Dict()

    ti[:sum1] = @elapsed @from i in df begin
                             @group i by i.id1 into g
                             @select {r=sum(g.v1)}
                             @collect DataFrame
                         end
    ti[:sum2] = @elapsed @from i in df begin
                             @group i by i.id1 into g
                             @select {r=sum(g.v1)}
                             @collect DataFrame
                         end
    ti[:sum3] = @elapsed @from i in df begin
                             @group i by (i.id1,i.id2) into g
                             @select {r=sum(g.v1)}
                             @collect DataFrame
                         end
    ti[:sum4] = @elapsed @from i in df begin
                             @group i by (i.id1,i.id2) into g
                             @select {r=sum(g.v1)}
                             @collect DataFrame
                         end
    ti[:sum_mean1] = @elapsed @from i in df begin
                             @group i by i.id3 into g
                             @select {s=sum(g.v1),m=mean(g.v3)}
                             @collect DataFrame
                         end
    ti[:sum_mean2] = @elapsed @from i in df begin
                             @group i by i.id3 into g
                             @select {s=sum(g.v1),m=mean(g.v3)}
                             @collect DataFrame
                         end
    ti[:mean7_9_by_id4_1] = @elapsed @from i in df begin
                             @group i by i.id4 into g
                             @select {m7=mean(g.v1),m8=mean(g.v2),m9=mean(g.v3)}
                             @collect DataFrame
                         end
    ti[:mean7_9_by_id4_2] = @elapsed @from i in df begin
                             @group i by i.id4 into g
                             @select {m7=mean(g.v1),m8=mean(g.v2),m9=mean(g.v3)}
                             @collect DataFrame
                         end
    ti[:sum7_9_by_id6_1] = @elapsed @from i in df begin
                             @group i by i.id6 into g
                             @select {m7=mean(g.v1),m8=mean(g.v2),m9=mean(g.v3)}
                             @collect DataFrame
                         end
    ti[:sum7_9_by_id6_2] = @elapsed @from i in df begin
                             @group i by i.id6 into g
                             @select {m7=mean(g.v1),m8=mean(g.v2),m9=mean(g.v3)}
                             @collect DataFrame
                         end
    return ti
end
function benches(df::IndexedTable)

    # timings
    ti = Dict()

    ti[:sum1] = @elapsed @from i in df begin
                             @group i by i.id1 into g
                             @select {r=sum(g.v1)}
                             @collect DataFrame
                         end
    ti[:sum2] = @elapsed @from i in df begin
                             @group i by i.id1 into g
                             @select {r=sum(g.v1)}
                             @collect DataFrame
                         end
    ti[:sum3] = @elapsed @from i in df begin
                             @group i by (i.id1,i.id2) into g
                             @select {r=sum(g.v1)}
                             @collect DataFrame
                         end
    ti[:sum4] = @elapsed @from i in df begin
                             @group i by (i.id1,i.id2) into g
                             @select {r=sum(g.v1)}
                             @collect DataFrame
                         end
    ti[:sum_mean1] = @elapsed @from i in df begin
                             @group i by i.id3 into g
                             @select {s=sum(g.v1),m=mean(g.v3)}
                             @collect DataFrame
                         end
    ti[:sum_mean2] = @elapsed @from i in df begin
                             @group i by i.id3 into g
                             @select {s=sum(g.v1),m=mean(g.v3)}
                             @collect DataFrame
                         end
    ti[:mean7_9_by_id4_1] = @elapsed @from i in df begin
                             @group i by i.id4 into g
                             @select {m7=mean(g.v1),m8=mean(g.v2),m9=mean(g.v3)}
                             @collect DataFrame
                         end
    ti[:mean7_9_by_id4_2] = @elapsed @from i in df begin
                             @group i by i.id4 into g
                             @select {m7=mean(g.v1),m8=mean(g.v2),m9=mean(g.v3)}
                             @collect DataFrame
                         end
    ti[:sum7_9_by_id6_1] = @elapsed @from i in df begin
                             @group i by i.id6 into g
                             @select {m7=mean(g.v1),m8=mean(g.v2),m9=mean(g.v3)}
                             @collect DataFrame
                         end
    ti[:sum7_9_by_id6_2] = @elapsed @from i in df begin
                             @group i by i.id6 into g
                             @select {m7=mean(g.v1),m8=mean(g.v2),m9=mean(g.v3)}
                             @collect DataFrame
                         end
    return ti
end

function DfMeta_benches(df::DataFrame)

    # timings
    ti = Dict()

    ti[:sum1] = @elapsed @linq df |>
                    @by(:id1,r = sum(:v1))

    ti[:sum2] = @elapsed @linq df |>
                    @by(:id1,r = sum(:v1))

    ti[:sum3] = @elapsed @linq df |>
                    @by([:id1,:id2],r = sum(:v1))

    ti[:sum4] = @elapsed @linq df |>
                    @by([:id1,:id2],r = sum(:v1))

    ti[:sum_mean1] = @elapsed @linq df |>
                    @by(:id3,s = sum(:v1),m=mean(:v1))

    ti[:sum_mean2] = @elapsed @linq df |>
                    @by(:id3,s = sum(:v1),m=mean(:v1))

    ti[:mean7_9_by_id4_1] = @elapsed @linq df |>
                        @by(:id4,m7=mean(:v1),m8=mean(:v2),m9=mean(:v3))

    ti[:mean7_9_by_id4_2] = @elapsed @linq df |>
                        @by(:id4,m7=mean(:v1),m8=mean(:v2),m9=mean(:v3))

    ti[:sum7_9_by_id6_1] = @elapsed @linq df |>
                        @by(:id6,m7=mean(:v1),m8=mean(:v2),m9=mean(:v3))

    ti[:sum7_9_by_id6_2] = @elapsed @linq df |>
                        @by(:id6,m7=mean(:v1),m8=mean(:v2),m9=mean(:v3))

    return ti
end

function run_benches(N=1_000_000;K=100)
    # get small data for JIT warmup
    d_ = createDataFrame(10,3)
    dt_ = createDataTable(10,3)
    di_ = createIndexedTable(10,3)

    # warm up julia benchmarks
    benches(d_);
    benches(dt_);
    benches(di_);
    DfMeta_benches(d_);

    # get real data
    d = createDataFrame(N,K)
    # measure DataFrames
    query_df = benches(d)
    meta = DfMeta_benches(d)
    d = 0
    gc()
    # measure DataTables
    dt = createDataTable(N,K)
    query_dt = benches(dt)
    dt = 0
    gc()
    # measure IndexedTables
    di = createIndexedTable(N,K)
    query_di = benches(di)
    di = 0
    gc()

    # get R data.table time
    R = R_datatable(N,K)

    # get R dplyr time
    Rdplyr = R_dplyr(N,K)

    # get
    k = collect(keys(query_df))
    out = DataFrame(bench = k,Query_DF = [query_df[kk] for kk in k],Query_DT= [query_dt[kk] for kk in k],Query_idxT= [query_di[kk] for kk in k],DataFramesMeta=[meta[kk] for kk in k], Rdplyr=[Rdplyr[kk] for kk in k], Rdatatable=[R[kk] for kk in k])
    sort!(out,cols=:bench)
    rel = deepcopy(out)
    rel = @transform(rel,Query_DF = :Query_DF./:Rdatatable, Query_DT = :Query_DT./:Rdatatable, Query_idxT = :Query_idxT./:Rdatatable, DataFramesMeta = :DataFramesMeta ./ :Rdatatable, Rdplyr = :Rdplyr ./ :Rdatatable, Rdatatable = 1.0)
    return (out,rel)
end

function run_all()
    d=Dict()
    for n in [10_000, 100_000, 1_000_000, 2_000_000]
        d[n] = run_benches(n)
        gc()
        info("results with $n rows:")
        info("-- absolute time in seconds:")
        println(d[n][1])
        info("-- time relative to R data.table:")
        println(d[n][2])
        println()
        println()
    end
    return d
end
end # module