# This file is a part of JuliaFEM.
# License is MIT: see https://github.com/JuliaFEM/JuliaFEM.jl/blob/master/LICENSE.md

using HDF5
using JuliaFEM

function aster_parse_nodes(section; strip_characters=true)
    nodes = Dict{Any, Vector{Float64}}()
    has_started = false
    for line in split(section, '\n')
        m = matchall(r"[\w.-]+", line)
        if (length(m) != 1) && (!has_started)
            continue
        end
        if length(m) == 1
            if (m[1] == "COOR_2D") || (m[1] == "COOR_3D")
                has_started = true
                continue
            end
            if m[1] == "FINSF"
                break
            end
        end
        if length(m) == 4
            nid = m[1]
            if strip_characters
                nid = matchall(r"\d", nid)
                nid = parse(Int, nid[1])
            end
            nodes[nid] = float(m[2:end])
        end
    end
    return nodes
end

function parse(mesh, ::Type{Val{:CODE_ASTER_MAIL}})
    model = Dict()
    header = nothing
    data = []
    for line in split(mesh, '\n')
        length(line) != 0 || continue
        info("line: $line")
        if is_aster_mail_keyword(strip(line))
            header = parse_aster_header(line)
            empty!(data)
            continue
        end
        if line == "FINSF"
            info(data)
            header = nothing
            process_aster_section!(model, join(data, ""), header, Val{header[1]})
        end
    end
    return model
end

""" Code Aster binary file (.med). """
type MEDFile
    data :: Dict
end

function MEDFile(fn)
    return MEDFile(h5read(fn, "/"))
end

function get_mesh_names(med::MEDFile)
    return sort(collect(keys(med.data["FAS"])))
end

function get_nodes(med::MEDFile, nsets, mesh_name)
    increments = keys(med.data["ENS_MAA"][mesh_name])
    @assert length(increments) == 1
    increment = first(increments)
    nodes = med.data["ENS_MAA"][mesh_name][increment]["NOE"]
    node_ids = nodes["NUM"]
    nset_ids = nodes["FAM"]
    nnodes = length(node_ids)
    node_coords = nodes["COO"]
    dim = round(Int, length(node_coords)/nnodes)
    node_coords = reshape(node_coords, nnodes, dim)'
    d = Dict{Int64}{Tuple{Symbol, Vector{Float64}}}()
    for i=1:nnodes
        nset = Symbol(nsets[nset_ids[i]])
        d[node_ids[i]] = (nset, node_coords[:, i])
    end
    return d
end

function get_node_sets(med::MEDFile, mesh_name)
    ns = Dict{Int64, Symbol}(0 => :NALL)
    if !haskey(med.data["FAS"], mesh_name)
        warn("Mesh $mesh_name not found from med file.")
        meshes = get_mesh_names(med)
        all_meshes = join(meshes, ", ")
        warn("Available meshes: $all_meshes")
        error("Mesh $mesh_name not found.")
    end
    haskey(med.data["FAS"][mesh_name], "NOEUD") || return ns
    nsets = med.data["FAS"][mesh_name]["NOEUD"]
    for nset in keys(nsets)
        k = split(nset, "_")
        nset_id = parse(Int, k[2])
        nset_name = ascii(pointer(convert(Vector{UInt8}, nsets[nset]["GRO"]["NOM"][1])))
        ns[nset_id] = Symbol(nset_name)
    end
    return ns
end

function get_element_sets(med::MEDFile, mesh_name)
    es = Dict{Int64, Symbol}()
    if !haskey(med.data["FAS"][mesh_name], "ELEME")
        return es
    end
    elsets = med.data["FAS"][mesh_name]["ELEME"]
    for elset in keys(elsets)
        k = split(elset, '_')
        elset_id = parse(Int, k[2])
        elset_name = ascii(pointer(convert(Vector{UInt8}, elsets[elset]["GRO"]["NOM"][1])))
        es[elset_id] = Symbol(elset_name)
    end
    return es
end

function get_connectivity(med::MEDFile, elsets, mesh_name)
    elsets[0] = :OTHER
    increments = keys(med.data["ENS_MAA"][mesh_name])
    @assert length(increments) == 1
    increment = first(increments)
    all_elements = med.data["ENS_MAA"][mesh_name][increment]["MAI"]
    d = Dict{Int64, Tuple{Symbol, Symbol, Vector{Int64}}}()
    for eltype in keys(all_elements)
        elements = all_elements[eltype]
        elset_ids = elements["FAM"]
        element_ids = elements["NUM"]
        nelements = length(element_ids)
        element_connectivity = elements["NOD"]
        element_dim = round(Int, length(element_connectivity)/nelements)
        element_connectivity = reshape(element_connectivity, nelements, element_dim)'
        for i=1:nelements
            eltype = Symbol(eltype)
            elco = element_connectivity[:, i]
            elset = Symbol(elsets[elset_ids[i]])
            d[element_ids[i]] = (eltype, elset, elco)
        end
    end
    return d
end

""" Parse code aster .med file.

Paramters
---------
fn
    file name to parse
mesh_name :: optional
    mesh name, if several meshes in one file

Returns
-------
Dict containing fields "nodes" and "connectivity".

"""
function parse_aster_med_file(fn, mesh_name=nothing; debug=false)
    med = MEDFile(fn)
    mesh_names = get_mesh_names(med::MEDFile)
    all_meshes = join(mesh_names, ", ")
    if mesh_name == nothing
        length(mesh_names) == 1 || error("several meshes found from med, pick one: $all_meshes")
        mesh_name = mesh_names[1]
    else
        mesh_name in mesh_names || error("Mesh $mesh_name not found from mesh file $fn. Available meshes: $all_meshes")
    end
    nsets = get_node_sets(med, mesh_name)
    elsets = get_element_sets(med, mesh_name)
    if debug
        elset_names = join(values(elsets), ", ")
        info("Code Aster .med reader: found $(length(elsets)) element sets: $elset_names")
        nset_names = join(values(nsets), ", ")
        info("Code ASter .med reader: found $(length(nsets)) node sets: $nset_names")
    end
    nodes = get_nodes(med, nsets, mesh_name)
    conn = get_connectivity(med, elsets, mesh_name)
    result = Dict("nodes" => nodes, "connectivity" => conn)
    result["nodes"] = nodes
    result["connectivity"] = conn
    return result
end

# some glues about ordering, this is still a mystery..
# http://onelab.info/pipermail/gmsh/2008/003850.html
# http://caelinux.org/wiki/index.php/Proj:UNVConvert

#global const med_connectivity = Dict{Symbol, Vector{Int}}(
#    :Tet4 => [3, 2, 1, 4],
#    :Hex8 => [4, 8, 7, 3, 1, 5, 6, 2],  # ..?
#    :Tet10 => [3, 2, 1, 4, 6, 5, 7, 10, 9, 8])

global const med_connectivity = Dict{Symbol, Vector{Int}}(
    :Tet4   => [4,3,1,2],
    :Tet10  => [4,3,1,2,10,7,8,9,6,5],
    :Wedge6 => [4,5,6,1,2,3],
    :Hex8   => [4,8,7,3,1,5,6,2],
    :Hex20  => [4,8,7,3,1,5,6,2,20,15,19,11,12,16,14,10,17,13,18,9],
    :Hex27  => [4,8,7,3,1,5,6,2,20,15,19,11,12,16,14,10,17,13,18,9,24,25,26,23,21,22,27])

# element names in CA -> element names in JuliaFEM
global const mapping = Dict(

    :PO1 => :Poi1,

    :SE2 => :Seg2,
    :SE3 => :Seg3,
    :SE4 => :Seg4,

    :TR3 => :Tri3,
    :TR6 => :Tri6,
    :TR7 => :Tri7,

    :QU4 => :Quad4,
    :QU8 => :Quad8,
    :QU9 => :Quad9,

    :TE4 => :Tet4,
    :T10 => :Tet10,

    :PE6 => :Wedge6,
    :P15 => :Wedge15,
    :P18 => :Wedge18,

    :HE8 => :Hex8,
    :H20 => :Hex20,
    :H27 => :Hex27,

    :PY5 => :Pyramid5,
    :P13 => :Pyramid13,

    )

function aster_read_mesh(fn, mesh_name=nothing; reorder_element_connectivity=true)
    result = parse_aster_med_file(fn, mesh_name)
    mesh = Mesh()
    for (nid, (nset, ncoords)) in result["nodes"]
        add_node!(mesh, nid, ncoords)
        add_node_to_node_set!(mesh, nset, nid)
    end
    for (elid, (eltype, elset, elcon)) in result["connectivity"]
        haskey(mapping, eltype) || error("Code Aster .med reader: element type $eltype not found from mapping")
        add_element!(mesh, elid, mapping[eltype], elcon)
        add_element_to_element_set!(mesh, elset, elid)
    end
    if reorder_element_connectivity
        reorder_element_connectivity!(mesh, med_connectivity)
    end
    return mesh
end

""" Code Aster result file (.rmed). """
type RMEDFile
    data :: Dict
end

function RMEDFile(fn)
    return RMEDFile(h5read(fn, "/"))
end

""" Return nodes from result med file. """
function aster_read_nodes(rmed::RMEDFile)
    increments = keys(rmed.data["ENS_MAA"]["MAIL"])
    @assert length(increments) == 1
    increment = first(increments)
    nodes = rmed.data["ENS_MAA"]["MAIL"][increment]["NOE"]
    node_names = nodes["NOM"]
    node_coords = nodes["COO"]
    nnodes = length(node_names)
    dim = round(Int, length(node_coords)/nnodes)
    node_coords = reshape(node_coords, nnodes, dim)'
    stripper(node_name) = strip(ascii(pointer(convert(Vector{UInt8}, node_name))))
    node_names = map(stripper, node_names)
    # INFO: quite safe assumption is that id is in node name, i.e. N1 => 1, N123 => 123
    node_id(node_name) = parse(matchall(r"\d+", node_name)[1])
    node_ids = map(node_id, node_names)
    nodes = Dict([j => node_coords[:,j] for j in node_ids])
    return nodes
end

""" Read nodal field from rmed file. """
function aster_read_data(rmed::RMEDFile, field_name; field_type=:NODE,
                         info_fields=true, node_ids=nothing)

    if contains(field_name, "ELGA")
        field_type = :GAUSS
    end

    if node_ids == nothing
        nodes = aster_read_nodes(rmed)
        node_ids = sort(collect(keys(nodes)))
    end

    if info_fields
        field_names = keys(rmed.data["CHA"])
        all_fields = join(field_names, ", ")
        info("results: $all_fields")
    end

    chdata = rmed.data["CHA"]["RESU____$field_name"]
    @assert length(chdata) == 1
    increment = chdata[first(keys(chdata))]
    if field_type == :NODE
        data = increment["NOE"]["MED_NO_PROFILE_INTERNAL"]["CO"]
        results = Dict([j => data[j] for j in node_ids])
    else
        error("Unable to read result of type $field_type: not implemented")
    end
    return results
end


# create NNN for an arbitrary lattice by bond distance
function getBondsNNNByBondDistance(
            unitcell :: U,
            labelfunction :: Function = (b1,b2) -> getDefaultLabel(L,"NNN")
        ) :: Vector{B} where {N,L,S,B<:AbstractBond{L,N},U<:AbstractUnitcell{S,B}}

    # obtain the organized bond lists
    bonds_organized_from = organizedBondsFrom(unitcell)
    bonds_organized_to   = organizedBondsTo(unitcell)

    # create all possible bonds by concanation of nearest neighbors
    bonds_NNN = B[
        concanateBonds(b1, b2, labelfunction)
        for i in 1:numSites(unitcell)
        for b1 in bonds_organized_to[i]
        for b2 in bonds_organized_from[i]
    ]
    # filter out the ones that connect to the same site
    filter!(b -> !(!isPeriodic(b) && from(b)==to(b)), bonds_NNN)

    # return the remaining bonds
    return bonds_NNN
end

# export the function
export getBondsNNNByBondDistance


using Test
using MultivariatePolynomials

# Allocating size for allocating a `BigInt`.
# Half size on 32-bit.
const BIGINT_ALLOC = Sys.WORD_SIZE == 64 ? 48 : 24

function alloc_test(f, n)
    f() # compile
    @test n == @allocated f()
end

struct CustomPoly{T, P<:AbstractPolynomial{T}} <: AbstractPolynomialLike{T}
    p::P
end
CustomPoly(p::AbstractPolynomial{T}) where T = CustomPoly{T, typeof(p)}(p)
MultivariatePolynomials.polynomial(p::CustomPoly) = p.p
MultivariatePolynomials.polynomial(p::CustomPoly, T::Type) = polynomial(p.p, T)
MultivariatePolynomials.variables(p::CustomPoly) = variables(p.p)
MultivariatePolynomials.monomialtype(::Type{<:CustomPoly{T, P}}) where {T, P} = monomialtype(P)
MultivariatePolynomials.constantmonomial(p::CustomPoly) = constantmonomial(p.p)

struct CustomTerms{T, P<:AbstractPolynomial{T}} <: AbstractPolynomialLike{T}
    p::P
end
CustomTerms(p::AbstractPolynomial{T}) where T = CustomTerms{T, typeof(p)}(p)
MultivariatePolynomials.terms(p::CustomTerms) = terms(p.p)
MultivariatePolynomials.variables(p::CustomTerms) = variables(p.p)
MultivariatePolynomials.monomialtype(::Type{<:CustomTerms{T, P}}) where {T, P} = monomialtype(P)
MultivariatePolynomials.constantmonomial(p::CustomPoly) = constantmonomial(p.p)

function _typetests(x, ::Type{T}) where T
    @test (@inferred coefficienttype(x)) == Int

    @test (@inferred monomialtype(x)) <: AbstractMonomial

    @test (@inferred termtype(x)) <: AbstractTerm{Int}
    @test (@inferred termtype(x, Float64)) <: AbstractTerm{Float64}

    @test (@inferred polynomialtype(x)) <: AbstractPolynomial{Int}
    @test (@inferred polynomialtype(x, Float64)) <: AbstractPolynomial{Float64}

    @test (@inferred monovectype(x)) <: AbstractArray{<:AbstractMonomial}
end

function typetests(x::Union{AbstractPolynomialLike{T}, Vector{<:AbstractPolynomialLike{T}}}) where T
    _typetests(x, T)
    _typetests(typeof(x), T)
end


import DataFrames
import StatsBase

function get_countmap(itr; skip_missings::Bool = true)
    if skip_missings
        result = get_countmap_skip_missings(itr)
    else
        result = get_countmap_include_missings(itr)
    end
    return result
end

function get_countmap_skip_missings(itr)
    result = StatsBase.countmap(collect(skipmissing(itr)))
    return result
end

function get_countmap_include_missings(itr)
    result = StatsBase.countmap(collect(itr))
    return result
end

function get_unique_values(itr; skip_missings::Bool = true)
    if skip_missings
        result = get_unique_values_skip_missings(itr)
    else
        result = get_unique_values_include_missings(itr)
    end
    return result
end

function get_unique_values_skip_missings(itr)
    result = keys(get_countmap_skip_missings(itr))
    return result
end

function get_unique_values_include_missings(itr)
    result = keys(get_countmap_include_missings(itr))
    return result
end

function get_number_of_unique_values(itr; skip_missings::Bool = true)::Int
    if skip_missings
        result = get_number_of_unique_values_skip_missings(itr)
    else
        result = get_number_of_unique_values_include_missings(itr)
    end
    return result
end

function get_number_of_unique_values_skip_missings(itr)::Int
    result::Int = length(get_unique_values_skip_missings(itr))
    return result
end

function get_number_of_unique_values_include_missings(itr)::Int
    result::Int = length(get_unique_values_include_missings(itr))
    return result
end

function find_constant_columns(
        df::DataFrames.AbstractDataFrame,
        )::Vector{Symbol}
    list_of_constant_column_names = Symbol[]
    for x in DataFrames.names(df)
        if get_number_of_unique_values(df[x]) < 2
            push!(list_of_constant_column_names, x)
        end
    end
    return list_of_constant_column_names
end

function check_no_constant_columns(
        df::DataFrames.AbstractDataFrame,
        )::Bool
    list_of_constant_column_names = find_constant_columns(df)
    if length(list_of_constant_column_names) > 0
        error(
            string(
                "Data frame contains the following constant columns: ",
                list_of_constant_column_names,
                "\"",
                )
            )
    else
        return true
    end
end


export ProdPreserveType, ProdPreserveTypeLeft, ProdPreserveTypeRight

import Base: prod

"""
    ProdPreserveType{T}

`ProdPreserveType` is one of the strategies for `prod` function. This strategy constraint an output of a prod to be in some specific form.
By default it fallbacks to a `ProdAnalytical` strategy and converts an output to a prespecified type but can be overwritten for some distributions
for better performance.

See also: [`prod`](@ref), [`ProdAnalytical`](@ref), [`ProdPreserveTypeLeft`](@ref), [`ProdPreserveTypeRight`](@ref)
"""
struct ProdPreserveType{T} end

ProdPreserveType(::Type{T}) where T = ProdPreserveType{T}()

prod(::ProdPreserveType{T}, left, right) where T = convert(T, prod(ProdAnalytical(), left, right))

"""
    ProdPreserveTypeLeft

`ProdPreserveTypeLeft` is one of the strategies for `prod` function. This strategy constraint an output of a prod to be in the functional form as `left` argument.
By default it fallbacks to a `ProdPreserveType` strategy and converts an output to a prespecified type but can be overwritten for some distributions
for better performance.

See also: [`prod`](@ref), [`ProdPreserveType`](@ref), [`ProdPreserveTypeRight`](@ref)
"""
struct ProdPreserveTypeLeft end

prod(::ProdPreserveTypeLeft, left::L, right) where L = prod(ProdPreserveType(L), left, right)

"""
    ProdPreserveTypeRight

`ProdPreserveTypeRight` is one of the strategies for `prod` function. This strategy constraint an output of a prod to be in the functional form as `right` argument.
By default it fallbacks to a `ProdPreserveType` strategy and converts an output to a prespecified type but can be overwritten for some distributions
for better performance.

See also: [`prod`](@ref), [`ProdPreserveType`](@ref), [`ProdPreserveTypeLeft`](@ref)
"""
struct ProdPreserveTypeRight end

prod(::ProdPreserveTypeRight, left, right::R) where R = prod(ProdPreserveType(R), left, right)


@doc """ Driver for compare_walks_parallel.jl.
This file sets the parameters to be run.
A command line for running this program:
julia -p 8 -L "compare_walks_driver.jl" -e "run_all_jobs()"
"""
include("../src/NKLandscapes.jl")
using NKLandscapes
using Base.Test
include("compare_walks_parallel.jl")

const BOUNDARIES = ["Nearest-neighbor", "Random"]

#const N_VALUES = [60, 100, 140, 180]
const N_VALUES = [220, 240]
const K_VALUE =  8
const Q_VALUE = 0  # q for NKq landscape, set to 0 for NK landscape
const B_VALUE = 2
const RUNS = 100  # number of runs for each setting of the parameters

function run_all_jobs()
  print_header()
  trial = 1
  for n_value in N_VALUES
    jobs = [Job(trial, n_value, K_VALUE, Q_VALUE, B_VALUE) for trial = 1:RUNS ]
    run_jobs(jobs)
    trial += 1
  end
end


## run.jl : entry point method for simulating an Estimator
#
# Runs the Estimator for the given tolerance, and returns a History object.
#
# This file is part of MultilevelEstimators.jl - A Julia toolbox for
# Multilevel Monte Carlo Methods (c) Pieterjan Robbe, 2019

"""
run(estimator::Estimator, ε::Real)
run(estimator::Estimator, ε::Vector{<:Real})

Run the estimator and compute the expected value of the quantity of interest up to the given tolerance(s) `ε`. Returns a [`History`](@ref)-object that contains usefull diagnostics about the simulation.

# Examples

An example detailing how to use MultilevelEstimators is provided in the [example in the documentation](@ref Example).

See also: [`Estimator`](@ref), [`History`](@ref)
"""
function run(estimator::Estimator, tols::AbstractVector{<:Real})

    # input checking
    check_larger_than("run", tols, "supplied tolerance(s)", 0)

    # make history
    history = History()

    # run the sequence of tolerances
    for tol in tols
        clear(estimator) # prepare new run
        t = @elapsed _run(estimator,tol)
        update_history!(history, estimator, tol, t) # log the results in history
    end

    return history
end

run(estimator::Estimator, tol::Real) = run(estimator, get_tols(estimator, tol))

## Main routine ##
function _run(estimator::Estimator{T1, T2}, ϵ::Real) where {T1<:AbstractIndexSet, T2<:AbstractSampleMethod}

    # print status
    estimator[:verbose] && print_header(estimator, ϵ)

    # start "level" is 0
    L = 0

    # initial MSE splitting parameter
    θ = T1 <: SL ? 0.5 : estimator[:min_splitting]

    # main loop
    is_converged = false
    while !is_converged

        # print level
        estimator[:verbose] && print_level(estimator, L)

        # update index set
        index_set = new_index_set(estimator, L)
        estimator[:verbose] && print_index_set(estimator, index_set)

        # obtain initial variance estimate
        ns = regress_nb_of_samples(estimator, index_set, ϵ, θ, L)
        for index in index_set
            if !has_samples_at_index(estimator, index)
                sample!(estimator, index, ns[index])
            end
        end

        # add new indices to the index set
        for index in index_set
            push!(estimator, index)
        end
        set_sz(estimator, L)

        # print status
        estimator[:verbose] && print_status(estimator)

        # update value of the MSE splitting parameter
        θ = T1 <: SL ? 0.5 : estimator[:do_mse_splitting] ? compute_splitting(estimator, ϵ) : estimator[:min_splitting]

        if T2 <: MC

            # evaluate optimal number of samples
            n_opt = Dict(τ => optimal_nb_of_samples(estimator, τ, ϵ, θ) for τ in keys(estimator))

            # print optimal number of samples
            estimator[:verbose] && print_optimal_nb_of_samples(estimator, n_opt)

            # take additional samples if required
            update_samples(estimator, n_opt)

        else

            while varest(estimator) > θ*ϵ^2
                max_index = find_index_with_max_var_over_cost(estimator)

                # increase the number of samples already taken
                n_opt = next_number_of_samples(estimator, max_index)

                # print optimal number of samples
                estimator[:verbose] && print_optimal_nb_of_samples(estimator, n_opt)

                # take additional samples
                update_samples(estimator, n_opt)

                # recompute splitting parameter
                θ = T1 <: SL ? 0.5 : estimator[:do_mse_splitting] ? compute_splitting(estimator, ϵ) : estimator[:min_splitting]

                # check next iteration
                estimator[:verbose] && print_qmc_convergence(estimator, ϵ, θ)
            end

        end

        # check for convergence
        if T1 <: SL
            is_converged = true
        else
            # show status
            estimator[:verbose] && print_status(estimator)
            estimator[:verbose] && L ≥ 2 && print_mse_analysis(estimator, ϵ, θ)

            # check if the new level exceeds the maximum level
			is_converged = L > estimator[:min_index_set_param] && converged(estimator, ϵ, θ)
            if !is_converged && max_level_exceeded(estimator)
                estimator[:verbose] && warn_max_level(estimator)
                is_converged = true
            else
                # update level
                L += 1
            end
        end
    end

    # update boundary in case of AD
    T1 <: AD && update_boundary(estimator)

    # print convergence status
    estimator[:verbose] && print_convergence(estimator, T1 <: SL ? true : converged(estimator, ϵ, θ))
end

## Unbiased estimator routine ##
function _run(estimator::Estimator{<:U, <:AbstractSampleMethod}, ϵ::Real)

    # print status
    estimator[:verbose] && print_header(estimator, ϵ)

    # update index set
    index_set = collect(keys(pmf(estimator)))
    estimator[:verbose] && print_index_set(estimator, index_set)

    # add new indices to the index set
    for index in index_set
        index ∈ current_index_set(estimator) || push!(estimator, index)
    end

    # main loop
    is_converged = varest(estimator) ≤ ϵ^2
    while !is_converged

        # print status
        estimator[:verbose] && print_status(estimator)

        # increase the number of samples already taken
        n_opt = next_number_of_samples(estimator)

        # print optimal number of samples
        estimator[:verbose] && print_optimal_nb_of_samples(estimator, n_opt)

        # take additional samples
        update_samples(estimator, n_opt)

        # update pmf
        update_pmf(estimator)
        estimator[:verbose] && print_pmf(estimator)

        # check next iteration
        estimator[:verbose] && print_unbiased_convergence(estimator, ϵ)

        is_converged = varest(estimator) ≤ ϵ^2

        # print index set
        estimator[:verbose] && print_index_set(estimator, index_set)
    end

    # print convergence status
    estimator[:verbose] && print_convergence(estimator, true)
end


#!/usr/bin/julia

# pangenome_make_table.jl - Reads a pangenome profile and generates core, new, and shared gene hits for the pangenome
# By: Shaun Adkins (sadkins@som.umaryland.edu)

using ArgParse

#############
# Type defs #
#############

type HitsProfile
    profile::Dict
    genome_names::Vector{String}
    num_genomes::Integer
end

HitsProfile() = HitsProfile(Dict(), Vector{String}(), 0)

###############
# HitsProfile #
###############

function read_in_profile(hp::HitsProfile, profile)
    # Read in tab-delimited file, and grab genome names
    (data, headers) = readdlm(profile, '\t', header=true, comments=false)
    hp.num_genomes = length(headers) - 2
    hp.genome_names = headers[3:end]
    hp.profile = parse_data(hp.genome_names, data)
end

function parse_data(headers::Vector{String}, data)
    # Iterate through each line of data
    profile = Dict{String, Any}()
    for i = 1:size(data,1)
        genome = data[i,1]
        gene = data[i,2]
        # Hits to each subject genome
        hits = data[i,3:end]
        g = Dict{String, Bool}()
        for j = 1:length(hits)
            if hits[j] == 1
                g[headers[j]] = true
            end
        end

        # Create typed dictionary for genome if one does not exist
        try
			profile[genome][gene] = g
		catch KeyError e
			profile[genome] = Dict{String, Any}(gene => g)
		end
    end
    return profile
end

########
# Main #
########

function determine_sampling_rate( hp::HitsProfile, args)
    if args["comparisons"] > 0
        return estimate_multiplicity(hp.num_genomes, args["comparisons"])
    else
        return args["multiplicity"]
    end
end

function estimate_multiplicity( genomes::Integer, comparisons::Integer)
    # Determine multiplex sampling rate based on specified comparisons
    lower_mult = 0
    lower_comp = 0
    lower_theor = 0
    upper_mult = 0
    upper_comp = 0
    upper_theor = 0
    ldiff = 0
    udiff = 0

	# Bound limits for estimating multiplicity rate
	lower_bounds = 5
	upper_bounds = 5000

    # Establish multiplex bounds
    for i = lower_bounds:upper_bounds
        (est_comps, theor_comps) = estimate_comparisons(genomes, i)
        if est_comps < comparisons
            lower_mult = i
            lower_comp = est_comps
            lower_theor = theor_comps
        else
            upper_mult = i
            upper_comp = est_comps
            upper_theor = theor_comps
            break
        end
    end
    if upper_mult == lower_bounds
        (est_comps, theor_comps) = estimate_comparisons(genomes, lower_bounds)
    elseif lower_mult == upper_bounds
        return upper_bounds
    else
        lower_diff = comparisons - lower_comp
        upper_diff = upper_comp - comparisons
        if ldiff <= udiff
            return lower_mult
        else
            return upper_mult
        end
    end
end

function estimate_comparisons(genomes::Integer, mult::Integer)
    # Estimate the number of comparisons necessary for good bootstrapping
    total_comp = 0
    theor_comp = 0
    for i = range(2,genomes+1)
        theor = get_theoretical_comps(genomes, i)
        real = mult * genomes
        theor_comp += theor
        total_comp += min(theor, real)
    end
    return (total_comp, theor_comp)
end

function get_theoretical_comps(genomes::Integer, i::Integer)
    # Get number of combinations for the given size pangenome
    factorial(genomes) / (factorial(genomes - i) * factorial(i-1))
end

function create_reference_string(reference_set)
    # Create a string from a random set of genomes
    seen_vec = [ reference_set[i] for i = 1:length(reference_set) ]
    sort!(seen_vec)
    join(seen_vec, ":")
end

function categorize_shared_gene(genes_by_category, comp_genome::String, gene::String)
    # Increment shared count for this gene if we have a hit
    get!(genes_by_category[comp_genome]["shared"], gene, 0)
    genes_by_category[comp_genome]["shared"][gene] += 1
    return genes_by_category
end

function categorize_core_gene(genes_by_category, comp_genome::String, gene::String)
    # Determine if the given gene is present in all other genomes up to this point.
    genes_by_category[comp_genome]["core"][gene] = 1
    return genes_by_category
end

function categorize_new_gene(genes_by_category, comp_genome::String, gene::String)
    # Determine if the given gene is present in no other genomes up to this point.
    genes_by_category[comp_genome]["new"][gene] = 1
    return genes_by_category
end

function write_single_genomes(hp::HitsProfile, fh)
    # Write the number of genes for the individual genomes
    genome_index = Dict{String, Integer}(hp.genome_names[i] => i for i in 1:hp.num_genomes)

    # For each genome, get number of genes and write to file (also write assigned index)
    for genome in hp.genome_names
        num_genes = length(keys(hp.profile[genome]))
        index = genome_index[genome]
        write(fh, "1\t0\t0\t$num_genes\t$index\t$genome\n")
    end
end

function write_results(genes_by_category, comp_genome::String, size::Integer, hp::HitsProfile, fh)
    index = findfirst(hp.genome_names,comp_genome)
    p_size = size + 1 # Pangenome size = Ref genomes + seed genome

    # Write core/shared/new results to file
    shared_count = length(keys(genes_by_category[comp_genome]["shared"]))
    new_count = length(keys(genes_by_category[comp_genome]["new"]))
    core_count = length(keys(genes_by_category[comp_genome]["core"]))

    write(fh, "$p_size\t$core_count\t$shared_count\t$new_count\t$index\t$comp_genome\n")
end

function do_analysis_with_sampling(args, hp::HitsProfile, fh)
    multiplex = determine_sampling_rate(hp, args)

    # Iterate through each possible pangenome size
    for i = 1:(hp.num_genomes-1)
        tic()
        print("Running analysis with $i genomes...")
        sampling_counter = Dict{String, Integer}()

        fac_numer = factorial(big(hp.num_genomes-1))    # Must convert to BigInt first
        fac_denom1 = factorial(big((i+1)-1))            # ... otherwise OverflowError
        fac_denom2 = factorial(big(hp.num_genomes-(i+1)))
        true_max = fac_numer / (fac_denom1 * fac_denom2)

        # Iterate through the all genomes in set
        for j = 1:hp.num_genomes
            seen_strings = Dict{String, Bool}()
            seed_genome = hp.genome_names[j]
            # Create group of reference genomes minus the seed genome
            ref_genomes = copy(hp.genome_names) # copying the variable creates a pointer
            splice!(ref_genomes, j)
            point_count = 0
            while point_count < min(multiplex, true_max)
                # Create of string of N-reference genomes for an N+1 sized pangenome
                reference_set = rand(ref_genomes, i)
                sort_string = create_reference_string(reference_set)

                # Proceed if this is a new string
                if haskey(seen_strings, sort_string) == false
                    get!(sampling_counter, seed_genome, 1)
                    genes_by_category = Dict()
                    genes_by_category[seed_genome] = Dict("shared" => Dict(), "new" => Dict(), "core" => Dict())

                    # Iterate through all the genes in the seed genome
                    for gene in keys(hp.profile[seed_genome])
                        for ref_g in reference_set
                            # If reference genome is a key in the HitsProfile, add to "shared" genes count
                            if haskey(hp.profile[seed_genome][gene], ref_g)
                                genes_by_category = categorize_shared_gene(genes_by_category, seed_genome, gene)
                            end
                        end
                        # Determine if gene is shared by all or none of the referece genomes
                        count = get(genes_by_category[seed_genome]["shared"], gene, 0)
                        if count == length(reference_set)
                            genes_by_category = categorize_core_gene(genes_by_category, seed_genome, gene)
                        elseif count == 0
                            genes_by_category = categorize_new_gene(genes_by_category, seed_genome, gene)
                        end
                    end
                    write_results(genes_by_category, seed_genome, i, hp, fh)
                    sampling_counter[seed_genome] += 1
                    seen_strings[sort_string] = true
                    point_count += 1
                end
            end
        end

        elapsed = toq()
        println("time elapsed: $elapsed secs")
    end

end

function do_analysis_without_sampling()
    println("Doing analysis without sampling is not implemented at this time")
end

function parse_commandline()
    s = ArgParseSettings(description = "Reads a pangenome profile matrix and generates total genes in the pangenome",
    prog = "pangenome_make_pangenome.jl",
	version = "1.0",
	add_version = true,
	add_help = true)

    # The macro to add a table of arguments and options to the ArgParseSettings object
    @add_arg_table s begin
        "--profile", "-p"
            help = "Path to profile matrix"
            metavar = "/path/to/pangenome/profile.txt"
            required = true
        "--output_dir", "-o"
            help = "Directory to write the output"
            metavar = "/path/to/pangenome/dir/"
            required = true
        "--comparisons", "-c"
            help = "The number of comparisons to make for any 1 value of N (sampling)"
            metavar = "100000"
			default = 0
			range_tester = (x->x>=0)
            arg_type = Int
        "--multiplicity", "-m"
            help = "Another option for sampling based on a multiplicity factor ((sum(m*n) for n=[2..n]) number of comparisons)"
            metavar = "20"
			default = 0
			range_tester = (x->x>=0)
            arg_type = Int
    # Will not add log_file or debug options for now
    end

    # Converts the ArgParseSettings object into key/value pairs
    return parse_args(s)
end

function main()
    args = parse_commandline()
    println("Parsed args:")
    for (arg,val) in args
        println("  $arg  =>  $val")
    end

    hp = HitsProfile()
    read_in_profile(hp, args["profile"])

    output_dir = args["output_dir"]
    output_file = "$output_dir/core_shared_new.tsv"
    out_fh = open(output_file, "w")

    ## our output table will take the form:
    ## genome_count  core    shared    new  root_genome
    write_single_genomes(hp, out_fh)

    if args["comparisons"] == 0 && args["multiplicity"] == 0
        do_analysis_without_sampling()
    else
        do_analysis_with_sampling(args, hp, out_fh)
    end

    close(out_fh)
end

main()
quit()


"
    gather(w::AbstractMatrix{T}, xs::OneHotArray) where

getting vector at the given onehot encoding.
"
gather(w::AbstractMatrix{T}, xs::OneHotArray) where T = gather(w, onehot2indices(xs))

"
    gather(w::AbstractMatrix{T}, xs) where

getting vector at the given indices, `xs` is a array of indices(`Int` type).
"
function gather(w::AbstractMatrix{T}, xs::AbstractArray{Int}) where T
    ys = similar(w, size(w, 1), size(xs)...)

    Threads.@threads for i = 1:length(xs)
        ind = Tuple(CartesianIndices(xs)[i])
        @inbounds ys[:, ind...] .= w[:, xs[i]]
    end
    return ys
end

"
    gather(w::AbstractArray{T}, xs) where

getting vector at the given indices, `xs` is a array of cartesian indices(`Tuple{Int}` type).
"
function gather(w::AbstractArray{T}, xs::AbstractArray{<:Tuple}) where T
    ys = similar(w, size(w, 1), size(xs)...)

    Threads.@threads for i = 1:length(xs)
        ind = Tuple(CartesianIndices(xs)[i])
        @inbounds ys[:, ind...] .= w[:, xs[i]...]
    end
    return ys
end

# cpu gather back
function ∇gather(Δ::AbstractArray{T}, w::AbstractMatrix{T}, xs::AbstractArray{Int}) where T
    ys = fill!(similar(w), zero(T))

    Threads.@threads for i = 1:length(xs)
        ind = Tuple(CartesianIndices(xs)[i])
        @inbounds ys[:, xs[i]] .+= Δ[:, ind...]
    end

    return ys
end

function ∇gather(Δ::AbstractArray{T}, w::AbstractArray{T}, xs::AbstractArray{<:Tuple}) where T
    ys = fill!(similar(w), zero(T))

    Threads.@threads for i = 1:length(xs)
        ind = Tuple(CartesianIndices(xs)[i])
        @inbounds ys[:, xs[i]] .+= Δ[:, ind...]
    end

    return ys
end

@init @require CuArrays="3a865a2d-5b23-5a0f-bc46-62713ec82fae" begin
    using .CuArrays
    using .CuArrays.CUDAnative

    # gpu gather
    gather(w::CuMatrix{T}, xs::OneHotArray) where T = gather(w, onehot2indices(xs))
    function gather(w::CuMatrix{T}, xs::CuArray{Int}) where T
        ys = CuArray{T}(undef, size(w, 1), size(xs)...)

        function kernel!(ys::CuDeviceArray{T}, w::CuDeviceArray{T}, xs)
            li = threadIdx().y + (blockIdx().y - 1) * blockDim().y
            i = threadIdx().x + (blockIdx().x - 1) * blockDim().x

            if li <= length(xs)
                ind = Tuple(CartesianIndices(xs)[li])
                ys[i, ind...] = w[i, xs[li]]
            end

            return
        end

        max_threads = 256
        threads_x = min(max_threads, size(ys,1))
        threads_y = min(max_threads ÷ threads_x, length(xs))
        threads = (threads_x, threads_y)
        blocks = ceil.(Int, (size(ys,1), length(xs)) ./ threads)

        CuArrays.@cuda blocks=blocks threads=threads kernel!(ys, w, xs)
        return ys
    end

    #gpu gather back
    function ∇gather(Δ::CuArray{T}, w::CuMatrix{T}, xs::CuArray{Int}) where T
        ys = fill!(similar(w), zero(T))

        function kernel!(ys::CuDeviceArray{T}, Δ::CuDeviceArray{T}, xs)
            xi = threadIdx().x + (blockIdx().x - 1) * blockDim().x

            if xi <= size(ys, 1)
                for i = 1:length(xs)
                    ind = Tuple(CartesianIndices(xs)[i])
                    ys[xi, xs[i]] += Δ[xi, ind...]
                end
            end

            return
        end

        max_threads = 256
        threads = max_threads
        blocks = ceil(Int, size(ys, 1) / threads)

        CuArrays.@cuda blocks=blocks threads=threads kernel!(ys, Δ, xs)
        return ys
    end

    # Not atomic to do this
    # function kernel!(ys::CuDeviceArray{T}, Δ::CuDeviceArray{T}, xs)
    #     li = threadIdx().y + (blockIdx().y - 1) * blockDim().y
    #     i = threadIdx().x + (blockIdx().x - 1) * blockDim().x
    #     if li <= length(xs)
    #         ind = Tuple(CartesianIndices(xs)[li])
    #         ys[i, xs[li]] += Δ[i, ind...]
    #     end

    #     return
    # end
    # max_threads = 256
    # threads_x = min(max_threads, size(ys,1))
    # threads_y = min(max_threads ÷ threads_x, length(xs))
    # threads = (threads_x, threads_y)
    # blocks = ceil.(Int, (size(ys,1), length(xs)) ./ threads)
    # @cuda blocks=blocks threads=threads kernel!(ys, Δ, xs)

    function gather(W::CuArray{T}, xs::CuArray{<:Tuple}) where T
        ys = CuArray{T}(undef, size(W, 1), size(xs)...)

        function kernel!(ys::CuDeviceArray{T}, w::CuDeviceArray{T}, xs)
            li = threadIdx().y + (blockIdx().y - 1) * blockDim().y
            i = threadIdx().x + (blockIdx().x - 1) * blockDim().x

            if li <= length(xs)
                ind = Tuple(CartesianIndices(xs)[li])
                ys[i, ind...] = w[i, xs[li]...]
            end

            return
        end

        max_threads = 256
        threads_x = min(max_threads, size(ys,1))
        threads_y = min(max_threads ÷ threads_x, length(xs))
        threads = (threads_x, threads_y)
        blocks = ceil.(Int, (size(ys,1), length(xs)) ./ threads)

        CuArrays.@cuda blocks=blocks threads=threads kernel!(ys, W, xs)
        return ys
    end

    function ∇gather(Δ::CuArray{T}, W::CuArray{T}, xs::CuArray{<:Tuple}) where T
        ys = fill!(similar(W), zero(T))

        function kernel!(ys::CuDeviceArray{T}, Δ::CuDeviceArray{T}, xs)
            xi = threadIdx().x + (blockIdx().x - 1) * blockDim().x

            if xi <= size(ys, 1)
                for i = 1:length(xs)
                    ind = Tuple(CartesianIndices(xs)[i])
                    ys[xi, xs[i]...] += Δ[xi, ind...]
                end
            end

            return
        end

        max_threads = 256
        threads = max_threads
        blocks = ceil(Int, size(ys, 1) / threads)

        CuArrays.@cuda blocks=blocks threads=threads kernel!(ys, Δ, xs)
        return ys
    end
end

using Flux: Tracker, TrackedArray, TrackedMatrix, data
using Flux.Tracker: @grad, track

gather(w::TrackedMatrix, xs::OneHotArray) = gather(w, onehot2indices(xs))
gather(w::TrackedMatrix, xs::AbstractArray{Int}) = track(gather, w, xs)
gather(w::TrackedArray, xs::AbstractArray{<:Tuple}) = track(gather, w, xs)

@grad gather(w::TrackedArray, xs) = gather(data(w), xs), Δ->(∇gather(Δ, data(w), xs),nothing)


using Distributed
@everywhere using Pkg
@everywhere Pkg.activate(@__DIR__)
@everywhere begin
    using CalibrateEDMF
    ce = pkgdir(CalibrateEDMF)
    gs = joinpath(ce, "experiments", "grid_search")
    include(joinpath(gs, "grid_search.jl"))
    include(joinpath(gs, "loss_map.jl"))
    include(joinpath(gs, "quick_plots.jl"))
end
using Test

config_path = joinpath(@__DIR__, "grid_search_integration_test_config.jl")
include(config_path)
config = get_config()  #
out_dir = joinpath(@__DIR__, "output", "grid_search")
rm(out_dir, recursive = true, force = true)  # delete old test output
@time begin
    grid_search(config, config_path, out_dir)
    @info "Grid search complete"
end
y_root = joinpath(out_dir, "general_stochastic_ent_params_{1}.general_stochastic_ent_params_{2}/0.3_0.2")
y_dirs = [
    joinpath(y_root, "TRMM_LBA.1/Output.TRMM_LBA.1"),
    joinpath(y_root, "Rico.1/Output.Rico.1"),
    joinpath(y_root, "TRMM_LBA.2/Output.TRMM_LBA.1"),
]
nc_files = joinpath.(y_dirs, ["stats/Stats.TRMM_LBA.nc", "stats/Stats.Rico.nc", "stats/Stats.TRMM_LBA.nc"])
@test all(isfile.(nc_files))

# update reference data paths
config["reference"]["y_dir"] = y_dirs

@time begin
    compute_loss_map(config, out_dir)
    @info "Loss map complete"
end

quick_plots(joinpath(out_dir, "loss.nc"), out_dir)


"""
`AbstractGeoArray` wraps an array (or location of an array) and metadata
about its contents. It may be memory ([`GeoArray`](@ref)) or disk-backed
([`NCDarray`](@ref), [`GDALarray`](@ref), [`GRDarray`](@ref)).

`AbstractGeoArray`s inherit from [`AbstractDimArray`]($DDarraydocs)
from DimensionalData.jl. They can be indexed as regular Julia arrays or with
DimensionalData.jl [`Dimension`]($DDdimdocs)s. They will plot as a heatmap in
Plots.jl with correct coordinates and labels, even after slicing with
`getindex` or `view`. `getindex` on a `AbstractGeoArray` will always return
a memory-backed `GeoArray`.
"""
abstract type AbstractGeoArray{T,N,D,A} <: AbstractDimensionalArray{T,N,D,A} end

# Marker singleton for lazy loaded arrays, only used for broadcasting.
# Can be removed when DiskArrays.jl is used everywhere
struct LazyArray{T,N} <: AbstractArray{T,N} end

# Interface methods ###########################################################

"""
    missingval(x)

Returns the value representing missing data in the dataset """ function missingval end
missingval(x) = missing
missingval(A::AbstractGeoArray) = A.missingval

"""
    crs(x)

Get the crs projection of a dim or for the `Lat`/`Lon` dims of an array.
"""
function crs end
crs(A::AbstractGeoArray) =
    if hasdim(A, Lat)
        crs(dims(A, Lat))
    elseif hasdim(A, Lon)
        crs(dims(A, Lon))
    else
        error("No Lat or Lon dimension, crs not available")
    end
crs(dim::Dimension) = crs(mode(dim), dim)

"""
    usercrs(x)

Get the user facing crs projection of a [`Projected`](@ref) mode
dim or for the `Lat`/`Lon` dims of an array.

This is used to convert [`Selector`]($DDselectordocs) values form the user defined
projection to the underlying projection, and to show plot axes in the user projection.
"""
function usercrs end
usercrs(A::AbstractGeoArray) =
    if hasdim(A, Lat)
        usercrs(dims(A, Lat))
    elseif hasdim(A, Lon)
        usercrs(dims(A, Lon))
    else
        error("No Lat or Lon dimension, usercrs not available")
    end
usercrs(dim::Dimension) = usercrs(mode(dim), dim)

"""
    dimcrs(x)

Get the index crs projection of a [`Converted`](@ref) mode dim or
for the `Lat`/`Lon` dims of an array.

This is used for NetCDF and other formats where the underlying projection
of the data may  be different to what is contained in the vector index.
"""
function dimcrs end
dimcrs(A::AbstractGeoArray) =
    if hasdim(A, Lat)
        dimcrs(dims(A, Lat))
    elseif hasdim(A, Lon)
        dimcrs(dims(A, Lon))
    else
        error("No Lat or Lon dimension, dimcrs not available")
    end
dimcrs(dim::Dimension) = dimcrs(mode(dim), dim)

# DimensionalData methods

units(A::AbstractGeoArray) = getmeta(A, :units, nothing)

# Rebuild all types of AbstractGeoArray as GeoArray
rebuild(A::AbstractGeoArray, data, dims::Tuple, refdims, name, metadata, missingval=missingval(A)) =
    GeoArray(data, dims, refdims, name, metadata, missingval)
rebuild(A::AbstractGeoArray; data=data(A), dims=dims(A), refdims=refdims(A), name=name(A), metadata=metadata(A), missingval=missingval(A)) =
    GeoArray(data, dims, refdims, name, metadata, missingval)

Base.parent(A::AbstractGeoArray) = data(A)

"""
Abstract supertype for all memory-backed GeoArrays where the data is an array.
"""
abstract type MemGeoArray{T,N,D,A} <: AbstractGeoArray{T,N,D,A} end

"""
Abstract supertype for all disk-backed GeoArrays.
For these the data is lazyily loaded from disk.
"""
abstract type DiskGeoArray{T,N,D,A} <: AbstractGeoArray{T,N,D,A} end

filename(A::DiskGeoArray) = A.filename

"""
    data(f, A::DiskGeoArray)

Run method `f` on the data source object for A, as passed by the
`withdata` method for the array. The only requirement of the
object is that it has an `Array` method that returns the data as an array.
"""
data(A::DiskGeoArray) = withsourcedata(Array, A)

# Base methods

Base.size(A::DiskGeoArray) = A.size

Base.getindex(A::DiskGeoArray, i1::StandardIndices, i2::StandardIndices, I::StandardIndices...) =
    rebuildgetindex(A, i1, i2, I...)
Base.getindex(A::DiskGeoArray, i1::Integer, i2::Integer, I::Vararg{<:Integer}) =
    rawgetindex(A, i1, i2, I...)
# Linear indexing returns Array
Base.@propagate_inbounds Base.getindex(A::DiskGeoArray, i::Union{Colon,AbstractVector{<:Integer}}) =
    rawgetindex(A, i)
# Exempt 1D DimArrays
Base.@propagate_inbounds Base.getindex(A::DiskGeoArray{<:Any,1}, i::Union{Colon,AbstractVector{<:Integer}}) =
    rebuildgetindex(A, i)

rawgetindex(A, I...) =
    withsourcedata(A) do data
        readwindowed(data, I...)
    end

rebuildgetindex(A, I...) =
    withsourcedata(A) do data
        dims_, refdims_ = slicedims(dims(A), refdims(A), I)
        data = readwindowed(data, I...)
        rebuild(A, data, dims_, refdims_)
    end

Base.write(A::T) where T <: DiskGeoArray = write(filename(A), A)
Base.write(filename::AbstractString, A::T) where T <: DiskGeoArray =
    write(filename, T, A)

# Concrete implementation ######################################################

"""
    GeoArray(A::AbstractArray{T,N}, dims::Tuple;
             refdims=(), name=Symbol(""), metadata=nothing, missingval=missing)
    GeoArray(A::AbstractArray{T,N};
             dims, refdims=(), name=Symbol(""), metadata=nothing, missingval=missing)
    GeoArray(A::AbstractGeoArray; [data=data(A), dims=dims(A), refdims=refdims(A),
             name=name(A), metadata=metadata(A), missingval=missingval(A)]) =

A generic, memory-backed spatial array type. All [`AbstractGeoArray`](@ref) are
converted to `GeoArray` when indexed or otherwise transformed.

# Keyword Arguments

- `name`: `Symbol` name for the array.
- `dims`: `Tuple` of `Dimension`s for the array.
- `refdims`: `Tuple of` position `Dimension`s the array was sliced from,
  defaulting to `()`.
- `missingval`: Value reprsenting missing values, defaulting to `missing`.
  can be passed it.
- `metadata`: [`Metadata`](@ref) object for the array, or `nothing`.

## Example

```julia
A = GRDarray(gdalarray; name="surfacetemp")
# Select Australia using lat/lon coords, whatever the crs is underneath.
A[Lat(Between(-10, -43), Lon(Between(113, 153)))
```
"""
struct GeoArray{T,N,D<:Tuple,R<:Tuple,A<:AbstractArray{T,N},Na<:Symbol,Me,Mi} <: MemGeoArray{T,N,D,A}
    data::A
    dims::D
    refdims::R
    name::Na
    metadata::Me
    missingval::Mi
end
GeoArray(A::AbstractArray, dims, refdims, name::String, metadata, missingval=missing) = begin
    @warn "The GeoArray `name` field is now a Symbol"
    GeoArray(A, dims, refdims, Symbol(name), metadata, missingval)
end
@inline GeoArray(A::AbstractArray, dims::Tuple;
                 refdims=(), name=Symbol(""), metadata=nothing, missingval=missing) =
    GeoArray(A, formatdims(A, dims), refdims, name, metadata, missingval)
@inline GeoArray(A::AbstractArray; dims, refdims=(), name=Symbol(""), metadata=nothing, missingval=missing) =
    GeoArray(A, formatdims(A, dims), refdims, name, metadata, missingval)
@inline GeoArray(A::AbstractGeoArray; data=data(A), dims=dims(A), refdims=refdims(A),
                 name=name(A), metadata=metadata(A), missingval=missingval(A)) =
    GeoArray(data, dims, refdims, name, metadata, missingval)

Base.@propagate_inbounds Base.setindex!(A::GeoArray, x, I::Vararg{DimensionalData.StandardIndices}) =
    setindex!(data(A), x, I...)

Base.convert(::Type{GeoArray}, array::AbstractGeoArray) = GeoArray(array)


module OceanographyCruises

using StaticArrays, Dates
using FieldMetadata, FieldDefaults
using PrettyTables
using Unitful
using Unitful: °
using Distances, TravelingSalesmanHeuristics
using RecipesBase, UnitfulRecipes

"""
    Station

Type containing `(lat,lon)` information.

(And optionally a `name` and a `date`).
"""
@default_kw struct Station <: FieldVector{2, Float64}
    lat::Float64   | NaN
    lon::Float64   | NaN
    name::String   | ""
    date::DateTime | Date(0)
end
pretty_data(st::Station) = [st.name (st.date == Date(0) ? "" : st.date) st.lat st.lon]
Base.show(io::IO, m::MIME"text/plain", st::Station) = println(io, string(st))
Base.show(io::IO, st::Station) = println(io, string(st))
function Base.string(st::Station)
    name = st.name == "" ? "Unnamed station " : "Station $(st.name) "
    date = st.date == Date(0) ? "" : "$(st.date) "
    return string(name, date, "($(latstring(st.lat)), $(lonstring(st.lon)))")
end
latstring(lat) = string(round(abs(lat)*10)/10, lat < 0 ? "S" : "N")
lonstring(lon) = string(round(abs(lon)*10)/10, lon < 0 ? "W" : "E")
# shift longitude for cruises that cross 0 to (-180,180)
shiftlon(lon; baselon=0) = mod(lon - baselon, 360) + baselon
shiftlon(st::Station; baselon=0) = Station(st.lat, shiftlon(st.lon, baselon=baselon), st.name, st.date)

"""
    CruiseTrack

Compact type containing cruise track information:
- cruise name
- stations
"""
@default_kw struct CruiseTrack
    name::String              | ""
    stations::Vector{Station} | Station[]
end
Base.length(ct::CruiseTrack) = length(ct.stations)
Base.isempty(ct::CruiseTrack) = isempty(ct.stations)
pretty_data(ct::CruiseTrack) = reduce(vcat, [pretty_data(st) for st in ct.stations])
function Base.show(io::IO, ::MIME"text/plain", ct::CruiseTrack)
    if isempty(ct)
        println("Empty cruise $(ct.name)")
    else
        println("Cruise $(ct.name)")
        pretty_table(pretty_data(ct), ["Station", "Date", "Lat", "Lon"])
    end
end
latitudes(ct::CruiseTrack) = [st.lat for st in ct.stations]
longitudes(ct::CruiseTrack) = [st.lon for st in ct.stations]
# shift longitude for cruises that cross 0 to (-180,180)
shiftlon(ct::CruiseTrack; baselon=0) = CruiseTrack(ct.name, shiftlon.(ct.stations, baselon=baselon))
function autoshift(ct::CruiseTrack)
    if any([0 ≤ st.lon < 90 for st in ct.stations]) && any([270 ≤ st.lon < 360 for st in ct.stations])
        shiftlon(ct, baselon=-180)
    else
        ct
    end
end

"""
    DepthProfile

A depth profile at a given station.
"""
@default_kw struct DepthProfile{V}
    station::Station        | Station()
    depths::Vector{Float64} | Float64[]
    values::Vector{V}       | Float64[]
    DepthProfile(st,d,v::Vector{V}) where {V} = (length(d) ≠ length(v)) ? error("`depths` and `values` must have same length") : new{V}(st,d,v)
end
Base.length(p::DepthProfile) = length(p.depths)
Base.isempty(p::DepthProfile) = isempty(p.depths)
pretty_data(p::DepthProfile) = [p.depths p.values]
pretty_data(p::DepthProfile{V}) where {V <: Quantity} = [p.depths ustrip.(p.values)]
function Base.show(io::IO, m::MIME"text/plain", p::DepthProfile)
    if isempty(p)
        println("Empty profile at ", string(p.station))
    else
        println("Depth profile at ", string(p.station))
        pretty_table(pretty_data(p), ["Depth", "Value"])
    end
end
function Base.show(io::IO, m::MIME"text/plain", p::DepthProfile{V}) where {V <: Quantity}
    if isempty(p)
        println("Empty profile at ", string(p.station))
    else
        println("Depth profile at ", string(p.station))
        pretty_table(pretty_data(p), ["Depth", "Value [$(unit(V))]"])
    end
end
Base.:*(pro::DepthProfile, q::Quantity) = DepthProfile(pro.station, pro.depths, pro.values .* q)
Base.:-(pro1::DepthProfile, pro2::DepthProfile) = DepthProfile(pro1.station, pro1.depths, pro1.values .- pro2.values)
Unitful.uconvert(u, pro::DepthProfile) = DepthProfile(pro.station, pro.depths, uconvert.(u, pro.values))
# shift longitude for cruises that cross 0 to (-180,180)
shiftlon(pro::DepthProfile; baselon=0) = DepthProfile(shiftlon(pro.station, baselon=baselon), pro.depths, pro.values)

"""
    Transect

A transect of depth profiles for a given tracer.
"""
@default_kw struct Transect{V}
    tracer::String                    | ""
    cruise::String                    | ""
    profiles::Vector{DepthProfile{V}} | DepthProfile{Float64}[]
end
Base.length(t::Transect) = length(t.profiles)
function Base.show(io::IO, m::MIME"text/plain", t::Transect)
    if isempty(t)
        println("Empty transect")
    else
        println("Transect of $(t.tracer)")
        show(io, m, CruiseTrack(t))
    end
end
CruiseTrack(t::Transect) = CruiseTrack(stations=[p.station for p in t.profiles], name=t.cruise)
Base.isempty(t::Transect) = isempty(t.profiles)
Base.:*(t::Transect, q::Quantity) = Transect(t.tracer, t.cruise, t.profiles .* q)
Base.:-(t1::Transect, t2::Transect) = Transect("$(t1.tracer)˟", t1.cruise, t1.profiles .- t2.profiles)
Unitful.uconvert(u, t::Transect) = Transect(t.tracer, t.cruise, uconvert.(u, t.profiles))
# shift longitude for cruises that cross 0 to (-180,180)
shiftlon(t::Transect; baselon=0) = Transect(t.tracer, t.cruise, shiftlon.(t.profiles, baselon=baselon))
function autoshift(t::Transect)
    if any([0 ≤ pro.station.lon < 90 for pro in t.profiles]) && any([270 ≤ pro.station.lon < 360 for pro in t.profiles])
        shiftlon(t, baselon=-180)
    else
        t
    end
end
latitudes(t::Transect) = [pro.station.lat for pro in t.profiles]
longitudes(t::Transect) = [pro.station.lon for pro in t.profiles]

"""
    Transects

A collection of transects for a given tracer.
"""
@default_kw struct Transects{V}
    tracer::String                 | ""
    cruises::Vector{String}        | String[]
    transects::Vector{Transect{V}} | Transect{Float64}[]
end
function Base.show(io::IO, m::MIME"text/plain", ts::Transects)
    println("Transects of $(ts.tracer)")
    print("(Cruises ")
    [print("$c, ") for c in ts.cruises[1:end-1]]
    println("and $(last(ts.cruises)).)")
end
Base.:*(ts::Transects, q::Quantity) = Transects(ts.tracer, ts.cruises, ts.transects .* q)
Base.:*(q::Quantity, ts::Transects) = ts * q
Base.:-(ts1::Transects, ts2::Transects) = Transects("$(ts1.tracer)˟", ts1.cruises, ts1.transects .- ts2.transects)
Unitful.uconvert(u, ts::Transects) = Transects(ts.tracer, ts.cruises, uconvert.(u, ts.transects ))
# shift longitude for cruises that cross 0 to (-180,180)
shiftlon(ts::Transects; baselon=0) = Transects(ts.tracer, ts.cruises, shiftlon.(t.transects, baselon=baselon))
autoshift(ts::Transects) = Transects(ts.tracer, ts.cruises, autoshift.(ts.transects))

import Base: sort, sortperm, getindex
getindex(ts::Transects, i::Int) = ts.transects[i]
"""
    sort(t::Transect)

Sorts the transect using a travelling salesman problem heuristic.
"""
sort(t::Transect; start=cruise_default_start(t)) = Transect(t.tracer, t.cruise, [t for t in t.profiles[sortperm(t; start=start)]])
sort(ct::CruiseTrack; start=cruise_default_start(ct)) = CruiseTrack(name=ct.name, stations=ct.stations[sortperm(ct; start=start)])
function sortperm(t::Union{CruiseTrack, Transect}; start=cruise_default_start(t))
    t = autoshift(t)
    n = length(t)
    lats = latitudes(t)
    lons = longitudes(t)
    pts = [lons lats]
    dist_mat = zeros(n+1, n+1)
    dist_mat[1:n,1:n] .= pairwise(Haversine(1), pts, dims=1)
    path, cost = solve_tsp(dist_mat)
    i = findall(path .== n+1)
    if length(i) == 1
        path = vcat(path[i[1]+1:end-1], path[1:i[1]-1])
    else
        path = path[2:end-1]
    end
    start == :south && pts[path[1],2] > pts[path[end],2] && reverse!(path)
    start == :west  && pts[path[1],1] > pts[path[end],1] && reverse!(path)
    return path
end
export sort, sortperm, getindex

function cruise_default_start(t)
    t = autoshift(t)
    extremalat = extrema(latitudes(t))
    extremalon = extrema(longitudes(t))
    Δlat = extremalat[2] - extremalat[1]
    Δlon = extremalon[2] - extremalon[1]
    Δlon > Δlat ? :west : :south
end

"""
    diff(t)

Computes the distance in km of each segment of the transect
"""
function Base.diff(t::Union{CruiseTrack,Transect})
    t = autoshift(t) # not sure this is needed since Haversine takes lat and lon?
    n = length(t)
    lats = latitudes(t)
    lons = longitudes(t)
    pts = [lons lats]
    return [Haversine(6371.0)(pts[i,:], pts[i+1,:]) for i in 1:n-1] * u"km"
end
export diff

function Base.range(departure::Station, arrival::Station; length::Int64, westmostlon=-180)
    lonstart = lonconvert(departure.lon, westmostlon)
    lonend = lonconvert(arrival.lon, westmostlon)
    lonstart - lonend >  180 && (lonend += 360)
    lonstart - lonend < -180 && (lonstart += 360)
    lats = range(departure.lat, arrival.lat, length=length)
    lons = range(lonstart, lonend, length=length)
    names = string.("$(departure.name) to $(arrival.name) ", 1:length)
    return [Station(lat=x[1], lon=lonconvert(x[2], westmostlon), name=x[3]) for x in zip(lats, lons, names)]
end

lonconvert(lon, westmostlon=-180) = mod(lon - westmostlon, 360) + westmostlon

export CruiseTrack, Station, DepthProfile, Transect, Transects
export latitudes, longitudes

Unitful.unit(t::Transect) = unit(t.profiles[1].values[1])

# nested functions
for f in (:maximum, :minimum)
    @eval begin
        import Base: $f
        """
            $($f)(t)

        Applies `$($f)` recursively.
        """
        $f(ts::Transects) = $f($f(t) for t in ts.transects)
        $f(t::Transect) = $f($f(pro) for pro in t.profiles)
        $f(pro::DepthProfile) = $f($f(v) for v in pro.values)
        export $f
    end
end

# recipe TODO split this file into a few different ones that make sens and include them all here
include("recipes.jl")

end # module


function vfilter!(f::F, x::Vector{T}, y::AbstractArray{T}) where {F,T <: NativeTypes}
    W, Wshift = VectorizationBase.pick_vector_width_shift(T)
    N = length(y)
    Nrep = N >>> Wshift
    Nrem = N & (W - 1)
    j = 0
    st = VectorizationBase.static_sizeof(T)
    zero_index = MM(W, Static(0), st)
    GC.@preserve x y begin
        ptr_x = pointer(x)
        ptr_y = pointer(y)
        for _ ∈ 1:Nrep
            vy = vload(ptr_y, zero_index)
            mask = f(vy)
            VectorizationBase.compressstore!(gep(ptr_x, VectorizationBase.lazymul(st, j)), vy, mask)
            ptr_y = gep(ptr_y, register_size())
            j = vadd_fast(j, count_ones(mask))
        end
        rem_mask = VectorizationBase.mask(T, Nrem)
        vy = vload(ptr_y, zero_index, rem_mask)
        mask = rem_mask & f(vy)
        VectorizationBase.compressstore!(gep(ptr_x, VectorizationBase.lazymul(st, j)), vy, mask)
        j = vadd_fast(j, count_ones(mask))
        Base._deleteend!(x, N-j) # resize!(x, j)
    end
    x
end
vfilter!(f::F, x::Vector{T}) where {F, T<:NativeTypes} = vfilter!(f, x, x)
vfilter(f::F, y::AbstractArray{T}) where {F, T<:NativeTypes} = vfilter!(f, Vector{T}(undef, length(y)), y)
vfilter(f::F, y) where {F} = filter(f, y)
vfilter!(f::F, y) where {F} = filter!(f, y)

"""
    vfilter(f, a::AbstractArray)

SIMD-vectorized `filter`, returning an array containing the elements of `a` for which `f` return `true`.

This function requires AVX512 to be faster than `Base.filter`, as it adds compressstore instructions.
"""
vfilter

"""
    vfilter!(f, a::AbstractArray)

SIMD-vectorized `filter!`, removing the element of `a` for which `f` is false.
"""
vfilter!


using DrWatson
@quickactivate "HyperspectraWithNeXL"

using NeXLSpectrum
using Gadfly

fes=loadspectrum(joinpath(datadir(),"exp_raw","FeS_and_FeS2","FeS std.msa"))
fes2=loadspectrum(joinpath(datadir(),"exp_raw","FeS_and_FeS2","FeS2 std.msa"))

set_default_plot_size(10inch,3inch)
p = plot(fes, fes2, klms=[n"Fe K-L3", n"Fe K-M3", n"Fe L3-M5", n"Fe L2-M1", n"S K-L3", n"S K-M3"], xmax=10.0e3)
mkpath(joinpath(papersdir(),"Figures", "Figure 1"))
p |> SVG(joinpath(papersdir(),"Figures", "Figure 1","FeS vs FeS2.svg"), 10inch, 3inch)


module AnnealingAPI

using CorrelationFunctions: Directional

"""
Abstract type for a rollback token returned by `update_corrfns!`.
"""
abstract type AbstractRollbackToken end

"""
    SimpleRollbackToken(value, index)

Create a simple rollback token which just holds an index and an old
value at that index.
"""
struct SimpleRollbackToken{T, N} <: AbstractRollbackToken
    value :: T
    index :: CartesianIndex{N}
end

"""
    update_corrfns!(tracker, value, index)

This function is equivalent to writing `tracker[index] = value` with
exception that it also returns a rollback handle which can fastly
bring the tracker to the previous state by calling `rollback!`.

See also: [`rollback!`](@ref).
"""
function update_corrfns! end

function update_corrfns!(array :: AbstractArray{T, N},
                         val,
                         index :: CartesianIndex{N}) where {T, N}
    token = SimpleRollbackToken(array[index], index)
    array[index] = val
    return token
end

"""
    rollback!(tracker, token)

Bring the system to the state before `update_corrfns!` was
called. `token` must be an object returned by `update_corrfns!`.

See also: [`update_corrfns!`](@ref).
"""
function rollback! end

rollback!(array :: AbstractArray, rollback :: AbstractRollbackToken) =
    array[rollback.index] = 1 - array[rollback.index]

rollback!(array    :: AbstractArray{T, N},
          rollback :: SimpleRollbackToken{T, N}) where {T, N} =
    array[rollback.index] = rollback.value

"""
    Generic type for correlation function tracker.

See also: [`L2Tracker`](@ref), [`S2Tracker`](@ref),
[`SSTracker`](@ref).
"""
abstract type AbstractTracker{T} end

"""
    L2Tracker(phase)

Descriptor for line-segment correlation function for the phase
`phase`.

See also: [`S2Tracker`](@ref), [`SSTracker`](@ref),
[`AbstractTracker`](@ref).
"""
struct L2Tracker{T} <: AbstractTracker{T}
    phase :: T
end

"""
    S2Tracker(phase)

Descriptor for two-point correlation function for the phase
`phase`.

See also: [`L2Tracker`](@ref), [`SSTracker`](@ref),
[`AbstractTracker`](@ref).
"""
struct S2Tracker{T} <: AbstractTracker{T}
    phase :: T
end

"""
    SSTracker(phase)

Descriptor for surface-surface correlation function for the phase
`phase`.

See also: [`S2Tracker`](@ref), [`L2Tracker`](@ref),
[`AbstractTracker`](@ref).
"""
struct SSTracker{T} <: AbstractTracker{T}
    phase :: T
end

"""
    SVTracker(phase)

Descriptor for surface-void correlation function for the phase
`phase`.

See also: [`S2Tracker`](@ref), [`L2Tracker`](@ref),
[`SSTracker`](@ref), [`AbstractTracker`](@ref).
"""
struct SVTracker{T} <: AbstractTracker{T}
    phase :: T
end

(tracked :: L2Tracker{T})(system :: AbstractArray{T}; kwargs...) where T =
    Directional.l2(system, tracked.phase; kwargs...)
(tracked :: S2Tracker{T})(system :: AbstractArray{T}; kwargs...) where T =
    Directional.s2(system, tracked.phase; kwargs...)
(tracked :: SSTracker{T})(system :: AbstractArray{T}; kwargs...) where T =
    Directional.surfsurf(system, tracked.phase; kwargs...)
(tracked :: SVTracker{T})(system :: AbstractArray{T}; kwargs...) where T =
    Directional.surfvoid(system, tracked.phase; kwargs...)

"""
    tracked_data(x :: AbstractArray)

Return an iterator over correlation function descriptors which are
tracked by the tracker `x`.
"""
function tracked_data end

"""
    tracked_length(x :: AbstractArray)

Return maximal tracked correlation length
"""
function tracked_length end

"""
    tracked_directions(x :: AbstractArray)

Return directions along which correlation functions are tracked.
"""
function tracked_directions end

export
    AbstractRollbackToken,
    update_corrfns!,
    rollback!,

    AbstractTracker,
    S2Tracker,
    L2Tracker,
    SSTracker,
    SVTracker,

    tracked_data,
    tracked_length,
    tracked_directions

end # module


export FactorBoundFreeEnergy

import Base: tail

struct FactorBoundFreeEnergy end

function score(::Type{T}, objective::BetheFreeEnergy, ::FactorBoundFreeEnergy, node::AbstractFactorNode, scheduler) where { T <: InfCountingReal }
    return score(T, objective, FactorBoundFreeEnergy(), sdtype(node), node, scheduler)
end

function score(::Type{T}, objective::BetheFreeEnergy, ::FactorBoundFreeEnergy, ::Deterministic, node::AbstractFactorNode, scheduler) where { T <: InfCountingReal }
    stream = combineLatest(map((interface) -> apply_skip_filter(messagein(interface), marginal_skip_strategy(objective)) |> schedule_on(scheduler), interfaces(node)), PushNew())

    vtag       = Val{ clustername(tail(interfaces(node))) }
    msgs_names = Val{ map(name, interfaces(node)) }

    mapping = let fform = functionalform(node), vtag = vtag, meta = metadata(node), msgs_names = msgs_names, node = node
        (messages) -> begin
            # We do not really care about (is_clamped, is_initial) at this stage, so it can be (false, false)
            marginal = Marginal(marginalrule(fform, vtag, msgs_names, messages, nothing, nothing, meta, node), false, false)
            return convert(T, -score(DifferentialEntropy(), marginal))
        end
    end

    return stream |> map(T, mapping)
end

function score(::Type{T}, objective::BetheFreeEnergy, ::FactorBoundFreeEnergy, ::Stochastic, node::AbstractFactorNode, scheduler) where { T <: InfCountingReal }
    stream = combineLatest(map((cluster) -> getmarginal!(node, cluster, marginal_skip_strategy(objective)) |> schedule_on(scheduler), localmarginals(node)), PushNew())

    mapping = let fform = functionalform(node), meta = metadata(node), marginal_names = Val{ localmarginalnames(node) }
        (marginals) -> begin
            average_energy   = score(AverageEnergy(), fform, marginal_names, marginals, meta)
            clusters_entropy = mapreduce(marginal -> score(DifferentialEntropy(), marginal), +, marginals)
            return convert(T, average_energy - clusters_entropy)
        end
    end

    return stream |> map(T, mapping)
end


function icn_benchmark_unit(params)
    @info "Running a benchmark unit with" params

    search_space_size = params[:domains_size]^params[:domains_size]
    if params[:search] == :complete
        if search_space_size > params[:complete_search_limit]
            @warn "Unit benchmark aborted (complete) search space is too large" search_space_size params[:complete_search_limit]
            return nothing
        end
        if search_space_size > params[:loss_sampling_threshold]
            if isnothing(params[:loss_sampler])
                @warn "Unit benchmark aborted (complete) search space is too large, and loss function is deterministically evaluated" search_space_size params[:loss_sampling_threshold] params[:loss_sampler]
                return nothing
            end
        end
    end
    if params[:search] == :partial
        if search_space_size < params[:partial_search_limit]
            @warn "Unit benchmark aborted (partial) search space is too small" search_space_size params[:partial_search_limit]
            return nothing
        end
        if search_space_size ≤ params[:loss_sampling_threshold]
            if !isnothing(params[:loss_sampler])
                @warn "Unit benchmark aborted (complete) search space is too small, and loss function is stochastically evaluated" search_space_size params[:loss_sampling_threshold] params[:loss_sampler]
                return nothing
            end
        end
    end
    json_name = savename(
        (
            con=string(params[:concept][1]),
            par=params[:concept][2],
            csl=params[:complete_search_limit],
            dom=params[:domains_size],
            gen=params[:generations],
            iter=params[:icn_iterations],
            lang=string(params[:language]),
            lst=params[:loss_sampling_threshold],
            ls=string(params[:loss_sampler]),
            metric=string(params[:metric]),
            psl=params[:partial_search_limit],
            pop=params[:population],
            sampling=params[:sampling],
            search=string(params[:search]),
            mem=params[:memoize],
        ),
        "json",
    )
    save_results = joinpath(datadir("compositions"), json_name)
    if isfile(save_results)
        @warn "The result file already exist" save_results
    else

        # Generate an appropriate parameter for the concept if relevant
        param = if isnothing(params[:concept][2])
            nothing
        elseif params[:concept][2] == 1
            rand(1:params[:domains_size])
        else
            rand(1:params[:domains_size], params[:concept][2])
        end

        # assign parameters
        constraint_concept = concept(BENCHED_CONSTRAINTS[params[:concept][1]])
        metric = params[:metric]
        domain_size = params[:domains_size]
        domains = fill(domain(1:domain_size), domain_size)

        # Time the data retrieval/generation
        t = @timed search_space(
            domain_size,
            constraint_concept,
            param;
            search=params[:search],
            complete_search_limit=params[:complete_search_limit],
            solutions_limit=params[:sampling],
        )
        solutions, non_sltns, has_data = t.value

        bench = @timed explore_learn_compose(
            domains,
            constraint_concept,
            param;
            global_iter=params[:icn_iterations],
            local_iter=params[:generations],
            metric,
            pop_size=params[:population],
            configurations=(solutions, non_sltns),
            sampler=params[:loss_sampler],
            memoize=params[:memoize],
        )
        _, _, all_compos = bench.value

        results = Dict{Any,Any}()

        # Global results
        push!(results, :data => has_data ? :loaded : :explored)
        push!(results, :data_time => t.time)
        push!(results, :data_bytes => t.bytes)
        push!(results, :data_gctime => t.gctime)
        push!(results, :data_gcstats => t.gcstats)
        push!(results, :icn_time => bench.time)
        push!(results, :icn_bytes => bench.bytes)
        push!(results, :icn_gctime => bench.gctime)
        push!(results, :icn_gcstats => bench.gcstats)
        push!(results, :total_time => t.time + bench.time)
        push!(results, :nthreads => Threads.nthreads())

        for (id, (compo, occurence)) in enumerate(pairs(all_compos))
            local_results = Dict{Symbol,Any}()

            # selection rate
            push!(local_results, :selection_rate => occurence / params[:icn_iterations])

            # Code composition
            for lang in (params[:language], :maths)
                push!(
                    local_results,
                    lang => CompositionalNetworks.code(
                        compo, lang; name=string(params[:concept][1])
                    ),
                )
            end
            push!(local_results, :symbols => CompositionalNetworks.symbols(compo))

            push!(results, :params => params)
            push!(results, id => local_results)
        end
        write(save_results, json(results, 2))
        @info "Temp results" results json_name
    end
    return nothing
end

function icn_benchmark(params=ALL_PARAMETERS; clear_results=false)
    # Ensure the folders for data output exist
    clear_results && rm(datadir(); recursive=true, force=true)
    mkpath(datadir("compositions"))

    # Run all the benchmarks for all the unit configuration from params
    configs = dict_list(params)
    @warn "Number of benchmark units is $(length(configs))"
    for (u, c) in enumerate(configs)
        @info "Starting the $u/$(length(configs)) benchmark unit"
        GC.gc()
        icn_benchmark_unit(c)
    end
    return nothing
end


#InspectDR: Base functionnality and types for Cairo layer
#-------------------------------------------------------------------------------

#=TODO
-Fix glyph size for "constant area" (appearance).  Currently, algorithms use
 "constant bounding box" to avoid repeating complex math (ex: h=sqrt(a^2+b^2)).
 Solution: Create glyph object that takes Vector{Point2D} - or something
 similar.  That way, complex math is done only once per curve.
=#

#==Constants
===============================================================================#

#==Main types
===============================================================================#
#2D plot canvas; basic info:
type PCanvas2D <: PlotCanvas
	ctx::CairoContext
	bb::BoundingBox #Entire canvas
	graphbb::BoundingBox #Graph portion
	ext::PExtents2D #Extents of graph portion
	xf::Transform2D #Transform used to render data
end
PCanvas2D(ctx, bb, graphbb, ext) =
	PCanvas2D(ctx, bb, graphbb, ext, Transform2D(ext, graphbb))

immutable CGlyph{GTYPE}; end #Dispatchable glyph object
CGlyph(gtype::Symbol) = CGlyph{gtype}()

#==Basic rendering
===============================================================================#

#Reset context to known state:
function _reset(ctx::CairoContext)
	Cairo.set_source(ctx, COLOR_BLACK)
	Cairo.set_line_width(ctx, 1);
	Cairo.set_dash(ctx, Float64[], 0)
end

#Clears a rectangle-shaped area with a solid color
function clear(ctx::CairoContext, bb::BoundingBox, color::Colorant=COLOR_WHITE)
	Cairo.set_source(ctx, color)
	Cairo.rectangle(ctx, bb)
	Cairo.fill(ctx)
end

#Apply transform to Cairo (when easier to do so):
function setxfrm(ctx::CairoContext, xf::Transform2D)
	xorig = xf.x0*xf.xs
	yorig = xf.y0*xf.ys
	Cairo.translate(ctx, xorig, yorig)
	Cairo.scale(ctx, xf.xs, xf.ys)
end

#==Rendering Glyphs
===============================================================================#

function getglyphcolor(glyph::GlyphAttributes, line::LineAttributes)
	c = glyph.color
	if nothing == c
		c = line.color
	end
	return c
end

function getglyphfill(glyph::GlyphAttributes)
	fill = glyph.fillcolor
	if COLOR_TRANSPARENT == fill #Small optimization
		fill = nothing
	end
	return fill
end

#Conditionnaly render fill (preserve path for stroke)
renderfill(ctx::CairoContext, fill::Void) = nothing
function renderfill(ctx::CairoContext, fill::Colorant)
	Cairo.save(ctx) #-----
	Cairo.set_source(ctx, fill)
	Cairo.fill_preserve(ctx)
	Cairo.restore(ctx) #-----
end

function drawglyph{T}(ctx::CairoContext, ::CGlyph{T}, pt::Point2D, size::DReal, fill)
	warn("Glyph shape not supported: $T")
end

#= Supported:
	:square, :diamond,
	:uarrow, :darrow, :larrow, :rarrow, #usually triangles
	:cross, :+, :diagcross, :x,
	:circle, :o, :star, :*,
=#

function drawglyph(ctx::CairoContext, ::CGlyph{:circle}, pt::Point2D, size::DReal, fill)
	radius = size/2
	Cairo.arc(ctx, pt.x, pt.y, radius, 0, 2pi)
	renderfill(ctx, fill)
	Cairo.stroke(ctx)
end
drawglyph(ctx::CairoContext, ::CGlyph{:o}, pt::Point2D, size::DReal, fill) =
	drawglyph(ctx, CGlyph{:circle}(), pt, size, fill)

function drawglyph(ctx::CairoContext, ::CGlyph{:square}, pt::Point2D, size::DReal, fill)
	elen = size #full edge length
	hlen = elen/2 #halflength
	Cairo.rectangle(ctx, pt.x-hlen, pt.y-hlen, elen, elen)
	renderfill(ctx, fill)
	Cairo.stroke(ctx)
end

function drawglyph(ctx::CairoContext, ::CGlyph{:cross}, pt::Point2D, size::DReal, fill)
	elen = size*sqrt(2) #full edge length
	hlen = elen/2 #halflength
	Cairo.move_to(ctx, pt.x, pt.y-hlen)
	Cairo.rel_line_to(ctx, 0, elen)
	Cairo.stroke(ctx)
	Cairo.move_to(ctx, pt.x-hlen, pt.y)
	Cairo.rel_line_to(ctx, elen, 0)
	Cairo.stroke(ctx)
end
drawglyph(ctx::CairoContext, ::CGlyph{:+}, pt::Point2D, size::DReal, fill) =
	drawglyph(ctx, CGlyph{:cross}(), pt, size, fill)

function drawglyph(ctx::CairoContext, ::CGlyph{:diagcross}, pt::Point2D, size::DReal, fill)
	elen = size #full edge length
	hlen = elen/2 #halflength
	Cairo.move_to(ctx, pt.x-hlen, pt.y-hlen)
	Cairo.rel_line_to(ctx, elen, elen)
	Cairo.stroke(ctx)
	Cairo.move_to(ctx, pt.x-hlen, pt.y+hlen)
	Cairo.rel_line_to(ctx, elen, -elen)
	Cairo.stroke(ctx)
end
drawglyph(ctx::CairoContext, ::CGlyph{:x}, pt::Point2D, size::DReal, fill) =
	drawglyph(ctx, CGlyph{:diagcross}(), pt, size, fill)

function drawglyph(ctx::CairoContext, ::CGlyph{:star}, pt::Point2D, size::DReal, fill)
	elen = size*sqrt(2) #full edge length
	hlen = elen/2 #halflength

	Cairo.move_to(ctx, pt.x, pt.y-hlen)
	Cairo.rel_line_to(ctx, 0, elen)
	Cairo.stroke(ctx)
	Cairo.move_to(ctx, pt.x-hlen, pt.y)
	Cairo.rel_line_to(ctx, elen, 0)
	Cairo.stroke(ctx)

	elen = size #full edge length
	hlen = elen/2 #halflength

	Cairo.move_to(ctx, pt.x-hlen, pt.y-hlen)
	Cairo.rel_line_to(ctx, elen, elen)
	Cairo.stroke(ctx)
	Cairo.move_to(ctx, pt.x-hlen, pt.y+hlen)
	Cairo.rel_line_to(ctx, elen, -elen)
	Cairo.stroke(ctx)
end
drawglyph(ctx::CairoContext, ::CGlyph{:*}, pt::Point2D, size::DReal, fill) =
	drawglyph(ctx, CGlyph{:star}(), pt, size, fill)

function drawglyph(ctx::CairoContext, ::CGlyph{:diamond}, pt::Point2D, size::DReal, fill)
	elen = size #full edge length
	hlen = elen/2 #halflength
	Cairo.move_to(ctx, pt.x, pt.y-hlen)
	Cairo.rel_line_to(ctx, hlen, hlen)
	Cairo.rel_line_to(ctx, -hlen, hlen)
	Cairo.rel_line_to(ctx, -hlen, -hlen)
	Cairo.close_path(ctx)
	renderfill(ctx, fill)
	Cairo.stroke(ctx)
end

#dir=1: up, -1: down
function drawglyph_varrow(ctx::CairoContext, pt::Point2D, size::DReal, dir::Int, fill)
	elen = size #full edge length
	hlen = elen/2 #halflength
	dlen = dir*elen #directional edge
	Cairo.move_to(ctx, pt.x, pt.y-dir*hlen)
	Cairo.rel_line_to(ctx, hlen, dlen)
	Cairo.rel_line_to(ctx, -elen, 0)
	Cairo.close_path(ctx)
	renderfill(ctx, fill)
	Cairo.stroke(ctx)
end

drawglyph(ctx::CairoContext, ::CGlyph{:uarrow}, pt::Point2D, size::DReal, fill) =
	drawglyph_varrow(ctx, pt, size, 1, fill)
drawglyph(ctx::CairoContext, ::CGlyph{:darrow}, pt::Point2D, size::DReal, fill) =
	drawglyph_varrow(ctx, pt, size, -1, fill)

#dir=1: right, -1: left
function drawglyph_harrow(ctx::CairoContext, pt::Point2D, size::DReal, dir::Int, fill)
	elen = size #full edge length
	hlen = elen/2 #halflength
	dlen = dir*elen #directional edge
	Cairo.move_to(ctx, pt.x+dir*hlen, pt.y)
	Cairo.rel_line_to(ctx, -dlen, hlen)
	Cairo.rel_line_to(ctx, 0, -elen)
	Cairo.close_path(ctx)
	renderfill(ctx, fill)
	Cairo.stroke(ctx)
end

drawglyph(ctx::CairoContext, ::CGlyph{:larrow}, pt::Point2D, size::DReal, fill) =
	drawglyph_harrow(ctx, pt, size, -1, fill)
drawglyph(ctx::CairoContext, ::CGlyph{:rarrow}, pt::Point2D, size::DReal, fill) =
	drawglyph_harrow(ctx, pt, size, 1, fill)

#Correctly displays a glyph, given wfrm properties.
function drawglyph_safe(ctx::CairoContext, wfrm::DWaveform, pt::Point2D)
	if wfrm.glyph.shape != :none
		_glyph = CGlyph(wfrm.glyph.shape)
		setlinestyle(ctx, :solid, Float64(wfrm.line.width))
		linecolor = getglyphcolor(wfrm.glyph, wfrm.line)
		Cairo.set_source(ctx, linecolor)
		fill = getglyphfill(wfrm.glyph)
		gsize = Float64(wfrm.glyph.size)

		drawglyph(ctx, _glyph, pt, gsize, fill)
	end
end

#==Rendering base plot elements
===============================================================================#

#Render main plot annotation (titles, axis labels, ...)
#-------------------------------------------------------------------------------
function render(canvas::PCanvas2D, a::Annotation, lyt::Layout)
	const ctx = canvas.ctx
	const bb = canvas.bb
	const graph = canvas.graphbb
	const TIMESTAMP_OFFSET = 3

	xcenter = (graph.xmin+graph.xmax)/2
	ycenter = (graph.ymin+graph.ymax)/2

	#Title
	pt = Point2D(xcenter, bb.ymin+lyt.htitle/2)
	render(ctx, a.title, pt, lyt.fnttitle, align=ALIGN_HCENTER|ALIGN_VCENTER)

	#X-axis
	pt = Point2D(xcenter, bb.ymax-lyt.haxlabel/2)
	render(ctx, a.xlabel, pt, lyt.fntaxlabel, align=ALIGN_HCENTER|ALIGN_VCENTER)

	#Y-axis
	pt = Point2D(bb.xmin+lyt.waxlabel/2, ycenter)
	render(ctx, a.ylabel, pt, lyt.fntaxlabel, align=ALIGN_HCENTER|ALIGN_VCENTER, angle=-π/2)

	#Time stamp
	if lyt.showtimestamp
		pt = Point2D(bb.xmax-TIMESTAMP_OFFSET, bb.ymax-TIMESTAMP_OFFSET)
		render(ctx, a.timestamp, pt, lyt.fnttime, align=ALIGN_RIGHT|ALIGN_BOTTOM)
	end
end

#Render frame around graph
#-------------------------------------------------------------------------------
function render_graphframe(canvas::PCanvas2D)
	const ctx = canvas.ctx
	Cairo.set_source(ctx, COLOR_BLACK)
	Cairo.set_line_width(ctx, 2);
	Cairo.rectangle(ctx, canvas.graphbb)
	Cairo.stroke(ctx)
end

#==Render grid
===============================================================================#
render_vlines(ctx::CairoContext, graphbb::BoundingBox, xf::Transform2D, lyt::Layout, xlines::AbstractGridLines) = nothing
function render_vlines(ctx::CairoContext, graphbb::BoundingBox, xf::Transform2D, lyt::Layout, xlines::GridLines)
	if lyt.grid.vmajor
		setlinestyle(ctx, :dash, GRID_MAJOR_WIDTH)
		Cairo.set_source(ctx, GRID_MAJOR_COLOR)
		for xline in xlines.major
			x = ptmap(xf, Point2D(xline, 0)).x
			drawline(ctx, Point2D(x, graphbb.ymin), Point2D(x, graphbb.ymax))
		end
	end
	if lyt.grid.vminor
		setlinestyle(ctx, :dash, GRID_MINOR_WIDTH)
		Cairo.set_source(ctx, GRID_MINOR_COLOR)
		for xline in xlines.minor
			x = ptmap(xf, Point2D(xline, 0)).x
			drawline(ctx, Point2D(x, graphbb.ymin), Point2D(x, graphbb.ymax))
		end
	end
end
render_hlines(ctx::CairoContext, graphbb::BoundingBox, xf::Transform2D, lyt::Layout, ylines::AbstractGridLines) = nothing
function render_hlines(ctx::CairoContext, graphbb::BoundingBox, xf::Transform2D, lyt::Layout, ylines::GridLines)
	if lyt.grid.hmajor
		setlinestyle(ctx, :dash, GRID_MAJOR_WIDTH)
		Cairo.set_source(ctx, GRID_MAJOR_COLOR)
		for yline in ylines.major
			y = ptmap(xf, Point2D(0, yline)).y
			drawline(ctx, Point2D(graphbb.xmin, y), Point2D(graphbb.xmax, y))
		end
	end
	if lyt.grid.hminor
		setlinestyle(ctx, :dash, GRID_MINOR_WIDTH)
		Cairo.set_source(ctx, GRID_MINOR_COLOR)
		for yline in ylines.minor
			y = ptmap(xf, Point2D(0, yline)).y
			drawline(ctx, Point2D(graphbb.xmin, y), Point2D(graphbb.xmax, y))
		end
	end
end

function render_grid(canvas::PCanvas2D, lyt::Layout, grid::GridRect)
	const ctx = canvas.ctx
	Cairo.save(ctx) #-----
	render_vlines(ctx, canvas.graphbb, canvas.xf, lyt, grid.xlines)
	render_hlines(ctx, canvas.graphbb, canvas.xf, lyt, grid.ylines)
	Cairo.restore(ctx) #-----
end

#==Render ticks
===============================================================================#

#Default label:
function render_ticklabel(ctx::CairoContext, val::DReal, pt::Point2D, font::Font, align::CAlignment, fmt::TickLabelFormatting, ::AxisScale)
	tstr = ""
	if false #Old display system.
		#TODO: deprecate
		tstr = "$val"
		if length(tstr) > 7 #HACK!
			tstr = @sprintf("%0.1e", val)
		end
	else
		tstr = formatted(val, fmt.fmt, showexp=!fmt.splitexp)
	end
	render(ctx, tstr, pt, font, align=align)
end

render_ticklabel(ctx::CairoContext, val::DReal, pt::Point2D, font::Font, align::CAlignment, fmt::TickLabelFormatting, scale::AxisScale{:log10}) =
	render_power(ctx, 10, val, pt, font, align)

function render_axisscalelabel(ctx::CairoContext, pt::Point2D, font::Font, align::CAlignment, fmt::TickLabelFormatting, ::AxisScale)
	tstr = formatted_exp(fmt.fmt)
	render(ctx, tstr, pt, font, align=align)
end

#Render ticks: Well-defined GridLines
#-------------------------------------------------------------------------------
function render_xticks(ctx::CairoContext, graphbb::BoundingBox, xf::Transform2D, lyt::Layout, xlines::GridLines)
	fmt = TickLabelFormatting(lyt.xlabelformat, xlines.rnginfo)
	yframe = graphbb.ymax
	ylabel = graphbb.ymax + lyt.vlabeloffset
	for xtick in xlines.major
		x = ptmap(xf, Point2D(xtick, 0)).x
		render_ticklabel(ctx, xtick, Point2D(x, ylabel), lyt.fntticklabel, ALIGN_TOP|ALIGN_HCENTER, fmt, xlines.scale)
		drawline(ctx, Point2D(x, yframe), Point2D(x, yframe-TICK_MAJOR_LEN))
	end
	for xtick in xlines.minor
		x = ptmap(xf, Point2D(xtick, 0)).x
		drawline(ctx, Point2D(x, yframe), Point2D(x, yframe-TICK_MINOR_LEN))
	end
	if fmt.splitexp
		xlabel = graphbb.xmax + lyt.tframe
		render_axisscalelabel(ctx, Point2D(xlabel, yframe), lyt.fntticklabel, ALIGN_BOTTOM|ALIGN_LEFT, fmt, xlines.scale)
	end
end
function render_yticks(ctx::CairoContext, graphbb::BoundingBox, xf::Transform2D, lyt::Layout, ylines::GridLines)
	fmt = TickLabelFormatting(lyt.ylabelformat, ylines.rnginfo)
	xframe = graphbb.xmin
	xlabel = graphbb.xmin - lyt.hlabeloffset
	for ytick in ylines.major
		y = ptmap(xf, Point2D(0, ytick)).y
		render_ticklabel(ctx, ytick, Point2D(xlabel, y), lyt.fntticklabel, ALIGN_RIGHT|ALIGN_VCENTER, fmt, ylines.scale)
		drawline(ctx, Point2D(xframe, y), Point2D(xframe+TICK_MAJOR_LEN, y))
	end
	for ytick in ylines.minor
		y = ptmap(xf, Point2D(0, ytick)).y
		drawline(ctx, Point2D(xframe, y), Point2D(xframe+TICK_MINOR_LEN, y))
	end
	if fmt.splitexp
		ylabel = graphbb.ymin - lyt.tframe
		render_axisscalelabel(ctx, Point2D(xframe, ylabel), lyt.fntticklabel, ALIGN_BOTTOM|ALIGN_LEFT, fmt, ylines.scale)
	end
end

#Render ticks: UndefinedGridLines
#-------------------------------------------------------------------------------
function render_xticks(ctx::CairoContext, graphbb::BoundingBox, xf::Transform2D, lyt::Layout, xlines::UndefinedGridLines)
	fmt = TickLabelFormatting(NoRangeDisplayInfo())
	yframe = graphbb.ymax
	ylabel = graphbb.ymax + lyt.vlabeloffset
	for (x, xlabel) in [(graphbb.xmin, xlines.minline), (graphbb.xmax, xlines.maxline)]
		render_ticklabel(ctx, xlabel, Point2D(x, ylabel), lyt.fntticklabel, ALIGN_TOP|ALIGN_HCENTER, fmt, AxisScale{:lin}())
		drawline(ctx, Point2D(x, yframe), Point2D(x, yframe-TICK_MAJOR_LEN))
	end
end
function render_yticks(ctx::CairoContext, graphbb::BoundingBox, xf::Transform2D, lyt::Layout, ylines::UndefinedGridLines)
	fmt = TickLabelFormatting(NoRangeDisplayInfo())
	xframe = graphbb.xmin
	xlabel = graphbb.xmin - lyt.hlabeloffset
	for (y, ylabel) in [(graphbb.ymax, ylines.minline), (graphbb.ymin, ylines.maxline)]
		render_ticklabel(ctx, ylabel, Point2D(xlabel, y), lyt.fntticklabel, ALIGN_RIGHT|ALIGN_VCENTER, fmt, AxisScale{:lin}())
		drawline(ctx, Point2D(xframe, y), Point2D(xframe+TICK_MAJOR_LEN, y))
	end
end

function render_ticks(canvas::PCanvas2D, lyt::Layout, grid::GridRect)
	render_xticks(canvas.ctx, canvas.graphbb, canvas.xf, lyt, grid.xlines)
	render_yticks(canvas.ctx, canvas.graphbb, canvas.xf, lyt, grid.ylines)
end

#==High-level rendering
===============================================================================#

#Render axis labels, ticks, ...
#-------------------------------------------------------------------------------
function render_axes(canvas::PCanvas2D, lyt::Layout, grid::GridRect)
	render_graphframe(canvas)
	render_ticks(canvas, lyt, grid)
end

#Render an actual waveform
#-------------------------------------------------------------------------------
function render(canvas::PCanvas2D, wfrm::DWaveform)
	ctx = canvas.ctx
	ds = wfrm.ds

	if length(ds) < 2
		return
	end

if wfrm.line.style != :none
	Cairo.set_source(ctx, wfrm.line.color)
	setlinestyle(ctx, wfrm.line.style, Float64(wfrm.line.width))
	pt = ptmap(canvas.xf, ds[1])
	Cairo.move_to(ctx, pt.x, pt.y)
	for i in 2:length(ds)
		pt = ptmap(canvas.xf, ds[i])
		Cairo.line_to(ctx, pt.x, pt.y)
	end
#	set_line_join(ctx, Cairo.CAIRO_LINE_JOIN_MITER)
	Cairo.stroke(ctx)
end

	if :none == wfrm.glyph.shape; return; end
	_glyph = CGlyph(wfrm.glyph.shape)

	#TODO: do not draw when outside graph extents.

	#Draw symbols:
	setlinestyle(ctx, :solid, Float64(wfrm.line.width))
	linecolor = getglyphcolor(wfrm.glyph, wfrm.line)
	Cairo.set_source(ctx, linecolor)
	fill = getglyphfill(wfrm.glyph)
	gsize = Float64(wfrm.glyph.size)
	for i in 1:length(ds)
		pt = ptmap(canvas.xf, ds[i])
		drawglyph(ctx, _glyph, pt, gsize, fill)
	end

	return
end
render(canvas::PCanvas2D, wfrmlist::Vector{DWaveform}) =
	map((w)->render(canvas, w), wfrmlist)

#Last line


"""
    calculateSWI(data::Data, options::Options=Options())

Returns the calculated SWI using 'data' and 'options'.

# Examples
```julia-repl
julia> TEs = [4,8,12]
julia> data = Data(mag, phase, header(mag), TEs);
julia> swi = calculateSWI(data);
```
# With Options
$(@doc Options)

# Examples
```julia-repl
julia> TEs = [4,8,12]
julia> data = Data(mag, phase, header(mag), TEs);
julia> options = Options(phase_hp_σ=[10,10,5], mag_softplus=false)
julia> swi = calculateSWI(data, options);
```
"""
function calculateSWI(data::Data, options::Options=Options())
    if !isnothing(options.writesteps) mkpath(options.writesteps) end
    getswimag(data, options) .* getswiphase(data, options)
end

"""
    createMIP(S::AbstractArray{<:Number,3}, d=7)

Creates minimum intensity projection of `S` over `d` slices.

# Examples
```julia-repl
julia> TEs = [4,8,12]
julia> data = Data(mag, phase, header(mag), TEs);
julia> swi = calculateSWI(data);
julia> mip = createMIP(swi);
```

See also [`createIntensityProjection`](@ref)
"""
createMIP(S::AbstractArray{<:Number,3}, d=7) = createIntensityProjection(S, minimum, d)

"""
    createIntensityProjection(S::AbstractArray{T,3}, func, d=7) where T

Calculates an intensity projection over the 3rd dimension of `S` with the function `func` over `d` slices.
Good function are `minimum`, `maximum`, `mean`, `std`, ...

# Examples
```julia-repl
julia> using Statistics
julia> a = rand(Float64, (64,64,20))
julia> createIntensityProjection(a, std)
```
"""
function createIntensityProjection(S::AbstractArray{T,3}, func, d=7) where T
    [func(S[x,y,z:z+d-1]) for x in 1:size(S,1), y in 1:size(S,2), z in 1:size(S,3)-d+1]
end


# This file is a part of Julia. License is MIT: https://julialang.org/license

abstract type IPAddr end

Base.isless(a::T, b::T) where {T<:IPAddr} = isless(a.host, b.host)
(dt::Type{<:Integer})(ip::IPAddr) = dt(ip.host)::dt

struct IPv4 <: IPAddr
    host::UInt32
    IPv4(host::UInt32) = new(host)
    IPv4(a::UInt8,b::UInt8,c::UInt8,d::UInt8) = new(UInt32(a)<<24|
                                                    UInt32(b)<<16|
                                                    UInt32(c)<<8|
                                                    d)
    function IPv4(a::Integer,b::Integer,c::Integer,d::Integer)
        if !(0<=a<=255 && 0<=b<=255 && 0<=c<=255 && 0<=d<=255)
            throw(ArgumentError("IPv4 field out of range (must be 0-255)"))
        end
        IPv4(UInt8(a),UInt8(b),UInt8(c),UInt8(d))
    end
end

"""
    IPv4(host::Integer) -> IPv4

Returns an IPv4 object from ip address `host` formatted as an [`Integer`](@ref).

```jldoctest
julia> IPv4(3223256218)
ip"192.30.252.154"
```
"""
function IPv4(host::Integer)
    if host < 0
        throw(ArgumentError("IPv4 address must be positive"))
    elseif typemax(typeof(host)) > typemax(UInt32) && host > typemax(UInt32)
        throw(ArgumentError("IPv4 address must fit within 32 bits"))
    else
        return IPv4(UInt32(host))
    end
end

# constructor: ("1.2.3.4")
IPv4(str::AbstractString) = parse(IPv4, str)

show(io::IO,ip::IPv4) = print(io,"ip\"",ip,"\"")
print(io::IO,ip::IPv4) = print(io,string((ip.host&(0xFF000000))>>24),".",
                                  string((ip.host&(0xFF0000))>>16),".",
                                  string((ip.host&(0xFF00))>>8),".",
                                  string(ip.host&0xFF))

struct IPv6 <: IPAddr
    host::UInt128
    IPv6(host::UInt128) = new(host)
    IPv6(a::UInt16,b::UInt16,c::UInt16,d::UInt16,
     e::UInt16,f::UInt16,g::UInt16,h::UInt16) = new(UInt128(a)<<(7*16)|
                            UInt128(b)<<(6*16)|
                            UInt128(c)<<(5*16)|
                            UInt128(d)<<(4*16)|
                            UInt128(e)<<(3*16)|
                            UInt128(f)<<(2*16)|
                            UInt128(g)<<(1*16)|
                            h)
    function IPv6(a::Integer,b::Integer,c::Integer,d::Integer,
          e::Integer,f::Integer,g::Integer,h::Integer)
        if !(0<=a<=0xFFFF && 0<=b<=0xFFFF && 0<=c<=0xFFFF && 0<=d<=0xFFFF &&
             0<=e<=0xFFFF && 0<=f<=0xFFFF && 0<=g<=0xFFFF && 0<=h<=0xFFFF)
            throw(ArgumentError("IPv6 field out of range (must be 0-65535)"))
        end
        IPv6(UInt16(a),UInt16(b),UInt16(c),UInt16(d),
             UInt16(e),UInt16(f),UInt16(g),UInt16(h))
    end
end

"""
    IPv6(host::Integer) -> IPv6

Returns an IPv6 object from ip address `host` formatted as an [`Integer`](@ref).

```jldoctest
julia> IPv6(3223256218)
ip"::c01e:fc9a"
```
"""
function IPv6(host::Integer)
    if host < 0
        throw(ArgumentError("IPv6 address must be positive"))
        # We allow passing bigger integer types, but need to be careful to avoid overflow
        # Let's hope promotion rules are sensible
    elseif typemax(typeof(host)) > typemax(UInt128) && host > typemax(UInt128)
        throw(ArgumentError("IPv6 address must fit within 128 bits"))
    else
        return IPv6(UInt128(host))
    end
end

IPv6(str::AbstractString) = parse(IPv6, str)

# Suppress leading '0's and "0x"
print_ipv6_field(io,field::UInt16) = print(io,string(field, base = 16))

print_ipv6_field(io,ip,i) = print_ipv6_field(io,ipv6_field(ip,i))
function ipv6_field(ip::IPv6,i)
    if i < 0 || i > 7
        throw(BoundsError())
    end
    UInt16(ip.host&(UInt128(0xFFFF)<<(i*16))>>(i*16))
end

show(io::IO, ip::IPv6) = print(io,"ip\"",ip,"\"")
# RFC 5952 compliant show function
# http://tools.ietf.org/html/rfc5952
function print(io::IO,ip::IPv6)
    i = 8
    m = 0
    longest_sub_i = -1
    while i!=0
        i-=1
        field = ipv6_field(ip,i)
        if field == 0 && longest_sub_i == -1
            # Find longest subsequence of 0
            longest_sub_i,j,m,c = i,i,1,1
            while j != 0
                j-=1
                if ipv6_field(ip,j) == 0
                    c += 1
                else
                    c = 0
                end
                if c > m
                    if j+c != longest_sub_i+1
                        longest_sub_i = j+c-1
                    end
                    m = c
                end
            end
            # Prevent single 0 from contracting to :: as required
            if m == 1
                longest_sub_i = 9
            end
        end
        if i == longest_sub_i
            print(io,":")
            i -= m-1
            if i == 0
                print(io,":")
                break
            end
        else
            if i != 7
                print(io,":")
            end
            print_ipv6_field(io,field)
        end
    end
end

# Parsing

const ipv4_leading_zero_error = """
Leading zeros in IPv4 addresses are disallowed due to ambiguity.
If the address is in octal or hexadecimal, convert it to decimal, otherwise remove the leading zero.
"""

function parse(::Type{IPv4}, str::AbstractString)
    fields = split(str,'.')
    i = 1
    ret = 0
    for f in fields
        if isempty(f)
            throw(ArgumentError("empty field in IPv4 address"))
        end
        if length(f) > 1 && f[1] == '0'
            throw(ArgumentError(ipv4_leading_zero_error))
        else
            r = parse(Int, f, base = 10)
        end
        if i != length(fields)
            if r < 0 || r > 255
                throw(ArgumentError("IPv4 field out of range (must be 0-255)"))
            end
            ret |= UInt32(r) << ((4-i)*8)
        else
            if r > ((UInt64(1)<<((5-length(fields))*8))-1)
                throw(ArgumentError("IPv4 field too large"))
            end
            ret |= r
        end
        i+=1
    end
    IPv4(ret)
end

function parseipv6fields(fields,num_fields)
    if length(fields) > num_fields
        throw(ArgumentError("too many fields in IPv6 address"))
    end
    cf = 7
    ret = UInt128(0)
    for f in fields
        if isempty(f)
            # ::abc:... and ..:abc::
            if cf != 7 && cf != 0
                cf -= num_fields-length(fields)
            end
            cf -= 1
            continue
        end
        ret |= UInt128(parse(Int, f, base = 16))<<(cf*16)
        cf -= 1
    end
    ret
end
parseipv6fields(fields) = parseipv6fields(fields,8)

function parse(::Type{IPv6}, str::AbstractString)
    fields = split(str,':')
    if length(fields) > 8
        throw(ArgumentError("too many fields in IPv6 address"))
    elseif length(fields) == 8
        return IPv6(parseipv6fields(fields))
    elseif in('.',fields[end])
        return IPv6((parseipv6fields(fields[1:(end-1)],6))
            | parse(IPv4, fields[end]).host )
    else
        return IPv6(parseipv6fields(fields))
    end
end

#
# This support IPv4 addresses in the common dot (IPv4) or colon (IPv6)
# separated formats. Most other common formats use a standard integer encoding
# of the appropriate size and should use the appropriate constructor
#

function parse(::Type{IPAddr}, str::AbstractString)
    if ':' in str
        return parse(IPv6, str)
    else
        return parse(IPv4, str)
    end
end

macro ip_str(str)
    return parse(IPAddr, str)
end

struct InetAddr{T<:IPAddr}
    host::T
    port::UInt16
end

InetAddr(ip::IPAddr, port) = InetAddr{typeof(ip)}(ip, port)