using DataStructures
using NPZ
using Statistics
using VCFTools

struct Segment{T}
    snp_start::T
    snp_end::T
    sample_start::T
    sample_end::T
end

"""
    build_prefix_and_divergence_arrays(H::Array{Int8, 2})

Compute the prefix and divergence arrays (n_samples x n_snps+1) for the
Positional Burrows-Wheeler Transform on the haplotype matrix H (n_samples x n_snps).
"""
function build_prefix_and_divergence_arrays(H::Array{Int8, 2})
    n_samples, n_snps = size(H)
    u = v = p = q = index = match_start = one(Int32)
    ppa_buffer = Vector{Int32}(undef, n_samples)
    div_buffer = Vector{Int32}(undef, n_samples)
    ppa = Matrix{Int32}(undef, n_samples, n_snps + 1)
    div = Matrix{Int32}(undef, n_samples, n_snps + 1)
    ppa[:, 1] = 1:1:n_samples
    div[:, 1] = ones(Int32, n_samples)
    @inbounds for snp_index in 1:n_snps
        u = v = 1
        p = q = snp_index + 1
        for sample_index in 1:n_samples
            index = ppa[sample_index, snp_index]
            match_start = div[sample_index, snp_index]
            if match_start > p
                p = match_start
            end
            if match_start > q
                q = match_start
            end
            if H[index, snp_index] == 0
                ppa[u, snp_index + 1] = index
                div[u, snp_index + 1] = p
                u += 1
                p = 0
            else
                ppa_buffer[v] = index
                div_buffer[v] = q
                v += 1
                q = 0
            end
        end
        for sample_index in u:n_samples
            ppa[sample_index, snp_index + 1] = ppa_buffer[sample_index - u + 1]
            div[sample_index, snp_index + 1] = div_buffer[sample_index - u + 1]
        end
    end
    return ppa, div
end

function build_reverse_prefix_array(ppa::Array{Int32, 2})
    M, N = size(ppa)
    reverse_ppa = similar(ppa)
    Threads.@threads for j in 1:N
        @inbounds for i in 1:M
            reverse_ppa[ppa[i, j], j] = i
        end
    end
    return reverse_ppa
end

function build_segments(div::Array{Int32, 2}, ppa::Array{Int32, 2}, reverse_ppa::Array{Int32, 2})
    n_samples = size(div, 1)
    n_snps = size(div, 2) - 1
    segments_by_thread = [Vector{Segment{Int32}}() for t in 1:min(Threads.nthreads(), n_snps)]
    Threads.@threads for snp_index in 1:n_snps
        segments = segments_by_thread[Threads.threadid()]
        last_segment = Segment{Int32}(snp_index, snp_index, convert(Int32, n_samples + 1), convert(Int32, n_samples + 1))
        split_index = convert(Int32, n_samples + 1)
        @inbounds for sample_index in n_samples:-1:1
            match_start = div[sample_index, snp_index + 1]
            if match_start == snp_index
                continue
            elseif last_segment.snp_start == match_start && last_segment.sample_start < sample_index < last_segment.sample_end
                continue
            elseif match_start == snp_index + 1
                segment = Segment{Int32}(snp_index, snp_index, convert(Int32, sample_index), convert(Int32, split_index - 1))
                push!(segments, segment)
                last_segment = segment
                split_index = sample_index
            elseif snp_index == n_snps || match_start != div[reverse_ppa[ppa[sample_index, snp_index + 1], snp_index + 2], snp_index + 2]
                sample_start = sample_index
                sample_end = n_samples
                for neighbor_index in sample_index-1:-1:1
                    if match_start < div[neighbor_index, snp_index + 1]
                        sample_start = neighbor_index
                        break
                    end
                end
                for neighbor_index in sample_index+1:n_samples
                    if match_start < div[neighbor_index, snp_index + 1]
                        sample_end = neighbor_index - 1
                        break
                    end
                end
                segment = Segment{Int32}(match_start, snp_index, convert(Int32, sample_start), convert(Int32, sample_end))
                push!(segments, segment)
                last_segment = segment
            end
        end
    end
    return cat(segments_by_thread..., dims=1)
end

function build_segment_overlaps(segments::Vector{Segment{Int32}}, ppa::Array{Int32, 2})
    n_segments = length(segments)
    n_snps = size(ppa, 2) - 1
    segment_index_by_snp = [Vector{Int32}() for snp_index in 1:n_snps]
    segment_overlaps = [Vector{Int32}() for segment_index in 1:n_segments]
    for segment_index in 1:n_segments
        segment = segments[segment_index]
        for snp_index in segment.snp_start:segment.snp_end
            push!(segment_index_by_snp[snp_index], segment_index)
        end
    end
    Threads.@threads for segment_index in 1:n_segments
        segment = segments[segment_index]
        candidate_overlap_set = Set{Int32}()
        original_form_sample_indices = Vector{Int32}()
        @inbounds for sample_index in segment.sample_start:segment.sample_end
            push!(original_form_sample_indices, ppa[sample_index, segment.snp_end + 1])
        end
        @inbounds for snp_index in segment.snp_start:segment.snp_end
            for candidate_index in segment_index_by_snp[snp_index]
                # if segment_index == candidate_index || in(candidate_index, candidate_overlap_set)
                if in(candidate_index, candidate_overlap_set)
                    continue
                end
                push!(candidate_overlap_set, candidate_index)
                candidate_segment = segments[candidate_index]
                for sample_index in candidate_segment.sample_start:candidate_segment.sample_end
                    if in(ppa[sample_index, candidate_segment.snp_end + 1], original_form_sample_indices)
                        push!(segment_overlaps[segment_index], candidate_index)
                        break
                    end
                end
            end
        end
    end
    return segment_overlaps
end

function calculate_scores(coverage::Array{Int8, 2}, ppa::Array{Int32, 2}, segments::Vector{Segment{Int32}}, indices::Vector{Int32})
    scores = Vector{Int32}(undef, length(indices))
    Threads.@threads for (score_index, segment_index) in collect(enumerate(indices))
        segment = segments[segment_index]
        @inbounds original_form_sample_indices = [ppa[sample_index, segment.snp_end + 1] for sample_index in segment.sample_start:segment.sample_end]
        score = zero(Int32)
        for snp_index in segment.snp_start:segment.snp_end
            for original_form_sample_index in original_form_sample_indices
                @inbounds score += coverage[original_form_sample_index, snp_index]
            end
        end
        @inbounds scores[score_index] = score
    end
    return scores
end

function calculate_scores(coverage::Array{Int8, 2}, ppa::Array{Int32, 2}, segments::Vector{Segment{Int32}})
    indices = convert(Vector{Int32}, 1:length(segments) |> collect)
    return calculate_scores(coverage, ppa, segments, indices)
end

function update_coverage!(coverage::Array{Int8, 2}, ppa::Array{Int32, 2}, segments::Vector{Segment{Int32}}, segment_index::Int)
    segment = segments[segment_index]
    original_form_sample_indices = [ppa[sample_index, segment.snp_end + 1] for sample_index in segment.sample_start:segment.sample_end]
    for snp_index in segment.snp_start:segment.snp_end
        for original_form_sample_index in original_form_sample_indices
            @inbounds coverage[original_form_sample_index, snp_index] = 0
        end
    end
end

function update_scores!(scores::MutableBinaryMaxHeap{Int32}, coverage::Array{Int8, 2}, ppa::Array{Int32, 2}, segments::Vector{Segment{Int32}}, indices::Vector{Int32})
    updated_scores = calculate_scores(coverage, ppa, segments, indices)
    for (segment_index, score) in zip(indices, updated_scores)
        update!(scores, convert(Int, segment_index), score)
    end
end

function build_encoding_haplotypes(H::Array{Int8, 2}, ppa::Array{Int32, 2}, encoding_segments::Vector{Segment{Int32}})
    n_segments = length(encoding_segments)
    encoding_haplotypes = Vector{Vector{Int8}}(undef, n_segments)
    Threads.@threads for segment_index in 1:n_segments
        @inbounds segment = encoding_segments[segment_index]
        @inbounds encoding_haplotypes[segment_index] = H[ppa[segment.sample_start, segment.snp_end + 1], segment.snp_start:segment.snp_end]
    end
    return encoding_haplotypes
end

function encode_haplotype_matrix(H::Array{Int8, 2}, ppa::Array{Int32, 2}, encoding_segments::Vector{Segment{Int32}})
    n_samples = size(H, 1)
    n_segments = length(encoding_segments)
    H_encoded = zeros(Int8, n_samples, n_segments)
    Threads.@threads for segment_index in 1:n_segments
        @inbounds segment = encoding_segments[segment_index]
        for sample_index in segment.sample_start:segment.sample_end
            @inbounds H_encoded[ppa[sample_index, segment.snp_end + 1], segment_index] = 1
        end
    end
    return H_encoded
end

function encode_haplotype_matrix(H::Array{Int8, 2}, encoding_positions::Vector{Int32}, encoding_haplotypes::Vector{Vector{Int8}})
    n_samples = size(H, 1)
    n_haplotypes = length(encoding_haplotypes)
    H_encoded = ones(Int8, n_samples, n_haplotypes)
    Threads.@threads for haplotype_index in 1:n_haplotypes
        @inbounds snp_start = encoding_positions[haplotype_index]
        @inbounds haplotype = encoding_haplotypes[haplotype_index]
        for sample_index in 1:n_samples
            for snp_index in 1:length(haplotype)
                @inbounds if H[sample_index, snp_start + snp_index - 1] != haplotype[snp_index]
                    @inbounds H_encoded[sample_index, haplotype_index] = 0
                    break
                end
            end
        end
    end
    return H_encoded
end

function build_encoding(H::Array{Int8, 2}, max_size::Int)
    encoding_segments = Vector{Segment{Int32}}(undef, 0)
    ppa, div = build_prefix_and_divergence_arrays(H)
    reverse_ppa = build_reverse_prefix_array(ppa)
    segments = build_segments(div, ppa, reverse_ppa)
    segment_overlaps = build_segment_overlaps(segments, ppa)
    coverage = ones(Int8, size(H, 1), size(H, 2))
    scores = MutableBinaryMaxHeap{Int32}(calculate_scores(coverage, ppa, segments))
    score, segment_index = top_with_handle(scores)
    while score > 0 && length(encoding_segments) < max_size
        push!(encoding_segments, segments[segment_index])
        update_coverage!(coverage, ppa, segments, segment_index)
        update_indices = [index for index in segment_overlaps[segment_index] if scores[convert(Int, index)] > 0]
        update_scores!(scores, coverage, ppa, segments, update_indices)
        score, segment_index = top_with_handle(scores)
    end
    encoding_positions = [segment.snp_start for segment in encoding_segments]
    encoding_haplotypes = build_encoding_haplotypes(H, ppa, encoding_segments)
    H_encoded = encode_haplotype_matrix(H, ppa, encoding_segments)
    return encoding_positions, encoding_haplotypes, H_encoded
end

function build_encoding(H::Array{Int8, 2})
    return build_encoding(H, length(H))
end

H =  Array{Int8, 2}([0 1 0 1 0 1; 1 1 0 0 0 1; 1 1 1 1 1 1; 0 1 1 1 1 0; 0 0 0 0 0 0; 1 0 0 0 1 0; 1 1 0 0 0 1; 0 1 0 1 1 0])
# H =  Array{Int8, 2}([1 0 0 0; 0 0 1 0; 0 0 1 0; 1 0 1 0])
path = "/media/storage/1000_genomes/GRCh38/variants/chr20/yri.chr20.GRCh38.vcf"
path = "/media/storage/1000_genomes/GRCh38/variants/chr10/LATINO.chr10.phase3_shapeit2_mvncall_integrated_v5a.20130502.genotypes.biallelic.vcf.gz"
path = "/home/rlaboulaye/Work/projects/genome-retrieval/admixed_american_2.vcf"
path = "/home/rlaboulaye/Work/projects/genome-retrieval/ashkenazi.vcf"
H = convert_ht(Int8, path)
# filter out maf <= 1%
H = H[:, Statistics.mean(H, dims=1)[1, :] .> .01]
@time encoding_positions, encoding_haplotypes, H_encoded = build_encoding(H)
npzwrite("ashkenazi.chr10.haplosoup.npy", H_encoded)
npzwrite("ashkenazi.chr10.encoding_positions.npy", encoding_positions)

size(encoding_positions)

ppa, div = build_prefix_and_divergence_arrays(H)
reverse_ppa = build_reverse_prefix_array(ppa)
segments = build_segments(div, ppa, reverse_ppa)
segment_overlaps = build_segment_overlaps(segments, ppa)
coverage = ones(Int8, size(H, 1), size(H, 2))
scores = MutableBinaryMaxHeap{Int32}(calculate_scores(coverage, ppa, segments))

segment_index = 15
segment = segments[segment_index]
H[ppa[:, segment.snp_end + 1], :]
for segment_index in segment_overlaps[segment_index]
    segment = segments[segment_index]
    println(segment)
    println(H[ppa[:, segment.snp_end + 1], 1:segment.snp_end])
end


## Basic functions
using DataStructures
using Flux: onehot
using PyCall
chem = pyimport_conda("rdkit.Chem", "rdkit", "conda-forge")

# currently supported atomic features and bond features
const atom_feat_list = ["symbol","degree","implicit_valence","formal_charge","radical_electrons","hybridization","aromaticity","total_H_num" ]
const bond_feat_list = ["bond_type","isConjugated","isInring"]

# Map atomic feature names to RDKit functions
const rdkit_atom_functions = Dict(
    "symbol"=>chem.Atom.GetSymbol,
    "degree"=>chem.Atom.GetDegree,
    "implicit_valence"=>chem.Atom.GetImplicitValence,
    "formal_charge"=>chem.Atom.GetFormalCharge,
    "radical_electrons"=>chem.Atom.GetNumRadicalElectrons,
    "hybridization"=>chem.Atom.GetHybridization,
    "aromaticity"=>chem.Atom.GetIsAromatic,
    "total_H_num"=>chem.Atom.GetTotalNumHs
)

# Map bond feature names to RDKit functions
const rdkit_bond_functions = Dict(
    "bond_type"=>chem.Bond.GetBondType,
    "isConjugated"=>chem.Bond.GetIsConjugated,
    "isInring"=>chem.Bond.IsInRing
)

# bins for atomic features
const atom_feat_bins = Dict(
    "symbol" => ["C","N","O","S","F","Si","P","Cl","Br","Mg","Na","Ca","Fe","As","Al","I","B","V","K","Tl","Yb","Sb","Sn","Ag","Pd","Co","Se","Ti","Zn",'H',"Li","Ge","Cu","Au","Ni","Cd","In","Mn","Zr","Cr","Pt","Hg","Pb","Unknown"],
    "degree" => [0,1,2,3,4,5,6,7,8,9,10],
    "implicit_valence" => [0,1,2,3,4,5,6],
    "formal_charge" => [nothing],  # [-1,0,1,2], # indicates using real value
    "radical_electrons" => [nothing],  # [0,1,2,3], # indicates using use real value
    "hybridization" => [chem.rdchem.HybridizationType.SP, chem.rdchem.HybridizationType.SP2,chem.rdchem.HybridizationType.SP3,chem.rdchem.HybridizationType.SP3D, chem.rdchem.HybridizationType.SP3D2],  # [2,3,4,5,6],
    "aromaticity" => [nothing],  # [0,1], # indicates using real value
    "total_H_num" => [0,1,2,3,4]
)

# bins for bond features
const bond_feat_bins = Dict(
    "bond_type" => [chem.rdchem.BondType.SINGLE, chem.rdchem.BondType.DOUBLE,chem.rdchem.BondType.TRIPLE, chem.rdchem.BondType.AROMATIC],  #[1,2,3,12],
    "isConjugated" => [nothing],  # [0,1],# indicates using real value
    "isInring" => [nothing]  #[0,1],# indicates using real value
)

"""
input:
    atom (RDKit.Atom)
    feature_list(Array{String}): list of atomic features of interest
output:
    atom_feat (OrderedDict): Dictionary of atomic features
"""
function smiles_atom_features(atom; feature_list::Array{String}=atom_feat_list)
    atom_feat = OrderedDict()
    for f in feature_list
        @assert f in atom_feat_list "Feature "*f*" is not in supported atomic features list: [symbol,degree,implicit_valence,formal_charge,radical_electrons,hybridization,aromaticity,total_H_num]"
        val = rdkit_atom_functions[f](atom)
        bins = atom_feat_bins[f]
        if isnothing(bins[1]) # use real value instead of one-hot
            atom_feat[f] = Float32(val)
        else
            onehot_vec = zeros(Float32,size(bins,1))
            try  # check if the value can be found in the given bins
                onehot_vec = Float32.(val .== bins)
            catch
                @warn f*" value out of range: "*string(val)*", use the max accpeted value instead"
                onehot_vec = Float32.(bins[end] .== bins)
            end
            atom_feat[f] = onehot_vec
        end
    end
    return atom_feat
end

"""
input:
    bond (RDKit.Bond)
    feature_list(Array{String}): list of bond features of interest
output:
    bond_feat (OrderedDict): Dictionary of bond features
"""
function smiles_bond_features(bond; feature_list::Array{String}=bond_feat_list)
    bond_feat = OrderedDict()
    for f in feature_list
        @assert f in bond_feat_list "Feature "*f*" is not in supported bond features list: [bond_type, isConjugated, isInring]"
        val = rdkit_bond_functions[f](bond)
        bins = bond_feat_bins[f]
        if isnothing(bins[1]) # use real value instead of one-hot
            bond_feat[f] = Float32(val)
        else
            onehot_vec = zeros(Float32,size(bins,1))
            try  # check if the value can be found in the given bins
                onehot_vec = Float32.(val .== bins)
            catch
                @warn f*" value out of range: "*string(val)*", use the max accpeted value instead"
                onehot_vec = Float32.(bins[end] .== bins)
            end
            bond_feat[f] = onehot_vec
        end
    end
    return bond_feat
end

"""
Pair features that will be used in weave_featurize()
input:
    mol (RDKit SMILES mol): mol coverted from SIMLES
    bond_feat(OrderedDict): list of bond features of mol
    adj_list(Array{Array{Int}}): adjacency list
    bf_len(Int): bond feature length
    max_distance(Int): cut-off distance
output:
    features (Array{Float32,2}): pair features for mol
"""
function pair_features(mol, bond_feats::OrderedDict, adj_list; bf_len = 6, max_distance = 7)
    N = mol.GetNumAtoms()  # number of atoms
    features = zeros(Float32,N,N,bf_len+max_distance+1) # declear feature matrix
    if length(bond_feats) == 0
        @warn "No bonds found in "*string(chem.MolToSmiles(mol))*"! Pair feature matrix filled with zeros!"
        return reshape(features,(N*N,bf_len+max_distance+1))
    end
    rings = mol.GetRingInfo().AtomRings()
    for a1 in 1:N
        for a2 in adj_list[a1]
            # first bf_len features are bond features
            index = a1<a2 ? (a1,a2) : (a2,a1)
            features[a1, a2, 1:bf_len] = bond_feats[index]
        end
        for ring in rings
            ring = ring .+ 1  # Julia index starts from 1
            if a1 in ring
                # `bf_len+1`-th feature is if the pair of atoms are in the same ring
                features[a1, collect(ring), bf_len+1] .= 1
                features[a1, a1, bf_len+1] = 0
            end
        end
        distance_matrix = zeros(Float32,N,max_distance)
        distances = _get_distance!(distance_matrix, a1, adj_list, max_distance)
        features[a1,:,bf_len+2:end] = distance_matrix
    end
    features = vcat([features[i,:,:] for i=1:size(features,1)]...)
    return features
end

"""
Helper function for pair_features()
Finds distance between the input atom and all other atoms using BFS

input:
    distance_matrix(Array{Int,2}): predefined matrix to be filled in with distances
    a1(Int): index for current atom
    adj_list(Array{Int}): adjacency list for a1
    max_distance(Int): cut-off distance
"""
function _get_distance!(distance_matrix,a1, adj_list, max_distance = 7)
    visited = falses(length(adj_list))
    visited[a1] = true
    cur_list = adj_list[a1]
    cur_distance = 1
    while length(cur_list)!=0 && cur_distance<=max_distance
        next_list = []
        for a2 in cur_list
            if visited[a2]
                continue
            end
            visited[a2] = true
            distance_matrix[a2,cur_distance] = 1
            for a3 in adj_list[a2]
                if ! visited[a3]
                    push!(next_list,a3)
                end
            end
        end
        cur_list = next_list
        cur_distance += 1
    end
end


function prefetchisagoodidea(ls::LoopSet, op::Operation, td::UnrollArgs)
    # return false
    @unpack u₁, u₁loopsym, u₂loopsym, vloopsym, vstep, u₂max, suffix = td
    length(loopdependencies(op)) ≤ 1 && return 0
    isvectorized(op) || return 0
    ((u₁ > 1) & (u₂max > 1)) || return 0
    u₂loopsym === Symbol("##undefined##") && return 0
    # @show cache_lnsze(ls) reg_size(ls) pointer_from_objref(ls.register_size)
    dontskip = (cache_lnsze(ls) ÷ reg_size(ls)) - 1
    # u₂loopsym is vectorized
    # u₁vectorized = vectorized === u₁loopsym
    u₂vectorized = vloopsym === u₂loopsym
    ((suffix ≠ -1) && u₂vectorized && !iszero(suffix & dontskip)) && return 0
    innermostloopsym = first(names(ls))
    loopedindex = op.ref.loopedindex
    if length(loopedindex) > 1 && first(loopedindex)
        indices = getindices(op)
        if first(indices) === vloopsym && first(indices) !== innermostloopsym
            (isone(vstep) && isone(first(getstrides(op)))) || return 0
            # We want at least 4 reuses per load
            uses = Core.ifelse(isu₁unrolled(op), 1, u₁)
            uses = Core.ifelse(isu₂unrolled(op), uses, uses * u₂max)
            uses < 4 && return 0
            innermostloopindv = findall(map(isequal(innermostloopsym), getindices(op)))
            isone(length(innermostloopindv)) || return 0
            innermostloopind = first(innermostloopindv)
            if prod(s -> Float64(looplength(ls, s)), @view(indices[1:innermostloopind-1])) ≥ 120.0 && length(getloop(ls, innermostloopsym)) ≥ 120
                if getoffsets(op)[innermostloopind] < 120
                    for opp ∈ operations(ls)
                        if iscompute(opp) && (innermostloopsym ∈ loopdependencies(opp)) &&
                            load_constrained(opp, u₁loopsym, u₂loopsym, innermostloopsym, true)
                            return 0
                        end
                    end
                    return innermostloopind
                end
            end
        end
    end
    0
end
function add_prefetches!(q::Expr, ls::LoopSet, op::Operation, td::UnrollArgs, prefetchind::Int)
    @unpack u₁, u₁loopsym, u₂loopsym, vloopsym, u₂max = td
    # we should only be here if `unitsride(vloop)`
    dontskip = (cache_lnsze(ls) ÷ reg_size(ls)) - 1
    ptr = vptr(op)
    innermostloopsym = first(names(ls))
    us = ls.unrollspecification[]
    prefetch_distance = u₁loopsym === innermostloopsym ? us.u₁ : ( u₂loopsym === innermostloopsym ? us.u₂ : 1 )
    # prefetch_distance = u₁loopsym === innermostloopsym ? u₁ : ( u₂loopsym === innermostloopsym ? u₂max : 1 )
    prefetch_multiplier = 5
    prefetch_distance *= prefetch_multiplier
    offsets = getoffsets(op) # what a hack
    inner_offset = offsets[prefetchind]
    prefetchstride = prefetch_distance * getstrides(op)[prefetchind]
    prefetchloop_step = step(getloop(ls, getindices(op)[prefetchind]))
    if isknown(prefetchloop_step)
        prefetchstride *= gethint(prefetchloop_step)
    end
    offsets[prefetchind] = inner_offset + prefetchstride
    gespinds = mem_offset_u(op, td, indices_calculated_by_pointer_offsets(ls, op.ref), false, 0)
    offsets[prefetchind] = inner_offset
    ptr = vptr(op)
    gptr = Symbol(ptr, "##GESPEDPREFETCH##")
    if !isknown(prefetchloop_step)
        for i ∈ eachindex(gespinds.args)
            if i == prefetchind
                gespinds.args[i] = mulexpr(getsym(prefetchloop_step), gespinds.args[i])
            end
            # gespinds.args[i] = Expr(:call, lv(:data), gespinds.args[i])
        end
    end
    push!(q.args, Expr(:(=), gptr, Expr(:call, lv(:gesp), ptr, gespinds)))

    inds = Expr(:tuple)
    indices = getindicesonly(op)

    i = 0
    for (j,ind) ∈ enumerate(indices)
        push!(inds.args, Expr(:call, lv(:Zero)))
        (ind == u₁loopsym) && (i = j)
    end
    push!(q.args, Expr(:call, lv(:prefetch0), gptr, copy(inds)))
    for u ∈ 1:u₁-1
        # for u ∈ umin:min(umin,U-1)
        # (u₁loopsym === vloopsym && !iszero(u & dontskip)) && continue
        if u₁loopsym === vloopsym
            iszero(u & dontskip) || continue
            # W = ls.vector_width[]
            # if W != 0
            #     inds.args[i] = staticexpr(W*u)
            # else
            if isone(u)
                inds.args[i] = VECTORWIDTHSYMBOL
            else
                inds.args[i] = mulexpr(VECTORWIDTHSYMBOL, u)
            end
        else
            inds.args[i] = staticexpr(u)
        end
        push!(q.args, Expr(:call, lv(:prefetch0), gptr, copy(inds)))
    end
    nothing
end
broadcastedname(mvar) = Symbol(mvar, "##broadcasted##")
function pushbroadcast!(q::Expr, mvar::Symbol)
    push!(q.args, Expr(:(=), broadcastedname(mvar), Expr(:call, lv(:vbroadcast), VECTORWIDTHSYMBOL, mvar)))
end
function lower_load_no_optranslation!(
    q::Expr, ls::LoopSet, op::Operation, td::UnrollArgs, mask::Bool, inds_calc_by_ptr_offset::Vector{Bool}
)
    @unpack u₁, u₁loopsym, u₂loopsym, vloopsym, suffix = td
    loopdeps = loopdependencies(op)
    # @assert isvectorized(op)
    opu₁ = isu₁unrolled(op)

    u = ifelse(opu₁, u₁, 1)
    mvar = Symbol(variable_name(op, Core.ifelse(isu₂unrolled(op), suffix,-1)), '_', u)
    falseexpr = Expr(:call, lv(:False)); rs = staticexpr(reg_size(ls))
    if all(op.ref.loopedindex) && !rejectcurly(op)
        inds = unrolledindex(op, td, mask, inds_calc_by_ptr_offset)
        loadexpr = Expr(:call, lv(:_vload), vptr(op), inds)
        add_memory_mask!(loadexpr, op, td, mask)
        push!(loadexpr.args, falseexpr, rs) # unaligned load
        push!(q.args, Expr(:(=), mvar, loadexpr))
    elseif u₁ > 1
        t = Expr(:tuple)
        for u ∈ 1:u₁
            inds = mem_offset_u(op, td, inds_calc_by_ptr_offset, true, u-1)
            loadexpr = Expr(:call, lv(:_vload), vptr(op), inds)
            domask = mask && (isvectorized(op) & ((u == u₁) | (vloopsym !== u₁loopsym)))
            add_memory_mask!(loadexpr, op, td, domask)
            push!(loadexpr.args, falseexpr, rs)
            push!(t.args, loadexpr)
            # push!(q.args, Expr(:(=), mvar, loadexpr))
        end
        push!(q.args, Expr(:(=), mvar, Expr(:call, lv(:VecUnroll), t)))
    else
        inds = mem_offset_u(op, td, inds_calc_by_ptr_offset, true, 0)
        loadexpr = Expr(:call, lv(:_vload), vptr(op), inds)
        add_memory_mask!(loadexpr, op, td, mask)
        push!(loadexpr.args, falseexpr, rs)
        push!(q.args, Expr(:(=), mvar, loadexpr))
    end
    if isvectorized(op)
        prefetchind = prefetchisagoodidea(ls, op, td)
        iszero(prefetchind) || add_prefetches!(q, ls, op, td, prefetchind)
    elseif any(isvectorized, children(op))
        pushbroadcast!(q, mvar)
    end
    nothing
end
# function lower_load_vectorized!(
#     q::Expr, ls::LoopSet, op::Operation, td::UnrollArgs, mask::Union{Nothing,Symbol,Unsigned} = nothing
# )
#     @unpack u₁, u₁loopsym, u₂loopsym, vectorized, suffix = td
#     loopdeps = loopdependencies(op)
#     @assert isvectorized(op)
#     opu₁ = isu₁unrolled(op)
#     inds_calc_by_ptr_offset = indices_calculated_by_pointer_offsets(ls, op.ref)
#     if opu₁
#         umin = 0
#         U = u₁
#     else
#         umin = -1
#         U = 0
#     end
#     # Urange = unrolled ∈ loopdeps ? 0:U-1 : 0
#     var = variable_name(op, suffix)
#     for u ∈ umin:U-1
#         td = UnrollArgs(td, u)
#         pushvectorload!(q, op, var, td, U, vectorized, mask, opu₁, inds_calc_by_ptr_offset, reg_size(ls))
#     end
#     prefetchind = prefetchisagoodidea(ls, op, td)
#     iszero(prefetchind) || add_prefetches!(q, ls, op, td, prefetchind, umin)
#     nothing
# end
function indisvectorized(ls::LoopSet, ind::Symbol)
    for op ∈ operations(ls)
        ((op.variable === ind) && isvectorized(op)) && return true
    end
    false
end
@inline firstunroll(vu::VecUnroll) = getfield(getfield(vu,:data),1,false)
@inline firstunroll(x) = x
@inline lastunroll(vu::VecUnroll) = last(getfield(vu,:data))
@inline lastunroll(x) = x
@inline unmm(x) = x
@inline unmm(x::MM) = getfield(x, :i)
function lower_load_for_optranslation!(
    q::Expr, op::Operation, posindicator::UInt8, ls::LoopSet, td::UnrollArgs, mask::Bool, translationind::Int
)
    @unpack u₁loop, u₂loop, vloop, u₁, u₂max, suffix = td

    # @unpack u₁, u₁loopsym, u₂loopsym, vloopsym, u₂max, suffix = td
    iszero(suffix) || return

    total_unroll = u₁ + u₂max - 1

    mref = op.ref
    inds_by_ptroff = indices_calculated_by_pointer_offsets(ls, mref)
    # initial offset pointer

    # Unroll directions can be + or -
    # we want to start at minimum position.
    step₁ = gethint(step(u₁loop))
    step₂ = gethint(step(u₂loop))

    # abs of steps are equal
    equal_steps = (step₁ == step₂) ⊻ (posindicator ≠ 0x03)
    # @show step₁, step₂, posindicator, equal_steps
    # _td = UnrollArgs(u₁loop, u₂loop, vloop, total_unroll, u₂max, Core.ifelse(equal_steps, 0, u₂max - 1))
    _td = UnrollArgs(u₁loop, u₂loop, vloop, u₁, u₂max, Core.ifelse(equal_steps, 0, u₂max - 1))
    gespinds = mem_offset(op, _td, inds_by_ptroff, false)
    ptr = vptr(op)
    gptr = Symbol(ptr, "##GESPED##")
    for i ∈ eachindex(gespinds.args)
        if i == translationind
            gespinds.args[i] = Expr(:call, lv(Core.ifelse(equal_steps, :firstunroll, :lastunroll)), gespinds.args[i])
        # else
        #     gespinds.args[i] = Expr(:call, lv(:unmm), gespinds.args[i])
        end
    end
    push!(q.args, Expr(:(=), gptr, Expr(:call, lv(:gesp), ptr, gespinds)))

    fill!(inds_by_ptroff, true)

    @unpack ref, loopedindex = mref
    indices = copy(getindices(ref))
    # old_translation_index = indices[translationind]
    # indices[translationind] = u₁loop.itersymbol
    # @show indices, translationind, vloop
    # getindicesonly returns a view of `getindices`
    dummyref = ArrayReference(ref.array, indices, zero(getoffsets(ref)), getstrides(ref))
    # loopedindex[translationind] = true
    dummymref = ArrayReferenceMeta(dummyref, fill!(similar(loopedindex), true), gptr)
    indonly = getindicesonly(dummyref)
    for i ∈ eachindex(indonly)
        if i == translationind
            indonly[i] = u₁loop.itersymbol
        elseif !loopedindex[i]
            ind = indonly[i]
            for indop ∈ operations(ls)
                if isvectorized(indop) & (name(indop) === ind)
                    indonly[i] = vloop.itersymbol
                    break
                end
            end
        end
    end
    # @show indices
    _td = UnrollArgs(u₁loop, u₂loop, vloop, total_unroll, u₂max, -1)
    op.ref = dummymref
    # @show isu₁unrolled(op), isu₂unrolled(op)
    _lower_load!(q, ls, op, _td, mask)
    # set old values
    op.ref = mref
    # loopedindex[translationind] = false
    # indices[translationind] = old_translation_index

    shouldbroadcast = (!isvectorized(op)) && any(isvectorized, children(op))

    # now we need to assign the `Vec`s from the `VecUnroll` to the correct name.
    variable_name_u = Symbol(variable_name(op, -1), '_', total_unroll)
    variable_name_data = Symbol(variable_name_u, "##data##")
    push!(q.args, :($variable_name_data = getfield($variable_name_u, 1)))
    if shouldbroadcast
        broadcasted_data = broadcastedname(variable_name_data)
        push!(q.args, :($broadcasted_data = getfield($(broadcastedname(variable_name_u)), 1)))
    end
    gf = GlobalRef(Core,:getfield)
    for u₂ ∈ 0:u₂max-1
        variable_name_u₂ = Symbol(variable_name(op, u₂), '_', u₁)
        t = Expr(:tuple)
        if shouldbroadcast
            tb = Expr(:tuple)
        end
        for u ∈ 1:u₁
            uu = if equal_steps
                u + u₂
            else
                u - u₂ + u₂max - 1
            end
            push!(t.args, :($gf($variable_name_data, $uu)))
            if shouldbroadcast
                push!(tb.args, :($gf($broadcasted_data, $uu)))
            end
        end
        push!(q.args, Expr(:(=), variable_name_u₂, Expr(:call, lv(:VecUnroll), t)))
        if shouldbroadcast
            push!(q.args, Expr(:(=), broadcastedname(variable_name_u₂), Expr(:call, lv(:VecUnroll), tb)))
        end
    end
    nothing
end

# TODO: this code should be rewritten to be more "orthogonal", so that we're just combining separate pieces.
# Using sentinel values (eg, T = -1 for non tiling) in part to avoid recompilation.
function lower_load!(
    q::Expr, op::Operation, ls::LoopSet, td::UnrollArgs, mask::Bool
)
    @unpack u₁, u₂max, u₁loopsym, u₂loopsym, vloopsym, suffix = td
    if (suffix != -1) && ls.loadelimination[]
        if (u₁ > 1) & (u₂max > 1)
            istr, ispl = isoptranslation(ls, op, UnrollSymbols(u₁loopsym, u₂loopsym, vloopsym))
            if istr ≠ 0x00
                return lower_load_for_optranslation!(q, op, ispl, ls, td, mask, istr)
            end
        end
        if (suffix > 0) && (u₂loopsym !== vloopsym)
            mno, id = maxnegativeoffset(ls, op, u₂loopsym)
            if -suffix < mno < 0 # already checked that `suffix != -1` above
                varnew = variable_name(op, suffix)
                varold = variable_name(operations(ls)[id], suffix + mno)
                opold = operations(ls)[id]
                u = isu₁unrolled(op) ? u₁ : 1
                push!(q.args, Expr(:(=), Symbol(varnew, '_', u), Symbol(varold, '_', u)))
                return
            end
        end
    end
    _lower_load!(q, ls, op, td, mask)
end
function _lower_load!(
    q::Expr, ls::LoopSet, op::Operation, td::UnrollArgs, mask::Bool, inds_calc_by_ptr_offset::Vector{Bool} = indices_calculated_by_pointer_offsets(ls, op.ref)
)
    if rejectinterleave(op)
        lower_load_no_optranslation!(q, ls, op, td, mask, inds_calc_by_ptr_offset)
    else
        omop = offsetloadcollection(ls)
        batchid, opind = omop.batchedcollectionmap[identifier(op)]
        if opind == 1
            collectionid, copind = omop.opidcollectionmap[identifier(op)]
            opidmap = offsetloadcollection(ls).opids[collectionid]
            idsformap = omop.batchedcollections[batchid]
            lower_load_collection!(q, ls, opidmap, idsformap, td, mask, inds_calc_by_ptr_offset)
        end
    end
end
function additive_vectorized_loopinductvar_only(op::Operation)
    isvectorized(op) || return true
    isloopvalue(op) && return true
    iscompute(op) || return false
    additive_instr = (:add_fast, :(+), :vadd, :identity, :sub_fast, :(-), :vsub)
    Base.sym_in(instruction(op).instr, additive_instr) || return false
    return all(additive_vectorized_loopinductvar_only, parents(op))
end
# Checks if we cannot use `Unroll`
function rejectcurly(ls::LoopSet, op::Operation, td::UnrollArgs)
    @unpack u₁loopsym, vloopsym = td
    rejectcurly(ls, op, u₁loopsym, vloopsym)
end
function rejectcurly(ls::LoopSet, op::Operation, u₁loopsym::Symbol, vloopsym::Symbol)
    indices = getindicesonly(op)
    li = op.ref.loopedindex
    AV = AU = false
    for (n,ind) ∈ enumerate(indices)
        # @show AU, op, n, ind, vloopsym, u₁loopsym
        if li[n]
            if ind === vloopsym
                AV && return true
                AV = true
            end
            if ind === u₁loopsym
                AU && return true
                AU = true
            end
        else
            opp = findop(parents(op), ind)
            # @show opp
            if isvectorized(opp)
                AV && return true
                AV = true
            end
            if (u₁loopsym === CONSTANTZEROINDEX) ? (CONSTANTZEROINDEX ∈ loopdependencies(opp)) : (isu₁unrolled(opp))
                AU && return true
                AU = true
            end
        end
    end
    false
end
function rejectinterleave(ls::LoopSet, op::Operation, vloop::Loop, idsformap::SubArray{Tuple{Int,Int}, 1, Vector{Tuple{Int,Int}}, Tuple{UnitRange{Int}}, true})
    vloopsym = vloop.itersymbol; strd = step(vloop)
    isknown(strd) || return true
    # TODO: reject if there is a vectorized !loopedindex index
    for (li,ind) ∈ zip(op.ref.loopedindex,getindicesonly(op))
        li && continue
        for indop ∈ operations(ls)
            if (name(indop) === ind) && isvectorized(indop)
                additive_vectorized_loopinductvar_only(indop) || return true # so that it is `MM`
            end
        end
    end
    (first(getindices(op)) === vloopsym) && (length(idsformap) ≠ first(getstrides(op)) * gethint(strd))
end
function lower_load_collection!(
    q::Expr, ls::LoopSet, opidmap::Vector{Int},
    idsformap::SubArray{Tuple{Int,Int}, 1, Vector{Tuple{Int,Int}}, Tuple{UnitRange{Int}}, true},
    ua::UnrollArgs, mask::Bool, inds_calc_by_ptr_offset::Vector{Bool}
)
    @unpack u₁, u₁loopsym, u₂loopsym, vloopsym, vloop, suffix = ua
    ops = operations(ls)
    nouter = length(idsformap)
    # ua = UnrollArgs(nouter, unrollsyms, u₂, 0)
    # idsformap contains (index, offset) pairs
    op = ops[opidmap[first(first(idsformap))]]
    opindices = getindices(op)
    interleave = first(opindices) === vloopsym
    # construct dummy unrolled loop
    offset_dummy_loop = Loop(first(opindices), MaybeKnown(1), MaybeKnown(1024), MaybeKnown(1), Symbol(""), Symbol(""))
    unrollcurl₂ = unrolled_curly(op, nouter, offset_dummy_loop, vloop, mask, 1) # interleave always 1 here
    inds = mem_offset_u(op, ua, inds_calc_by_ptr_offset, false)
    falseexpr = Expr(:call, lv(:False)); rs = staticexpr(reg_size(ls));
    if isu₁unrolled(op) && u₁ > 1 # both unrolled
        if interleave # TODO: handle this better than using `rejectinterleave`
            interleaveval = -nouter
        else
            interleaveval = 0
        end
        unrollcurl₁ = unrolled_curly(op, u₁, ua.u₁loop, vloop, mask, interleaveval)
        inds = Expr(:call, unrollcurl₁, inds)
    end
    uinds = Expr(:call, unrollcurl₂, inds)
    vp = vptr(op)
    loadexpr = Expr(:call, lv(:_vload), vp, uinds)
    # not using `add_memory_mask!(storeexpr, op, ua, mask)` because we checked `isconditionalmemop` earlier in `lower_load_collection!`
    (mask && isvectorized(op)) && push!(loadexpr.args, MASKSYMBOL)
    push!(loadexpr.args, falseexpr, rs)
    collectionname = Symbol(vp, "##collection##number#", opidmap[first(first(idsformap))], "#", suffix, "##size##", nouter, "##u₁##", u₁)
    # getfield to extract data from `VecUnroll` object, so we have a tuple
    push!(q.args, Expr(:(=), collectionname, Expr(:call, :getfield, loadexpr, 1)))
    u = Core.ifelse(isu₁unrolled(op), u₁, 1)
    for (i,(opid,o)) ∈ enumerate(idsformap)
        _op = ops[opidmap[opid]]
        mvar = Symbol(variable_name(_op, Core.ifelse(isu₂unrolled(_op), suffix, -1)), '_', u)
        push!(q.args, Expr(:(=), mvar, Expr(:call, :getfield, collectionname, i, false)))
    end
end


letters = ['a':'z';
		'A':'Z';
		'0':'9';
		' ';
		'	';
		'.';
		'"';
		'!';
		'?';
		'/';
		'(';
		')';
		'=';
		'-';
		'[';
		']';
		'{';
		'}';
		'\\']

function setupFontCharacters()
	glPixelStorei(GL_UNPACK_ALIGNMENT, 1)

	tex = GLuint[0]
	glGenTextures(1, tex)
	glActiveTexture(GL_TEXTURE1)
	glBindTexture(GL_TEXTURE_2D, tex[1])

	fontState.textAtlas = tex[1]

	w = 0
	h = 0
	@inbounds for c in letters
		img, metric = renderface(fontState.face, c)
		w += size(img, 1)
		h = max(h, size(img, 2))
	end
	fontState.atlasWidth = w
	fontState.atlasHeight = h

	# glTexImage2D(GL_TEXTURE_2D, 0, GL_RED, w, h, 0, GL_RED, GL_FLOAT, C_NULL)
	glTexImage2D(GL_TEXTURE_2D, 0, GL_RED, w, h, 0, GL_RED, GL_UNSIGNED_BYTE, C_NULL)
	glGenerateMipmap(GL_TEXTURE_2D)
	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE)
	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE)
	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR_MIPMAP_NEAREST)
	glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR)
	max_aniso = [Float32(0.0)]
	glGetFloatv(GL_TEXTURE_MAX_ANISOTROPY_EXT, max_aniso)
	glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAX_ANISOTROPY_EXT, 16)

	x = 0
	@inbounds for c in letters
		img, metric = renderface(fontState.face, c)

		w = size(img, 1)
		h = size(img, 2)
		# w1 = w
		# h1 = h

		# the following uses a modification of the signed distance transform
		# that comes from GLVisualize.jl. it follows some of the ideas from
		# https://www.mapbox.com/blog/text-signed-distance-fields/.
		# what we need to do is turn the glyph image into an array of Boolean
		# values. 'true' where the character is and 'false' where it is not.
		# then, we can run the sdf routine on that and save that as our
		# texture. the font shader then takes care of turning this into a
		# textual element.

		# this pads the image so that we can later downsample it properly
		# restrict_steps = 2
		# halfpad = 2*(2^restrict_steps)
		# w = w + 2*halfpad
		# h = h + 2*halfpad
		# in_or_out = zeros(Bool, w, h)

		# for x = 1:w
		#	for y = 1:h
		#		i = x-halfpad
		#		j = y-halfpad
		#		if checkbounds(Bool, size(img), i, j)
		#			in_or_out[i, j] = (img[i, j] >  0.5)
		#		else
		#			in_or_out[i, j] = false
		#		end
		#	end
		# end

		# sd = sdf(in_or_out)

		# for x = 1:restrict_steps
		#	w1, h1 = Images.restrict_size(w1), Images.restrict_size(h1)
		#	sd = Images.restrict(sd)
		# end

		# maxlen = norm([size(img, 1), size(img, 2)])

		# distfield = zeros(Float32, size(sd, 1), size(sd, 2))
		# for x = 1:size(sd, 1)
		#	for y = 1:size(sd, 2)
		#		distfield[x, y] = clamp(sd[x, y]/maxlen, -1, 1)
		#	end
		# end

		# glTexSubImage2D(GL_TEXTURE_2D, 0, x, 0, w, h, GL_RED, GL_FLOAT, distfield)
		glTexSubImage2D(GL_TEXTURE_2D, 0, x, 0, w, h, GL_RED, GL_UNSIGNED_BYTE, img)

        character = textCharacter([w, h],
					[metric.horizontal_bearing[1], metric.horizontal_bearing[2]],
					[metric.advance[1], metric.advance[2]],
					x / fontState.atlasWidth)
        fontState.characters[c] = character
        x += w
	end

    glPixelStorei(GL_UNPACK_ALIGNMENT, 0)
    FT_Done_Face(fontState.face[1])
end


#
# This file is a part of MolecularGraph.jl
# Licensed under the MIT License http://opensource.org/licenses/MIT
#

# TODO: radical

const SMARTS_ATOM_COND_SYMBOL = Dict(
    'X' => :connectivity,
    'D' => :nodedegree,
    'v' => :valence,
    'H' => :hydrogenconnected,
    'r' => :smallestsssr,
    'R' => :sssrcount
)

const SMARTS_CHARGE_SIGN = Dict(
    '+' => 1,
    '-' => -1
)

"""
    atomsymbol!(state::SmartsParserState) -> Union{Pair,Nothing}

Atomsymbol <- Br / Cl / [AaBCcNnOoPpSsFI*]
"""
function atomsymbol!(state::SmartsParserState)
    sym1 = read(state)
    sym2 = lookahead(state, 1)
    if sym1 == 'C' && sym2 == 'l'
        forward!(state, 2)
        return QueryFormula(
            :and, Set([QueryFormula(:atomsymbol, :Cl), QueryFormula(:isaromatic, false)]))
    elseif sym1 == 'B' && sym2 == 'r'
        forward!(state, 2)
        return QueryFormula(
            :and, Set([QueryFormula(:atomsymbol, :Br), QueryFormula(:isaromatic, false)]))
    elseif sym1 in "BCNOPSFI"
        forward!(state)
        return QueryFormula(
            :and, Set([QueryFormula(:atomsymbol, Symbol(sym1)), QueryFormula(:isaromatic, false)]))
    elseif sym1 in "bcnops"
        forward!(state)
        return QueryFormula(
            :and, Set([
                QueryFormula(:atomsymbol, Symbol(uppercase(sym1))),
                QueryFormula(:isaromatic, true)
            ]))
    elseif sym1 == 'A'
        forward!(state)
        return QueryFormula(:isaromatic, false)
    elseif sym1 == 'a'
        forward!(state)
        return QueryFormula(:isaromatic, true)
    elseif sym1 == '*'
        forward!(state)
        return QueryFormula(:any, true)
    end
    # return nothing
end

"""
    atomprop!(state::SmartsParserState) -> Union{Pair,Nothing}

AtomProp <- '\$(' RecursiveQuery ')' / Mass / Symbol / AtomNum / Stereo / CHG / [DHRrvX]
"""
function atomprop!(state::SmartsParserState)
    sym1 = read(state)
    sym2 = lookahead(state, 1)
    if haskey(ATOMSYMBOLMAP, string(uppercase(sym1), sym2))
        # Two letter atoms
        forward!(state, 2)
        if string(uppercase(sym1), sym2) in ("As", "Se")
            return QueryFormula(
                :and, Set([
                    QueryFormula(:atomsymbol, Symbol(uppercase(sym1), sym2)),
                    QueryFormula(:isaromatic, islowercase(sym1))
                ]))
        end
        @assert isuppercase(sym1)
        return QueryFormula(:atomsymbol, Symbol(sym1, sym2))
    elseif haskey(SMARTS_ATOM_COND_SYMBOL, sym1)
        # Neighbor and ring conditions
        forward!(state)
        if isdigit(sym2)
            num = parse(Int, sym2)
            forward!(state)
        else
            sym1 in ('r', 'R') && return QueryFormula(:not, QueryFormula(:sssrcount, 0))
            num = 1
        end
        return QueryFormula(SMARTS_ATOM_COND_SYMBOL[sym1], num)
    elseif sym1 in ('A', 'a', '*')
        forward!(state)
        sym1 == 'A' && return QueryFormula(:isaromatic, false)
        sym1 == 'a' && return QueryFormula(:isaromatic, true)
        sym1 == '*' && return QueryFormula(:any, true)
    elseif haskey(ATOMSYMBOLMAP, string(uppercase(sym1)))
        # Single letter atoms
        forward!(state)
        if uppercase(sym1) in "BCNOPS"
            return QueryFormula(
                :and, Set([
                    QueryFormula(:atomsymbol, Symbol(uppercase(sym1))),
                    QueryFormula(:isaromatic, islowercase(sym1))
                ]))
        end
        @assert isuppercase(sym1)
        return QueryFormula(:atomsymbol, Symbol(sym1))
    elseif sym1 == '#'
        # Atomic number
        forward!(state)
        start = state.pos
        while isdigit(lookahead(state, 1))
            forward!(state)
        end
        num = parse(Int, SubString(state.input, start, state.pos))
        forward!(state)
        return QueryFormula(:atomsymbol, atomsymbol(num))
    elseif sym1 in keys(SMARTS_CHARGE_SIGN)
        # Charge
        forward!(state)
        if isdigit(sym2)
            chg = parse(Int, sym2)
            forward!(state)
        else
            chg = 1
            while read(state) == sym1
                forward!(state)
                chg += 1
            end
        end
        return QueryFormula(:charge, chg * SMARTS_CHARGE_SIGN[sym1])
    elseif sym1 == '@'
        # Stereo
        # @ -> anticlockwise, @@ -> clockwise, ? -> or not specified
        forward!(state)
        cw = false
        if read(state) == '@'
            forward!(state)
            cw = true
        end
        if read(state) == '?'
            forward!(state)
            return QueryFormula(:not, QueryFormula(:stereo, cw ? :anticlockwise : :clockwise))
        else
            return QueryFormula(:stereo, cw ? :clockwise : :anticlockwise)
        end
    elseif isdigit(sym1)
        # Isotope
        start = state.pos
        while isdigit(lookahead(state, 1))
            forward!(state)
        end
        num = SubString(state.input, start, state.pos)
        forward!(state)
        return QueryFormula(:mass, parse(Int, num))
    elseif sym1 == '$' && sym2 == '('
        # Recursive
        forward!(state, 2)
        start = state.pos
        toclose = 1
        while true
            if read(state) == ')'
                if toclose == 1
                    break
                else
                    toclose -= 1
                end
            elseif read(state) == '('
                toclose += 1
            end
            forward!(state)
        end
        q = SubString(state.input, start, state.pos - 1)
        forward!(state)
        return QueryFormula(:recursive, q)
    end
    # return nothing
end

"""
    atom!(state::SmilesParser) -> Vector{SmilesAtom}

Atom <- '[' AtomProp+ ']' / AtomSymbol
"""
function atom!(state::SmilesParser)
    sym1 = read(state)
    if sym1 == '['
        forward!(state)
        fml = lghighand!(state, atomprop!)
        fml = tidyformula(fml)
        symcls = read(state)
        @assert symcls == ']' "(atom!) unexpected token: $(symcls)"
        forward!(state)
        @assert findformula(fml, :atomsymbol) !== nothing || findformula(fml, :hydrogenconnected) >= 1
        atoms = [SmilesAtom(
            something(findformula(fml, :atomsymbol), :H),
            something(findformula(fml, :charge), 0),
            1,  # TODO: multiplicity
            findformula(fml, :mass),
            something(findformula(fml, :isaromatic), false),
            something(findformula(fml, :stereo), :unspecified)
        )]
        hcnt = something(findformula(fml, :hydrogenconnected), 0)
        if findformula(fml, :atomsymbol) === nothing
            hcnt -= 1
        end
        for h in 1:hcnt
            push!(atoms, SmilesAtom(:H))
        end
        return atoms
    else
        fml = atomsymbol!(state)
        if fml === nothing
            return SmilesAtom[]
        else
            asym = findformula(fml, :atomsymbol)
            arom = findformula(fml, :isaromatic)
            return [SmilesAtom(asym, 0, 1, nothing, arom, :unspecified)]
        end
    end
end

"""
    atom!(state::SmartsParser) -> Vector{SmartsAtom}

Atom <- '[' (AtomProp / LogicalOperator)+ ']' / AtomSymbol
"""
function atom!(state::SmartsParser)
    sym1 = read(state)
    if sym1 == '['
        forward!(state)
        fml = lglowand!(state, atomprop!)
        @assert fml !== nothing "(atom!) empty atomprop"
        fml = tidyformula(fml)
        symcls = read(state)
        @assert symcls == ']' "(atom!) unexpected token: $(symcls)"
        forward!(state)
        return [SmartsAtom(fml)]
    else
        fml = atomsymbol!(state)
        if fml === nothing
            return SmartsAtom[]
        else
            return [SmartsAtom(fml)]
        end
    end
end


module QuelqunQuiTest

1+1==2
# Just being sure I changed sth to have to use git

end # module


#=
The Agfa ColorReference is designed according to the IT8.7 standard.

The spectral reflectance of samples were measured with Minolta spectro-
photometer with option SCE (Specular component excluded). The CM-2002
utilizes d/8 (diffuse illumination/8o viewing) geometry which conforms to
ISO and DIN standards (and also to CIE d/0 recommendations).

Measuring equipment: Minolta spectrofotometer CM-2002.
Wavelength interval: 400 nm - 700 nm.
Wavelength resolution: 10 nm.

Measurer: ela@ee.oulu.fi (University of Oulu, Electrical Eng.Dept)

Further information: ela@ee.oulu.fi (University of Oulu, Electrical Eng.Dept)

References:

   AgfaFotoreference, Trademark of Agfa-Gevaert A.G.,
   Septestraat 27 2640 Mortsel - Belgium,1992.
=#
"""
`agfait_specs()`
"""
function agfait_specs()
    filename = "agfait872.dat"
    fp=@__DIR__
    cd(fp)
    cd("..\\spectra")
    readlines(filename)
end

"""
`agfait(idx::Int)`

Returns the reflectance spectrum of the selected Agfait872 color patch.
Spectral data are from 400 to 700 nm in 10 nm steps.

1st group: 1-264

2nd group: Neutral scale samples (265-286)

3rd group:

* black sample (287)
* white sample (288)
* Minolta white calibration sample (289)

# Examples
```jldoctest
julia> agfait(123)
RSpec(Real[400.0, 410.0, 420.0, 430.0, 440.0, 450.0, 460.0, 470.0, 480.0, 490.0  …  610.0, 620.0, 630.0, 640.0, 650.0, 660.0, 670.0, 680.0, 690.0, 700.0], Real[0.1282, 0.3004, 0.4806, 0.5796, 0.6429, 0.6818000000000001, 0.7078, 0.7159, 0.7071999999999999, 0.6896  …  0.259, 0.2318, 0.21309999999999998, 0.2, 0.1932, 0.19210000000000002, 0.19699999999999998, 0.2095, 0.2311, 0.2594])

julia> agfait(0)
ERROR: DomainError with 0:
Color index out of range. Expecting 0 ≤ index ≤ 289.
```

Measurer: ela@ee.oulu.fi (University of Oulu, Electrical Eng.Dept)
"""
function agfait(idx::Int)
    if 1 ≤ idx ≤ 289
        file = agfait_specs()
        start = 119 * (idx - 1) + 2
        lam = collect(400.0:10.0:700.0)
        spd = map(x -> (parse(Float64, x) / 100.0), file[start:start+30])
        reflectance_spec(lam, spd)
    else
        throw(DomainError(idx, "Color index out of range. Expecting 0 ≤ index ≤ 289."))
    end
end


isdefined(Base, :__precompile__) && __precompile__()

#__precompile__()

module StatLib

using Compat, DataArrays, GLM, DataFrames, Distributions, NLopt, Showoff, StatsBase, DataStructures, StatsFuns, JuMP, DBAPI, JavaCall, JDBC
using JSON, Requests, HttpParser

import DataArrays
import DBAPI, JavaCall, JDBC
import StatsBase: coef, coeftable, df, deviance, fit!, fitted, loglikelihood, model_response, nobs, vcov
import DataFrames: @~

using JSON, Compat, DataArrays, GLM, DataFrames, Distributions, NLopt, Showoff, StatsBase, DataStructures, StatsFuns, JuMP, MixedModels, NamedArrays, NLsolve
#using JSON, Requests, HttpParser

import DataArrays
#import DBAPI, JavaCall, JDBC
import StatsBase: coef, coeftable, df, deviance, fit!, fitted, loglikelihood, model_response, nobs, vcov
import Base: cond, std
import Distributions: Bernoulli, Binomial, Poisson, Gamma
import GLM: LogitLink, LogLink, InverseLink
import DataFrames: @~

export
       @~,
       rest

import Base: ==, *

include("rest.jl")

end # module


# This file holds the definition of the Recruitment type. This type is used to
#   describe a recruitment scheme for a specific part of the organisation and
#   defines which attributes need to be initialised and what the distributions
#   of those attributes are.

export Recruitment
"""
The `Recruitment` type described a recruitment scheme, allowing new personnel members to enter the organisation into a specific nodes.

The type contains the following fields:
* `name::String`: the name of the recruitment scheme.
* `freq::Float64`: the period of the recruitment cycle. Default = 1.0
* `offset::Float64`: the offset of the recruitment cycle w.r.t. the start of the simulation. Default = 0.0
* `targetNode::BaseNode`: the target node of the recruitment scheme.
* `minRecruitment::Int`: the minimum number of personnel members to recruit during a single recruitment step. This is only used for adaptive recruitment schemes. Default = 0
* `maxRecruitment::Int`: the maximum number of personnel members to recruit during a single recruitment step, where -1 stands for no upper limit. This is only used for adaptive recruitment schemes. Default = -1
* `recruitmentDistType::Symbol`: the type of the distribution of the number of people to recruit. Supported types are `:disc` (pointwise), `:pUnif` (piecewise uniform), and `:pLin` (piecewise linear). This is only used for non-adaptive recruitment schemes. Default = `:disc`
* `recruitmentDistNodes::Dict{Int,Float64}`: the nodes of the distribution of the number of people to recruit. These are given as number/weight pairs. Default: Dict( 0 => 1.0 )
* `ageDistType::Symbol`: the type of the distribution of the ages of the recruited people. Supported types are `:disc` (pointwise), `:pUnif` (piecewise uniform), and `:pLin` (piecewise linear). This is only used for non-adaptive recruitment schemes. Default = `:disc`
* `ageDistNodes::Dict{Float64,Float64}`: the nodes of the distribution of the ages of the recruited people. These are given as age/weight pairs. Default: Dict( 0.0 => 1.0 )

Several additional fields are used to speed up computations:
* `isAdaptive::Bool`: a flag indicating whether the recruitment scheme is adaptive or not.
* `recruitmentDist::Function`: the function that draws a sample from the distribution of the number of people to recruit.
* `ageDist::Function`: the function that draws a sample from the distrubiton of the recruitment age.
* `priority::Int`: the priority of the SimJulia process of this recruitment scheme.
* `recRNG::MersenneTwister`: the random stream that generates the number of people to recruit during each recruitment cycle.
* `ageRNG::MersenneTwister`: the random stream that generates the ages of the newly recruited personnel members.

Constructor:
```
Recruitment( name::String )
```
This constructor creates a `Recruitment` object with name `name`, and zero recruitment defined.
"""
mutable struct Recruitment

    name::String
    freq::Float64
    offset::Float64
    targetNode::String
    minRecruitment::Int
    maxRecruitment::Int
    recruitmentDistType::Symbol
    recruitmentDistNodes::Dict{Int,Float64}
    ageDistType::Symbol
    ageDistNodes::Dict{Float64,Float64}

    isAdaptive::Bool
    recruitmentDist::Function
    ageDist::Function
    priority::Int
    recRNG::MersenneTwister
    ageRNG::MersenneTwister

    function Recruitment( name::String )

        newRec = new()
        newRec.name = name
        newRec.freq = 1.0
        newRec.offset = 0.0
        newRec.targetNode = "dummy"
        newRec.minRecruitment = 0
        newRec.maxRecruitment = -1
        newRec.recruitmentDistType = :disc
        newRec.recruitmentDistNodes = Dict( 0 => 1.0 )
        newRec.ageDistType = :disc
        newRec.ageDistNodes = Dict( 0.0 => 1.0 )

        newRec.isAdaptive = false
        newRec.recruitmentDist = function() return 0 end
        newRec.ageDist = function( n::Integer ) return zeros( Float64, n ) end
        newRec.priority = 1
        newRec.recRNG = MersenneTwister()
        newRec.ageRNG = MersenneTwister()
        return newRec

    end  # Recruitment( name )

end  # mutable struct Recruitment


module LULESH

using CEnum
using MPI
using Parameters
using Printf
using Random
using StaticArrays
using LinearAlgebra

abstract type AbstractDomain end
# Device vector types
# const VD{T} = Union{Vector{T}, CuVector{T}} where T
const VD{T} = Vector{T} where T

# Index type
const IndexT = Int
# MPI Message Tags
const MSG_COMM_SBN      = 1024
const MSG_SYNC_POS_VEL  = 2048
const MSG_MONOQ         = 3072
const MAX_FIELDS_PER_MPI_COMM = 6
# Assume 128 byte coherence
# Assume Float64 is an "integral power of 2" bytes wide
const CACHE_COHERENCE_PAD_REAL = div(128, sizeof(Float64))

# Only supported configuration
const SEDOV_SYNC_POS_VEL_EARLY = true
const ALLOW_UNPACKED_PLANE = false

struct LuleshProblem
    num_iters::Int
    structured::Bool
    nx::Int
    nr::Int
    balance::Int
    cost::Int
    col::Int
    row::Int
    plane::Int
    side::Int
    devicetype
    floattype
    comm::Union{MPI.Comm, Nothing}
    function LuleshProblem(num_iters, structured, nx, nr, balance, cost, devicetype, floattype, comm)
        col, row, plane, side = initMeshDecomp(comm)
        return new(num_iters, structured, nx, nr, balance, cost, col, row, plane, side, devicetype, floattype, comm)
    end
end

include("bc.jl")
include("types.jl")
include("domain.jl")
include("mpi.jl")
include("comm.jl")
include("utils.jl")

export printUsage, IndexT, Domain, LuleshProblem, getMyRank, getNumRanks, getWtime,
       lagrangeLeapFrog, comm_max, timeIncrement!, nodalMass, commRecv, MSG_COMM_SBN, verifyAndWriteFinalOutput
export commSend, commRecv, commSBN

function initMeshDecomp(comm)
    # Assume cube processor layout for now
    numRanks = getNumRanks(comm)
    myRank = getMyRank(comm)
    testProcs = floor(cbrt(numRanks+0.5))
    if (testProcs*testProcs*testProcs != numRanks)
        error("Num processors must be a cube of an integer (1, 8, 27, ...)")
    end
    # TODO This is not good for the padding
    if (MAX_FIELDS_PER_MPI_COMM > getCacheCoherencePad(testProcs))
        error("corner element comm buffers too small. MAX_FIELDS_PER_MPI_COMM > CACHE_COHERENCE_PAD_REAL ($MAX_FIELDS_PER_MPI_COMM > $(getCacheCoherencePad(testProcs))")
    end
    testProcs = convert(Int, testProcs)
    dx = convert(Int, testProcs)
    dy = convert(Int, testProcs)
    dz = convert(Int, testProcs)

    # temporary test
    if dx*dy*dz != numRanks
        error("error -- must have as many domains as procs\n") ;
    end
    remainder = (dx*dy*dz) % numRanks
    myDom = Int(0)
    if (myRank < remainder)
        myDom = myRank*( 1+ (div(dx*dy*dz, numRanks)))
    else
      myDom = remainder*( 1+ (div(dx*dy*dz, numRanks))) +
         (myRank - remainder)*(div(dx*dy*dz, numRanks))
    end
    col = myDom % dx
    row = div(myDom, dx) % dy
    plane = div(myDom, (dx*dy))
    side = testProcs

    return col, row, plane, side
end
export InitMeshDecomp

function printNormField(domain, fields)
    if getMyRank(domain.comm) == 0
        for field in fields
            # println("$field[", length(getfield(domain, field)), "]: ", norm(getfield(domain, field)))
        end
    end
end

function printField(domain, fields)
    if getMyRank(domain.comm) == 0
        for field in fields
            # println("$field[", length(getfield(domain, field)), "]: ", getfield(domain, field))
        end
    end
end

function printNormAllFields(domain, location="")
    if getMyRank(domain.comm) == 0
        # println("Location: ", location)
    end
    # fields = [:x, :xd, :fx, :nodalMass, :symmX, :dxx, :delv_xi]
    # fields = [:x, :commDataSend, :commDataRecv]
    # printNormField(domain, fields)
end

function printAllFields(domain, location="")
    if getMyRank(domain.comm) == 0
        # println("Location: ", location)
    end
    # fields = [:x, :xd, :fx, :nodalMass, :symmX, :nodelist]
    # fields = [:commDataSend, :commDataRecv]
    # printField(domain, fields)
end

export printField, printNormField, printAllFields, printNormAllFields
end # module


Base.isequal(x::NanoDate, y::NanoDate) =
    isequal(x.datetime, y.datetime) && isequal(x.nanosecs, y.nanosecs)

Base.isequal(x::NanoDate, y::Date) =
    isequal(x.datetime, y) && iszero(x.nanosecs)
Base.isequal(x::Date, y::NanoDate) =
    isequal(x, y.datetime) && iszero(y.nanosecs)

Base.isequal(x::NanoDate, y::DateTime) =
    isequal(x.datetime, y) && iszero(x.nanosecs)
Base.isequal(x::DateTime, y::NanoDate) =
    isequal(x, y.datetime) && iszero(y.nanosecs)

Base.isless(x::NanoDate, y::NanoDate) =
    isless(x.datetime, y.datetime) ||
    (isequal(x.datetime, y.datetime) && isless(x.nanosecs, y.nanosecs))

Base.isless(x::NanoDate, y::Date) = isless(x.datetime, y)
Base.isless(x::Date, y::NanoDate) =
    isless(x, y.datetime) ||
    (isequal(x, y.datetime) && !iszero(y.nanosecs))

Base.isless(x::NanoDate, y::DateTime) = isless(x.datetime, y)
Base.isless(x::DateTime, y::NanoDate) =
    isless(x, y.datetime) ||
    (isequal(x, y.datetime) && !iszero(y.nanosecs))

Base.:(==)(x::NanoDate, y::NanoDate) =
    (x.datetime == y.datetime) && (x.nanosecs == y.nanosecs)
Base.:(<)(x::NanoDate, y::NanoDate) =
    (x.datetime < y.datetime) || (x.datetime == y.datetime && x.nanosecs < y.nanosecs)
Base.:(<=)(x::NanoDate, y::NanoDate) =
    (x.datetime < y.datetime) || (x.datetime == y.datetime && x.nanosecs <= y.nanosecs)
Base.:(>)(x::NanoDate, y::NanoDate) =
    (x.datetime > y.datetime) || (x.datetime == y.datetime && x.nanosecs > y.nanosecs)
Base.:(>=)(x::NanoDate, y::NanoDate) =
    (x.datetime > y.datetime) || (x.datetime == y.datetime && x.nanosecs >= y.nanosecs)

for (T) in (:DateTime, :Date)
  @eval begin
    Base.:(==)(x::NanoDate, y::$T) = (x.datetime == y) && iszero(x.nanosecs)
    Base.:(==)(x::$T, y::NanoDate) = (x == y.datetime) && iszero(y.nanosecs)

    Base.:(<)(x::NanoDate, y::$T) = (x.datetime < y)
    Base.:(<)(x::$T, y::NanoDate) = (x < y.datetime)
    Base.:(>)(x::NanoDate, y::$T) = (x.datetime > y) || (x.datetime == y && !iszero(x.nanosecs))
    Base.:(>)(x::$T, y::NanoDate) = (x > y.datetime)

    Base.:(<=)(x::NanoDate, y::$T) = (x < y) || (x == y)
    Base.:(<=)(x::$T, y::NanoDate) = (x < y) || (x == y)
    Base.:(>=)(x::NanoDate, y::$T) = (x > y) || (x == y)
    Base.:(>=)(x::$T, y::NanoDate) = (x > y) || (x == y)
  end
end


### Latent models ####
include("prior.jl")
include("posterior.jl")

## Exact Gaussian Process
struct LatentGP{T,Tpr<:GPPrior,Tpo<:Posterior{T},O} <: AbstractLatent{T,Tpr,Tpo}
    prior::Tpr
    post::Tpo
    opt::O
end

function LatentGP(T::DataType, dim::Int, kernel::Kernel, mean::PriorMean, opt)
    return LatentGP(
        GPPrior(deepcopy(kernel), deepcopy(mean)),
        Posterior(dim, zeros(T, dim), cholesky(Matrix{T}(I(dim)))),
        deepcopy(opt),
    )
end

@traitimpl IsFull{LatentGP}

## AbstractVarLatent

abstract type AbstractVarLatent{T,Tpr,Tpo} <: AbstractLatent{T,Tpr,Tpo} end

## Variational Gaussian Process
mutable struct VarLatent{T,Tpr<:GPPrior,Tpo<:VarPosterior{T},O} <:
               AbstractVarLatent{T,Tpr,Tpo}
    prior::Tpr
    post::Tpo
    opt::O
end

function VarLatent(T::DataType, dim::Int, kernel::Kernel, mean::PriorMean, opt)
    return VarLatent(
        GPPrior(deepcopy(kernel), deepcopy(mean)), VarPosterior{T}(dim), deepcopy(opt)
    )
end

@traitimpl IsFull{VarLatent}

## Sparse Variational Gaussian Process

mutable struct SparseVarLatent{
    T,Tpr<:GPPrior,Tpo<:VarPosterior{T},Topt,TZ<:AbstractVector,TZopt
} <: AbstractVarLatent{T,Tpr,Tpo}
    prior::Tpr
    post::Tpo
    Z::TZ
    opt::Topt
    Zopt::TZopt
end

function SparseVarLatent(
    T::DataType,
    Z::AbstractVector,
    kernel::Kernel,
    mean::PriorMean,
    opt=nothing,
    Zopt=nothing,
)
    dim = length(Z)
    return SparseVarLatent(
        GPPrior(deepcopy(kernel), deepcopy(mean)),
        VarPosterior{T}(dim),
        deepcopy(Z),
        deepcopy(opt),
        deepcopy(Zopt),
    )
end

@traitimpl IsSparse{SparseVarLatent}

## Monte-Carlo Gaussian Process

struct SampledLatent{T,Tpr<:GPPrior,Tpo<:SampledPosterior{T}} <: AbstractLatent{T,Tpr,Tpo}
    prior::Tpr
    post::Tpo
end

function SampledLatent(T::DataType, dim::Int, kernel::Kernel, mean::PriorMean)
    return SampledLatent(
        GPPrior(deepcopy(kernel), deepcopy(mean)),
        SampledPosterior(dim, zeros(T, dim), Symmetric(Matrix{T}(I(dim)))),
    )
end

@traitimpl IsFull{SampledLatent}

## Online Sparse Variational Process

mutable struct OnlineVarLatent{
    T,
    Tpr<:GPPrior,
    Tpo<:AbstractVarPosterior{T},
    Topt,
    TZ<:AbstractVector,
    TZalg<:InducingPoints.OnIPSA,
    TZopt,
} <: AbstractVarLatent{T,Tpo,Tpr}
    prior::Tpr
    post::Tpo
    Z::TZ
    Zₐ::TZ
    Zalg::TZalg
    Zupdated::Bool
    opt::Topt
    Zopt::TZopt
end

function OnlineVarLatent(
    T::DataType,
    dim::Int,
    Z::AbstractVector,
    Zalg::InducingPoints.OnIPSA,
    kernel::Kernel,
    mean::PriorMean,
    opt=nothing,
    Zopt=nothing,
)
    return OnlineVarLatent(
        GPPrior(deepcopy(kernel), deepcopy(mean)),
        OnlineVarPosterior{T}(dim),
        Z,
        deepcopy(Z),
        Zalg,
        false,
        deepcopy(opt),
        deepcopy(Zopt),
    )
end

@traitimpl IsSparse{OnlineVarLatent}

## Variational Student-T Process

mutable struct TVarLatent{T<:Real,Tpr<:TPrior,Tpo<:VarPosterior{T},O} <:
               AbstractVarLatent{T,Tpr,Tpo}
    prior::Tpr
    post::Tpo
    opt::O
end

function TVarLatent(T::DataType, ν::Real, dim::Int, kernel::Kernel, mean::PriorMean, opt)
    return TVarLatent(
        TPrior(deepcopy(kernel), deepcopy(mean), ν, rand(T), rand(T)),
        VarPosterior{T}(dim),
        deepcopy(opt),
    )
end

@traitimpl IsFull{TVarLatent}

### Functions

prior(gp::AbstractLatent) = gp.prior
kernel(gp::AbstractLatent) = kernel(prior(gp))
setkernel!(gp::AbstractLatent, kernel::Kernel) = setkernel!(prior(gp), kernel)
pr_mean(gp::AbstractLatent) = mean(prior(gp))
pr_mean(gp::AbstractLatent, X::AbstractVector) = mean(prior(gp), X)
setpr_mean!(gp::AbstractLatent, μ₀::PriorMean) = setmean!(prior(gp), μ₀)

posterior(gp::AbstractLatent) = gp.post
Distributions.dim(gp::AbstractLatent) = dim(posterior(gp))
Distributions.mean(gp::AbstractLatent) = mean(posterior(gp))
Distributions.cov(gp::AbstractLatent) = cov(posterior(gp))
Distributions.var(gp::AbstractLatent) = var(posterior(gp))
nat1(gp::AbstractVarLatent) = nat1(posterior(gp))
nat2(gp::AbstractVarLatent) = nat2(posterior(gp))

@traitfn function mean_f(gp::T, kernel_matrices) where {T <: AbstractLatent; IsFull{T}}
    return mean_f(mean(gp))
end
@traitfn function mean_f(gp::T, kernel_matrices) where {T <: AbstractLatent; !IsFull{T}}
    return mean_f(mean(gp), kernel_matrices.κ)
end

mean_f(μ::AbstractVector) = μ
mean_f(μ::AbstractVector, κ::AbstractMatrix) = κ * μ

@traitfn function var_f(gp::T, kernel_matrices) where {T <: AbstractLatent; IsFull{T}}
    return var_f(cov(gp))
end
@traitfn function var_f(gp::T, kernel_matrices) where {T <: AbstractLatent; !IsFull{T}}
    return var_f(cov(gp), kernel_matrices.κ, kernel_matrices.K̃)
end

var_f(Σ::AbstractMatrix) = diag(Σ)
var_f(Σ::AbstractMatrix, κ::AbstractMatrix, K̃::AbstractVector) = diag_ABt(κ * Σ, κ) + K̃

Zview(gp::SparseVarLatent) = gp.Z
Zview(gp::OnlineVarLatent) = gp.Z

setZ!(gp::AbstractLatent, Z::AbstractVector) = gp.Z = Z

opt(gp::AbstractLatent) = gp.opt
Zopt(::AbstractLatent) = nothing
Zopt(gp::SparseVarLatent) = gp.Zopt
Zopt(gp::OnlineVarLatent) = gp.Zopt

function compute_K(gp::AbstractLatent, X::AbstractVector, jitt::Real)
    return cholesky(kernelmatrix(kernel(gp), X) + jitt * I)
end

function compute_K(gp::AbstractLatent, jitt::Real)
    return cholesky(kernelmatrix(kernel(gp), gp.Z) + jitt * I)
end

function compute_κ(gp::SparseVarLatent, X::AbstractVector, K, jitt::Real)
    Knm = kernelmatrix(kernel(gp), X, gp.Z)
    κ = copy(Knm / K)
    K̃ = kernelmatrix_diag(kernel(gp), X) .+ jitt - diag_ABt(κ, Knm)
    all(K̃ .> 0) || error("K̃ has negative values")
    return (; Knm, κ, K̃)
end

function compute_κ(gp::OnlineVarLatent{T}, X::AbstractVector, K, jitt::Real) where {T}
    k = dim(gp)
    # Covariance with the model at t-1
    if isempty(gp.Zₐ)
        Kab = zeros(T, k, k)
        κₐ = I
        K̃ₐ = zero(Kab)
    else
        Kab = kernelmatrix(kernel(gp), gp.Zₐ, gp.Z)
        κₐ = Kab / K
        Kₐ = Symmetric(kernelmatrix(kernel(gp), gp.Zₐ) + jitt * I)
        K̃ₐ = Kₐ - κₐ * transpose(Kab)
    end

    # Covariance with a new batch
    Knm = kernelmatrix(kernel(gp), X, gp.Z)
    κ = Knm / K
    K̃ = kernelmatrix_diag(kernel(gp), X) .+ jitt - diag_ABt(κ, Knm)
    all(K̃ .> 0) || error("K̃ has negative values")
    return (; Kab, κₐ, K̃ₐ, Knm, κ, K̃)
end


using SeisIO, SeisIO.Quake, SeisIO.SeisHDF
import Printf

# US FDSN example: 5 stations, 2 networks, all channels, last 600 seconds
println(stdout, "Some real data acquisition examples...\n\n")

println(stdout, "preliminary: change directory")
path = Base.source_dir()
println(stdout, "cd(", path, ")")
cd(path)

printstyled(stdout, "FDSN get_data\n", color=:green, bold=true)
CHA = "CC.PALM, UW.HOOD, CC.TIMB, CC.HIYU, UW.TDH"
s = -600
t = u2d(time())
printstyled(stdout, "Command: ", color=:green)
println(stdout, "S_fdsn = get_data(\"FDSN\", \"", CHA, "\", src=\"IRIS\", s=", s, ", t=", t, ")")
S_fdsn = get_data("FDSN", CHA, src="IRIS", s=s, t=t)
printstyled(stdout, "Results: ", color=:green)
println(stdout, "S_fdsn")
show(S_fdsn)

# IRIS example: 6 channels, 30 minutes
printstyled(stdout, "\n\nIRIS get_data\n", color=:green, bold=true)
STA = ["CC.TIMB..BHE", "CC.TIMB..BHN", "CC.TIMB..BHZ", "UW.HOOD..HHE", "UW.HOOD..HHN", "UW.HOOD..HHZ"]
st = -3600
en = -1800
printstyled(stdout, "Command: ", color=:green)
println(stdout, "S_iris = get_data(\"IRIS\", ", STA, ", s=", st, ", t=", en, ")")
S_iris = get_data("IRIS", STA, s=st, t=en)
printstyled(stdout, "Results: ", color=:green)
println(stdout, "S_iris")
show(S_iris)

# The Tohoku-Oki great earthquake, from IRIS FDSN, recorded by boreholes in WA (USA)
printstyled(stdout, "\n\nFDSNevt\n", color=:green, bold=true)
printstyled(stdout, "Command: ", color=:green)
println(stdout, "S_evt = FDSNevt(\"201103110547\", \"PB.B004..EH?,PB.B004..BS?,PB.B001..BS?,PB.B001..EH?\")")
S_evt = FDSNevt("201103110547", "PB.B004..EH?,PB.B004..BS?,PB.B001..BS?,PB.B001..EH?")
printstyled(stdout, "Results: ", color=:green)
println(stdout, "S_evt")
show(S_evt)

# SeisComp3 SeedLink session, IRIS server, TIME mode
printstyled(stdout, "\n\nSeedLink\n", color=:green, bold=true)
sta = "UW.GRUT,UW.H1K,UW.MDW"
s1 = -120
t1 = 120

printstyled(stdout, "Commands: ", color=:green)

println(stdout, "S_sl = seedlink(\"TIME\", \"", sta, "\", s=", s1, ", t=", t1, ")")
S_sl = seedlink("TIME", sta, s=s1, t=t1)

println(stdout, "          seedlink!(S_sl, \"DATA\", \"SampleFiles/SL_long_test.conf\")")
seedlink!(S_sl, "DATA", "SampleFiles/SL_long_test.conf")

println(stdout, "          sleep(30)")
sleep(30)

println(stdout, "          for conn in S_sl.c; close(conn); end")
for conn in S_sl.c; close(conn); end

printstyled(stdout, "Results: ", color=:green)
println(stdout, "S_sl")
show(S_sl)

printstyled(stdout, "\n\n\nNote: ", color=:white, bold=true)
println(stdout, "ALL data from these examples can be written to file.")
printstyled(stdout, "wseis(\"fname.seis\", S)", color=7)
println(stdout, "       write S to low-level SeisIO native format in file fname.seis")
printstyled(stdout, "writesac(S)", color=7)
println(stdout, "                  write S to SAC files with auto-generated names")
printstyled(stdout, "write_hdf5(\"fname.h5\", S)", color=7)
println(stdout, "    write S to ASDF (HDF5) file \"fname.h5\"")


#############################################################################
# ECOS.jl
# Wrapper around the ECOS solver https://github.com/ifa-ethz/ecos
# See http://github.com/JuliaOpt/ECOS.jl
#############################################################################
# ECOS.jl
# Contains the wrapper itself
#############################################################################

module ECOS

# Try to load the binary dependency
if isfile(joinpath(Pkg.dir("ECOS"),"deps","deps.jl"))
    include("../deps/deps.jl")
else
    error("ECOS not properly installed. Please run Pkg.build(\"ECOS\")")
end

macro ecos_ccall(func, args...)
    f = "ECOS_$(func)"
    quote
        @unix_only ret = ccall(($f,ECOS.ecos), $(args...))
        @windows_only ret = ccall(($f,ECOS.ecos), stdcall, $(args...))
        ret
    end
end

include("ECOSSolverInterface.jl")  # MathProgBase interface
include("types.jl")  # All the types and constants defined in ecos.h

export setup, solve, cleanup

# setup  (direct interface)
# Provide ECOS with a problem in the form
# min c'x
# st  A*x = b
#     G*x in_K h,
#       or equivalently  h - G*x in K
#       or equivalently  G*x + s = h, s in K
# K is the product of R+ and multiple second-order cones, with the
# positive orthant first.
# Inputs:
#   n       Number of variables == length(x)
#   m       Number of inequality constraints == length(h)
#   p       Number of equality constraints == length(b)  (can be 0)
#   l       Dimension of the positive orthant,
#               i.e. the first l elements of s are >= 0
#   ncones  Number of second-order cones present in K
#   q       ncones-long vector of the length of the index sets of the SOCs
#               e.g. cone 1 is indices 4:6, cone 2 is indices 7:10
#                    ->  q = [3, 4]
#   Gpr, Gjc, Gir
#           Non-zeros, column indices, and the row index arrays, respectively,
#           for the matrix G represented in column compressed storage (CCS) format
#           which is equivalent to Julia's SparseMatrixCSC
#   Apr, Ajc, Air
#           Equivalent to above for the matrix A.
#           Can be all nothing if no equalities are present.
#   c       Objective coefficients, length(c) == n
#   h       RHS for inequality constraints, length(h) == m
#   b       RHS for equality constraints, length(b) == b (can be nothing)
# Returns a pointer to the ECOS pwork structure (Cpwork in ECOS.jl). See
# types.jl for more information.
function setup(n::Int, m::Int, p::Int, l::Int, ncones::Int, q::Union(Vector{Int},Nothing),
                Gpr::Vector{Float64}, Gjc::Vector{Int}, Gir::Vector{Int},
                Apr::Union(Vector{Float64},Nothing), Ajc::Union(Vector{Int},Nothing), Air::Union(Vector{Int},Nothing),
                c::Vector{Float64}, h::Vector{Float64}, b::Union(Vector{Float64},Nothing))
    # Convert to canonical forms
    q = (q == nothing) ? convert(Ptr{Clong}, C_NULL) : convert(Vector{Clong},q)
    Apr = (Apr == nothing) ? convert(Ptr{Cdouble}, C_NULL) : Apr
    Ajc = (Ajc == nothing) ? convert(Ptr{Cdouble}, C_NULL) : convert(Vector{Clong},Ajc)
    Air = (Air == nothing) ? convert(Ptr{Cdouble}, C_NULL) : convert(Vector{Clong},Air)
    b = (b == nothing) ? convert(Ptr{Cdouble}, C_NULL) : b
    problem = ccall((:ECOS_setup, ECOS.ecos), Ptr{Cpwork},
        (Clong, Clong, Clong, Clong, Clong, Ptr{Clong},
         Ptr{Cdouble}, Ptr{Clong}, Ptr{Clong},
         Ptr{Cdouble}, Ptr{Clong}, Ptr{Clong},
         Ptr{Cdouble}, Ptr{Cdouble}, Ptr{Cdouble}),
        n, m, p, l, ncones, q,
        Gpr, convert(Vector{Clong},Gjc), convert(Vector{Clong},Gir),
        Apr, Ajc, Air,
        c, h, b)
end

# setup  (more general interface)
# A more tolerant version of the above method that doesn't require
# user to fiddle with internals of the sparse matrix format
# User can pass nothing as argument for A, b, and q
function setup(n, m, p, l, ncones, q, G, A, c, h, b)
    if A == nothing
        if b != nothing
            @assert length(b) == 0
            b = nothing
        end
        @assert p == 0
        Apr = nothing
        Ajc = nothing
        Air = nothing
    else
        numrow, numcol = size(A)
        @assert numcol == n
        @assert numrow == length(b)
        sparseA = sparse(A)
        Apr = convert(Vector{Float64}, sparseA.nzval)
        Ajc = sparseA.colptr - 1  # C is 0-based
        Air = sparseA.rowval - 1
    end

    sparseG = sparse(G)
    Gpr = convert(Vector{Float64}, sparseG.nzval)
    Gjc = sparseG.colptr - 1  # C is 0-based
    Gir = sparseG.rowval - 1

    setup(  n, m, p, l, ncones, q,
            Gpr, Gjc, Gir,
            Apr, Ajc, Air,
            c, h, b)
end

# solve
# Solves the provided problem. Results are stored inside the structure,
# but currently there is no convenient interface-provided way to access
# this - use MathProgBase interface.
function solve(problem::Ptr{Cpwork})
    exitflag = ccall((:ECOS_solve, ECOS.ecos), Clong, (Ptr{Cpwork},), problem)
end

# cleanup
# Frees memory allocated by ECOS for the problem.
# The optional keepvars argument is number of variables to NOT free.
function cleanup(problem::Ptr{Cpwork}, keepvars::Int = 0)
    ccall((:ECOS_cleanup, ECOS.ecos), Void, (Ptr{Cpwork}, Clong), problem, keepvars)
end

# ver  [not exported]
# Returns the version of ECOS in use
function ver()
    ver_ptr = ccall((:ECOS_ver, ECOS.ecos), Ptr{Uint8}, ())
    return bytestring(pointer_to_array(ver_ptr, 7))
end

end # module


# This code is part of QuantumCircuits.
#
# (C) Copyright Rafał Pracht 2022.
#
# This code is licensed under the Apache License, Version 2.0. You may
# obtain a copy of this license in the LICENSE.txt file in the root directory
# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
#
# Any modifications or derivative works of this code must retain this
# copyright notice, and modified files need to carry a notice indicating
# that they have been altered from the originals.

module Graph

using QuantumCircuits.QCircuits.QBase
using QuantumCircuits.QCircuits.Gates
using QuantumCircuits.QCircuits.Gates: onlyParameters
import QuantumCircuits.QCircuits.QBase: add!

export DirectedGraph, to_vector

"Circuit direct graph structure"
struct DirectedGraph{IndexT<:Integer}
    qubits::Int
    start_nodes::Vector{IndexT}
    end_nodes::Vector{IndexT}
    vertices::Vector{QuantumGate}
    vertices_map::Dict{QuantumGate, IndexT}
    edges::Vector{Pair{IndexT, IndexT}}
    #sources::Vector{QuantumGate}
    #targets::Vector{QuantumGate}
    in_edges::Vector{Set{IndexT}}
    out_edges::Vector{Set{Pair{IndexT, IndexT}}}
    inline_optimization::Bool
end

"Create new empyt graph"
function DirectedGraph{IndexT}(qubits::Int, inline_optimization) where IndexT<:Integer
    g = DirectedGraph{IndexT}(
                      qubits,
                      zeros(IndexT, qubits),
                      zeros(IndexT, qubits),
                      QuantumGate[],
                      Dict{QuantumGate, IndexT}(),
                      Vector{Pair{IndexT, IndexT}}(),
                      Set{IndexT}[],
                      Vector{Set{Pair{Integer, IndexT}}}[],
                      inline_optimization)

    return g
end

"The index of start and end node."
const StartEndNode = 0
"Mapping from qubit number to line number."
qubitToLine(q) = q + 1

"Add gate to circuit"
function add!(g::DirectedGraph{IndexT}, gate::QuantumGate) where IndexT<:Integer
    qubits = getqubitsids(gate)
    @assert all(q < g.qubits for q in qubits) "Qubits in gate $gate is out of DAG qubits range $(g.qubits)."

    @assert length(g.vertices) == length(g.in_edges) "Inexpected inconsistency between length of vertices and in edges"
    @assert length(g.vertices) == length(g.out_edges) "Inexpected inconsistency between length of vertices and out edges"

    # Chceck if this is and the previous are u3 gates with prameters
    # In that case we can skip adding this gate
    if g.inline_optimization &&         # inline optimization is enabled
       length(qubits) == 1 &&         # only single qubit
       typeof(gate) == U3 &&          # u3 gate
       onlyParameters(gate)           # use all 3 parameters
       # get last qubit
        q = qubitToLine(qubits[1]) # qiskit qubit start indexes from 0
        last_end = g.end_nodes[q]
        if last_end != StartEndNode &&                      # if there is a previous
           typeof(g.vertices[last_end]) == U3 &&            # u3 gate
           onlyParameters(g.vertices[last_end])             # use all 3 parameters
            # We can skip add this gate
            return nothing
        end
    end

    # calculate gate index
    push!(g.vertices, gate)
    idx = length(g.vertices)
    g.vertices_map[gate] = idx

    # add in/out edges sets
    push!(g.in_edges, Set{IndexT}())
    push!(g.out_edges, Set{Pair{Integer, IndexT}}())

    # actualize edge
    for q in qubits
        q = qubitToLine(q) # qiskit qubit start indexes from 0

        last_end = g.end_nodes[q]

        # check if this is first gate on given qubits
        if last_end == StartEndNode
            g.start_nodes[q] = idx
        else
            # Add the output edge
            push!(g.out_edges[last_end], q => idx)
            # TODO perfomance
            if (last_end => idx) ∉ g.edges
                push!(g.edges, last_end => idx)
            end
        end

        g.end_nodes[q] = idx
        push!(g.in_edges[idx], last_end)
    end

    nothing
end

function to_vector(g::DirectedGraph{IndexT})::Vector{QuantumGate} where IndexT<:Integer
    gates = Set{IndexT}([x for x in 1:length(g.vertices)])

    lines = g.start_nodes[1:end]
    code = QuantumGate[]

    while any(!iszero, lines)
        for l in lines
            # if there is somethink to do :)
            if l != StartEndNode && isempty(intersect(gates, g.in_edges[l]))
                # we process all source edges, so we put the gate
                push!(code, g.vertices[l])
                # remove vertice from gates
                pop!(gates, l)
                # Update lines
                expected_qubits = Set(qubitToLine(n) for n in getqubitsids(g.vertices[l]))
                for (q, idx) in g.out_edges[l]
                    lines[q] = idx
                    pop!(expected_qubits, q)
                end
                # cleare the qubits without out edge
                for q in expected_qubits
                    lines[q] = StartEndNode
                end

                break
            end
        end
    end

    return code
end

# "Get index of the vertex in graph"
# vertex_index(vertex::QuantumGate, g::DirectedGraph) = g.verticesmap[vertex]

# AbstractGraph interface
# vertices(g::DirectedGraph) = g.vertices
# edges(g::DirectedGraph) = g.edges
# in_edges(vertex::QuantumGate, g::DirectedGraph{V, E}) where {V, E} = g.inedges[vertex_index(vertex)]
# out_edges(vertex::QuantumGate, g::DirectedGraph{V, E}) where {V, E} = g.outedges[vertex_index(vertex)]

end  # module Graph


# CUFFT API function return values
const cufftStatus_t = UInt32
const CUFFT_STATUS_SUCCESS        = 0  #  The cuFFT operation was successful
const CUFFT_STATUS_INVALID_PLAN   = 1  #  cuFFT was passed an invalid plan handle
const CUFFT_STATUS_ALLOC_FAILED   = 2  #  cuFFT failed to allocate GPU or CPU memory
const CUFFT_STATUS_INVALID_TYPE   = 3  #  No longer used
const CUFFT_STATUS_INVALID_VALUE  = 4  #  User specified an invalid pointer or parameter
const CUFFT_STATUS_INTERNAL_ERROR = 5  #  Driver or internal cuFFT library error
const CUFFT_STATUS_EXEC_FAILED    = 6  #  Failed to execute an FFT on the GPU
const CUFFT_STATUS_SETUP_FAILED   = 7  #  The cuFFT library failed to initialize
const CUFFT_STATUS_INVALID_SIZE   = 8  #  User specified an invalid transform size
const CUFFT_STATUS_UNALIGNED_DATA = 9  #  No longer used
const CUFFT_STATUS_INCOMPLETE_PARAMETER_LIST = 10 #  Missing parameters in call
const CUFFT_STATUS_INVALID_DEVICE = 11 #  Execution of a plan was on different GPU than plan creation
const CUFFT_STATUS_PARSE_ERROR    = 12 #  Internal plan database error
const CUFFT_STATUS_NO_WORKSPACE   = 13  #  No workspace has been provided prior to plan execution
const CUFFT_STATUS_NOT_IMPLEMENTED = 14 # Function does not implement functionality for parameters given.
const CUFFT_STATUS_LICENSE_ERROR  = 15 # Used in previous versions.
const CUFFT_STATUS_NOT_SUPPORTED  = 16  # Operation is not supported for parameters given.

const cufftReal = Float32
const cufftDoubleReal = Float64

const cufftComplex = ComplexF32
const cufftDoubleComplex = ComplexF64

# CUFFT transform directions
const CUFFT_FORWARD = -1 # Forward FFT
const CUFFT_INVERSE =  1 # Inverse FFT

# CUFFT supports the following transform types
const cufftType = Cint
const CUFFT_R2C = 0x2a     # Real to Complex
const CUFFT_C2R = 0x2c     # Complex to Real
const CUFFT_C2C = 0x29     # Complex to Complex
const CUFFT_D2Z = 0x6a     # Double to Double-Complex
const CUFFT_Z2D = 0x6c     # Double-Complex to Double
const CUFFT_Z2Z = 0x69     # Double-Complex to Double-Complex

const cufftCompatibility = Cint
const   CUFFT_COMPATIBILITY_NATIVE          = 0x00
const   CUFFT_COMPATIBILITY_FFTW_PADDING    = 0x01
const   CUFFT_COMPATIBILITY_FFTW_ASYMMETRIC = 0x02
const   CUFFT_COMPATIBILITY_FFTW_ALL        = 0x03

const cufftHandle_t = Cint


using SHA

# Script to download the leap seconds list from NIST, and format it into julia
# code for use in Chrono.jl.

#-------------------------------------------------------------------------------
# Download leap-seconds.list
# Official URL (?)
url = "ftp://time.nist.gov/pub/leap-seconds.list"
# IETF has a mirror if the NIST site is unresponsive, which it commonly is.
# The mirror is not as up to date though.
# url = "https://www.ietf.org/timezones/data/leap-seconds.list"
local_name = "leap-seconds.list"
if length(ARGS) < 1 || ARGS[1] != "false"
    download(url, local_name)
end
lines = readlines(local_name)

#-------------------------------------------------------------------------------
# Parse file.
rawdata = [] # for hash computation
last_update_ntp = 0
expiry_ntp = 0
leap_epochs_ntp = Int[]
leap_secs = Int[]
expected_sha = ""
for line in lines
    # Mostly, lines starting with # are comments.  However, due to
    # terribleness, lines starting with "#@", "#$" or "#h" also contain valid
    # data which needs to be parsed for hash verification.
    if startswith(line, "#\$")
        datastr = match(r"^#\$\s*(\d+)\s*$", line)[1]
        last_update_ntp = parse(Int64, datastr)
        push!(rawdata, datastr)
    elseif startswith(line, "#@")
        datastr = match(r"^#@\s*(\d+)\s*$", line)[1]
        expiry_ntp = parse(Int64, datastr)
        push!(rawdata, datastr)
    elseif startswith(line, "#h")
        # The specified hash method for leap-seconds.list is super broken,
        # since it removes semantically meaningful whitespace.  At least we can
        # guard against transmission errors...
        #
        # There's also a bug in writing out the expected hash!  Work around it
        # by parsing and reformatting, ugh.
        pieces = map(x->@sprintf("%08x", parse(Int,x,16)), split(line[3:end]))
        expected_sha = join(pieces)
    elseif !startswith(line, "#")
        m = match(r"(\d+)\s*(\d+)\s*#.*", line)
        push!(leap_epochs_ntp, parse(Int64, m[1]))
        push!(leap_secs, parse(Int, m[2]))
        push!(rawdata, m[1])
        push!(rawdata, m[2])
    end
end

actual_sha = bytes2hex(SHA.sha1(join(rawdata)))
@show expected_sha actual_sha
@assert expected_sha == actual_sha
@assert all(diff(leap_secs) .== 1)

#-------------------------------------------------------------------------------
# Generate julia code
function format_ntp_secs_as_date(ntp_epoch_secs)
    d = Dates.Second(ntp_epoch_secs) + Dates.DateTime(1900)
    @assert Dates.hour(d) == 0 && Dates.minute(d) == 0 &&
            Dates.second(d) == 0 && Dates.millisecond(d) == 0
    "Date($(Dates.year(d)), $(Dates.month(d)), $(Dates.day(d)))"
end

genname = "leap_seconds.jl"
open(genname, "w") do file
    write(file,
        """
        # Automatically generated by update_leap_seconds.jl - do not edit!

        const leap_index_offset = $(leap_secs[1]-1) # For use with searchsortedlast()

        const utc_offsets = [
            typemin(Date),  # For use with searchsortedlast()
            $(join(map(format_ntp_secs_as_date, leap_epochs_ntp[2:end]), ",\n    "))
        ]

        const leap_secs_expiry_date = $(format_ntp_secs_as_date(expiry_ntp))
        """
    )
end

print(readstring(genname))


using Diversity.API
using EcoBase: AbstractAssemblage, AbstractThings, AbstractPlaces

"""
    gettypes(m::AbstractAssemblage)

Returns the AbstractTypes component of the metacommunity.
"""
function gettypes(m::AbstractAssemblage)
    return _gettypes(m)
end

"""
    getpartition(m::AbstractAssemblage)

Returns the AbstractPartition component of the metacommunity.

"""
function getpartition(m::AbstractAssemblage)
        return _getpartition(m)
end

"""
    counttypes(m::AbstractAssemblage[, raw::Bool = false])
    counttypes(t::AbstractTypes[, raw::Bool = false])

Returns number of types in an `AbstractTypes` object or the
`AbstractAssemblage` containing it. `raw` determines whether to
count the number of raw or processed types, which varies, for instance,
when the types are determined by a phylogeny.
"""
function counttypes end

function counttypes(m::AbstractAssemblage, raw::Bool = false)
    return _counttypes(_gettypes(m), raw)
end

function counttypes(t::AbstractThings, raw::Bool = false)
    return _counttypes(t, raw)
end

"""
    countsubcommunities(m::AbstractAssemblage)
    countsubcommunities(p::AbstractPartition)

Returns number of subcommunities in an `AbstractPartition` object or the
`AbstractAssemblage` containing it.
"""
function countsubcommunities end

function countsubcommunities(m::AbstractAssemblage)
    return _countsubcommunities(_getpartition(m))
end

function countsubcommunities(p::AbstractPartition)
    return _countsubcommunities(p)
end

"""
    gettypenames(m::AbstractAssemblage[, raw::Bool = false])
    gettypenames(t::AbstractTypes[, raw::Bool = false])

Returns the names of the types of the `AbstractTypes` object or the
`AbstractAssemblage` containing it. `raw` determines whether to
count the number of raw or processed types, which varies, for instance,
when the types are determined by a phylogeny.
"""
function gettypenames end

function gettypenames(m::AbstractAssemblage, raw::Bool = false)
    return _gettypenames(_gettypes(m), raw)
end

function gettypenames(t::AbstractTypes, raw::Bool = false)
    return _gettypenames(t, raw)
end

"""
    getdiversityname(m::AbstractAssemblage)
    getdiversityname(t::AbstractTypes)

Returns the name of the diversity type used.
"""
function getdiversityname end

getdiversityname(t::AbstractThings) = _getdiversityname(t)
getdiversityname(m::AbstractAssemblage) = _getdiversityname(_gettypes(m))

"""
    addedoutputcols(m::AbstractAssemblage)
    addedoutputcols(t::AbstractTypes)

Returns the name of any additional columns needed to disambiguate the
diversity type used.
"""
function addedoutputcols end

addedoutputcols(t::AbstractThings) = _addedoutputcols(t)
addedoutputcols(m::AbstractAssemblage) = addedoutputcols(_gettypes(m))

"""
    getaddedoutput(::AbstractTypes)

Returns the contents of any additional columns to be added to outputs.
"""
function getaddedoutput end

getaddedoutput(t::AbstractTypes) = _getaddedoutput(t)
getaddedoutput(m::AbstractAssemblage) = _getaddedoutput(_gettypes(m))

"""
    getsubcommunitynames(m::AbstractAssemblage)
    getsubcommunitynames(p::AbstractPartition)

Returns the names of the subcommunities in an `AbstractPartition` object or the
`AbstractAssemblage` containing it.
"""
function getsubcommunitynames end

function getsubcommunitynames(m::AbstractAssemblage)
    return _getsubcommunitynames(_getpartition(m))
end

function getsubcommunitynames(p::AbstractPartition)
    return _getsubcommunitynames(p)
end

"""
    getabundance(m::AbstractAssemblage, raw::Bool)

Returns the abundances array of the metacommunity.
"""
function getabundance(m::AbstractAssemblage, raw::Bool = false)
    return _getabundance(m, raw)
end

"""
    getmetaabundance(m::AbstractAssemblage)

Returns the metacommunity abundances of the metacommunity.
"""
function getmetaabundance(m::AbstractAssemblage, raw::Bool = false)
    return _getmetaabundance(m, raw)
end

"""
    getweight(m::AbstractAssemblage)

Returns the subcommunity weights of the metacommunity.
"""
function getweight(m::AbstractAssemblage)
    return _getweight(m)
end

"""
    getordinariness!(m::AbstractAssemblage)

Returns (and possibly calculates) the ordinariness array of the subcommunities.
"""
function getordinariness!(m::AbstractAssemblage)
    return _getordinariness!(m)
end

"""
    getmetaordinariness!(m::AbstractAssemblage)

Returns (and possibly calculates) the ordinariness of the
metacommunity as a whole.
"""
function getmetaordinariness!(m::AbstractAssemblage)
    return _getmetaordinariness!(m)
end

"""
    hassimilarity(t::AbstractAssemblage)
    hassimilarity(t::AbstractThings)

Is there similarity of some non-trivial type in the object?
"""
function hassimilarity end
hassimilarity(t::AbstractThings) = _hassimilarity(t)
hassimilarity(asm::AbstractAssemblage) = hassimilarity(gettypes(asm))

"""
    calcsimilarity(t::AbstractTypes, scale::Real)

Retrieves (and possibly calculates) a similarity matrix from t.
"""
function calcsimilarity(t::AbstractTypes, scale::Real)
    return _calcsimilarity(t, scale)
end

function createsummaryline(vec::AbstractVector{<:AbstractString})
    linefunc(vec) = mapreduce(x->x*", ", *, vec[1:(end-1)])*vec[end]
    length(vec) == 1 && return vec[1]
    length(vec) < 6 && return linefunc(vec)
    linefunc(vec[1:3])*"..."*linefunc(vec[(end-1):end])
end

import Base.show
function show(io::IO, mc::AbstractMetacommunity)
    sp = createsummaryline(gettypenames(mc))
    si = createsummaryline(getsubcommunitynames(mc))
    println(io, "$(typeof(mc)) with $(counttypes(mc)) species in $(countsubcommunities(mc)) sites measuring $(getdiversityname(gettypes(mc))) diversity.\n\nSpecies names:\n$(sp)\n\nSite names:\n$(si)")
end


T = Float32
_pkey = :z
n = 10
tbl = table((x=rand(T, n),
             y=rand(T, n),
             z=collect(Int, 1:n),
             xx=vcat(ones(T, round(Int, n/2)),
                     zeros(T, round(Int, n/2))),
             zz=[randstring(2) for _ in 1:n]),
            pkey=_pkey)
nds = ndsparse(deepcopy(tbl))

@testset "DB: filtering, sorting" begin
    all_ids = collect(Int, 1:n)
    no_ids = Int[]
    # Test many missing, empty values
    @test indexfilter(tbl, Dict(), id_key=_pkey) == all_ids
    @test indexfilter(tbl, Dict(), sort_keys=(), id_key=_pkey) == all_ids
    @test indexfilter(tbl, Dict(), sort_keys=(), id_key=:UNK) == no_ids
    @test indexfilter(tbl, Dict(), sort_keys=(:UNK, :UNK2), id_key=_pkey) == all_ids
    @test indexfilter(tbl, Dict(), sort_keys=(:x,), id_key=_pkey) == sortperm(columns(tbl, :x))
    @test indexfilter(tbl, Dict(:missing=>1), sort_keys=(), id_key=_pkey) == all_ids
    @test indexfilter(tbl, Dict(:missing=>1), id_key=_pkey) == all_ids

    # Test interval filters
    @test indexfilter(tbl, Dict(:z=>[1,5]), id_key=_pkey) == collect(1:5)
    @test indexfilter(tbl, Dict(:z=>[1,5]), sort_keys=(), id_key=_pkey) == collect(1:5)
    @test indexfilter(tbl, Dict(:z=>[1,5]), sort_keys=(:z,), id_key=_pkey) == collect(1:5)
    @test indexfilter(tbl, Dict(:z=>[1,5]), sort_keys=(:z,), sort_reverse=true, id_key=_pkey) == collect(5:-1:1)
    # Test sort_keys isa Vector
    @test indexfilter(tbl, Dict(:z=>[1,5]), sort_keys=[:z], sort_reverse=true, id_key=_pkey) == collect(5:-1:1)

    _xcol = columns(tbl, :x)
    _zcol = columns(tbl, :z)

    # Test exact value filters
    tidx = rand(1:n)
    @test indexfilter(tbl, Dict(:z=>_zcol[tidx]), id_key=_pkey) == [tidx]
    @test indexfilter(tbl, Dict(:z=>_zcol[tidx]), sort_keys=(:z,), id_key=_pkey) == [tidx]
    tidx = rand(1:n)
    @test indexfilter(tbl, Dict(:x=>_xcol[tidx]), id_key=_pkey) == [tidx]
    @test indexfilter(tbl, Dict(:x=>_xcol[tidx]), sort_keys=(:z,), id_key=_pkey) == [tidx]

    # Test set values filters
    tidxs = unique([rand(1:n) for _ in 1:5])
    @test indexfilter(tbl, Dict(:z=>Tuple(tidxs)), sort_keys=(:z,), sort_reverse=true, id_key=_pkey) == sort(tidxs, rev=true)

    # Test set value filters, different sort key
    idxs = indexfilter(tbl, Dict(:z=>(1,2,5,7)), sort_keys=(:x,), sort_reverse=true, id_key=_pkey)
    _zord = _zcol[sortperm(_xcol, rev=true)]
    @test idxs == filter(in([1,2,5,7]), _zord)
end