module SymbolFactorGraphs

using LightGraphs

import LightGraphs: nv, ne, vertices, is_directed, has_edge, edges, inneighbors, outneighbors, rem_vertex!, neighbors, connected_components

import ...DistributedFactorGraphs: DFGNode

export
    SymbolFactorGraph,
    SymbolEdge
    # is_directed,
    # has_edge,
    # rem_vertex!,
    # addVariable!,
    # addFactor!

# import DistributedFactorGraphs: DFGNode
# const AbstractNodeType = DFGNode
import DistributedFactorGraphs: AbstractDFGVariable, AbstractDFGFactor
const AbstractVariableType = AbstractDFGVariable
const AbstractFactorType = AbstractDFGFactor

import Base: zero
const UnionNothingSymbol = Union{Nothing, Symbol}
zero(UnionNothingSymbol) = nothing

include("symboledge.jl")

"""
    SymbolFactorGraph{V, F}

A type representing an undirected bipartite factor graph based on `label::Symbol`.
"""
mutable struct SymbolFactorGraph{V <: AbstractVariableType, F <: AbstractFactorType} <: AbstractGraph{UnionNothingSymbol}
    ne::Int
    fadjdict::Dict{Symbol,Vector{Symbol}} # [variable src id]: (dst, dst, dst)
    variables::Dict{Symbol,V}
    factors::Dict{Symbol,F}
end

################################################################################
# overwrites from SimpleGraphs.jl

nv(g::SymbolFactorGraph) = length(g.fadjdict)
ne(g::SymbolFactorGraph) = g.ne

vertices(g::SymbolFactorGraph) = keys(g.fadjdict)

inneighbors(g::SymbolFactorGraph, v::Symbol) = (badj(g, v))

outneighbors(g::SymbolFactorGraph, v::Symbol) = (fadj(g, v))

neighbors(g::SymbolFactorGraph, v::Symbol) = (fadj(g, v))

"""
    is_directed(g)

Return `true` if `g` is a directed graph.
"""
is_directed(::Type{SymbolFactorGraph{V,F}}) where {V,F} = false

is_directed(::Type{SymbolFactorGraph}) = false

is_directed(g::SymbolFactorGraph) = false

##############################################################################

add_edge!(g::SymbolFactorGraph, x, y) = add_edge!(g, edgetype(g)(x, y))
eltype(x::SymbolFactorGraph) = Symbol

fadj(g::SymbolFactorGraph) = g.fadjdict
fadj(g::SymbolFactorGraph, v::Symbol) = g.fadjdict[v]

edgetype(::SymbolFactorGraph) = SymbolEdge

# eltype(x::SymbolFactorGraph{V,F}) where {T,V,F} = T

# Graph{UInt8}(6), Graph{Int16}(7), Graph{UInt8}()
"""
    SymbolFactorGraph{V,F}(nv=0, nf=0)

Construct an empty `SymbolFactorGraph{V,F}` with `nv,nf` varaibles `sizehint!`, factors and 0 edges.
If not specified, the element type `T` is the type of `nv,nf`.

## Examples
```jldoctest
# SymbolFactorGraph{AbstractVariableType,AbstractFactorType}()
```
"""
function SymbolFactorGraph{V, F}(nv::Int=100, nf::Int=100) where {V <: AbstractVariableType, F <: AbstractFactorType}
    fadjdict = Dict{Symbol,Vector{Symbol}}()
        sizehint!(fadjdict, nv + nf)
    variables = Dict{Symbol,V}()
        sizehint!(variables, nv)
    factors = Dict{Symbol,F}()
        sizehint!(factors, nf)
    return SymbolFactorGraph{V, F}(0, fadjdict, variables, factors)
end

# SymbolFactorGraph{V, F}() where {V<:AbstractVariableType,F<:AbstractFactorType}= SymbolFactorGraph{Int, V, F}()

# SimpleGraph(UInt8)
"""
    SymbolFactorGraph(::Type{T},::Type{V}, ::Type{F})

Construct an empty `SymbolFactorGraph{T}` with 0 vertices and 0 edges.

## Examples
```jldoctest
julia> SymbolFactorGraph(Int,AbstractVariableType, AbstractFactorType)
SymbolFactorGraph{Int64,AbstractVariableType,AbstractFactorType}(0, 0, Union{Nothing, FGNode{AbstractVariableType,Int64}}[], Union{Nothing, FGNode{AbstractFactorType,Int64}}[])
```
"""
# SymbolFactorGraph(::Type{T}, ::Type{V}, ::Type{F}) where {T <: Integer, V<:AbstractVariableType,F<:AbstractFactorType} = SymbolFactorGraph{T, V, F}(zero(T), zero(T))

# SymbolFactorGraph(Int,AbstractVariableType, AbstractFactorType)

#= #############################################################################
# SimpleGraph of a SimpleGraph
"""
    SimpleGraph{T}(g::SimpleGraph)

Construct a copy of g.
If the element type `T` is specified, the vertices of `g` are converted to this type.
Otherwise the element type is the same as for `g`.

## Examples
```jldoctest
julia> g = complete_graph(5)
julia> SimpleGraph{UInt8}(g)
{5, 10} undirected simple UInt8 graph
```
"""
SimpleGraph(g::SimpleGraph) = copy(g)
=#

"""
    badj(g::SymbolFactorGraph[, v::Symbol])

Return the backwards adjacency list of a graph. If `v` is specified,
return only the adjacency list for that vertex.

"""
badj(g::SymbolFactorGraph) = fadj(g)
badj(g::SymbolFactorGraph, v::Symbol) = fadj(g, v)

"""
    adj(g[, v])

Return the adjacency list of a graph. If `v` is specified, return only the
adjacency list for that vertex.

"""
adj(g::SymbolFactorGraph) = fadj(g)
adj(g::SymbolFactorGraph, v::Symbol) = fadj(g, v)

#TODO
#=
# copy(g::SymbolFactorGraph) =  SymbolFactorGraph(g.ne, deepcopy_adjlist(g.fadjlist))

==(g::SimpleGraph, h::SimpleGraph) =
vertices(g) == vertices(h) &&
ne(g) == ne(h) &&
fadj(g) == fadj(h)
=#

function has_edge(g::SymbolFactorGraph, s::Symbol, d::Symbol)
    verts = vertices(g)
    (s in verts && d in verts) || return false  # edge out of bounds
    @inbounds list_s = g.fadjdict[s]
    @inbounds list_d = g.fadjdict[d]
    if length(list_s) > length(list_d)
        d = s
        list_s = list_d
    end
    # return insorted(d, list_s)
    return in(d, list_s)
end

function has_edge(g::SymbolFactorGraph, e::SymbolEdge)
    s, d = Symbol.(Tuple(e))
    return has_edge(g, s, d)
end

"""
    add_edge!(g, e)

Add an edge `e` to graph `g`. Return `true` if edge was added successfully,
otherwise return `false`.

# Examples
```jldoctest
julia> using LightGraphs

julia> g = SimpleGraph(2);

julia> add_edge!(g, 1, 2)
true

julia> add_edge!(g, 2, 3)
false
```
"""
function add_edge!(g::SymbolFactorGraph, e::SymbolEdge)
    s, d = Tuple(e)
    s == d && error("Self loops not allowed")

    verts = vertices(g)
    (s in verts && d in verts) || return false  # edge out of bounds
    @inbounds list = g.fadjdict[s]
    index = searchsortedfirst(list, d)
    @inbounds (index <= length(list) && list[index] == d) && return false  # edge already in graph
    insert!(list, index, d)

    g.ne += 1

    @inbounds list = g.fadjdict[d]
    index = searchsortedfirst(list, s)
    insert!(list, index, s)
    return true  # edge successfully added
end

"""
    rem_edge!(g, e)

Remove an edge `e` from graph `g`. Return `true` if edge was removed successfully,
otherwise return `false`.

### Implementation Notes
If `rem_edge!` returns `false`, the graph may be in an indeterminate state, as
there are multiple points where the function can exit with `false`.

# Examples
```jldoctest
julia> using LightGraphs

julia> g = SimpleGraph(2);

julia> add_edge!(g, 1, 2);

julia> rem_edge!(g, 1, 2)
true

julia> rem_edge!(g, 1, 2)
false
```
"""
function rem_edge!(g::SymbolFactorGraph, e::SymbolEdge)
    s, d = Tuple(e)
    verts = vertices(g)
    (s in verts && d in verts) || return false  # edge out of bounds
    @inbounds list = g.fadjdict[s]
    index = searchsortedfirst(list, d)
    @inbounds (index <= length(list) && list[index] == d) || return false  # edge not in graph
    deleteat!(list, index)

    g.ne -= 1
    s == d && return true  # selfloop

    @inbounds list = g.fadjdict[d]
    index = searchsortedfirst(list, s)
    deleteat!(list, index)
    return true  # edge successfully removed
end

"""
    add_vertex!(g)

Add a new vertex to the graph `g`. Return `true` if addition was successful.

# Examples
```jldoctest
julia> using LightGraphs

julia> g = SimpleGraph(Int8(typemax(Int8) - 1))
{126, 0} undirected simple Int8 graph

julia> add_vertex!(g)
true

julia> add_vertex!(g)
false
```
"""
# function add_vertex!(g::SymbolFactorGraph, node::) where T
#     (nv(g) + one(T) <= nv(g)) && return false       # test for overflow
#     push!(g.fadjlist, Vector{T}())
#     return true
# end

function addVariable!(g::SymbolFactorGraph{V, F}, variable::V) where {V, F}
    (nv(g) + one(Int) <= nv(g)) && return false       # test for overflow
    # !isa(node,V) && (@warn "not a variable type"; return false)
    label = variable.label

    haskey(g.fadjdict, label) && (@error "Label already in fg"; return false) #TODO debug error or exception?
    # haskey(g.labelIndex, label) && error("Label already in fg")

    push!(g.fadjdict, label=>Symbol[])

    push!(g.variables, label=>variable)

    return true
end

function addFactor!(g::SymbolFactorGraph{V, F}, variableLabels::Vector{Symbol}, factor::F)::Bool where {V, F}

    haskey(g.fadjdict, factor.label) && (@error "Label $(factor.label) already in fg"; return false)

    for label in variableLabels
        !haskey(g.fadjdict, label) && (@error "Variable '$(label)' not found in graph when creating Factor '$(factor.label)'"; return false) #TODO debug error or exception?
    end

    (nv(g) + one(Int) <= nv(g)) && return false       # test for overflow
    # !isa(node,V) && (@warn "not a variable type"; return false)

    # factor._variableOrderSymbols = copy(variableLabels)

    push!(g.fadjdict, factor.label=>Symbol[])

    push!(g.factors, factor.label=>factor)

    # add the edges
    for label in variableLabels
        add_edge!(g, label, factor.label) || return false
    end

    return true
end

function edges(g::SymbolFactorGraph)

    fadjdict = fadj(g)
    n = nv(g)

    edges = SymbolEdge[]
    sizehint!(edges, n)

    for p = pairs(fadjdict)
        for l = p.second
            push!(edges,SymbolEdge(p.first=>l))
        end
    end

    edges

end

"""
    rem_vertex!(g, v)

Remove the vertex `v` from graph `g`. Return `false` if removal fails
(e.g., if vertex is not in the graph); `true` otherwise.

# Examples
```jldoctest
julia> using LightGraphs

julia> g = SimpleGraph(2);

julia> rem_vertex!(g, 2)
true

julia> rem_vertex!(g, 2)
false
```
"""
function rem_vertex!(g::SymbolFactorGraph, v::Symbol)
    v in vertices(g) || return false
    n = nv(g)

    # TODO behoort nie moontelik te wees nie...
    # self_loop_n = false  # true if n is self-looped (see #820)

    # remove the in_edges from v
    srcs = copy(inneighbors(g, v))
    @inbounds for s in srcs
        rem_edge!(g, edgetype(g)(s, v))
    end

    # # remove the in_edges from the last vertex
    # neigs = copy(inneighbors(g, n))
    # @inbounds for s in neigs
    #     rem_edge!(g, edgetype(g)(s, n))
    # end
    # if v != n
    #     # add the edges from n back to v
    #     @inbounds for s in neigs
    #         if s != n  # don't add an edge to the last vertex - see #820.
    #             add_edge!(g, edgetype(g)(s, v))
    #         else
    #             self_loop_n = true
    #         end
    #     end
    # end

    # if self_loop_n
    #     add_edge!(g, edgetype(g)(v, v))
    # end
    delete!(g.fadjdict, v)
    delete!(g.variables, v)
    delete!(g.factors, v)

    return true
end

function _connected_components!(label::Dict{Symbol, T}, g::SymbolFactorGraph) where T <: UnionNothingSymbol
    nvg = nv(g)

    for u in vertices(g)
        label[u] != nothing && continue
        label[u] = u
        Q = Vector{T}()
        push!(Q, u)
        while !isempty(Q)
            src = popfirst!(Q)
            for vertex in neighbors(g, src)
                if label[vertex] == zero(T)
                    push!(Q, vertex)
                    label[vertex] = u
                end
            end
        end
    end
    return label
end

function _components(labels::Dict{Symbol, UnionNothingSymbol})

    d = Dict{Symbol,Int}()
    c = Vector{Vector{Symbol}}()
    i = 1 #one(T)
    for (v, l) in enumerate(labels)
        index = get!(d, l.second, i)
        if length(c) >= index
            push!(c[index], l.first)
        else
            push!(c, [l.first])
            i += 1
        end
    end
    return c, d
end

function connected_components(g::SymbolFactorGraph)
    # label = zeros(T, nv(g))
    label = Dict{Symbol, UnionNothingSymbol}(l=>nothing for l in vertices(g))
    _connected_components!(label, g)
    c, d = _components(label)
    return c
end

end


# Objective:  Find the size of the components of the neutral network of a goal.
# Methodology:  Do multiple neutral random walks starting at a random circuit that maps to the goal.
# neutral_walk() accumulate all neighbors of all neutral circuits encountered on the random neutral walk.
# Then run_neutral_walk() combines the circuit lists returned by multiple random walks
#      if they have a circuit in common.
# Thus, run_neutral_walk() returns circuit lists that have not been shown to be in the same connected
#      component.

using Statistics, Printf

# steps is the number of random walk steps.
# maxsteps is the number of steps in mut_evolve().
# maxtries is the number of attempts to find the next chromosome in the random_neutral_walk.
# Assumes a single output goal
# Returns circuit_code_list of neutral circuits discovered on the neutral walk.
# Includes both walk circuits and neutral neighbors of walk circuits.
# Note that sometimes the function fails when the neutral walk cannot be extended.
function neutral_walk( g::Goal, p::Parameters, steps::Int64, maxsteps::Int64, maxtries::Int64 )
  @assert p.numoutputs == 1
  funcs = default_funcs( p.numinputs )
  circuit_ints = Int64[]
  n_repeats = 10  # maximum number of tries to find c that maps to g
  complexity_sum = 0.0
  complexity_count = 0
  res = mut_evolve_repeat(n_repeats, p, [g], funcs, maxsteps )
  c = res[1]
  #println("complexity(c): ",complexity5(c))
  complexity_sum += complexity5(c)
  complexity_count += 1
  #@assert sort(output_values(c)) == sort(g)
  @assert output_values(c) == g    # Assumes p.numoutputs==1
  push!( circuit_ints, circuit_int(c) )
  for i = 1:steps
    (new_c,active) = mutate_chromosome!( deepcopy(c), funcs )
    outputs = output_values(new_c)
    attempts = 1
    while attempts < maxtries && outputs != g  # try to find a c to extend walk
      (new_c,active) = mutate_chromosome!( deepcopy(c), funcs )
      outputs = output_values(new_c)
      #println("attempts: ",attempts,"  outputs: ",outputs,"  goal: ",g)
      attempts += 1
    end
    if attempts == maxtries
      # Alternatively, try all mutations of c in our attempt to extend walk
      all_chromes = map( x->x[2], mutate_all( deepcopy(c), funcs, output_chromosomes=true ))
      filter!( x->output_values(x)==g, all_chromes )  # only chromosomes that map to g
      #println(" attempts == maxtries  len(all_chromes): ",length(all_chromes))
      if length(all_chromes) == 0
        println("unable to extend random_neutral_walk in function neutral_walk() at step: ",i)
        break
      else
        @assert output_values(all_chromes[1]) == g
        c = rand(all_chromes)
      end
    else
      c = new_c   # neutral walk extended
    end
    complexity_sum += complexity5(c)
    complexity_count += 1
    @assert output_values(c) == g
    # mutate_all() returns a list of pairs where the second element of the pair is the chromosome
    all_chromes = map( x->x[2], mutate_all( c, funcs, output_chromosomes=true ))
    filter!( x->output_values(c)==g, all_chromes )
    for ch in all_chromes
      push!( circuit_ints, circuit_int(ch) )
    end
    circuit_ints = unique(circuit_ints)
  end # for loop
  (Set(circuit_ints), complexity_sum/complexity_count )
end

# Does multiple random walks and combines the returned circuit code lists if they have circuits in common.
function run_neutral_walk( g::Goal, p::Parameters, n_walks::Int64, steps::Int64, maxsteps::Int64, maxtries::Int64, int_list_file::IOStream )
  walk_list = Int64[]
  circuit_int_list = Set{Int64}[]
  walk_results = pmap(x->neutral_walk( g, p, steps, maxsteps, maxtries), collect(1:n_walks))
  println("after pmap()")
  #walk_results = map(x->neutral_walk( g, p, steps, maxsteps, maxtries), collect(1:n_walks))
  #println("walk_results[1]: ",walk_results[1])
  complexity_avg = 0.0
  for w = 1:n_walks
    #cclist = Set(neutral_walk( g, p, steps, maxsteps, maxtries))
    cclist = walk_results[w][1]
    complexity_avg += walk_results[w][2]
    #println("length(cclist): ",length(cclist))
    to_combine = Int64[]   # indices of circuit_int list to combine
    for i = 1:length(circuit_int_list)
      if length(intersect(cclist,circuit_int_list[i])) > 0
        push!(to_combine,i)
      end
    end
    if length(to_combine) > 0
      println("w: ",w,"  to_combine: ",to_combine)
      combined_list = cclist
      for i = length(to_combine):-1:1
        #println("combining circuit_int_list[",to_combine[i],"]")
        combined_list = union(combined_list,circuit_int_list[to_combine[i]])
        if i > 1
          deleteat!( circuit_int_list, to_combine[i] )
          #println("removing index: ",to_combine[i]," from circuit_int_list")
          #println("walk_list before: ",walk_list)
          #walk_list = setdiff(walk_list,to_combine[i])
          deleteat!( walk_list, to_combine[i] )
          #println("removing index: ",to_combine[i]," from walk list")
          #println("walk_list after: ",walk_list)
        end
      end
      circuit_int_list[to_combine[1]] = combined_list
    else
      push!(walk_list,w)
      push!(circuit_int_list,cclist)
      #println("walk list: ",walk_list)
      #println("   lengths circuit_int_list: ", [length(circuit_int_list[k]) for k = 1:length(circuit_int_list)])
    end
    #println("w: ",w,"  walk_list after add: ",walk_list)
    println("w: ",w,"  len: ",length(circuit_int_list),"  lengths circuit_int_list: ", [length(circuit_int_list[k]) for k = 1:length(circuit_int_list)])
    if w == n_walks
      println(int_list_file,"goal: ",g,"  len: ",length(circuit_int_list),"  lengths circuit_int_list: ", [length(circuit_int_list[k]) for k = 1:length(circuit_int_list)])
    end
    #print("w: ",w,"  length(circuit_int_list) after add: ",length(circuit_int_list))
    #println("  lengths circuit_int_list: ",[length(ccl) for ccl in circuit_int_list])
    for i = 1:length(circuit_int_list)
      for j = 1:(i-1)
        if !isempty(intersect(circuit_int_list[i],circuit_int_list[j]))
          println("XXXXX i: ",i,"  j: ",j,"  len intersect: ",length(intersect(circuit_int_list[i],circuit_int_list[j])))
        end
      end
    end
  end  # for w = 1:n_walks
  #walk_list
  (g,length(circuit_int_list),[length(ccl) for ccl in circuit_int_list],complexity_avg/n_walks)
end

# Do multiple runs of run_neutral_walk() defined above for each goal in goallist gl.
# To do repeated runs on a goal, include it multiple times in the gl.
function run_neutral_walk( gl::GoalList, p::Parameters, n_walks::Int64, steps::Int64, maxsteps::Int64, maxtries::Int64;
      csvfile::String="" )
  df = DataFrame()
  df.goal = Vector{MyInt}[]
  df.numinputs = Int64[]
  df.numoutputs = Int64[]
  df.numints = Int64[]
  df.numlevsback = Int64[]
  df.n_walks = Int64[]
  df.steps = Int64[]
  df.maxsteps = Int64[]
  df.maxtries = Int64[]
  df.n_combined = Float64[]
  df.complexity = Float64[]
  if length(csvfile) > 0
    println("file: ","$(csvfile[1:(end-4)])_ints.txt")
    int_list_file=open("$(csvfile[1:(end-4)])_ints.txt","w")
  end
  result = map(g->run_neutral_walk( g, p, n_walks, steps, maxsteps, maxtries, int_list_file ), gl )
  #println("after map()")
  for r in result
    push!(df,(r[1], p.numinputs, p.numoutputs, p.numinteriors, p.numlevelsback, n_walks, steps, maxsteps, maxtries, r[2], r[4] ))
  end
  hostname = chomp(open("/etc/hostname") do f read(f,String) end)
  println("# date and time: ",Dates.now())
  println("# host: ",hostname," with ",nprocs()-1,"  processes: " )
  println("# funcs: ", Main.CGP.default_funcs(p.numinputs))
  print_parameters(p,comment=true)
  println("# n_walks: ",n_walks)
  println("# steps: ",steps)
  println("# maxsteps: ",maxsteps)
  println("# maxtries: ",maxtries)
  println("# ngoals: ",length(gl))
  if length(csvfile) > 0
    close(int_list_file)
    open( csvfile, "w" ) do f
      println(f,"# date and time: ",Dates.now())
      println(f,"# host: ",hostname," with ",nprocs()-1,"  processes: " )
      println(f,"# funcs: ", Main.CGP.default_funcs(p.numinputs))
      print_parameters(f,p,comment=true)
      println(f,"# steps: ",steps)
      println(f,"# maxsteps: ",maxsteps)
      println(f,"# maxtries: ",maxtries)
      println(f,"# ngoals: ",length(gl))
      CSV.write(f, df, append=true, writeheader=true )
    end
  end
  return df
end

# Distribution of complexities on a neutral walk.
function neutral_walk_complexity( g::Goal, p::Parameters, steps::Int64, maxsteps::Int64, maxtries::Int64 )
  @assert p.numoutputs == 1
  funcs = default_funcs( p.numinputs )
  walk_complexities = Float64[]
  walk_count = 0
  neighbor_complexities = Float64[]
  neighbor_count = 9
  n_repeats = 10  # maximum number of tries to find c that maps to g
  res = mut_evolve_repeat(n_repeats, p, [g], funcs, maxsteps )
  c = res[1]
  complexity = complexity5(c)
  #println("w complexity(c): ",complexity)
  push!(walk_complexities,complexity)
  #@assert sort(output_values(c)) == sort(g)
  @assert output_values(c) == g    # Assumes p.numoutputs==1
  for i = 1:steps
    (new_c,active) = mutate_chromosome!( deepcopy(c), funcs )
    outputs = output_values(new_c)
    attempts = 1
    while attempts < maxtries && outputs != g  # try to find a c to extend walk
      (new_c,active) = mutate_chromosome!( deepcopy(c), funcs )
      outputs = output_values(new_c)
      #println("attempts: ",attempts,"  outputs: ",outputs,"  goal: ",g)
      attempts += 1
    end
    if attempts == maxtries
      # Alternatively, try all mutations of c in our attempt to extend walk
      all_chromes = map( x->x[2], mutate_all( deepcopy(c), funcs, output_chromosomes=true ))
      filter!( x->output_values(x)==g, all_chromes )  # only chromosomes that map to g
      #println(" attempts == maxtries  len(all_chromes): ",length(all_chromes))
      if length(all_chromes) == 0
        println("unable to extend random_neutral_walk in function neutral_walk() at step: ",i)
        continue
      else
        @assert output_values(all_chromes[1]) == g
        c = rand(all_chromes)
      end
    else
      c = new_c   # neutral walk extended
    end
    @assert output_values(c) == g
    complexity = complexity5(c)
    #println("w complexity(c): ",complexity)
    push!(walk_complexities,complexity)
    walk_count += 1
    # mutate_all() returns a list of pairs where the second element of the pair is the chromosome
    all_chromes = map( x->x[2], mutate_all( c, funcs, output_chromosomes=true ))
    filter!( x->output_values(c)==g, all_chromes )
    for ch in all_chromes
      complexity = complexity5(ch)
      #println("n complexity(c): ",complexity)
      push!(neighbor_complexities,complexity)
      neighbor_count += 1
    end
  end
  #println("steps: ",steps,"  length(walk_c): ",length(walk_complexities),"  length(nbr_c): ",length(neighbor_complexities))
  (walk_complexities,neighbor_complexities)
end

# Distribution of complexities on a neutral walk.
function run_neutral_walk_complexity( goallist::GoalList, p::Parameters, n_walks::Int64, steps::Int64,
      maxsteps::Int64, maxtries::Int64; csvfile::String="" )
  if length(csvfile) > 0
    f = open(csvfile,"w")
    print_parameters(f,p)
    println(f,"n_walks: ",n_walks)
    println(f,"steps: ",steps)
    println(f,"maxtries: ",maxtries)
  end
  for g in goallist
    w_cmplx = fill(Float64[],n_walks)
    n_cmplx = fill(Float64[],n_walks)
    walk_complexities = Float64[]
    neighbor_complexities = Float64[]
    for w = 1:n_walks
      (w_cmplx[w],n_cmplx[w]) = neutral_walk_complexity( g, p, steps, maxsteps, maxtries )
      #println("w: ",w,"  length(w_cmplx): ",length(w_cmplx[w]),"  length(n_cmplx): ",length(n_cmplx[w]))
    end
    walk_complexities = vcat( w_cmplx... )
    neighbor_complexities = vcat( n_cmplx... )
    println("length(walk_complexities): ",length(walk_complexities),"  length(neighbor_complexities): ",length(neighbor_complexities))
    # Statistics
    w_mean = mean(walk_complexities)
    w_max = maximum(walk_complexities)
    w_min = minimum(walk_complexities)
    w_std = std(walk_complexities)
    w_q = quantile(walk_complexities,[0.9,0.95,0.99])
    n_mean = mean(neighbor_complexities)
    n_max = maximum(neighbor_complexities)
    n_min = minimum(neighbor_complexities)
    n_std = std(neighbor_complexities)
    n_q = quantile(neighbor_complexities,[0.9,0.95,0.99])
    @printf("goal: [0x%04x]",g[1])
    @printf("  w_count:%8d",length(walk_complexities))
    @printf("  w_mean:%6.3f",w_mean)
    @printf("  w_max:%6.3f",w_max)
    @printf("  w_min:%6.3f",w_min)
    @printf("  w_std:%6.3f",w_std)
    @printf("  w_q90:%6.3f",w_q[1])
    @printf("  w_q95:%6.3f",w_q[2])
    @printf("  w_q99:%6.3f\n",w_q[3])
    @printf("goal: [0x%04x]",g[1])
    @printf("  n_count:%8d",length(neighbor_complexities))
    @printf("  n_mean:%6.3f",n_mean)
    @printf("  n_max:%6.3f",n_max)
    @printf("  n_min:%6.3f",n_min)
    @printf("  n_std:%6.3f",n_std)
    @printf("  n_q90:%6.3f",n_q[1])
    @printf("  n_q95:%6.3f",n_q[2])
    @printf("  n_q99:%6.3f\n",n_q[3])
    if length(csvfile) > 0
      @printf(f,"goal: [0x%04x]",g[1])
      @printf(f,"  w_count:%8d",length(walk_complexities))
      @printf(f,"  w_mean:%6.3f",w_mean)
      @printf(f,"  w_max:%6.3f",w_max)
      @printf(f,"  w_min:%6.3f",w_min)
      @printf(f,"  w_std:%6.3f",w_std)
      @printf(f,"  w_q90:%6.3f",w_q[1])
      @printf(f,"  w_q95:%6.3f",w_q[2])
      @printf(f,"  w_q99:%6.3f\n",w_q[3])
      @printf(f,"goal: [0x%04x]",g[1])
      @printf(f,"  n_count:%8d",length(neighbor_complexities))
      @printf(f,"  n_mean:%6.3f",n_mean)
      @printf(f,"  n_max:%6.3f",n_max)
      @printf(f,"  n_min:%6.3f",n_min)
      @printf(f,"  n_std:%6.3f",n_std)
      @printf(f,"  n_q90:%6.3f",n_q[1])
      @printf(f,"  n_q95:%6.3f",n_q[2])
      @printf(f,"  n_q99:%6.3f\n",n_q[3])
    end
  end
  close(f)
  #(walk_complexities,neighbor_complexities)
end

function neutral_walk_connectivity( p::Parameters,  g::Goal, max_steps::Int64, max_evolve_steps::Int64; c2_walk_length::Int64=200 )
  funcs = default_funcs( p.numinputs )
  res1 = mut_evolve( random_chromosome( p, funcs ), [g], funcs, max_evolve_steps )
  if res1[2] < max_evolve_steps
    c1 = res1[1]
  else
    error("failed to find circuit that maps to goal: ",g)
  end
  #print_build_chromosome( c1 )
  res2 = mut_evolve( random_chromosome( p, funcs ), [g], funcs, max_evolve_steps )
  if res2[2] < max_evolve_steps
    c2 = res2[1]
  else
    error("failed to find circuit that maps to goal: ",g)
  end
  #print_build_chromosome( c2 )
  neutral_walk_connectivity( c1, c2, c2_walk_length, max_steps )
end

function neutral_walk_connectivity( c1::Chromosome, c2::Chromosome, c2_walk_length::Int64, max_steps::Int64 )
  max_attempts = 10
  @assert c1.params.numoutputs == 1
  funcs = default_funcs( p.numinputs )
  c2_walk_list = random_neutral_walk( c2, c2_walk_length )
  g = output_values(c1)
  out2 = output_values(c2)
  @assert g==out2
  c_code = circuit_code(c1)
  c_dist = circuit_distance_to_list( c1, c2_walk_list )
  println("original c_dist: ",c_dist)
  if c_dist == 0
    return 0
  end
  c = deepcopy(c1)
  @assert output_values(c) == g    # Assumes p.numoutputs==1
  for step = 1:max_steps
    c_code = circuit_code(c)
    (new_c,active) = mutate_chromosome!( deepcopy(c), funcs )
    outputs = output_values(new_c)
    attempts = 1
    # try to find a c to extend walk
    while attempts < max_attempts && (outputs != g || circuit_distance_to_list(new_c,c2_walk_list) > c_dist || circuit_code(new_c)==c_code )
      (new_c,active) = mutate_chromosome!( deepcopy(c), funcs )
      outputs = output_values(new_c)
      #println("step: ",step,"  attempts: ",attempts,"  outputs: ",outputs,"  new_c_dist: ",circuit_distance(new_c,c2))
      attempts += 1
    end
    if attempts == max_attempts
      # Alternatively, try all mutations of c in our attempt to extend walk
      c_code = circuit_code(c)
      all_chromes = map( x->x[2], mutate_all( deepcopy(c), funcs, output_chromosomes=true ))
      # filter to only chromosomes that map to g and do not increase circuit distance and change the circuit code
      filter!( (x->output_values(x)==g && circuit_distance_to_list(x,c2_walk_list) <= c_dist && circuit_code(x)!=c_code), all_chromes )
      println(" attempts == maxtries  len(all_chromes): ",length(all_chromes))
      if length(all_chromes) == 0
        println("failed to extend random_neutral_walk in function neutral_walk() at step: ",step)
        return -1
      else
        @assert output_values(all_chromes[1]) == g
        c = rand(all_chromes)
        c_dist = circuit_distance_to_list(c,c2_walk_list)
        println("mx step: ",step,"  neutral walk extended new_cdist: ",c_dist,"  new_circuit_code: ",circuit_code(c))
        @assert circuit_code(c) != c_code
      end
    else
      c = new_c   # neutral walk extended
      c_dist = circuit_distance_to_list(c,c2_walk_list)
      println("at step: ",step,"  neutral walk extended new_cdist: ",c_dist,"  new_circuit_code: ",circuit_code(c))
      @assert circuit_code(c) != c_code
    end
    @assert output_values(c) == g
    new_c_dist = circuit_distance_to_list( c, c2_walk_list )
    @assert new_c_dist <= c_dist
    c_dist = new_c_dist < c_dist ? new_c_dist : c_dist
    if c_dist == 0.0
      println("found path c1 to c2 after ",step," steps")
      return step
    end
  end
  println("failed to find path from c1 to c2")
  return -1
end

function random_neutral_walk( c::Chromosome, length::Int64, max_attempts::Int64=100 )
  funcs = default_funcs( c.params.numinputs )
  chrome_list = Chromosome[c]
  g = output_values(c)
  c_code = circuit_code(c)
  new_c = Chromosome(p,[InputNode(1)],[],[OutputNode(1)],0.0,0.0)  # dummy chromosome to establish scope
  outputs = output_values(new_c)
  len = 0
  while len < length
    attempts = 0
    while attempts < max_attempts && (outputs != g || circuit_code(new_c) == c_code )
      (new_c,active) = mutate_chromosome!( deepcopy(c), funcs )
      outputs = output_values(new_c)
      #println("step: ",step,"  attempts: ",attempts,"  outputs: ",outputs )
      attempts += 1
    end
    if attempts < max_attempts
      c = new_c
      c_code = circuit_code(c)
      len += 1
      push!( chrome_list, new_c )
    else
      break
    end
  end
  return map( c->circuit_code(c), chrome_list )
end

function circuit_distance_to_list( c::Chromosome, c_code_list::Vector{Vector{Int64}} )
  circuit_distance_to_list( circuit_code(c), c_code_list )
end

function circuit_distance_to_list( c_code::Vector{Int64}, c_code_list::Vector{Vector{Int64}} )
  distance = length(c_code)
  for cc in c_code_list
    @assert length(c_code) == length(cc)
    diff_count = 0
    for i = 1:length(c_code)
      diff_count += c_code[i] == cc[i] ? 0 : 1
    end
    distance = diff_count < distance ? diff_count : distance
  end
  distance/length(c_code)
end

#=
function neutral_walk_connectivity( c1::Chromosome, c2::Chromosome, max_steps::Int64 )
  @assert c1.params == c2.params
  funcs = default_funcs( c1.params.numinputs )
  out1 = output_values(c1)
  out2 = output_values(c2)
  @assert out1==out2
  code1 = circuit_code(c1)
  code2 = circuit_code(c2)
  c_dist = circuit_distance( c1, c2 )
  println("original c_dist: ",c_dist)
  if c_dist == 0
    return 0
  end
  new_c1 = mutate_chromosome!( deepcopy(c1), funcs )[1]
  new_c2 = mutate_chromosome!( deepcopy(c2), funcs )[1]
  dist1 = circuit_distance( new_c1, c2 )
  dist2 = circuit_distance( new_c2, c1 )
  println("orig dist1: ",dist1,"  orig dist2: ",dist2)
  step = 1
  while step < max_steps && c1 != c2
    while step < max_steps && dist1 > c_dist && dist2 > c_dist
      new_c1 = mutate_chromosome!( c1, funcs )[1]
      new_c2 = mutate_chromosome!( c2, funcs )[1]
      dist1 = circuit_distance( new_c1, c2 )
      dist2 = circuit_distance( new_c2, c1 )
      step += 1
    end
    println("after inner while loop.  step: ",step)
    if dist1 < c_dist
      c1 = new_c1
      c_dist = circuit_distance( c1, c2 )
      println("c1 = new_c1  c_dist: ",c_dist)
    elseif dist2 < c_dist
      c2 = new_c2
      c_dist = circuit_distance( c1, c2 )
      println("c2 = new_c2  c_dist: ",c_dist)
    end
    if c_dist == 0
      println("found path from c1 to c2 in ",step," steps")
      return step
    end
    if dist1 == c_dist
      c1 = new_c1
      println("c1 = new_c1")
    elseif dist2 == c_dist && circuit_distance( new_c2, c1 ) <= c_dist
      c2 = new_c2
      println("c2 = new_c2")
    end
    new_c1 = mutate_chromosome!( c1, funcs )[1]
    new_c2 = mutate_chromosome!( c2, funcs )[1]
    dist1 = circuit_distance( new_c1, c2 )
    dist2 = circuit_distance( new_c2, c1 )
    println("new dist1: ",dist1,"  new dist2: ",dist2)
    step += 1
  end
  if c1 == c2
    println("found path from c1 to c2 in ",step," steps")
  else
    println("failed to find path from c1 to c2 in ",step," steps")
  end
  return step
end

=#


# Scalar operations

# @inline Base.@pure vshr(a::Int64, b) = llvmcall("%res = ashr i64 %0, %1\nret i64 %res", Int64, Tuple{Int64,Int64}, a, b % Int64)
# @inline Base.@pure vshr(a::Int32, b) = llvmcall("%res = ashr i32 %0, %1\nret i32 %res", Int32, Tuple{Int32,Int32}, a, b % Int32)

# @inline Base.@pure vshr(a::Int16, b) = llvmcall("%res = ashr i16 %0, %1\nret i16 %res", Int16, Tuple{Int16,Int16}, a, b % Int16)
# @inline Base.@pure vshr(a::Int8, b) = llvmcall("%res = ashr i8 %0, %1\nret i8 %res", Int8, Tuple{Int8,Int8}, a, b % Int8)

# @inline Base.@pure vshr(a::UInt64, b) = llvmcall("%res = lshr i64 %0, %1\nret i64 %res", UInt64, Tuple{UInt64,UInt64}, a, b % UInt64)
# @inline Base.@pure vshr(a::UInt32, b) = llvmcall("%res = lshr i32 %0, %1\nret i32 %res", UInt32, Tuple{UInt32,UInt32}, a, b % UInt32)

# @inline Base.@pure vshr(a::UInt16, b) = llvmcall("%res = lshr i16 %0, %1\nret i16 %res", UInt16, Tuple{UInt16,UInt16}, a, b % UInt16)
# @inline Base.@pure vshr(a::UInt8, b) = llvmcall("%res = lshr i8 %0, %1\nret i8 %res", UInt8, Tuple{UInt8,UInt8}, a, b % UInt8)

function binary_op(op, W, @nospecialize(UNUSED::Type{T}), W2 = W) where {T}
    @assert W == W2
    ty = LLVM_TYPES[T]
    if isone(W)
        V = T
    else
        ty = "<$W x $ty>"
        V = NTuple{W,VecElement{T}}
    end
    instrs = "%res = $op $ty %0, %1\nret $ty %res"
    call = :(llvmcall($instrs, $V, Tuple{$V,$V}, data(v1), data(v2)))
    W > 1 && (call = Expr(:call, :Vec, call))
    Expr(:block, Expr(:meta, :inline), call)
end
# @generated function binary_operation(::Val{op}, v1::V1, v2::V2) where {op, V1, V2}
#     M1, N1, W1, T1 = description(V1)
#     M2, N2, W2, T2 = description(V2)

#     lc = Expr(:call, :llvmcall, join(instrs, "\n"), )
# end
# function binary_op(op, W, @nospecialize(_::Type{T})) where {T}
#     ty = 'i' * string(8*sizeof(T))
#     binary_op(op, W, T, ty)
# end

# Integer
    # vop = Symbol('v', op)
for (op,f) ∈ [("add",:+),("sub",:-),("mul",:*),("shl",:<<)]
    ff = Symbol('v', op); _ff = Symbol(:_, ff)
    @eval @generated $_ff(v1::T, v2::T) where {T<:Integer} = binary_op($op * (T <: Signed ? " nsw" : " nuw"), 1, T)
    @eval @generated $_ff(v1::Vec{W1,T}, v2::Vec{W2,T}) where {W1,W2,T<:Integer} = binary_op($op * (T <: Signed ? " nsw" : " nuw"), W1, T, W2)
    @eval @generated Base.$f(v1::Vec{W1,T}, v2::Vec{W2,T}) where {W1,W2,T<:Integer} = binary_op($op, W1, T, W2)
    @eval Base.@pure @inline $ff(v1::T, v2::T) where {T} = $_ff(v1, v2)
    @eval @inline $ff(v1, v2) = ((v3, v4) = promote(v1, v2); $ff(v3, v4))
end
for (op,f) ∈ [("div",:÷),("rem",:%)]
    ff = Symbol('v', op); _ff = Symbol(:_, ff)
    @eval @generated Base.$f(v1::Vec{W1,T}, v2::Vec{W2,T}) where {W1,W2,T<:Integer} = binary_op((T <: Signed ? 's' : 'u') * $op, W1, T, W2)
    @eval @generated $_ff(v1::T, v2::T) where {T<:Integer} = binary_op((T <: Signed ? 's' : 'u') * $op, 1, T)
    @eval @inline $_ff(v1::AbstractSIMD, v2::AbstractSIMD) = Base.$f(v1, v2)
    @eval Base.@pure @inline $ff(v1::T, v2::T) where {T} = $_ff(v1, v2)
    @eval @inline $ff(v1, v2) = ((v3, v4) = promote(v1, v2); $ff(v3, v4))
end
@inline vcld(x, y) = vadd(vdiv(vsub(x,one(x)), y), one(x))
@inline function vdivrem(x, y)
    d = vdiv(x, y)
    r = vsub(x, vmul(d, y))
    d, r
end
for (op,f,s) ∈ [("ashr",:>>,0x01),("lshr",:>>,0x02),("lshr",:>>>,0x03),("and",:&,0x03),("or",:|,0x03),("xor",:⊻,0x03)]
    _ff = Symbol(:_, 'v', op)
    fdef = Expr(:where, :(Base.$f(v1::Vec{W1,T}, v2::Vec{W2,T})), :W1, :W2)
    ffdef = Expr(:where, :($_ff(v1::T, v2::T)))
    if iszero(s & 0x01)
        push!(fdef.args, :(T <: Unsigned))
        push!(ffdef.args, :(T <: Unsigned))
    elseif iszero(s & 0x02)
        push!(fdef.args, :(T <: Signed))
        push!(ffdef.args, :(T <: Signed))
    else
        push!(fdef.args, :T)
        push!(ffdef.args, :T)
    end
    @eval @generated $fdef = binary_op($op, W1, T, W2)
    @eval @generated $ffdef = binary_op($op, 1, T)
    if op === "ashr" # replace first iteration with <<
        f = :(<<)
    end
    @eval begin
        @inline Base.$f(v::Vec, i::Integer) = ((x, y) = promote(v, i); $f(x, y))
        @inline Base.$f(i::Integer, v::Vec) = ((x, y) = promote(i, v); $f(x, y))
    end
end
for (op,f) ∈ [("lshr",:>>),("ashr",:>>),("and",:&),("or",:|),("xor",:⊻)]
    ff = Symbol('v', op); _ff = Symbol(:_, ff)
    @eval Base.@pure @inline $ff(v1::T, v2::T) where {T} = $_ff(v1, v2)
    @eval @inline $ff(v1, v2) = ((v3, v4) = promote(v1, v2); $ff(v3, v4))
end

for (op,f,ff) ∈ [("fadd",:+,:vadd),("fsub",:-,:vsub),("fmul",:*,:vmul),("fdiv",:/,:vdiv),("frem",:%,:vrem)]
    @eval begin
        @generated Base.$f(v1::Vec{W1,T}, v2::Vec{W2,T}) where {W1,W2,T<:Union{Float32,Float64}} = binary_op($(op * " nsz arcp contract afn reassoc"), W1, T, W2)
        @generated $ff(v1::Vec{W1,T}, v2::Vec{W2,T}) where {W1,W2,T<:Union{Float32,Float64}} = binary_op($op, W1, T, W2)
        @inline $ff(a::T, b::T) where {T <: Union{Float32,Float64}} = $f(a, b)
    end
end

@inline Base.:(/)(a::Vec{W,<:Integer}, b::Vec{W,<:Integer}) where {W} = float(a) / float(b)

for op ∈ [:⊻, :&, :|]
    @eval begin
        @inline function Base.$op(v1::AbstractSIMDVector{W}, v2::AbstractSIMDVector{W}) where {W}
            v3, v4 = promote(v1, v2)
            $op(v3, v4)
        end
    end
end

function promote_shift_quote(op::Symbol, ::Type{T1}, ::Type{T2}) where {T1, T2}
    s1 = sizeof(T1); s2 = sizeof(T2);
    @assert s1 != s2 "op: $op, T1 === $T1, T2 === $T2"
    if s1 < s2
        newT = T1 <: Signed ? signed(T2) : unsigned(T2)
        f = Expr(:call, op, Expr(:call, :convert, newT, :v1), :v2)
    else
        newT = T2 <: Signed ? signed(T1) : unsigned(T1)
        f = Expr(:call, op, :v1, Expr(:call, :convert, newT, :v2))
    end
    Expr(:block, Expr(:meta, :inline), f)
end
@inline function Base.:(<<)(v1::AbstractSIMDVector{W,T1}, v2::AbstractSIMDVector{W,T2}) where {W,T1<:SignedHW,T2<:UnsignedHW}
    convert(T1, convert(T2, v1) << v2)
end
@inline function Base.:(<<)(v1::AbstractSIMDVector{W,T1}, v2::AbstractSIMDVector{W,T2}) where {W,T1<:UnsignedHW,T2<:SignedHW}
    convert(T1, convert(T2, v1) << v2)
end
@inline function Base.:(>>)(v1::AbstractSIMDVector{W,T1}, v2::AbstractSIMDVector{W,T2}) where {W,T1<:SignedHW,T2<:UnsignedHW}
    convert(T1, convert(T2, v1) >> v2)
end
@inline function Base.:(>>)(v1::AbstractSIMDVector{W,T1}, v2::AbstractSIMDVector{W,T2}) where {W,T1<:UnsignedHW,T2<:SignedHW}
    convert(T1, convert(T2, v1) >> v2)
end
@inline function Base.:(>>>)(v1::AbstractSIMDVector{W,T1}, v2::AbstractSIMDVector{W,T2}) where {W,T1<:SignedHW,T2<:UnsignedHW}
    convert(T1, convert(T2, v1) >>> v2)
end
@inline function Base.:(>>>)(v1::AbstractSIMDVector{W,T1}, v2::AbstractSIMDVector{W,T2}) where {W,T1<:UnsignedHW,T2<:SignedHW}
    convert(T2, v1 >>> convert(T1, v2))
end
for op ∈ [:(<<), :(>>), :(>>>), :(&), :(|)]
    @eval begin
        @generated function Base.$op(v1::AbstractSIMDVector{W,T1}, v2::AbstractSIMDVector{W,T2}) where {W,T1<:IntegerTypes,T2<:IntegerTypes}
            promote_shift_quote($(QuoteNode(op)), T1, T2)
        end
    end
end


blockbandwidths(A::AdjOrTrans) = reverse(blockbandwidths(parent(A)))
subblockbandwidths(A::AdjOrTrans) = reverse(subblockbandwidths(parent(A)))


struct QUInt{N}
  bits::Bits{N}
end

QUInt{N}(v::Integer = 0) where N = QUInt{N}(Bits{N}(v))

const uint_types = [UInt8,UInt16,UInt32,UInt64]
@generated uint_type(::Type{QUInt{N}}) where N = uint_types[ceil(Int,N/8)]

bits(x::QUInt) = x.bits
repr(::Type{QUInt{N}}, bits) where N = uint_type(QUInt{N})(int(bits))
Base.show(io::IO, x::QUInt) = print(io, "QUInt{$(nbits(x.bits))}()")

for N in [4,8,16,32,64]
  QUIntN = Symbol(:QUInt,N)
  @eval export $QUIntN
  @eval const $QUIntN = QUInt{$N}
end


"""
    BlockSlice(indices)

Represent an AbstractUnitRange of indices that attaches a block.

Upon calling `to_indices()`, Blocks are converted to BlockSlice objects to represent
the indices over which the Block spans.

This mimics the relationship between `Colon` and `Base.Slice`.
"""
struct BlockSlice{BB} <: AbstractUnitRange{Int}
    block::BB
    indices::UnitRange{Int}
end

Block(bs::BlockSlice{<:Block}) = bs.block

for f in (:axes, :unsafe_indices, :axes1, :first, :last, :size, :length,
          :unsafe_length, :start)
    @eval $f(S::BlockSlice) = $f(S.indices)
end

getindex(S::BlockSlice, i::Integer) = getindex(S.indices, i)
show(io::IO, r::BlockSlice) = print(io, "BlockSlice(", r.block, ",", r.indices, ")")
next(S::BlockSlice, s) = next(S.indices, s)
done(S::BlockSlice, s) = done(S.indices, s)

function _unblock(cum_sizes, I::Tuple{Block{1, T},Vararg{Any}}) where {T}
    B = first(I)
    b = first(B.n)

    range = cum_sizes[b]:cum_sizes[b + 1] - 1

    BlockSlice(B, range)
end

function _unblock(cum_sizes, I::Tuple{BlockRange{1,R}, Vararg{Any}}) where {R}
    B = first(I)
    b_start = first(first(B.indices))
    b_stop = last(first(B.indices))

    range = cum_sizes[b_start]:cum_sizes[b_stop + 1] - 1

    BlockSlice(B, range)
end

_sub_cumul_sizes(cs, inds) = _sub_cumul_sizes(cs, inds[1], tail(inds))
_sub_cumul_sizes(::Tuple{}, ::Tuple{}) = ()

function _sub_cumul_sizes(cs, inds1::BlockSlice{Block{1,Int}}, inds)
    B = Int(inds1.block)
    ret = view(cs[1], B:B+1)
    (ret .- ret[1] .+ 1, _sub_cumul_sizes(tail(cs), inds)...)
end

function _sub_cumul_sizes(cs, inds1::BlockSlice{BlockRange{1,Tuple{UnitRange{Int}}}}, inds)
    ret = view(cs[1], inds1.block.indices[1][1]:(inds1.block.indices[1][end]+1))
    (ret .- ret[1] .+ 1, _sub_cumul_sizes(tail(cs), inds)...)
end

blocksizes(V::SubArray) = BlockSizes(_sub_cumul_sizes(cumulsizes(parent(V)), parentindices(V)))

"""
    unblock(block_sizes, inds, I)

Returns the indices associated with a block as a `BlockSlice`.
"""
function unblock(A::AbstractArray{T,N}, inds, I) where {T, N}
    if length(inds) == 0
        # Allow `ones(2)[Block(1)[1:1], Block(1)[1:1]]` which is
        # similar to `ones(2)[1:1, 1:1]`.
        _unblock(Base.OneTo(2), I)
    else
        _unblock(cumulsizes(A, N - length(inds) + 1), I)
    end
end

to_index(::Block) = throw(ArgumentError("Block must be converted by to_indices(...)"))
to_index(::BlockRange) = throw(ArgumentError("BlockRange must be converted by to_indices(...)"))

@inline to_indices(A, inds, I::Tuple{Block{1}, Vararg{Any}}) =
    (unblock(A, inds, I), to_indices(A, _maybetail(inds), tail(I))...)

# splat out higher dimensional blocks
# this mimics view of a CartesianIndex
@inline to_indices(A, inds, I::Tuple{Block, Vararg{Any}}) =
    to_indices(A, inds, (Block.(I[1].n)..., tail(I)...))

@inline to_indices(A, inds, I::Tuple{BlockRange{1,R}, Vararg{Any}}) where R =
    (unblock(A, inds, I), to_indices(A, _maybetail(inds), tail(I))...)

# splat out higher dimensional blocks
# this mimics view of a CartesianIndex
@inline to_indices(A, inds, I::Tuple{BlockRange, Vararg{Any}}) =
    to_indices(A, inds, (BlockRange.(tuple.(I[1].indices))..., tail(I)...))

# In 0.7, we need to override to_indices to avoid calling linearindices
@inline to_indices(A, I::Tuple{Block, Vararg{Any}}) =
    to_indices(A, axes(A), I)

@inline to_indices(A, I::Tuple{BlockRange, Vararg{Any}}) =
    to_indices(A, axes(A), I)

# The first argument for `reindex` is removed as of
# https://github.com/JuliaLang/julia/pull/30789 in Julia `Base`.  So,
# we define 2-arg `reindex` for Julia 1.2 and later.
if VERSION >= v"1.2-"

# BlockSlices map the blocks and the indices
# this is loosely based on Slice reindex in subarray.jl
reindex(idxs::Tuple{BlockSlice{<:BlockRange}, Vararg{Any}},
        subidxs::Tuple{BlockSlice{<:BlockRange}, Vararg{Any}}) =
    (@_propagate_inbounds_meta; (BlockSlice(BlockRange(idxs[1].block.indices[1][Int.(subidxs[1].block)]),
                                            idxs[1].indices[subidxs[1].indices]),
                                 reindex(tail(idxs), tail(subidxs))...))

reindex(idxs::Tuple{BlockSlice{BlockRange{1,Tuple{UnitRange{Int}}}}, Vararg{Any}},
        subidxs::Tuple{BlockSlice{Block{1,Int}}, Vararg{Any}}) =
    (@_propagate_inbounds_meta; (BlockSlice(Block(idxs[1].block.indices[1][Int(subidxs[1].block)]),
                                            idxs[1].indices[subidxs[1].indices]),
                                 reindex(tail(idxs), tail(subidxs))...))

function reindex(idxs::Tuple{BlockSlice{Block{1,Int}}, Vararg{Any}},
        subidxs::Tuple{BlockSlice{Block{1,Int}}, Vararg{Any}})
    (idxs[1], reindex(tail(idxs), tail(subidxs))...)
end

else  # if VERSION >= v"1.2-"

reindex(V, idxs::Tuple{BlockSlice{<:BlockRange}, Vararg{Any}},
        subidxs::Tuple{BlockSlice{<:BlockRange}, Vararg{Any}}) =
    (@_propagate_inbounds_meta; (BlockSlice(BlockRange(idxs[1].block.indices[1][Int.(subidxs[1].block)]),
                                            idxs[1].indices[subidxs[1].indices]),
                                    reindex(V, tail(idxs), tail(subidxs))...))

reindex(V, idxs::Tuple{BlockSlice{BlockRange{1,Tuple{UnitRange{Int}}}}, Vararg{Any}},
        subidxs::Tuple{BlockSlice{Block{1,Int}}, Vararg{Any}}) =
    (@_propagate_inbounds_meta; (BlockSlice(Block(idxs[1].block.indices[1][Int(subidxs[1].block)]),
                                            idxs[1].indices[subidxs[1].indices]),
                                    reindex(V, tail(idxs), tail(subidxs))...))

function reindex(V, idxs::Tuple{BlockSlice{Block{1,Int}}, Vararg{Any}},
        subidxs::Tuple{BlockSlice{Block{1,Int}}, Vararg{Any}})
    subidxs[1].block == Block(1) || throw(BoundsError(V, subidxs[1].block))
    (idxs[1], reindex(V, tail(idxs), tail(subidxs))...)
end

end  # if VERSION >= v"1.2-"

#################
# support for pointers
#################

const BlockOrRangeIndex = Union{RangeIndex, BlockSlice}

function unsafe_convert(::Type{Ptr{T}},
                        V::SubArray{T, N, BlockArray{T,N,AT,BS}, NTuple{N, BlockSlice{Block{1,Int}}}}) where {AT <: AbstractArray{<:AbstractArray{T,N},N}, BS <: AbstractBlockSizes{N}} where {T,N}
    unsafe_convert(Ptr{T}, parent(V).blocks[Int.(Block.(parentindices(V)))...])
end

unsafe_convert(::Type{Ptr{T}}, V::SubArray{T,N,PseudoBlockArray{T,N,AT},<:Tuple{Vararg{BlockOrRangeIndex}}}) where {T,N,AT} =
    unsafe_convert(Ptr{T}, V.parent) + (Base.first_index(V)-1)*sizeof(T)


using DistCtrl4DistMan: runExp
using Plots

DEP_params = Dict{String,Any}("maxDist" => (300e-6, 550e-6));
MAG_params = Dict{String,Any}("maxDist" => (50e-3, 75e-3));
ACU_params = Dict{String,Any}("maxDist" => (45e-3, 65e-3));

## DEP - two microparticles, 16x16 array of electrodes
runExp(platform=:DEP, N_iter=25, N_agnts=2, N_acts=(16, 16), convanalysis=false, saveplots=true, figfilename="DEParray_small.pdf", params=DEP_params);

## DEP - six microparticles, 16x16 array of electrodes
runExp(platform=:DEP, N_iter=25, N_agnts=6, N_acts=(16, 16),convanalysis=false, saveplots=true, figfilename="DEParray.pdf", params=DEP_params);

## MAG - five steel balls, 8x8 array of coils
runExp(platform=:MAG, N_iter=25, N_agnts=5, N_acts=(8, 8), convanalysis=false, saveplots=false, figfilename="MAGarray_8x8.pdf", plotActuatorCommands=true);

## MAG - five steel balls, 12x12 array of coils
runExp(platform=:MAG, N_iter=25, N_agnts=5, N_acts=(12, 12), convanalysis=false, saveplots=true, figfilename="MAGarray_12x12.pdf", plotActuatorCommands=true);

## ACU - six pressure points, 16x16 array of actuators
runExp(platform=:ACU, N_iter=25, N_agnts=6, N_acts=(16, 16),convanalysis=false, saveplots=true, figfilename="ACUarray.pdf");

## DEP - six microparticles, 16x16 array of electrodes, Algorithm 1 vs Algorithm 2
N_agnts=2;
N_iter=25;
N_acts = 16;

# run the experiment with Algorithm 2 (i.e. with different c_i and d_i sets)
params, exp_data_A2 = runExp(platform=:DEP, N_iter=N_iter, N_agnts=N_agnts, N_acts=(N_acts,N_acts),
                            convanalysis=true,
                            saveplots=false,
                            showplots=false,
                            params=DEP_params);

# Extract the randomly generated agents' positions and desdired DEP forces
agnts = exp_data_A2[1]["Agents"];
agent_positions = [agnts[i].pos for i in 1:length(agnts)];
Fdes = [agnts[i].Fdes .* agnts[i].Fdes_sc for i in 1:length(agnts)];

# run the experiment with Algorithm 1 (c_i = d_i)
DEP_params_A1 = copy(DEP_params);
DEP_params_A1["maxDist"] = (300e-6, 300e-6);
params_A1, exp_data_A1 = runExp(platform=:DEP, N_iter=N_iter, N_agnts=N_agnts, N_acts=(N_acts,N_acts),
                            convanalysis=true,
                            saveplots=false,
                            showplots=false,
                            agent_positions = agent_positions,
                            Fdes = Fdes,
                            params = DEP_params_A1);

# Compute the developed force by the phase shifts found by Alg. 2
Fdev_arr_A2 = Array{NTuple{params["force_dim"], Float64}, 1}();
controls_A2 = exp_data_A2[1]["ActuatorCommands"];
actuatorArray_A2 = exp_data_A2[1]["ActuatorArray"];
for agnt_pos in agent_positions
    Fdev = params["calcForce"](actuatorArray_A2, agnt_pos, controls_A2);
    push!(Fdev_arr_A2, Fdev);
end

# Compute the developed force by the phase shifts found by Alg. 1
Fdev_arr_A1 = Array{NTuple{params["force_dim"], Float64}, 1}();
controls_A1 = exp_data_A1[1]["ActuatorCommands"];
actuatorArray_A1 = exp_data_A1[1]["ActuatorArray"];
for agnt_pos in agent_positions
    Fdev2 = params["calcForce"](actuatorArray_A1, agnt_pos, controls_A1);
    push!(Fdev_arr_A1, Fdev2);
end

# Compare the developed forces in a plot
Plots.display(plot(actuatorArray_A2, agents=agnts,
                    dev_forces=Fdev_arr_A2,
                    aux_forces=Fdev_arr_A1,
                    showReqForces=true,
                    Fscale=params_A1["Fscale"]*1.5,
                    legend=nothing,
                    size=(500,500)))
##
savefig("DEP_Alg1_vs_Alg2.pdf")


import Pkg
Pkg.activate(joinpath(@__DIR__, "../../"))  # environment: CytofResearch

using BSON
using CSV
using CytofResearch
import DataFrames; const DF = DataFrames
using Distributions
import Random


module ModernHopfieldNetworks

include("functions.jl")
using LinearAlgebra

#
export

end


using SimulacoesPedro
using Test

@testset "SimulacoesPedro.jl" begin
    data = Data()
    p = initial_point(data)
    @test utotal(p,data) ≈ utotal2(p,data)
end


module Variational

import Base: start, done, next

export transposecalls!

# aux methods to hide the separation in head and args
numchds(xp) = length(xp.args) + 1
child(xp, idx) = idx == 1 ? xp.head : xp.args[idx-1]

type DepthFirst
    xp::Expr
end

"""
    depthfirst(xp)

Returns an iterator that visits all nodes in an Expr depth first. head and
args are visited before the Expr they define.
"""
depthfirst(xp) = DepthFirst(xp)

type DepthFirstState
    val
    par
    idx
end

function start(itr::DepthFirst)
    head = DepthFirstState(itr.xp, nothing, -1)
    return DepthFirstState(itr.xp, head, 0)
end

done(itr::DepthFirst, state::DepthFirstState) = (state.par == nothing)

function next(itr::DepthFirst, state::DepthFirstState)

    # if all children processed, move up on level
    if state.idx == numchds(state.val)
        return state.val, state.par
    end

    # move to next child
    state = DepthFirstState(state.val, state.par, state.idx+1)

    # if the next child is a leaf, return it
    chd = child(state.val, state.idx)
    if !isa(chd, Expr)
        return chd, state
    end

    # the next child is an expression; descend into it
    # and find the next valid state. There will always
    # be at least one, pointing to the child itself.
    state = DepthFirstState(chd, state, 0)
    return next(itr, state)
end

"""
    transposecall!(xp, skip=[])

Goes through the syntax tree and replace all function calls
`f(x,p1,p2,...)` with `x(f,p1,p2,...)`.
"""
function transposecalls!(xp, skip=[])
    isa(xp, Expr) || return xp
    for x in depthfirst(xp)
        if isa(x, Expr) && x.head == :call && !(x.args[1] in skip)
            @assert length(x.args) >= 2
            tmp = x.args[1]
            x.args[1] = x.args[2]
            x.args[2] = tmp
        end
    end
    return xp
end

import Base.==

export hilbertspace, @hilbertspace
export @varform, Equation
export LinForm, LinTerm, BilForm, BilTerm

import Base: +, -, *, dot, getindex, ^, print

type HilbertVector
    idx
    space
    opstack
end

type LinForm
  test_space
  terms
end

type LinTerm
  test_id
  test_ops
  coeff
  functional
end

type BilForm
  test_space
  trial_space
  terms
end

type BilTerm
  test_id
  trial_id
  test_ops
  trial_ops
  coeff
  kernel
end

type Equation
    lhs
    rhs
end

"""
    ==(lhs::BilForm, rhs::LinForm)

Build an equation from a left hand and right hand side
"""
==(lhs::BilForm, rhs::LinForm) = Equation(lhs, rhs)

"""
    hilbert_space(type, g1, g2, ...)

Returns generators defining a Hilbert space of field `type`
"""
hilbertspace(vars::Symbol...) = [HilbertVector(i, [vars...], []) for i in 1:length(vars)]

macro hilbertspace(syms...)

    for sym in syms
        @assert isa(sym, Symbol) "@hilbertspace takes a list of Symbols"
    end

    rhs = :(hilbertspace())
    for sym in syms
        push!(rhs.args, QuoteNode(sym))
    end

    vars = gensym()
    xp = quote
        $vars = $rhs
    end
    for (i,s) in enumerate(syms)
        push!(xp.args, :(global $s = $vars[$i]))
    end

    xp
end

"""
    call(u::HilbertVector, f, params...)
    u(f, params...)

Add another operation to the opstack of `u`.
"""
(u::HilbertVector)(f, params...) = HilbertVector(u.idx, u.space, [(f, params...); u.opstack])

"""
    getindex(f, v::HilbertVector)
    f[v]

Return a LinForm corresponding to f[v]
"""
getindex(f, v::HilbertVector) = LinForm(v.space, [LinTerm(v.idx, v.opstack, 1, f)])

"""
    getindex(A, v::HilbertVector, u::HilbertVector)

Create a BilForm corresponding to A[v,u]
"""
function getindex(A, v::HilbertVector, u::HilbertVector)
    terms = [ BilTerm(v.idx, u.idx, v.opstack, u.opstack, 1, A) ]
    BilForm(v.space, u.space, terms)
end

"Add two BilForms together"
function +(a::BilForm, b::BilForm)
    @assert a.test_space == b.test_space
    @assert a.trial_space == b.trial_space
    BilForm(a.test_space, a.trial_space, [a.terms; b.terms])
end

function+(a::LinForm, b::LinForm)
    @assert a.test_space == b.test_space
    LinForm(a.test_space, [a.terms; b.terms])
end

function *(α::Number, a::BilForm)
  b = deepcopy(a)
  for t in b.terms t.coeff *= α end
  return b
end

function *(α::Number, a::LinForm)
    b = deepcopy(a)
    for t in b.terms t.coeff *= α end
    return b
end

-(a::BilForm) = (-1 * a)
-(a::BilForm, b::BilForm) = a + (-b)

-(a::LinForm) = (-1 * a)
-(a::LinForm, b::LinForm) = a + (-b)

function print(io::IO, v::HilbertVector)
    sym = v.space[v.idx]
    ops = v.opstack
    for op in ops
        print(io, op[1], "(")
    end
    print(io, sym)
    for op in reverse(ops)
        for p in op[2:end]
            print(io, ", ", p)
        end
        print(io, ")")
    end
end

function print(io::IO, a::LinForm)
    N = length(a.terms)
    #T = typeof(a.terms[1].coeff)
    for (n,t) in enumerate(a.terms)
      u = HilbertVector(t.test_id, a.test_space, t.test_ops)
      t.coeff != 1 && print(io, t.coeff, "*")
      print(io, t.functional, "[", u, "]")
      n == N || print(io, " + ")
  end
end

function print(io::IO, f::BilForm)
    N = length(f.terms)
    #T = typeof(f.terms[1].coeff)

    for (n,t) in enumerate(f.terms)
        u = HilbertVector(t.test_id, f.test_space, t.test_ops)
        v = HilbertVector(t.trial_id, f.trial_space, t.trial_ops)
        t.coeff != 1 && print(io, t.coeff, "*")
        print(io, t.kernel, "[", u, ", ", v, "]")
        n == N || print(io, " + ")
    end
end

function print(io::IO, eq::Equation)
    print(io, eq.lhs)
    print(io, " == ")
    print(io, eq.rhs)
end

"""
    @varform <form-definition>

The Julia form compiler uses the Julia parser and meta-programming
based traversal of the AST to create a structure containing all
information required for the description of a variational formulation
from an Expr that follows closely widespread mathematical convention.

E.g:

    EFIE = @varform T[k,j] = e[k]
    MFIE = @varform 0.5*I[k,j] + K[k,j] = h[k]
    PMCH = @varform M[k,j] - η*T[k,m] + 1/η*T[l,j] + M[l,m] = e[k] + h[l]
"""
macro varform(x)
    y = transposecalls!(x, [:+, :-, :*, :(==)])
    esc(y)
end

end


using Printf, GMT, JLD2

west = -122
east = -119.5
south = 35
north = 37

basemapregion ="$west/$south/$east/$(north)r"
frame=(axes=:NSEW, annot=1, ticks="30m",grid="15m")
center = [-120.0 36.0]
#proj = (name=:Cassini, center=center)
proj = (name=:Mercator)
Ispace = "0.1m"
color="./GMT_relief.cpt"

figsize = 14

#make large xyz file
#grdlist = ["grdn36w120", "grdn36w121"]
grdlist = ["grdn36w120", "grdn36w121", "grdn36w122", "grdn37w120", "grdn37w121", "grdn37w122"]
#
# for gg = grdlist
#    println(gg)
#    #convert it to grd file
#    fi = "./Arcgrid/xyzfiles/$gg"
#    G1 = xyz2grd(fi, region=basemapregion, proj=proj, I=Ispace)
#
#    #---process grd data---#
#    #remove inf in z
#    replace!(G1.z, -Inf=>NaN)
#    #replace!(G1.z, 0=>-2000) this cause bug
#
#    #======================#
#
#    #save into grd file
#    gmtwrite("./$gg.grd", G1)
# end

#
# for gg = grdlist
#
#    #convert it to grd file
#    fi = "./$gg.grd"
#    G1 = gmtread(fi, grd=true)
#
#    #save into grd file
#    gmtwrite("./$(gg)_lim.grd", G1)
# end

# plot

icount = 0
for gg = grdlist
   global icount += 1
   fi = "./$(gg).grd"
   println(fi)
   if icount == 1
      grdimage(fi, region=basemapregion, proj=proj, frame=frame,figsize=figsize,  color=color, coast=true, shade=(auto=true,), nan_alpha=true)
   elseif icount == length(grdlist)
      grdimage!(fi, region=basemapregion, proj=proj, frame=frame,figsize=figsize,  color=color, coast=true, shade=(auto=true,), nan_alpha=true)
   else
      grdimage!(fi, region=basemapregion, proj=proj, frame=frame,figsize=figsize,  color=color, coast=true, shade=(auto=true,), nan_alpha=true)
   end
end

# plot fault
t = jldopen("sanandreasfault.jld2")
fault = t["fault"]

for i = 1:length(fault)
#for i = 1:10

   f1 = fault[i]

   x = Float64[]
   y = Float64[]
   for j = 1:length(f1)
      push!(x, f1[j][1])
      push!(y, f1[j][2])
   end

   # select faults
   if  all(x -> x >= west, x) && all(x -> x <= east, x) && all(x -> x >= south, y) && all(x -> x <= north, y)
      println(x, y)
      xy = mat2ds(y, x=x)
      lines!(xy, pen=(1.5,:red)) # you can ignore error with this line
   end
end

#colorbar!(pos=(width=10, justify=:TC, outside=true, offset=(-6, 11), horizontal=true),
#   frame=(annot=1000, ticks=500),  truncate=(0, 3000), cmap=color, show=1)

# plot stations
station = jldopen("stationcoords.jld2")

# NC
NC = station["NC"]

for i = 1:length(NC)
   GMT.plot!([NC[i][1] NC[i][2]], marker="d", markersize=0.35, pen=(0.6),
   markeredgecolor=:black, markerfacecolor="135/206/250")
end

# BP
BP = station["BP"]

for i = 1:length(BP)
   GMT.plot!([BP[i][1] BP[i][2]], marker="v", markersize=0.35, pen=(0.8),
   markeredgecolor=:black, markerfacecolor="251/224/0")
end

#OFFICIAL WEB SITE SAYS INCOLECT VARIABLE NOM: scale_lat -> scale_at_lat
mapscale=(map=true, anchor=[-121.5, 35.2], scale_at_lat=36,  length=50, fancy=true, label="[km]")
p = coast!(region=basemapregion, shore=(pen=(0.8,:black)), L=mapscale,
 frame=frame, S="0/141/203", stamp="", show=true)


function VectorizationBase.vectorizable(ScA::ChunkedArray{E,M,T,N,Np2}) where {E,M,T,N,Np2}
    VectorizedChunkedArray{E,M,T,N,Np2}(pointer(ScA.data), size(ScA.data))
end

const ChunkedVector{E,M,T} = ChunkedArray{E,M,T,1,2}
const ChunkedMatrix{E,M,T} = ChunkedArray{E,M,T,2,3}

function chunked_ind_base_quote(T)
    W, Wshift = VectorizationBase.pick_vector_width_shift(T)
    quote
        $(Expr(:meta, :inline))
        i1 = 1+((i[1]-1) & $(W-1))
        i2 = 1+((i[1]-1) >> $Wshift)
    end
end

function CcA_setindex_quote(N, L, T)
    q = chunked_ind_base_quote(T)
    inds = [:(i[$j]) for j ∈ 2:N]
    for j ∈ 1:L
        push!(q.args, Expr(:call, :setindex!, :(ScA.data), :(v[LinearStorage(), $j]), :i1, inds..., j,  :i2))
    end
    q
end

@generated function Base.setindex!(ScA::ChunkedArray{E,M,T,N,Np2}, v::T,
                                   # i::Vararg{<:Integer,N}) where {T,E,M,N,Np2}
                                   i::Vararg{<:Integer,N}) where {E,M,T,N,Np2}
    CcA_setindex_quote(N, type_length(T), E)
end

@generated function Base.setindex!(ScA::ChunkedArray{E,M,T,N,Np2}, v::T,
                                    # i::CartesianIndex{N}) where {T,E,M,N,Np2}
                                    i::MultiDimIndex{N}) where {E,M,T,N,Np2}
    CcA_setindex_quote(N, type_length(T), E)
end

@generated function Base.getindex(ScA::ChunkedArray{E,M,T,1,3}, i::Integer, j::Integer) where {E,M,T}
    q = chunked_ind_base_quote(E)
    # inds = [:(i[$n]) for n in 2:N]
    ind_expr = [Expr(:call, :getindex, :(ScA.data), :i1, k, :i2) for k ∈ 1:type_length(T)]
    push!(q.args, construct_expr(T, ind_expr))
    q
end
@generated function Base.getindex(ScA::ChunkedArray{E,M,T,N,Np2}, i::Vararg{<:Integer,N}) where {E,M,T,N,Np2}
    q = chunked_ind_base_quote(E)
    inds = [:(i[$n]) for n in 2:N]
    ind_expr = [Expr(:call, :getindex, :(ScA.data), :i1, inds..., j, :i2) for j ∈ 1:type_length(T)]
    push!(q.args, construct_expr(T, ind_expr))
    q
end

@generated function Base.getindex(ScA::ChunkedArray{E,M,T,N,Np2},
                        i::MultiDimIndex{N}) where {E,M,T,N,Np2}
    q = chunked_ind_base_quote(E)
    inds = [:(i[$n]) for n in 2:N]
    ind_expr = [Expr(:call, :getindex, :(ScA.data), :i1, inds..., j, :i2) for j ∈ 1:type_length(T)]
    push!(q.args, construct_expr(T, ind_expr))
    q
end

@noinline function calc_stride(sv::Core.SimpleVector)
    N = length(sv)
    X = Vector{Int}(undef,N)
    X[1] = 1
    for n in 2:N
        X[n] = X[n-1] * (sv[n])::Int
    end
    X
end

@noinline function chunked_array_view_inds(V, N, T)
    W, Wshift = VectorizationBase.pick_vector_width_shift(T)
    inds = Vector{Expr}(undef, N)
    if V.parameters[1] == Colon
        ind1 = :(i[1])
    else
        ind1 = :(ScA.view[1])
    end
    inds[1] = :( 1 + (($ind1-1) & $(W-1)) )
    i = 1
    for n ∈ 2:N
        if V.parameters[n] == Colon
            i += 1
            inds[n] = :(i[$i])
        else
            inds[n] = :(ScA.view[$n])
        end
    end
    inds, :(1 + (($ind1-1) >> $Wshift))
end

@generated function Base.getindex(ScA::ChunkedArrayView{E,M,T,new_N,N,Np2,V},
                        i::Vararg{<:Any,new_N}) where {E,M,T,new_N,N,Np2,V}
    inds, indslast = chunked_array_view_inds(V, N, E)
    # The strategy is to calculate the first index, as well as the stride between indices.
    # After the first index is calculated, the remaining indices can be calcualted much more
    # efficiently via the stide.
    q = quote
        $(Expr(:meta, :inline))
        stride = $(Expr(:call, :*, [:(ScA.full_size[$n]) for n ∈ 1:N]...))
        ind_1 = sub2ind(ScA.full_size, $(Expr(:tuple, inds..., 1, indslast)))
    end
    for j ∈ 2:type_length(T)
        push!(q.args, :($(Symbol(:ind_,j)) = ind_1 + stride * $(j-1)))
    end
    push!(q.args, construct_expr(T, [:(unsafe_load(ScA.ptr, $(Symbol(:ind_,j)))) for j ∈ 1:type_length(T)]))
    q
end
@generated function Base.setindex!(ScA::ChunkedArrayView{E,M,T,new_N,N,Np2,V}, v,
                        i::Vararg{<:Any,new_N}) where {E,M,T,new_N,N,Np2,V}
    inds, indslast = chunked_array_view_inds(V, N, E)
    q = quote
        $(Expr(:meta, :inline))
        stride = $(Expr(:call, :*, [:(ScA.full_size[$n]) for n ∈ 1:N]...))
        ind_1 = sub2ind(ScA.full_size, $(Expr(:tuple, inds..., 1, indslast)))
        unsafe_store!(ScA.ptr, v[LinearStorage(), 1], ind_1)
    end
    for j ∈ 2:type_length(T)
        push!(q.args, quote
            unsafe_store!(ScA.ptr, v[LinearStorage(), $j], ind_1 + stride * $(j-1))
        end)
    end
    push!(q.args, :(v))
    q
end
@generated function Base.setindex!(
                    ScA::ChunkedArrayView{E,M,T,new_N,N,Np2,V}, v,
                    i::Vararg{<:Any,new_N}
                ) where {E,M,S,T<:PaddedMatrices.AbstractConstantFixedSizeArray{S,E,M},new_N,N,Np2,V}
    inds, indslast = chunked_array_view_inds(V, N, E)
    q = quote
        $(Expr(:meta, :inline))
        stride = $(Expr(:call, :*, [:(ScA.full_size[$n]) for n ∈ 1:N]...))
        ind_1 = sub2ind(ScA.full_size, $(Expr(:tuple, inds..., 1, indslast)))
        # unsafe_store!(ScA.ptr, v[1], ind_1)
    end
    sv = S.parameters
    for (j,ci) ∈ enumerate(ntuple(i -> 1:sv[i], Val(M)))
        push!(q.args, quote
            unsafe_store!(ScA.ptr, v[$ci], ind_1 + stride * $(j-1))
        end)
    end
    push!(q.args, :(v))
    q
end
@inline function Base.view(ScA::ChunkedArray{E,M,T,N,Np2}, i::Vararg{<:Any,N}) where {E,M,T,N,Np2}
    ChunkedArrayView(ScA, i)
end
@generated function ChunkedArrayView(ScA::ChunkedArray{E,M,T,N,Np2,U}, i::V) where {E,M,T,N,Np2,V,U}
    new_N = 0
    new_size = Expr(:tuple)
    for n ∈ 1:N
        if V.parameters[n] == Colon
            new_N += 1
            push!(new_size.args, :(full_size[$n]))
        end
    end
    quote
        full_size = size(ScA.data)
        ChunkedArrayView{$E,$M,$T,$new_N,$N,$Np2,$V,$U}(pointer(ScA.data), $new_size, full_size, i, ScA.mask)
    end
end

# @inline function SArray{Tuple{S},T,N,L}(vs::Vararg{SIMDPirates.SVec{W,T},L}) where {S,T,N,L,W}
#     SArray{S}(vs...)
# end
@inline function Base.:+(i::Integer, v::VectorizedChunkedArray{E,M,T,N,Np2}) where {E,M,T,N,Np2}
    VectorizedChunkedArray{E,M,T,N,Np2}(v.ptr + sizeof(T) * i, v.fullsize)
end
@inline function Base.:+(v::VectorizedChunkedArray{E,M,T,N,Np2}, i::Integer) where {E,M,T,N,Np2}
    VectorizedChunkedArray{E,M,T,N,Np2}(v.ptr + sizeof(T) * i, v.fullsize)
end
@generated function SIMDPirates.vload(::Type{V}, vScA::VectorizedChunkedArray{E,M,T,N,Np2}) where {E,M,T,N,Np2,W, V <: Union{SIMDPirates.SVec{W,E},SIMDPirates.Vec{W,E}}}
    W_full, Wshift_full = VectorizationBase.pick_vector_width_shift(E)
    quote
        $(Expr(:meta,:inline))
        $(construct_expr(T, [Expr(:call, :vload, V, :(vScA.ptr + $(W_full*sizeof(E)*j))) for j ∈ 0:type_length(T)-1]))
    end
end
@generated function SIMDPirates.vload(
                    ::Type{V}, vScA::VectorizedChunkedArray{E,M,T,N,Np2}
                ) where {E,M,S,T<:PaddedMatrices.AbstractFixedSizeArray{S},N,Np2,W, V <: Union{SIMDPirates.SVec{W,E},SIMDPirates.Vec{W,E}}}
    W_full, Wshift_full = VectorizationBase.pick_vector_width_shift(E)
    @assert W == W_full
    sv = S.parameters
    R = calc_stride(sv)
    :(PaddedMatrices.PtrArray{$S,Vec{$W_full,$E},$(length(sv)),$R,$(PaddedMatrices.simple_vec_prod(sv)),true}(vScA.ptr))
end
@generated function vload!(
                A::PaddedMatrices.AbstractMutableFixedSizeArray{S,NTuple{W,Core.VecElement{E}},N3,P2,L2},
                vScA::VectorizedChunkedArray{E,M,ConstantFixedSizeArray{S,E,N3,P1,L1},N,Np2}
            ) where {E,M,N,Np2,W,S,P1,L1,P2,L2,N3}
    W_full, Wshift_full = VectorizationBase.pick_vector_width_shift(E)
    quote
        $(Expr(:meta,:inline))
        # $(construct_expr(T, [Expr(:call, :vload, V, :(vScA.ptr + $(W_full*sizeof(E)*j))) for j ∈ 0:type_length(T)-1]))
        ptrvScA = vScA.ptr
        ptrA = pointer(A)
        @inbounds for l ∈ 0:$(L2-1)
            SIMDPirates.vstore!(
                ptrA + $(W_full*sizeof(E))*l,
                SIMDPirates.vload(NTuple{$W,Core.VecElement{$E}}, ptrvScA + $(W_full*sizeof(E))*l)
            )
        end
        nothing
    end
end

function chunked_vload_quote(N, W, E, T, V)
    W_full, Wshift_full = VectorizationBase.pick_vector_width_shift(E)
    inds = [:(i[$n]) for n in 2:N]
    quote
        $(Expr(:meta,:inline))
        $(construct_expr(T, [Expr(:call, :vload, V, :(vScA.ptr),
            Expr(:call, :sub2ind, :(vScA.fullsize), Expr(:tuple, 1, inds..., j, 1 + :( (i[1]-1) >> $Wshift_full ) ))) for j in 1:type_length(T)]))
    end
end

@generated function SIMDPirates.vload(::Type{V}, vScA::VectorizedChunkedArray{E,M,T,N,Np2},
                                                # i::CartesianIndex{N}) where {E,M,T,N,Np2,W}
                                                i::Vararg{<:Integer,N}) where {E,M,T,N,Np2,W, V <: Union{SIMDPirates.SVec{W,E},SIMDPirates.Vec{W,E}}}
    chunked_vload_quote(N, W, E, T, V)
end
@generated function SIMDPirates.vload(::Type{V}, vScA::VectorizedChunkedArray{E,M,T,N,Np2},
                                                # i::Vararg{<:Integer,N}) where {E,M,T,N,Np2,W}
                                                i::MultiDimIndex{N}) where {E,M,T,N,Np2,W, V <: Union{SIMDPirates.SVec{W,E},SIMDPirates.Vec{W,E}}}
    chunked_vload_quote(N, W, E, T, V)
end
"""
For vload(::Type{<:SVec}, ::VectorizedChunkedArray), we assume the loads are alligned with chunk boundaries.
"""
@generated function SIMDPirates.vload(::Type{V}, vScA::VectorizedChunkedArray{E,M,T,N,Np2},
                                                # i::Vararg{<:Integer,N}) where {E,M,T,N,Np2,W}
                                                i::Integer) where {E,M,T,N,Np2,W, V <: Union{SIMDPirates.SVec{W,E},SIMDPirates.Vec{W,E}}}
    W_full, Wshift_full = VectorizationBase.pick_vector_width_shift(E)
    WE = W_full * sizeof(E)
    TL = type_length(T)
    quote
        $(Expr(:meta,:inline))
        ind = $(TL*sizeof(E)) * (i-1)
        $(construct_expr(T, [Expr(:call, :vload, V, :(vScA.ptr + ind + $(WE*j))) for j in 0:TL-1]))
    end
end

function Base.copyto!(ScA::ChunkedArray{E,M,T,N,Np2}, A::AbstractArray{T}) where {E,M,T,N,Np2}
    @boundscheck size(ScA) == size(A) || throwdimensionmismatcherror(size(ScA), size(A))
    @inbounds for i in eachindex(A)
        ScA[i] = A[i]
    end
    ScA
end

@generated function ChunkedArray(A::AbstractArray{T,N}) where {E,M,T <: AbstractArray{E,M},N}
    isbitstype(T) || thrownotisbitserror(T)
    TL = type_length(T)
    W, Wshift = VectorizationBase.pick_vector_width_shift(E)
    U = VectorizationBase.mask_type(T)
    quote
        L = size(A,1)
        mask = VectorizationBase.mask_from_remainder($T, L & $(W-1))
        ScA = ChunkedArray{$E,$M,$T,$N,$(N+2),$U}(
            Array{$E}(undef, $(Expr(:tuple, W, [:(size(A,$n)) for n ∈ 2:N]..., TL, :((L+$(W-1)) >> $Wshift) )) ),
            size(A), mask
        )
        @inbounds copyto!(ScA, A)
    end
end
@generated function ChunkedArray(A::AbstractArray{T,N}) where {E,M,S,T <: PaddedMatrices.AbstractFixedSizeArray{S,E,M},N}
    isbitstype(T) || thrownotisbitserror(T)
    TL = type_length(T)
    W, Wshift = VectorizationBase.pick_vector_width_shift(E)
    U = VectorizationBase.mask_type(T)
    sv = S.parameters
    X = calc_stride(sv)
    quote
        L = size(A,1)
        mask = VectorizationBase.mask_from_remainder($T, L & $(W-1))
        ScA = ChunkedArray{$E,$M,PaddedMatrices.ConstantFixedSizeArray{$S,$E,$M,$(Tuple{X...}),$(PaddedMatrices.simple_vec_prod(sv))},$N,$(N+2),$U}(
            Array{$E}(undef, $(Expr(:tuple, W, [:(size(A,$n)) for n ∈ 2:N]..., TL, :((L+$(W-1)) >> $Wshift) )) ),
            size(A), mask
        )
        @inbounds copyto!(ScA, A)
    end
end
@generated function ChunkedArray{E,M,T,N,Np2}(::UndefInitializer, I::Vararg{<:Integer,N}) where {E,M,T <: AbstractArray{E,M},N,Np2}
    isbitstype(T) || thrownotisbitserror(T)
    N + 2 == Np2 || throw("N + 1 = $(N + 1) != Np2 = $Np2")
    # @show T
    TL = type_length(T)
    W, Wshift = VectorizationBase.pick_vector_width_shift(E)
    U = VectorizationBase.mask_type(T)
    quote
        L = I[1]
        mask = VectorizationBase.mask_from_remainder($T, L & $(W-1))
        ChunkedArray{$E,$M,$T,$N,$Np2,$U}(
            Array{$E}(undef, $(Expr(:tuple, W, [:(I[$n]) for n ∈ 2:N]..., TL, :(I[1] >> $Wshift))) ),
            I, mask
        )
    end
end

# @generated function Base.size(ScA::ChunkedArray{E,M,T,N,Np2}) where {E,M,T,N,Np2}
#     quote
#         $(Expr(:meta, :inline))
#         s = size(ScA.data)
#         @inbounds $(Expr(:tuple, :(s[1]*s[end]), [:(s[$n]) for n ∈ 2:N]...))
#     end
# end
@inline Base.size(ScA::ChunkedArray) = ScA.size
@inline function Base.size(ScA::ChunkedArray{E,M,T,N}, n::Integer) where {E,M,T,N}
    @boundscheck n > N && ThrowBoundsError("$n > N, the number of dimensions.")
    @inbounds ScA.size[n]
end
# @inline Base.size(ScA::ChunkedArray{E,M,T,N,Np2}, n::Integer) where {E,M,T,N,Np2} = n == 1 ? size(ScA.data, 1)*size(ScA.data, Np2) : size(ScA.data, n)
@inline function Base.length(ScA::ChunkedArray{E,M,T,N,Np2}) where {E,M,T,N,Np2}
    s = ScA.size
    @inbounds l = s[1]
    @inbounds for n ∈ 2:N
        l *= s[n]
    end
    l
end


"""
	grdpaste(cmd0::String="", arg1=nothing, arg2=nothing, kwargs...)

Combine grids ``grid1`` and ``grid2`` into ``grid3`` by pasting them together along their common edge.
Both grids must have the same dx, dy and have one edge in common.

Full option list at [`grdpaste`]($(GMTdoc)grdpaste.html)

Parameters
----------

- **G** | **outgrid** :: [Type => Str]

    Output grid file name. Note that this is optional and to be used only when saving
    the result directly on disk. Otherwise, just use the G = grdpaste(....) form.
    ($(GMTdoc)grdpaste.html#g)
- $(GMT.opt_V)
- $(GMT.opt_f)
"""
function grdpaste(cmd0::String="", arg1=nothing, arg2=nothing; kwargs...)

	d = KW(kwargs)
	cmd = parse_common_opts(d, "", [:V_params :f :params])
	cmd = add_opt(cmd, 'G', d, [:G :outgrid])

	cmd, got_fname, arg1, arg2 = find_data(d, cmd0, cmd, arg1, arg2)
	return common_grd(d, "grdpaste " * cmd, arg1, arg2)		# Finish build cmd and run it
end

# ---------------------------------------------------------------------------------------------------
grdpaste(arg1, arg2=nothing, cmd0::String=""; kw...) = grdpaste(cmd0, arg1, arg2; kw...)


#!/Applications/Julia-0.3.9.app/Contents/Resources/julia/bin/julia

using Color

function compare_colors(color_a, color_b)
    # compare two colors, looking just at their LUV luminance values
    rx, gx, bx = color_a
    ry, gy, by = color_b
    hslx = convert(LUV, RGB(rx, gx, bx))
    hsly = convert(LUV, RGB(ry, gy, by))
    hslx.l > hsly.l
end

function draw_swatch(colorname, x, y, swatchwidth, swatchheight, color_values)
    r, g, b = color_values
    toplabel = """<text x="$(x)" y="$(y - 3)" fill="black" font-size="5px" font-family="Helvetica-Bold">$colorname</text>\n"""
    bottomlabel = """<text x="$(x)" y="$(y + swatchheight + 3)" font-size="3px" font-family="Helvetica">($r, $g, $b)</text>\n"""
    return  toplabel *
            bottomlabel *
            """<rect rx="2" ry="2" x="$(x)" y="$(y)" width="$(swatchwidth)" height="$(swatchheight)" fill="rgb($r, $g, $b)" /> \n """
end

function draw_synonym(colorname, x, y, swatchwidth, swatchheight)
    # add a synonym below the swatch
    return """<text x="$(x)" y="$(y + swatchheight + 3)" fill="black" font-size="3px" font-family="Helvetica-Bold">$colorname</text>\n"""
end

function make_color_table(image_width, image_height, output_file)
    # prepare synonym lists
    synonyms = Dict{Tuple, Array}()
    for color in Color.color_names
        col_name  = color[1]
        col_value = color[2]
        if haskey(synonyms, col_value)
            push!(synonyms[col_value], col_name)
        else
            synonyms[col_value] = [col_name]
        end
    end

    # calculate sizes of swatches
    aspect = image_width/image_height
    if aspect > 1
        n_rows = ceil(sqrt(length(synonyms)/aspect))
    else
        n_rows = ceil(sqrt(length(synonyms) * aspect))
    end
    n_cols = ceil(length(synonyms)/n_rows)

    margin = 30
    row_space = 25
    col_space = 15

    swatchwidth  = ((image_width  - margin - margin - (col_space * n_cols )) / n_cols)
    swatchheight = ((image_height - margin - margin - (row_space * n_rows )) / n_rows)

    # send output to file
    f = open(output_file, "w")

    # write svg header
    println(f, """<svg xmlns="http://www.w3.org/2000/svg" version="1.1" width="$(image_width)" height="$(image_height)"> \n
    <rect x="0" y="0" width="$(image_width)" height="$(image_height)" fill="white" /> \n """)

    # title
    println(f, """<text x="$(margin)" y="$(margin - margin/2)" font-size="14px" font-family="Helvetica-Bold">Color names in Color.jl, sorted by luminance</text>\n""")

    # x and y track position
    x = margin
    y = margin

    for color_values in sort(collect(keys(synonyms)), lt = (x,y) -> compare_colors(x,y))
        color_synonyms = sort(synonyms[color_values])
        print(f, draw_swatch(color_synonyms[1], x, y, swatchwidth, swatchheight, color_values))
        oldy = y # save the current position during excursion
        for synonym in color_synonyms[2:end]
            # draw the other names for this color
            y += 3
            print(f, draw_synonym(string("also: " * synonym), x, y, swatchwidth, swatchheight))
        end
        y = oldy
        if x > (image_width - swatchwidth - margin - margin)
            x = margin # next row
            y += swatchheight + row_space # gap between rows
        else
            x += swatchwidth + col_space  # gap between columns
        end
    end
    println(f, "</svg>")
    close(f)
end

make_color_table(1400, 1400, "/tmp/color_names_sorted.svg")


"""
	potree2trie(potree::String) -> trie.children

Trie data structures for Potree hierarchy:
 - each file in octree directory is stored in a node,
 - first node is `r`, root of potree.
"""
function potree2trie(potree::String)
	metadata = CloudMetadata(potree) # metadata of potree
	tree = joinpath(potree,metadata.octreeDir,"r") # path to octree files
	trie = DataStructures.Trie{String}()

	# search all files, si potrebbe fare con un'unica function
	if ( metadata.pointAttributes[1] == "LAS" )
		files = searchfile(tree,".las")
	elseif ( metadata.pointAttributes[1] == "LAZ" )
		files = searchfile(tree,".laz")
	elseif ( metadata.pointAttributes[1] != "LAZ" )	# modo poco elegante di individuare != las/laz
		files = searchfile(tree,".bin")
	else
		throw(DomainError(metadata.pointAttributes,"Format not yet allowed"))
	end

	# build trie from filename
	for file in files
		name = rsplit(splitdir(file)[2],".")[1]
		trie[name] = file
	end

	# root of potree
	return trie.children['r']
end

"""
	max_depth(trie::DataStructures.Trie{String}) -> Int

Maximum depth of trie.
"""
function max_depth(trie::DataStructures.Trie{String})::Int
	if length(trie.children) == 0
		return 0
	else
		depth = Int[]
		for key in collect(keys(trie.children))
			push!(depth,max_depth(trie.children[key]))
		end
		return max(depth...) + 1
	end
end

"""
	cut_trie!(trie::DataStructures.Trie{String}, LOD::Int, l = 0::Int)

Return truncated trie.
"""
function cut_trie!(trie::DataStructures.Trie{String}, LOD::Int, l = 0::Int)
	if l >= LOD
		empty!(trie.children) #delete all children
	end
	for key in collect(keys(trie.children))
		cut_trie!(trie.children[key], LOD, l+1) # next level
	end
end

"""
	get_all_values(trie::DataStructures.Trie{String}) -> Array{String,1}

Return a collection of all values in trie.
"""
function get_all_values(trie::DataStructures.Trie{String})::Array{String,1}
	key_s = collect(keys(trie))
	data = Array{String,1}(undef,length(key_s))
	for i in 1:length(key_s)
		data[i] = trie[key_s[i]]
	end
	return data
end

"""
	sub_trie(t::DataStructures.Trie{String}, root::AbstractString) -> DataStructures.Trie

Return the subtrie with defined `root` node.
"""
function sub_trie(t::DataStructures.Trie{String}, root::AbstractString)::DataStructures.Trie
	return subtrie(t, root[2:end])
end

# =====================
"""
get_files_in_potree_folder(potree::String, LOD::Int, all_prev=true::Bool) -> Array{String,1}

Return all files at that level of potree
"""
function get_files_in_potree_folder(potree::String, LOD::Int, all_prev=true::Bool)::Array{String,1}
	trie = potree2trie(potree)
	return get_files(trie, LOD, String[], 0, all_prev)
end

"""
get_files(trie::DataStructures.Trie{String}, LOD::Int, data=String[]::Array{String,1}, l = 0::Int, all_prev = true::Bool) -> Array{String,1}

Accumulate all values from root to defined level `LOD`.
"""
function get_files(trie::DataStructures.Trie{String}, LOD::Int, data=String[]::Array{String,1}, l = 0::Int, all_prev = true::Bool)::Array{String,1}
	if all_prev
		if l<=LOD
			push!(data,trie.value)
			for key in collect(keys(trie.children))
				get_files(trie.children[key],LOD,data,l+1,all_prev)
			end
		end
	else
		if l==LOD
			push!(data,trie.value)
		end

		if l<LOD
			for key in collect(keys(trie.children))
				get_files(trie.children[key],LOD,data,l+1,all_prev)
			end
		end
	end
	return data
end

"""
readhrc(potree::String)

Reads the hrc file of the potree
"""
function readhrc(potree::String)

	cloudmetadata = CloudMetadata(potree) # useful parameters togli quelli che non usi
	tree = joinpath(potree,cloudmetadata.octreeDir,"r") # path to directory "r"
	hrcs = searchfile(tree,".hrc")

	for hrc in hrcs
		raw = read(hrc)
		treehrc = reshape(raw, (5, div(length(raw), 5)))

		for i in 1:size(treehrc,2)
			children = bitstring(UInt8(treehrc[1,i]))
			npoints = parse(Int, bitstring(UInt8(treehrc[5,i]))*bitstring(UInt8(treehrc[4,i]))*bitstring(UInt8(treehrc[3,i]))*bitstring(UInt8(treehrc[2,i])); base=2)
			#struct da finire
			@show children, npoints
		end
	end

end


using MultiwayNumberPartitioning, HiGHS
using Test

@testset "MultiwayNumberPartitioning.jl" begin
    @testset "Example" begin
        include("../example/example.jl")
    end
end


using Parameters

const AnalysisData = ImmutableDict{Type{<:AbstractAnalysis}, Any}

mutable struct EClass
    id::Int64
    nodes::OrderedSet{ENode}
    parents::OrderedDict{ENode, Int64}
    data::Union{Nothing, AnalysisData}
end
EClass(id) = EClass(id, OrderedSet{ENode}(), OrderedDict{ENode, Int64}(), nothing)
EClass(id, nodes, parents) = EClass(id, nodes, parents, nothing)

# Interface for indexing EClass
Base.getindex(a::EClass, i) = a.nodes[i]
Base.setindex!(a::EClass, v, i) = setindex!(a.nodes, v, i)
Base.firstindex(a::EClass) = firstindex(a.nodes)
Base.lastindex(a::EClass) = lastindex(a.nodes)

# Interface for iterating EClass
Base.iterate(a::EClass) = iterate(a.nodes)
Base.iterate(a::EClass, state) = iterate(a.nodes, state)

# Showing
# Base.show(io::IO, a::EClass) = Base.show(io, a.nodes)
#
# function addparent!(a::EClass, n::ENode, p::EClass)
#     a.parents[n] = p
# end

function addparent!(a::EClass, n::ENode, id::Int64)
    a.parents[n] = id
end

function Base.union(to::EClass, from::EClass)
    EClass(to.id, from.nodes ∪ to.nodes, from.parents ∪ to.parents, from.data ∪ to.data)
end

function Base.union!(to::EClass, from::EClass)
    union!(to.nodes, from.nodes)
    merge!(to.parents, from.parents)
    if to.data != nothing && from.data != nothing
        # merge!(to.data, from.data)
        # to.data = join_analysis_data(to.data, from.data)
        to.data = join_analysis_data(to.data, from.data)
    elseif to.data == nothing
        to.data = from.data
    end
    return to
end

function join_analysis_data(d::AnalysisData, dsrc::AnalysisData)
    for (an, val_b) in dsrc
        if haskey(d, an)
            val_a = d[an]
            nv = join(an, val_a, val_b)
            # d[an] = nv
            # WARNING immutable version
            d = Base.ImmutableDict(d,an=>nv)
        end
    end
    return d
end

# mutable version
function join_analysis_data(d::AnalysisData, dsrc::AnalysisData)
    for (an, val_b) in dsrc
        if haskey(d, an)
            val_a = d[an]
            nv = join(an, val_a, val_b)
        end
    end
    return d
end

# Thanks to Shashi Gowda
function hasdata(a::EClass, x::Type{<:AbstractAnalysis})
    a.data == nothing && (return false)
    return haskey(a.data, x)
end

function getdata(a::EClass, x::Type{<:AbstractAnalysis})
    !hasdata(a, x) && error("EClass does not contain analysis data for $x")
    return a.data[x]
end

function getdata(a::EClass, x::Type{<:AbstractAnalysis}, default)
    hasdata(a, x) ? a.data[x] : default
end

function setdata!(a::EClass, x::Type{<:AbstractAnalysis}, value)
    # lazy allocation
    a.data == nothing && (a.data = AnalysisData())
    # a.data[x] = value
    a.data = AnalysisData(a.data, x, value)
end


######################
# Include Statements #
######################
    include("quarray.jl")
    include("arraymath.jl")
    include("constructors.jl")
    include("ladderops.jl")
    include("spinops.jl")


fi = open("../fv.bin", "r")
const pc_on_sq_old = read!( fi, Array{Int16}(tuple(pos_n, NumSQ)))
const kkp_old = read!( fi, Array{Int16}(tuple(kkp_end, NumSQ, NumSQ)))
#println("type of pc_on_sq = ", typeof(pc_on_sq))
#println("type of kkp = ", typeof(kkp))
close(fi)

const pc_on_sq = pc_on_sq_old
kkp = kkp_old
# kkp = zeros(Int16,NumSQ,NumSQ,kkp_end)
# for i = 1:kkp_end
#     for j = 1:NumSQ
#         for k = 1:NumSQ
#             kkp[k,j,i] = kkp_old[i,j,k]
#         end
#     end
# end

# println("pc_on_sq[0,0] = ", pc_on_sq[1,1])
# println("pc_on_sq[0,1] = ", pc_on_sq[1,2])
# println("pc_on_sq[1,0] = ", pc_on_sq[2,1])
# println("pc_on_sq[1,1] = ", pc_on_sq[2,2])
# println("kkp[0,0,0] = ", kkp[1,1,1])
# println("kkp[0,0,1] = ", kkp[1,1,2])
# println("kkp[0,1,0] = ", kkp[1,2,1])
# println("kkp[0,1,1] = ", kkp[1,2,2])


using PureSeq
using Base.Test

## BamReader

# test forward reads
reader = BamReader("data/small.bam", :forward, ReferenceContigs_hg38)
@test position(reader) == 10544
advance!(reader)
@test position(reader) == 11247
close(reader)

# test reverse reads
reader = BamReader("data/small.bam", :reverse, ReferenceContigs_hg38)
@test position(reader) == 10056
advance!(reader)
@test position(reader) == 10102
close(reader)

# test reading through a whole file
reader = BamReader("data/small.bam", :any, ReferenceContigs_hg38)
lastPos = -1
while !eof(reader)
	lastPos = position(reader)
	advance!(reader)
end
close(reader)

# test reading through a whole file with an iterator
reader = BamReader("data/small.bam", :reverse, ReferenceContigs_hg38)
for pos in eachposition(reader)
end
close(reader)

## FeatureMap

# test forward reads
reader = BamReader("data/small.bam", :forward, ReferenceContigs_hg38)
fm = FeatureMap(reader, ones(2001))
@test position(fm) == 9544 # where data first enters the window
@test value(fm) == 1.0
while !eof(fm)
	if position(fm) == 10544
		@test value(fm) == 3.0
	end
	advance!(fm)
end
close(fm)

## DenseBlockIterator

# test forward reads
fm1 = FeatureMap(BamReader("data/small.bam", :forward, ReferenceContigs_hg38), ones(21)*2)
fm2 = FeatureMap(BamReader("data/small.bam", :forward, ReferenceContigs_hg38), ones(21))
count = 0
for block in denseblocks([fm1, fm2], 100)
	if count == 105
		@test block[33,1] == 0.0
		@test block[33,2] == 0.0
		@test block[34,1] == 2.0
		@test block[34,2] == 1.0
		@test block[54,1] == 2.0
		@test block[54,2] == 1.0
		@test block[55,1] == 0.0
		@test block[55,2] == 0.0
	end
	count += 1
	if count > 120 break end
end
close(fm1)
close(fm2)

# test infinite looping of blocks
fm1 = FeatureMap(BamReader("data/small.bam", :forward, ReferenceContigs_hg38), ones(21)*2)
fm2 = FeatureMap(BamReader("data/small.bam", :forward, ReferenceContigs_hg38), ones(21))
count = 0
for block in denseblocks([fm1, fm2], 1000, loop=true)
	count += 1
	if count > 12000 break end
end
@test count == 12001
close(fm1)
close(fm2)

## SamWriter

f = open("data/tmp.sam", "w")
sw = SamWriter(f, ReferenceContigs_hg38)
writeRead(sw, 10, 16)
close(f)

## write_binned and BinnedReader

# test forward reads
write_binned("data/small.bam", 1000, :reverse, skipDup=false)
reader = BinnedReader("data/small.bam.rbin1000")
@test position(reader) == 11 # where data first enters the window
@test value(reader) == 3.0

while !eof(reader)
	if position(reader) == 12
		@test value(reader) == 5.0
	end
	advance!(reader)
end

# now skip duplicate reads
write_binned("data/small.bam", 1000, :reverse, skipDup=true)
reader = BinnedReader("data/small.bam.rbin1000")
@test position(reader) == 11 # where data first enters the window
@test value(reader) == 3.0
while !eof(reader)
	if position(reader) == 12
		@test value(reader) == 4.0
	end
	advance!(reader)
end
close(reader)

## ContextMap

# test forward reads
reader = BamReader("data/small.bam", :forward, ReferenceContigs_hg38)
cm = ContextMap(reader, 1000, 1000)
@test position(cm) == 9544 # where data first enters the window
@test value(cm) == 1.0
while !eof(cm)
	if position(cm) == 10544
		@test value(cm) == 3.0
	end
	advance!(cm)
end
close(cm)

# test bins
reader = BinnedReader("data/small.bam.fbin1000")
cm = ContextMap(reader, 2, 2)
@test position(cm) == 9 # where data first enters the window
@test value(cm) == 1.0
while !eof(cm)
	if position(cm) == 12
		@test value(cm) == 13.0
	end
	advance!(cm)
end
close(cm)


using MPSensitivity
using Test

@testset "MPSensitivity.jl" begin
    # Write your tests here.
    @test get_efficiency(4,2) == 2.0
end


function _bls_by_shape(bls::AbstractBL{SIDE},i::Interval) where SIDE
	if SIDE == 1
		stop = i.start
		start = stop + bls.depth
	else
		stop = i.stop
		start = stop - bls.depth
	end
	return bls(start=start,stop=stop)
end

################################################################################
# Plotting

@recipe function f(bnd::Boundary{1})
	bnd.shape
end


using CUDA, Base.Test

#Initialize device
dev = CuDevice(0)

#Create context
ctx = create_context(dev)

#Modules include all functions
md = CuModule("thing.ptx")

#Load function
vadd = CuFunction(md, "vadd")
vmul = CuFunction(md, "vmul")
vsub = CuFunction(md, "vsub")
vdiv = CuFunction(md, "vdiv")

#Init
a = rand(Float32, 10)
b = rand(Float32, 10)
ad = CuArray(a)
bd = CuArray(b)

#Addition
c = zeros(Float32, 10)
cd = CuArray(c)
CUDA.launch(vadd, 10, 1, (ad,bd,cd))
c = to_host(cd)
@test_approx_eq c a+b

#Subtraction
c = zeros(Float32, 10)
cd = CuArray(c)
CUDA.launch(vsub, 10, 1, (ad,bd,cd))
c = to_host(cd)
@test_approx_eq c a-b

#Multiplication
c = zeros(Float32, 10)
cd = CuArray(c)
CUDA.launch(vmul, 10, 1, (ad,bd,cd))
c = to_host(cd)
@test_approx_eq c a.*b

#Division
c = zeros(Float32, 10)
cd = CuArray(c)
CUDA.launch(vdiv, 10, 1, (ad,bd,cd))
c = to_host(cd)
@test_approx_eq c a./b

#Negative test cases
a = rand(Float32, 10)
ad = CuArray(Float32, 5)
@test_throws ArgumentError copy!(ad, a)
@test_throws ArgumentError copy!(a, ad)

#Utility
@test CUDA.ndims(ad) == 1
@test CUDA.eltype(ad) == Float32

#Not sure what this does
push(ctx)
pop(ctx)

#Count number of devices
CUDA.devcount()

#Device info
println("Name of device = $(name(dev))")
println("Device capability = $(capability(dev))")
println("Device memory = $(totalmem(dev))")

#Free memory and finish up
free(ad)
free(bd)
free(cd)
unload(md)
destroy(ctx)


using NetCDF

# Add array with time in netcdfs with forcing data

path_st = "/data04/jmg/fsm_simulations/netcdf/forcings_st"

files = readdir(path_st)

for file in files

    file = joinpath(path_st, file)

    dim_time = ncread(file, "dim_time")

    time_str = ncread(file, "time_str")

    time = DateTime.(time_str, "yyyy-mm-dd HH:MM:SS")

    time_array = [Dates.year.(time) Dates.month.(time) Dates.day.(time) Dates.hour.(time)]

    nccreate(file, "time_array", "ymdh", 4, Dict("ymdh" => "header"), "dim_time")

    ncclose()

    ncwrite(time_array', file, "time_array")

    ncclose()

    @show file

end


####### TO MOVE
import ProximalOperators: prox, prox!, gradient, gradient!

using RecursiveArrayTools
function prox(f::ProximableFunction, x::ArrayPartition, args...)
  y, fy = prox(f, x.x, args...)
  return ArrayPartition(y), fy
end

prox!(y::ArrayPartition, f::ProximableFunction, x::ArrayPartition, args...) =
prox!(y.x, f, x.x, args...)

function gradient(f::ProximableFunction, x::ArrayPartition)
  gradx, fy = gradient(f, x.x)
  return ArrayPartition(gradx), fy
end

gradient!(y::ArrayPartition, f::ProximableFunction, x::ArrayPartition) =
gradient!(y.x, f, x.x)
###############


__precompile__()

module SPTK

# A thin Julia wrapper for Speech signal Processing Toolkit (SPTK).
#
# The original SPTK: http://sp-tk.sourceforge.net/
# Note that the SPTK.jl is based on the slightly modified SPTK
# https://github.com/r9y9/SPTK
#
# NOTE1: no function is exported; i.e., you should explicitly import functions
# if you need.
#
# NOTE2: Function interfaces might be different betweeen C and Julia,
# as the following convensions.
#
# Conventions for wrapping C interface
#
# 1. Avoid really short names for variables (e.g. a, b, c, aa, bb, dd) .
#
#    Variable names should be informative. If the C functions have such short
#    names, use self-descriptive names instead for Julia interfaces, unless
#    they have clear meanings in their context.
#
# 2. Avoid too many function arguments.
#
#    Less is better. If the C functions have too many function arguments, use
#    keyword arguments with proper default values for optional ones in Julia.
#
# 3. Handle errors in Julia
#
#    Since C functions might `exit` (unfortunately) inside thier functions for
#    unexpected inputs, it should be check if the inputs are supported or not
#    in Julia before `ccall`.

deps = joinpath(dirname(@__FILE__), "..", "deps", "deps.jl")
if isfile(deps)
    include(deps)
else
    error("SPTK not properly installed. Please run Pkg.build(\"SPTK\")")
end

for fname in [
              "common",
              "lib",
              "adaptive",
              "conversions",
              "excite",
              "f0",
              "mfcc",
              "mgcep",
              "synthesis_filters",
              "utils",
              "window",
              "extend"
    ]
    include(string(fname, ".jl"))
end

end # module SPTK


function sanitise_epcs(project)
    stockdata = project.dat["master"]

    # Find street names
    parseaddr(str) = split(str,", ")
    findstreet(str_arr) = str_arr[1+minimum([findfirst(length.(str_arr)[2:end].>3),length(str_arr)-3])]
    streetnames = findstreet.(parseaddr.(stockdata.Full_postal_address))
    streetlist = sort(unique(streetnames))

    # Find indices of all dwellings linked to each street in `streetlist`
    findall_arr(arr,val) = findall(arr.==val)
    i = findall_arr.((streetnames,),streetlist)

    # Util functions for propagating most common occurrences from `groupby`
    countoccur(arr,val) = sum(skipmissing(arr).==val)
    mostcommon_occur(arr,groupby) = length(arr)==sum(ismissing.(arr)) ? "tbc" : groupby[findmax(countoccur.((arr,),groupby))[2]]
    # Propagate based on most common occurrence on street
    propagate_mostcommon(newArr,refArr,groupby,i) = newArr[i].=mostcommon_occur(refArr[i],groupby)

    # Apply propagation method: Age bands
    stockdata[!,:ParaDwell_Age_Band] = fill!(Array{String}(undef,nrow(stockdata)),"")
    groupby = ["before 1919","1919-1929","1930-1949","1950-1964","1965-1975","1976-1983","1984-1991","1992-1998","1999-2002","2003-2007","2008 onwards"]
    propagate_mostcommon.((stockdata.ParaDwell_Age_Band,), (stockdata.EPC_Part_1_Construction_Age_Band,), (groupby,), i)

end

@time sanitise_epcs(project)


JuMP.num_variables(m::AbstractBilevelModel) = JuMP.num_variables(bilevel_model(m))
_plural(n) = (isone(n) ? "" : "s")
function JuMP.show_constraints_summary(io::IO, model::BilevelModel)
    JuMP.show_constraints_summary(io, Upper(model))
    JuMP.show_constraints_summary(io, Lower(model), true)
end
function JuMP.show_constraints_summary(io::IO, model::UpperModel)
    n = JuMP.num_constraints(model)
    println(io, "Upper Constraint", _plural(n), ": ", n)
end
function JuMP.show_constraints_summary(io::IO, model::LowerModel, line_break = false)
    n = JuMP.num_constraints(model)
    print(io, "Lower Constraint", _plural(n), ": ", n, ifelse(line_break, "\n",""))
end
JuMP.show_backend_summary(io::IO, model::InnerBilevelModel) = JuMP.show_backend_summary(io, model.m)
function JuMP.show_backend_summary(io::IO, model::BilevelModel)
    println(io, "Bilevel Model")
    println(io, "Solution method: ", model.mode)
    if model.solver === nothing
        print(io, "No solver attached")
    else
        name = JuMP.solver_name(model::BilevelModel)
        print(io, "Solver name: ", name)
    end
end
function JuMP.solver_name(model::BilevelModel)
    name = try
        MOI.get(model.solver, MOI.SolverName())::String
    catch ex
        if isa(ex, ArgumentError)
            return "SolverName() attribute not implemented by the optimizer."
        else
            rethrow(ex)
        end
    end
end

function JuMP.show_objective_function_summary(io::IO, model::UpperModel)
    println(io, "Upper objective function type: \n",
            JuMP.objective_function_type(model))
end
function JuMP.show_objective_function_summary(io::IO, model::LowerModel)
    println(io, "Lower objective function type: \n",
            JuMP.objective_function_type(model))
end
function JuMP.show_objective_function_summary(io::IO, model::BilevelModel)
    JuMP.show_objective_function_summary(io, Upper(model))
    JuMP.show_objective_function_summary(io, Lower(model))
end