export UCEvo, UCInd, evaluate, populate, generation, mean_fitness

using Cambrian
using Statistics

import Cambrian.populate, Cambrian.evaluate, Cambrian.log_gen, Cambrian.save_gen

"""
Individuals definition
"""
mutable struct UCInd <: Cambrian.Individual
    e_chromosome::Vector{Float64}
    c_chromosome::Vector{Float64}
    fitnesses::Vector{Float64}
    lifetime::Int64
    reached_frames::Int64
end

function UCInd(e_chromosome::Vector{Float64}, c_chromosome::Vector{Float64})
    UCInd(e_chromosome, c_chromosome, Vector{Float64}(), 0, 0)
end

function UCInd(
	config::NamedTuple,
	e_chromosome::Vector{Float64},
	c_chromosome::Vector{Float64}
)
    UCInd(e_chromosome, c_chromosome)
end

"""
Evolution framework
"""
mutable struct UCEvo{T} <: Cambrian.AbstractEvolution
    config::NamedTuple
    logid::String
    logger::CambrianLogger
	resdir::String
    population::Vector{T}
    fitness::Function
    gen::Int64
    atari_games::Vector{Game}
end

function UCEvo{T}(
	config::NamedTuple,
    resdir::String,
	fitness::Function,
    init_population::Function,
    rom_name::String
) where T
    logid = config.id
    log_path = joinpath(resdir, logid, "logs/logs.csv")
    logger = CambrianLogger(log_path)
    population = init_population(T, config)
	# n_children = config.n_population - config.n_elite
	atari_games = [Game(rom_name, 0) for _ in 1:length(population)]
    UCEvo(config, logid, logger, resdir, population, fitness, 0, atari_games)
end

evaluate(e::UCEvo{T}) where T = fitness_evaluate(e, e.fitness)
populate(e::UCEvo{T}) where T = tournament_populate(e)

ucb(m::Float64, n_total::Int64, n_eval::Int64) = m + sqrt(2.0 * log(n_total) / n_eval)

function ucb(ind::UCInd)
	n_eval = length(ind.fitnesses)
	if n_eval > 0
		return ucb(mean(ind.fitnesses), ind.lifetime, n_eval)
	else
		return Inf
	end
end

function mean_fitness(ind::UCInd)
	if length(ind.fitnesses) > 0
		return mean(ind.fitnesses)
	else
		return -Inf
	end
end

function increment_lifetime!(e::UCEvo{T}, n::Int64) where T
	for ind in e.population
		ind.lifetime += n
	end
end

"""
Elite selection function
"""
function generation(e::UCEvo{T}) where T
    if e.config.greedy_elitism
        sort!(e.population, by=ind->mean_fitness(ind), rev=true)
    else
        sort!(e.population, by=ind->ucb(ind), rev=true)
    end
    e.population = e.population[1:e.config.n_elite]
end

"""
Evaluation function
"""
function fitness_evaluate(e::UCEvo{T}, fitness::Function) where T
	if e.config.surrogate
		if e.gen == 1
			exact_fitness_evaluate(e, fitness)
		else
			surrogate_fitness_evaluate(e, fitness)
		end
	else
		exact_fitness_evaluate(e, fitness)
	end
end

"""
An individual is not evaluated if enough evaluations have been made.
"""
function evaluation_utility(n_eval_max::Int64, ind::UCInd) where T
	if n_eval_max > 0
		if length(ind.fitnesses) < n_eval_max
			ucb(ind)
		else
			-Inf
		end
	else
		ucb(ind)
	end
end

function evaluation_utility(
	n_eval_max::Int64,
	m::Float64,
	n_total::Int64,
	n_eval::Int64
) where T
	if n_eval_max > 0
		if n_eval < n_eval_max
			ucb(m, n_total, n_eval)
		else
			-Inf
		end
	else
		ucb(m, n_total, n_eval)
	end
end

function exact_fitness_evaluate(e::UCEvo{T}, fitness::Function) where T
	# 1. Evaluate all new individuals in parallel
	n_children = e.config.n_population - e.config.n_elite
	sort!(e.population, by=ind->ucb(ind), rev=true) # Children are 1st
	n = e.gen == 1 ? length(e.population) : n_children
	#=
	for i in 1:n
		push!(e.population[i].fitnesses,
			  fitness(e.population[i], e.gen, e.atari_games[i]))
	end
	=#
	lck = ReentrantLock()
	@sync for i in 1:n
        Threads.@spawn begin
  			f = fitness(e.population[i], e.gen, e.atari_games[i])
  			lock(lck)
  			try
  				push!(e.population[i].fitnesses, f)
  			finally
  				unlock(lck)
  			end
        end
    end
	increment_lifetime!(e, n)
	@assert all([length(ind.fitnesses) > 0 for ind in e.population])
	# 2. Finish evaluation budget sequentially
	n_remaining_eval = e.config.n_eval - n
	for _ in 1:n_remaining_eval
		sort!(e.population, by=ind->evaluation_utility(e.config.n_eval_max, ind), rev=true)
		push!(e.population[1].fitnesses,
			  fitness(e.population[1], e.gen, e.atari_games[1]))
		increment_lifetime!(e, 1)
	end
end

function surrogate_fitness_evaluate(e::UCEvo{T}, fitness::Function) where T
	# 1. Evaluate all new individuals in parallel
	n_children = e.config.n_population - e.config.n_elite
	sort!(e.population, by=ind->ucb(ind), rev=true) # Children are 1st
	lck = ReentrantLock()
	@sync for i in 1:n_children
        Threads.@spawn begin
			# OUTDATED unsafe (I thought it was safe)
			#=push!(e.population[i].fitnesses,
				fitness(e.population[i], e.gen, e.atari_games[i]))=#
			# NEW safe
			f = fitness(e.population[i], e.gen, e.atari_games[i])
			lock(lck)
			try
				push!(e.population[i].fitnesses, f)
			finally
				unlock(lck)
			end
        end
    end
	increment_lifetime!(e, n_children)
	@assert all([length(ind.fitnesses) > 0 for ind in e.population])
	# 2. Determine individuals to evaluate next
	n_remaining_eval = e.config.n_eval - n_children
	f = [mean_fitness(ind) for ind in e.population]
	N = [ind.lifetime for ind in e.population]
	n = [length(ind.fitnesses) for ind in e.population]
	to_eval = Vector{Int64}()
	for i in 1:n_remaining_eval
		utilities = [evaluation_utility(e.config.n_eval_max, f[i], N[i], n[i]) for i in eachindex(f)]
		index = argmax(utilities)
		push!(to_eval, index)
		N .+= 1
		n[index] += 1
	end
	# 3. Evaluate remaining individuals
	lck = ReentrantLock()
	@sync for k in eachindex(to_eval)
        Threads.@spawn begin
			i = to_eval[k]
			# OUTDATED unsafe
			#=push!(e.population[i].fitnesses,
				fitness(e.population[i], e.gen, e.atari_games[i]))=#
			# NEW safe
			f = fitness(e.population[i], e.gen, e.atari_games[i])
			lock(lck)
			try
				push!(e.population[i].fitnesses, f)
			finally
				unlock(lck)
			end
		end
    end
	increment_lifetime!(e, n_remaining_eval)
end

"""
Populate function
"""
function tournament_populate(e::UCEvo{T}) where T
    children = Vector{T}()
    for _ in e.config.n_elite+1:e.config.n_population
        parent = ucb_tournament_selection(e.population, e.config.tournament_size)
        push!(children, mutate(parent))
    end
    push!(e.population, children...)
end

function ucb_tournament_selection(
    pop::Vector{T},
    tournament_size::Int64
) where T
    inds = shuffle!(collect(1:length(pop)))
    sort(pop[inds[1:tournament_size]], by=ind->ucb(ind))[end]
end

function fitness_tournament_selection(
    pop::Vector{T},
    tournament_size::Int64
) where T
    inds = shuffle!(collect(1:length(pop)))
    sort(pop[inds[1:tournament_size]], by=ind->ind.fitness)[end]
end

"""
Logging
"""
function log_gen(e::UCEvo{T}) where T
	sep = ";"
    if e.gen == 1
        f = open(joinpath(e.resdir, e.logid, "logs/logs.csv"), "w+")
        write(f, string("gen_number", sep, "fitnesses", sep,
			"reached_frames", sep, "dna_id\n"))
        close(f)
    end
	enco_path = joinpath(e.resdir, e.logid, Formatting.format("gens/encoder_{1:04d}", e.gen))
    cont_path = joinpath(e.resdir, e.logid, Formatting.format("gens/controller_{1:04d}", e.gen))
    mkpath(enco_path)
    mkpath(cont_path)
    for i in eachindex(e.population)
        dna_id = Formatting.format("{1:04d}", i)
		# Log results
		f = open(joinpath(e.resdir, e.logid, "logs/logs.csv"), "a+")
        write(f, Formatting.format(string(e.gen, sep,
			   e.population[i].fitnesses, sep,
			   e.population[i].reached_frames, sep, dna_id, "\n")))
        close(f)
		# Log individuals
		ind_fit = [mean_fitness(e.population[i])]
		enco = IPCGPInd(e.config.e_config, e.population[i].e_chromosome)
		enco.fitness .= ind_fit
		f = open(string(enco_path, "/", dna_id, ".dna"), "w+")
        write(f, string(enco))
        close(f)
        cont = CGPInd(e.config.c_config, e.population[i].c_chromosome)
		cont.fitness .= ind_fit
        f = open(string(cont_path, "/", dna_id, ".dna"), "w+")
        write(f, string(cont))
        close(f)
    end
end


using Makie, VideoIO, Dates, Observables

include("testvideo.jl")

# create the test video if it doesn't exist
if !isfile(testvideo)
    createtestvideo(testvideo)
end

# open the video
f = VideoIO.openvideo(testvideo)

# read one image to get its type <-- this might be unecessary since we might know that in advance for all videos
img = Observable(read(f))
timestamp = Observable(Time(0))

# reset the pointer to the begining of the file <-- there might be a more elegant way to achieve this
f = VideoIO.openvideo(testvideo)

# create a buffer of 2 seconds
nframesbuffer = round(Int, 2f.framerate)
buff = Channel{Pair{Time, typeof(img[])}}(nframesbuffer)
# fill up the buffer with the next 15 seconds worth of frames
task = @async begin
    while !eof(f)
        t = Time(0) + Millisecond(round(Int, 1000gettime(f)))
        put!(buff, Pair(t, read(f)))
        yield() # needed
    end
end

# all the stuff needed for display
pixelaspectratio = VideoIO.aspect_ratio(f)
h = f.height
w = round(typeof(h), f.width * pixelaspectratio)
AbstractPlotting.set_theme!(backgroundcolor = :black)
scene = Scene(resolution = (w, h))
Makie.image!(scene, 1:h, 1:w, img, show_axis = false, scale_plot = false)
Makie.rotate!(scene, -0.5π)
update_limits!(scene)
update_cam!(scene)
update!(scene)

# everything starts at this observable, the current frame
old_slider_pos = Observable(1)

# get an estimate for the number of frames, this could be very innacurate for VFR videos
nframes = round(Int, VideoIO.get_duration(f.avin.io)*f.framerate)

# the slider, updating currentframe and getting its string from timestamps
slider_h = slider(1:nframes, start = 1, valueprinter = i -> "no, this will be wrong", textcolor = :white)

lift(slider_h[end][:value]) do frame
    while old_slider_pos[] ≤ frame
        timestamp[], img[] = take!(buff)
        old_slider_pos[] += 1
    end
    0
    # currentframe[] = frame
end

# this is for a separate display of the time stamp, due to the fact that we can not calculate with the time stamp is from the frame number (which is what the slider has)
timestamp_h = text(lift(string, timestamp), color = :white, raw = true, camera = campixel!)

# I have to have a forward button becuase apparently the stuff in the lift gets evaluated, so to have the movie start in frame #1 I need to go forward once and the backwards...
fwdbutton = button(">", textcolor = :white)
lift(fwdbutton[end][:clicks]) do _
    # if currentframe[] < nframes # check we are not in the last frame
    timestamp[], img[] = take!(buff)
    slider_h[end][:value][] += 1
        # so now the movie is in frame #2
    # end
    0
end
#=bckbutton = button!(rsc(), "<", textcolor = :white)
lift(bckbutton[end][:clicks]) do _
    if currentframe[] > 1 # check we're not in frame #1
        currentframe[] -= 1
        slider_h[end][:value][] -= 1
        # and now we're back in frame #1
    end
    0
end=#

# done!
hbox(vbox(slider_h, fwdbutton, timestamp_h), scene)

# played = Observables.async_latest(play, play_bubtton[end][:clicks])

# hbox(vbox(play_button, next_button, slider_h), scene)


# This file is a part of JuliaFEM.
# License is MIT: see https://github.com/JuliaFEM/MortarContact2D.jl/blob/master/LICENSE

mutable struct Contact2D <: BoundaryProblem
    master_elements :: Vector{Element}
    rotate_normals :: Bool
    dual_basis :: Bool
    max_distance :: Float64
    iteration :: Int
    contact_state_in_first_iteration :: Symbol
    store_fields :: Vector{String}
end

function Contact2D()
    return Contact2D([], false, true, Inf, 0, :AUTO, [])
end

function FEMBase.add_elements!(::Problem{Contact2D}, ::Any)
    error("use `add_slave_elements!` and `add_master_elements!` to add ",
          "elements to the Contact2D problem.")
end

function FEMBase.get_elements(problem::Problem{Contact2D})
    return [problem.elements; problem.properties.master_elements]
end

function FEMBase.add_slave_elements!(problem::Problem{Contact2D}, elements)
    for element in elements
        push!(problem.elements, element)
    end
end

function FEMBase.add_master_elements!(problem::Problem{Contact2D}, elements)
    for element in elements
        push!(problem.properties.master_elements, element)
    end
end

function FEMBase.get_slave_elements(problem::Problem{Contact2D})
    return problem.elements
end

function FEMBase.get_master_elements(problem::Problem{Contact2D})
    return problem.properties.master_elements
end

function get_slave_dofs(problem::Problem{Contact2D})
    dofs = Int64[]
    for element in get_slave_elements(problem)
        append!(dofs, get_gdofs(problem, element))
    end
    return sort(unique(dofs))
end

function get_master_dofs(problem::Problem{Contact2D})
    dofs = Int64[]
    for element in get_master_elements(problem)
        append!(dofs, get_gdofs(problem, element))
    end
    return sort(unique(dofs))
end

function create_rotation_matrix(element::Element{Seg2}, time::Float64)
    n = element("normal", time)
    R = [0.0 -1.0; 1.0 0.0]
    t1 = R'*n[1]
    t2 = R'*n[2]
    Q1 = [n[1] t1]
    Q2 = [n[2] t2]
    Z = zeros(2, 2)
    Q = [Q1 Z; Z Q2]
    return Q
end

function create_contact_segmentation(problem::Problem{Contact2D}, slave_element::Element{Seg2}, master_elements::Vector, time::Float64; deformed=false)

    result = []

    max_distance = problem.properties.max_distance

    x1 = slave_element("geometry", time)

    if deformed && haskey(slave_element, "displacement")
        x1 = map(+, x1, slave_element("displacement", time))
    end

    slave_midpoint = 1/2*(x1[1] + x1[2])
    slave_length = norm(x1[2]-x1[1])

    for master_element in master_elements

        x2 = master_element("geometry", time)

        if deformed && haskey(master_element, "displacement")
            x2 = map(+, x2, master_element("displacement", time))
        end

        master_midpoint = 1/2*(x2[1] + x2[2])
        master_length = norm(x2[2]-x2[1])
        # charasteristic length
        dist = norm(slave_midpoint - master_midpoint)
        cl = dist/max(slave_length, master_length)
        if cl > max_distance
            continue
        end

        # 3.1 calculate segmentation
        x21, x22 = x2
        xi1a = project_from_master_to_slave(slave_element, x21, time)
        xi1b = project_from_master_to_slave(slave_element, x22, time)
        xi1 = clamp.([xi1a; xi1b], -1.0, 1.0)
        l = 1/2*abs(xi1[2]-xi1[1])
        if isapprox(l, 0.0)
            continue # no contribution in this master element
        end
        push!(result, (master_element, xi1, l))
    end
    return result
end

function FEMBase.assemble_elements!(problem::Problem{Contact2D}, assembly::Assembly,
                                    elements::Vector{Element{Seg2}}, time::Float64)

    props = problem.properties
    props.iteration += 1

    slave_elements = get_slave_elements(problem)

    field_dim = get_unknown_field_dimension(problem)
    field_name = get_parent_field_name(problem)

    # 1. calculate nodal normals for slave element nodes j ∈ S
    normals = calculate_normals([slave_elements...], time; rotate_normals=props.rotate_normals)
    tangents = Dict(j => [t2, -t1] for (j, (t1,t2)) in normals)
    update!(slave_elements, "normal", time => normals)
    update!(slave_elements, "tangent", time => tangents)

    Rn = 0.0

    # 2. loop all slave elements
    for slave_element in slave_elements

        nsl = length(slave_element)
        X1 = slave_element("geometry", time)
        x1 = slave_element("geometry", time)
        if haskey(slave_element, "displacement")
            u1 = slave_element("displacement", time)
            x1 = map(+, x1, u1)
        end
        la1 = slave_element("lambda", time)
        n1 = slave_element("normal", time)
        t1 = slave_element("tangent", time)

        contact_area = 0.0
        contact_error = 0.0
        Q2 = create_rotation_matrix(slave_element, time)

        master_elements = get_master_elements(problem)
        segmentation = create_contact_segmentation(problem, slave_element, master_elements, time)
        if length(segmentation) == 0 # no overlapping in master and slave surfaces with this slave element
            continue
        end

        Ae = Matrix(I, nsl, nsl)
        if props.dual_basis
            De = zeros(nsl, nsl)
            Me = zeros(nsl, nsl)
            for (master_element, xi1, l) in segmentation
                for ip in get_integration_points(slave_element, 3)
                    detJ = slave_element(ip, time, Val{:detJ})
                    w = ip.weight*detJ*l
                    xi = ip.coords[1]
                    xi_s = dot([1/2*(1-xi); 1/2*(1+xi)], xi1)
                    N1 = vec(get_basis(slave_element, xi_s, time))
                    De += w*Matrix(Diagonal(N1))
                    Me += w*N1*N1'
                end
                Ae = De*inv(Me)
            end
        end

        # loop all segments
        for (master_element, xi1, l) in segmentation

            nm = length(master_element)
            X2 = master_element("geometry", time)
            x2 = master_element("geometry", time)
            if haskey(master_element, "displacement")
                u2 = master_element("displacement", time)
                x2 = map(+, x2, u2)
            end

            # 3.3. loop integration points of one integration segment and calculate
            # local mortar matrices
            De = zeros(nsl, nsl)
            Me = zeros(nsl, nsl)
            Ne = zeros(nsl, 2*nsl)
            Te = zeros(nsl, 2*nsl)
            He = zeros(nsl, 2*nsl)
            ce = zeros(nsl)
            ge = zeros(nsl)
            for ip in get_integration_points(slave_element, 3)
                detJ = slave_element(ip, time, Val{:detJ})
                w = ip.weight*detJ*l
                xi = ip.coords[1]
                xi_s = dot([1/2*(1-xi); 1/2*(1+xi)], xi1)
                N1 = vec(get_basis(slave_element, xi_s, time))
                Phi = Ae*N1

                # project gauss point from slave element to master element in direction n_s
                X_s = interpolate(N1, X1) # coordinate in gauss point
                n_s = interpolate(N1, n1) # normal direction in gauss point
                t_s = interpolate(N1, t1) # tangent condition in gauss point
                n_s /= norm(n_s)
                t_s /= norm(t_s)
                xi_m = project_from_slave_to_master(master_element, X_s, n_s, time)
                N2 = vec(get_basis(master_element, xi_m, time))
                X_m = interpolate(N2, X2)

                u_s = interpolate(N1, u1)
                u_m = interpolate(N2, u2)
                x_s = map(+, X_s, u_s)
                x_m = map(+, X_m, u_m)
                la_s = interpolate(Phi, la1)

                # contact virtual work
                De += w*Phi*N1'
                Me += w*Phi*N2'

                # contact constraints
                Ne += w*reshape(kron(N1, n_s, Phi), 2, 4)
                Te += w*reshape(kron(N2, n_s, Phi), 2, 4)
                He += w*reshape(kron(N1, t_s, Phi), 2, 4)
                ge += w*Phi*dot(n_s, x_m-x_s)
                ce += w*N1*dot(n_s, la_s)
                Rn += w*dot(n_s, la_s)

                contact_area += w
                contact_error += 1/2*w*dot(n_s, x_s-x_m)^2
            end

            sdofs = get_gdofs(problem, slave_element)
            mdofs = get_gdofs(problem, master_element)

            # add contribution to contact virtual work
            for i=1:field_dim
                lsdofs = sdofs[i:field_dim:end]
                lmdofs = mdofs[i:field_dim:end]
                add!(problem.assembly.C1, lsdofs, lsdofs, De)
                add!(problem.assembly.C1, lsdofs, lmdofs, -Me)
            end

            # add contribution to contact constraints
            add!(problem.assembly.C2, sdofs[1:field_dim:end], sdofs, Ne)
            add!(problem.assembly.C2, sdofs[1:field_dim:end], mdofs, -Te)
            add!(problem.assembly.D, sdofs[2:field_dim:end], sdofs, He)
            add!(problem.assembly.g, sdofs[1:field_dim:end], ge)
            add!(problem.assembly.c, sdofs[1:field_dim:end], ce)

        end # master elements done

        if "contact area" in props.store_fields
            update!(slave_element, "contact area", time => contact_area)
        end

        if "contact error" in props.store_fields
            update!(slave_element, "contact error", time => contact_error)
        end

    end # slave elements done, contact virtual work ready

    S = sort(collect(keys(normals))) # slave element nodes
    weighted_gap = Dict{Int64, Vector{Float64}}()
    contact_pressure = Dict{Int64, Vector{Float64}}()
    complementarity_condition = Dict{Int64, Vector{Float64}}()
    is_active = Dict{Int64, Int}()
    is_inactive = Dict{Int64, Int}()
    is_slip = Dict{Int64, Int}()
    is_stick = Dict{Int64, Int}()

    la = problem.assembly.la
    # FIXME: for matrix operations, we need to know the dimensions of the
    # final matrices
    ndofs = 2*length(S)
    ndofs = max(ndofs, length(la))
    ndofs = max(ndofs, size(problem.assembly.K, 2))
    ndofs = max(ndofs, size(problem.assembly.C1, 2))
    ndofs = max(ndofs, size(problem.assembly.C2, 2))
    ndofs = max(ndofs, size(problem.assembly.D, 2))
    ndofs = max(ndofs, size(problem.assembly.g, 2))
    ndofs = max(ndofs, size(problem.assembly.c, 2))

    C1 = sparse(problem.assembly.C1, ndofs, ndofs)
    C2 = sparse(problem.assembly.C2, ndofs, ndofs)
    D = sparse(problem.assembly.D, ndofs, ndofs)
    g = Vector(sparse(problem.assembly.g, ndofs, 1)[:])
    c = Vector(sparse(problem.assembly.c, ndofs, 1)[:])

    for j in S
        dofs = [2*(j-1)+1, 2*(j-1)+2]
        weighted_gap[j] = g[dofs]
    end

    state = problem.properties.contact_state_in_first_iteration
    if problem.properties.iteration == 1
        @info("First contact iteration, initial contact state = $state")

        if state == :AUTO
            avg_gap = mean([weighted_gap[j][1] for j in S])
            std_gap = std([weighted_gap[j][1] for j in S])
            if (avg_gap < 1.0e-12) && (std_gap < 1.0e-12)
                state = :ACTIVE
            else
                state = :UNKNOWN
            end
            @info("Average weighted gap = $avg_gap, std gap = $std_gap, automatically determined contact state = $state")
        end

    end

    # active / inactive node detection
    for j in S
        dofs = [2*(j-1)+1, 2*(j-1)+2]
        weighted_gap[j] = g[dofs]

        if length(la) != 0
            p = dot(normals[j], la[dofs])
            t = dot(tangents[j], la[dofs])
            contact_pressure[j] = [p, t]
        else
            contact_pressure[j] = [0.0, 0.0]
        end

        complementarity_condition[j] = contact_pressure[j] - weighted_gap[j]
        if complementarity_condition[j][1] < 0
            is_inactive[j] = 1
            is_active[j] = 0
            is_slip[j] = 0
            is_stick[j] = 0
        else
            is_inactive[j] = 0
            is_active[j] = 1
            is_slip[j] = 1
            is_stick[j] = 0
        end
    end

    if (problem.properties.iteration == 1) && (state == :ACTIVE)
        for j in S
            is_inactive[j] = 0
            is_active[j] = 1
            is_slip[j] = 1
            is_stick[j] = 0
        end
    end

    if (problem.properties.iteration == 1) && (state == :INACTIVE)
        for j in S
            is_inactive[j] = 1
            is_active[j] = 0
            is_slip[j] = 0
            is_stick[j] = 0
        end
    end

    if "weighted gap" in props.store_fields
        update!(slave_elements, "weighted gap", time => weighted_gap)
    end
    if "contact pressure" in props.store_fields
        update!(slave_elements, "contact pressure", time => contact_pressure)
    end
    if "complementarity condition" in props.store_fields
        update!(slave_elements, "complementarity condition", time => complementarity_condition)
    end
    if "active nodes" in props.store_fields
        update!(slave_elements, "active nodes", time => is_active)
    end
    if "inactive nodes" in props.store_fields
        update!(slave_elements, "inactive nodes", time => is_inactive)
    end
    if "stick nodes" in props.store_fields
        update!(slave_elements, "stick nodes", time => is_stick)
    end
    if "slip nodes" in props.store_fields
        update!(slave_elements, "slip nodes", time => is_slip)
    end

    @info("# | A | I | St | Sl | gap | pres | comp")
    for j in S
        str1 = "$j | $(is_active[j]) | $(is_inactive[j]) |  $(is_stick[j]) |  $(is_slip[j]) | "
        str2 = "$(round(weighted_gap[j][1]; digits=3)) | $(round(contact_pressure[j][1]; digits=3)) | $(round(complementarity_condition[j][1]; digits=3))"
        @info(str1 * str2)
    end

    # solve variational inequality

    # remove inactive nodes from assembly
    for j in S
        dofs = [2*(j-1)+1, 2*(j-1)+2]
        n, t = dofs
        if is_inactive[j] == 1
            @debug("$j is inactive, removing dofs $dofs")
            C1[dofs,:] .= 0.0
            C2[dofs,:] .= 0.0
            D[dofs,:] .= 0.0
            g[dofs,:] .= 0.0
        elseif (is_active[j] == 1) && (is_slip[j] == 1)
            @debug("$j is in active/slip, removing tangential constraint $t")
            C2[t,:] .= 0.0
            g[t] = 0.0
            D[t, dofs] .= tangents[j]
        end
    end

    problem.assembly.C1 = C1
    problem.assembly.C2 = C2
    problem.assembly.D = D
    problem.assembly.g = sparsevec(g)

end


################################################################################
#  Copyright 2020, Tom Van Acker                                               #
################################################################################
# MultiStateSystems.jl                                                         #
# A Julia package to solve multi-state system models.                          #
# See http://github.com/timmyfaraday/MultiStateSystems.jl                      #
################################################################################

"""
The [Anholt Wind Farm](https://en.wikipedia.org/wiki/Anholt_Offshore_Wind_Farm)

"""

# Load pkgs
using Plots
using Unitful
using MultiStateSystems

# Pkg const
const _MSS = MultiStateSystems

# Settings for a specific analysis
cbl = true
mss = true
wtr = true
number_of_clusters = 8

# Include the stochastic models for the wind turbines and cables
include(joinpath(_MSS.BASE_DIR,"examples/anholt_wind_farm/elements/wind_turbine.jl"))
include(joinpath(_MSS.BASE_DIR,"examples/anholt_wind_farm/elements/sea_cable.jl"))

# Feeder 1 - Nodes 30, 62:65, 76, 109:111
ntwᶠ¹ = Network()
add_user!(ntwᶠ¹, node = 1)
add_sources!(ntwᶠ¹, node = 2:10, ntw = (ntwʷᵗ,1))
cbl ? add_components!(ntwᶠ¹, edge = [(1,2),(2,3),(3,4),(4,5),(5,6),(5,7),(3,8),(8,9),(9,10)],
                             std = [stdᶜᵇˡ[(8.500u"km",500u"mm^2")],stdᶜᵇˡ[(1.678u"km",500u"mm^2")],
                                    stdᶜᵇˡ[(0.603u"km",150u"mm^2")],stdᶜᵇˡ[(0.603u"km",150u"mm^2")],
                                    stdᶜᵇˡ[(0.603u"km",150u"mm^2")],stdᶜᵇˡ[(0.809u"km",150u"mm^2")],
                                    stdᶜᵇˡ[(1.464u"km",150u"mm^2")],stdᶜᵇˡ[(0.603u"km",150u"mm^2")],
                                    stdᶜᵇˡ[(0.603u"km",150u"mm^2")]]) :
     add_components!(ntwᶠ¹, edge = [(1,2),(2,3),(3,4),(4,5),(5,6),(5,7),(3,8),(8,9),(9,10)]) ;

# Feeder 2 - Nodes 23:29, 42, 61
ntwᶠ² = Network()
add_user!(ntwᶠ², node = 1)
add_sources!(ntwᶠ², node = 2:10, ntw = (ntwʷᵗ,1))
cbl ? add_components!(ntwᶠ², edge = [(1,2),(2,3),(3,4),(4,5),(5,6),(6,7),(7,8),(8,9),(9,10)],
                             std = [stdᶜᵇˡ[(4.500u"km",500u"mm^2")],stdᶜᵇˡ[(0.603u"km",500u"mm^2")],
                                    stdᶜᵇˡ[(0.603u"km",240u"mm^2")],stdᶜᵇˡ[(0.603u"km",240u"mm^2")],
                                    stdᶜᵇˡ[(0.603u"km",240u"mm^2")],stdᶜᵇˡ[(0.603u"km",150u"mm^2")],
                                    stdᶜᵇˡ[(0.603u"km",150u"mm^2")],stdᶜᵇˡ[(0.809u"km",150u"mm^2")],
                                    stdᶜᵇˡ[(0.809u"km",150u"mm^2")]]) :
      add_components!(ntwᶠ², edge = [(1,2),(2,3),(3,4),(4,5),(5,6),(6,7),(7,8),(8,9),(9,10)]) ;

# Feeder 3 - Nodes 22, 59, 60 102:108,
ntwᶠ³ = Network()
add_user!(ntwᶠ³, node = 1)
add_sources!(ntwᶠ³, node = 2:11, ntw = (ntwʷᵗ,1))
cbl ? add_components!(ntwᶠ³, edge = [(1,2),(2,3),(3,4),(4,5),(5,6),(6,7),(7,8),(8,9),(5,10),(10,11)],
                             std = [stdᶜᵇˡ[(4.000u"km",500u"mm^2")],stdᶜᵇˡ[(2.100u"km",500u"mm^2")],
                                    stdᶜᵇˡ[(0.691u"km",500u"mm^2")],stdᶜᵇˡ[(1.678u"km",240u"mm^2")],
                                    stdᶜᵇˡ[(0.603u"km",150u"mm^2")],stdᶜᵇˡ[(0.603u"km",150u"mm^2")],
                                    stdᶜᵇˡ[(0.603u"km",150u"mm^2")],stdᶜᵇˡ[(0.603u"km",150u"mm^2")],
                                    stdᶜᵇˡ[(0.603u"km",150u"mm^2")],stdᶜᵇˡ[(0.603u"km",150u"mm^2")]]) :
      add_components!(ntwᶠ³, edge = [(1,2),(2,3),(3,4),(4,5),(5,6),(6,7),(7,8),(8,9),(5,10),(10,11)]) ;

# Feeder 4 - Nodes 19:21, 56:58, 99:101
ntwᶠ⁴ = Network()
add_user!(ntwᶠ⁴, node = 1)
add_sources!(ntwᶠ⁴, node = 2:10, ntw = (ntwʷᵗ,1))
cbl ? add_components!(ntwᶠ⁴, edge = [(1,2),(2,3),(3,4),(4,5),(5,6),(3,7),(7,8),(2,9),(9,10)],
                             std = [stdᶜᵇˡ[(2.000u"km",500u"mm^2")],stdᶜᵇˡ[(2.100u"km",500u"mm^2")],
                                    stdᶜᵇˡ[(2.100u"km",150u"mm^2")],stdᶜᵇˡ[(0.603u"km",150u"mm^2")],
                                    stdᶜᵇˡ[(0.603u"km",150u"mm^2")],stdᶜᵇˡ[(0.873u"km",150u"mm^2")],
                                    stdᶜᵇˡ[(0.873u"km",150u"mm^2")],stdᶜᵇˡ[(0.691u"km",150u"mm^2")],
                                    stdᶜᵇˡ[(1.272u"km",150u"mm^2")]]) :
      add_components!(ntwᶠ⁴, edge = [(1,2),(2,3),(3,4),(4,5),(5,6),(3,7),(7,8),(2,9),(9,10)]) ;

# Feeder 5 - Nodes 17,18, 41, 54,55, 75, 96:98
ntwᶠ⁵ = Network()
add_user!(ntwᶠ⁵, node = 1)
add_sources!(ntwᶠ⁵, node = 2:10, ntw = (ntwʷᵗ,1))
cbl ? add_components!(ntwᶠ⁵, edge = [(1,2),(2,3),(3,4),(4,5),(5,6),(6,7),(6,8),(4,9),(2,10)],
                             std = [stdᶜᵇˡ[(2.000u"km",500u"mm^2")],stdᶜᵇˡ[(1.464u"km",240u"mm^2")],
                                    stdᶜᵇˡ[(1.095u"km",240u"mm^2")],stdᶜᵇˡ[(1.095u"km",150u"mm^2")],
                                    stdᶜᵇˡ[(1.095u"km",150u"mm^2")],stdᶜᵇˡ[(0.603u"km",150u"mm^2")],
                                    stdᶜᵇˡ[(0.603u"km",150u"mm^2")],stdᶜᵇˡ[(0.873u"km",150u"mm^2")],
                                    stdᶜᵇˡ[(0.691u"km",150u"mm^2")]]) :
      add_components!(ntwᶠ⁵, edge = [(1,2),(2,3),(3,4),(4,5),(5,6),(6,7),(6,8),(4,9),(2,10)]) ;

# Feeder 6 - Nodes 15,16, 40, 52,53, 74, 92:95
ntwᶠ⁶ = Network()
add_user!(ntwᶠ⁶, node = 1)
add_sources!(ntwᶠ⁶, node = 2:11, ntw = (ntwʷᵗ,1))
cbl ? add_components!(ntwᶠ⁶, edge = [(1,2),(2,3),(3,4),(4,5),(5,6),(6,7),(7,8),(6,9),(4,10),(2,11)],
                             std = [stdᶜᵇˡ[(2.000u"km",500u"mm^2")],stdᶜᵇˡ[(1.272u"km",500u"mm^2")],
                                    stdᶜᵇˡ[(1.095u"km",240u"mm^2")],stdᶜᵇˡ[(1.272u"km",240u"mm^2")],
                                    stdᶜᵇˡ[(1.464u"km",150u"mm^2")],stdᶜᵇˡ[(0.691u"km",150u"mm^2")],
                                    stdᶜᵇˡ[(0.809u"km",150u"mm^2")],stdᶜᵇˡ[(0.603u"km",150u"mm^2")],
                                    stdᶜᵇˡ[(0.873u"km",150u"mm^2")],stdᶜᵇˡ[(0.691u"km",150u"mm^2")]]) :
      add_components!(ntwᶠ⁶, edge = [(1,2),(2,3),(3,4),(4,5),(5,6),(6,7),(7,8),(6,9),(4,10),(2,11)]) ;

# Feeder 7 - Nodes 14, 39, 51, 73, 83,84, 88:91
ntwᶠ⁷ = Network()
add_user!(ntwᶠ⁷, node = 1)
add_sources!(ntwᶠ⁷, node = 2:11, ntw = (ntwʷᵗ,1))
cbl ? add_components!(ntwᶠ⁷, edge = [(1,2),(2,3),(3,4),(4,5),(5,6),(6,7),(7,8),(8,9),(9,10),(6,11)],
                             std = [stdᶜᵇˡ[(3.000u"km",500u"mm^2")],stdᶜᵇˡ[(1.272u"km",500u"mm^2")],
                                    stdᶜᵇˡ[(1.272u"km",500u"mm^2")],stdᶜᵇˡ[(1.464u"km",240u"mm^2")],
                                    stdᶜᵇˡ[(1.095u"km",240u"mm^2")],stdᶜᵇˡ[(1.272u"km",150u"mm^2")],
                                    stdᶜᵇˡ[(0.691u"km",150u"mm^2")],stdᶜᵇˡ[(0.691u"km",150u"mm^2")],
                                    stdᶜᵇˡ[(0.691u"km",150u"mm^2")],stdᶜᵇˡ[(0.809u"km",150u"mm^2")]]) :
      add_components!(ntwᶠ⁷, edge = [(1,2),(2,3),(3,4),(4,5),(5,6),(6,7),(7,8),(8,9),(9,10),(6,11)]) ;

# Feeder 8 - Nodes 13, 50, 72, 80:82, 85:87
ntwᶠ⁸ = Network()
add_user!(ntwᶠ⁸, node = 1)
add_sources!(ntwᶠ⁸, node = 2:10, ntw = (ntwʷᵗ,1))
cbl ? add_components!(ntwᶠ⁸, edge = [(1,2),(2,3),(3,4),(4,5),(5,6),(6,7),(7,8),(8,9),(9,10)],
                             std = [stdᶜᵇˡ[(3.000u"km",500u"mm^2")],stdᶜᵇˡ[(2.100u"km",500u"mm^2")],
                                    stdᶜᵇˡ[(1.464u"km",240u"mm^2")],stdᶜᵇˡ[(1.272u"km",240u"mm^2")],
                                    stdᶜᵇˡ[(0.809u"km",240u"mm^2")],stdᶜᵇˡ[(0.809u"km",150u"mm^2")],
                                    stdᶜᵇˡ[(1.464u"km",150u"mm^2")],stdᶜᵇˡ[(0.809u"km",150u"mm^2")],
                                    stdᶜᵇˡ[(0.691u"km",150u"mm^2")]]) :
      add_components!(ntwᶠ⁸, edge = [(1,2),(2,3),(3,4),(4,5),(5,6),(6,7),(7,8),(8,9),(9,10)]) ;

# Feeder 9 - Nodes 12, 38, 49, 69:71, 77:79
ntwᶠ⁹ = Network()
add_user!(ntwᶠ⁹, node = 1)
add_sources!(ntwᶠ⁹, node = 2:10, ntw = (ntwʷᵗ,1))
cbl ? add_components!(ntwᶠ⁹, edge = [(1,2),(2,3),(3,4),(4,5),(5,6),(6,7),(7,8),(8,9),(9,10)],
                             std = [stdᶜᵇˡ[(4.000u"km",500u"mm^2")],stdᶜᵇˡ[(1.464u"km",500u"mm^2")],
                                    stdᶜᵇˡ[(1.272u"km",240u"mm^2")],stdᶜᵇˡ[(1.678u"km",240u"mm^2")],
                                    stdᶜᵇˡ[(0.873u"km",240u"mm^2")],stdᶜᵇˡ[(0.873u"km",150u"mm^2")],
                                    stdᶜᵇˡ[(0.952u"km",150u"mm^2")],stdᶜᵇˡ[(0.809u"km",150u"mm^2")],
                                    stdᶜᵇˡ[(0.809u"km",150u"mm^2")]]) :
      add_components!(ntwᶠ⁹, edge = [(1,2),(2,3),(3,4),(4,5),(5,6),(6,7),(7,8),(8,9),(9,10)]) ;

# Feeder 10 - Nodes 11, 37, 45:48, 66:68
ntwᶠ¹⁰ = Network()
add_user!(ntwᶠ¹⁰, node = 1)
add_sources!(ntwᶠ¹⁰, node = 2:10, ntw = (ntwʷᵗ,1))
cbl ? add_components!(ntwᶠ¹⁰, edge = [(1,2),(2,3),(3,4),(4,5),(5,6),(6,7),(6,8),(8,9),(9,10)],
                              std = [stdᶜᵇˡ[(4.500u"km",500u"mm^2")],stdᶜᵇˡ[(1.678u"km",500u"mm^2")],
                                     stdᶜᵇˡ[(1.464u"km",240u"mm^2")],stdᶜᵇˡ[(0.952u"km",240u"mm^2")],
                                     stdᶜᵇˡ[(0.952u"km",240u"mm^2")],stdᶜᵇˡ[(0.952u"km",150u"mm^2")],
                                     stdᶜᵇˡ[(0.873u"km",150u"mm^2")],stdᶜᵇˡ[(0.809u"km",150u"mm^2")],
                                     stdᶜᵇˡ[(0.809u"km",150u"mm^2")]]) :
      add_components!(ntwᶠ¹⁰, edge = [(1,2),(2,3),(3,4),(4,5),(5,6),(6,7),(6,8),(8,9),(9,10)]) ;

# Feeder 11 - Nodes 10, 31:36, 43,44
ntwᶠ¹¹ = Network()
add_user!(ntwᶠ¹¹, node = 1)
add_sources!(ntwᶠ¹¹, node = 2:10, ntw = (ntwʷᵗ,1))
cbl ? add_components!(ntwᶠ¹¹, edge = [(1,2),(2,3),(3,4),(4,5),(5,6),(6,7),(7,8),(6,9),(9,10)],
                              std = [stdᶜᵇˡ[(5.000u"km",500u"mm^2")],stdᶜᵇˡ[(1.678u"km",500u"mm^2")],
                                     stdᶜᵇˡ[(0.873u"km",240u"mm^2")],stdᶜᵇˡ[(0.873u"km",240u"mm^2")],
                                     stdᶜᵇˡ[(0.873u"km",240u"mm^2")],stdᶜᵇˡ[(0.873u"km",150u"mm^2")],
                                     stdᶜᵇˡ[(0.809u"km",150u"mm^2")],stdᶜᵇˡ[(0.691u"km",150u"mm^2")],
                                     stdᶜᵇˡ[(0.809u"km",150u"mm^2")]]) :
      add_components!(ntwᶠ¹¹, edge = [(1,2),(2,3),(3,4),(4,5),(5,6),(6,7),(7,8),(6,9),(9,10)]) ;

# Feeder 12 - Nodes
ntwᶠ¹² = Network()
add_user!(ntwᶠ¹², node = 1)
add_sources!(ntwᶠ¹², node = 2:10, ntw = (ntwʷᵗ,1))
cbl ? add_components!(ntwᶠ¹², edge = [(1,2),(2,3),(3,4),(4,5),(5,6),(6,7),(7,8),(8,9),(9,10)],
                              std = [stdᶜᵇˡ[(5.750u"km",500u"mm^2")],stdᶜᵇˡ[(0.603u"km",500u"mm^2")],
                                     stdᶜᵇˡ[(0.603u"km",240u"mm^2")],stdᶜᵇˡ[(0.603u"km",240u"mm^2")],
                                     stdᶜᵇˡ[(0.603u"km",240u"mm^2")],stdᶜᵇˡ[(0.603u"km",150u"mm^2")],
                                     stdᶜᵇˡ[(0.603u"km",150u"mm^2")],stdᶜᵇˡ[(0.603u"km",150u"mm^2")],
                                     stdᶜᵇˡ[(0.603u"km",150u"mm^2")]]) :
      add_components!(ntwᶠ¹², edge = [(1,2),(2,3),(3,4),(4,5),(5,6),(6,7),(7,8),(8,9),(9,10)]) ;

# Feeders 1-6
ntwᶠ¹⁻⁶ = Network()
add_user!(ntwᶠ¹⁻⁶, node = 1)
add_sources!(ntwᶠ¹⁻⁶, node = 1, ntw = [(ntwᶠ¹,1),(ntwᶠ²,1),(ntwᶠ³,1),(ntwᶠ⁴,1),(ntwᶠ⁵,1),(ntwᶠ⁶,1)])

# Feeders 7-12
ntwᶠ⁷⁻¹² = Network()
add_user!(ntwᶠ⁷⁻¹², node = 1)
add_sources!(ntwᶠ⁷⁻¹², node = 1, ntw = [(ntwᶠ⁷,1),(ntwᶠ⁸,1),(ntwᶠ⁹,1),(ntwᶠ¹⁰,1),(ntwᶠ¹¹,1),(ntwᶠ¹²,1)])

# Overall network
ntw = Network()
add_user!(ntw, node = 1, ind = [:GRA,:EENS])
add_sources!(ntw, node = 1, ntw = [(ntwᶠ¹⁻⁶,1),(ntwᶠ⁷⁻¹²,1)])

# Solve the problem
@time solve!(ntw)

# Plot the probability distribution (p>=1e-5)
ugf = ntw.usr[1][:ugf]
scatter(ustrip.(ugf.val[ugf.prb.>=1e-5]),ugf.prb[ugf.prb.>=1e-5],legend=false)


baremodule Monoids
    using ..Types
end
MonoidUnion = Union{AbstractMonoid, libgb.GrB_Monoid}
function monoidnames(name)
    simple = splitconstant(name)[2]
    containername = Symbol(simple, "_MONOID_T")
    exportedname = Symbol(simple * "_MONOID")
    return containername, exportedname
end
function createmonoids()
    builtins = ["GrB_MIN", "GrB_MAX", "GrB_PLUS", "GrB_TIMES", "GxB_ANY", "GrB_LOR", "GrB_LAND", "GrB_LXOR", "GrB_LXNOR", "GxB_EQ", "GxB_BOR", "GxB_BAND", "GxB_BXOR", "GxB_BXNOR"]
    for name ∈ builtins
        containername, exportedname = monoidnames(name)
        structquote = quote
            struct $containername <: AbstractMonoid
                pointers::Dict{DataType, libgb.GrB_Monoid}
                name::String
                $containername() = new(Dict{DataType, libgb.GrB_Monoid}(), $name)
            end
        end
        @eval(Types, $structquote)
        constquote = quote
            const $exportedname = Types.$containername()
            export $exportedname
        end
        @eval(Monoids, $constquote)
    end
end

function load(monoid::AbstractMonoid)
    booleans = ["GxB_ANY", "GrB_LOR", "GrB_LAND", "GrB_LXOR", "GrB_LXNOR", "GxB_EQ"]
    integers = ["GrB_MIN", "GrB_MAX", "GrB_PLUS", "GrB_TIMES", "GxB_ANY"]
    unsignedintegers = ["GrB_MIN", "GrB_MAX", "GrB_PLUS", "GrB_TIMES", "GxB_ANY", "GxB_BOR", "GxB_BAND", "GxB_BXOR", "GxB_BXNOR"]
    floats = ["GrB_MIN", "GrB_MAX", "GrB_PLUS", "GrB_TIMES", "GxB_ANY"]
    name = monoid.name

    if name ∈ booleans
        constname = name * ((isGxB(name) ? "_BOOL_MONOID" : "_MONOID_BOOL"))
        monoid.pointers[Bool] = load_global(constname)
    end

    if name ∈ integers
        monoid.pointers[Int8] = load_global(name * (isGxB(name) ? "_INT8_MONOID" : "_MONOID_INT8"))
        monoid.pointers[Int16] = load_global(name * (isGxB(name) ? "_INT16_MONOID" : "_MONOID_INT16"))
        monoid.pointers[Int32] = load_global(name * (isGxB(name) ? "_INT32_MONOID" : "_MONOID_INT32"))
        monoid.pointers[Int64] = load_global(name * (isGxB(name) ? "_INT64_MONOID" : "_MONOID_INT64"))
    end

    if name ∈ unsignedintegers
        monoid.pointers[UInt8] = load_global(name * (isGxB(name) ? "_UINT8_MONOID" : "_MONOID_UINT8"))
        monoid.pointers[UInt16] = load_global(name * (isGxB(name) ? "_UINT16_MONOID" : "_MONOID_UINT16"))
        monoid.pointers[UInt32] = load_global(name * (isGxB(name) ? "_UINT32_MONOID" : "_MONOID_UINT32"))
        monoid.pointers[UInt64] = load_global(name * (isGxB(name) ? "_UINT64_MONOID" : "_MONOID_UINT64"))
    end

    if name ∈ floats
        monoid.pointers[Float32] = load_global(name * (isGxB(name) ? "_FP32_MONOID" : "_MONOID_FP32"))
        monoid.pointers[Float64] = load_global(name * (isGxB(name) ? "_FP64_MONOID" : "_MONOID_FP64"))
    end
end

operator(monoid::MonoidUnion) = libgb.GxB_Monoid_operator(monoid)
xtype(monoid::MonoidUnion) = xtype(operator(monoid))
ytype(monoid::MonoidUnion) = ytype(operator(monoid))
ztype(monoid::MonoidUnion) = ztype(operator(monoid))


module MegTools

using AxisKeys
using Infiltrator
using LinearAlgebra
using MAT
using Measures
using NamedDims
using Statistics

# Paths are based on where this module is
# Once you're done with your script and feel you want to generalise it,
# then the functions are put here or 'include'-ed  here
# E.g. include("../scripts/ReadContEps.jl")

include("IO_Meg.jl")
export load_besa_av, load_cont_epochs, load_BSepochs

include("analysis_recipes.jl")
export average_across_trials, select_channels, baseline_correction, find_peaks, collect_peaks,
    find_mean_amplitude, collect_mean_amps, get_averaged_baseline

include("filters.jl")
export highlow_butterworth_filter

include("trigger_keys.jl")
export load_trigger_values

end


module kNNRecommenders

using CSV
using Random
using SparseArrays

export MovieLens, load_data, split_data, transform_to_rating_matrix
export kNNRecommender, compute_similarity_matrix, predict, evaluate_recall

include("MovieLens.jl")
include("ItemkNN.jl")

end # module


"""
Return default marking value based on `PNTD`. Has meaning of empty, as in `zero`.
"""
function default_term end
default_term(::PNTD) where {PNTD <: PnmlType} = zero(Integer) #!
default_term(::PNTD) where {PNTD <: AbstractContinuousCore} = zero(Float64) #!
default_term(::PNTD) where {PNTD <: AbstractHLCore} = Term() #!

"""
$(TYPEDEF)
$(TYPEDFIELDS)

Part of the many-sorted algebra attached to nodes on a Petri Net Graph.

 ast variants:
  - variable
  - operator
"""
struct Term{T<:AbstractDict}  <: AbstractTerm #TODO make mutable?
  tag::Symbol
  dict::T
  #TODO xml
end

Term() = Term(:empty, PnmlDict())

Term(p::Pair{Symbol,PnmlDict}; kw...) = Term(p.first, p.second)

convert(::Type{Maybe{Term}}, pdict::PnmlDict) = Term(pdict)

tag(t::Term) = t.tag
dict(t::Term) = t.dict
#xml(t::Term) = t.xml

"""
Evaluate a term by returning the ':value' in `dict`.
Assumes that `dict` is an `AbstractDictionary`.
"""
(t::Term)() = get(t.dict, :value, 1)


numeric_predicates = [:(Base.iseven), :(Base.ispow2), :(Base.isfinite), :(Base.isinf), :(Base.isodd)]
isdefined(Base, :isprime) && push!(numeric_predicates, :(Base.isprime))

type_predicates = (:(Base.isinteger), :(Base.isreal))

container_predicates = (:(Base.isempty),)

for p in numeric_predicates
  @eval begin
    ($p)(v::NAtype) = NA
  end
end

for p in type_predicates
  @eval begin
    ($p)(v::NAtype) = NA
  end
end


module BackpropNeuralNet

export
    # types
    NeuralNetwork,

    # functions
    init_network,
    train,
    net_eval

include("neural_net.jl")

end # module


module Flist                    # feature matrices
using KUparser: SFeature, DFeature # sparse and dense features

gn13cpv = DFeature["s0v","s1v","n0v","n1v",
	           "s0c","s1c","n0c","n1c","s1rc","s1lc","s0lc","s0rc","n0lc",
	           "s0p","s1p","n0p","n1p","s1rp","s1lp","s0lp","s0rp","n0lp"]

# ZN11 dense variants.  The original has:
# (w,p) for (s,n,n1,n2,sh,sh2,sl,sr,nl,sl2,sr2,nl2): 24 features, 1200 dims for v,c; 540 dims for p
# L for (s,sh,sl,sr,nl,sl2,sr2,nl2): 8 features, 96 dims
# sd, sa, sb, sA, sB, na, nA: 7 features, 69 dims
# We replace (w,p) with 8 subsets of (v,c,p)

zn11n = DFeature["sA","nA","sa","na","sB","sb","sd",
                 "sL","shL","slL","srL","nlL","sl2L","sr2L","nl2L"]

zn11p = DFeature["sA","nA","sa","na","sB","sb","sd",
                 "sL","shL","slL","srL","nlL","sl2L","sr2L","nl2L",
                 "sp","np","n1p","n2p","shp","sh2p","slp","srp","nlp","sl2p","sr2p","nl2p"]

zn11v = DFeature["sA","nA","sa","na","sB","sb","sd",
                 "sL","shL","slL","srL","nlL","sl2L","sr2L","nl2L",
                 "sv","nv","n1v","n2v","shv","sh2v","slv","srv","nlv","sl2v","sr2v","nl2v"]

zn11c = DFeature["sA","nA","sa","na","sB","sb","sd",
                 "sL","shL","slL","srL","nlL","sl2L","sr2L","nl2L",
                 "sc","nc","n1c","n2c","shc","sh2c","slc","src","nlc","sl2c","sr2c","nl2c"]

zn11pv = DFeature["sA","nA","sa","na","sB","sb","sd",
                  "sL","shL","slL","srL","nlL","sl2L","sr2L","nl2L",
                  "sp","np","n1p","n2p","shp","sh2p","slp","srp","nlp","sl2p","sr2p","nl2p",
                  "sv","nv","n1v","n2v","shv","sh2v","slv","srv","nlv","sl2v","sr2v","nl2v"]

zn11cv = DFeature["sA","nA","sa","na","sB","sb","sd",
                  "sL","shL","slL","srL","nlL","sl2L","sr2L","nl2L",
                  "sc","nc","n1c","n2c","shc","sh2c","slc","src","nlc","sl2c","sr2c","nl2c",
                  "sv","nv","n1v","n2v","shv","sh2v","slv","srv","nlv","sl2v","sr2v","nl2v"]

zn11cp = DFeature["sA","nA","sa","na","sB","sb","sd",
                  "sL","shL","slL","srL","nlL","sl2L","sr2L","nl2L",
                  "sc","nc","n1c","n2c","shc","sh2c","slc","src","nlc","sl2c","sr2c","nl2c",
                  "sp","np","n1p","n2p","shp","sh2p","slp","srp","nlp","sl2p","sr2p","nl2p"]

zn11cpv = DFeature["sA","nA","sa","na","sB","sb","sd",
                   "sL","shL","slL","srL","nlL","sl2L","sr2L","nl2L",
                   "sc","nc","n1c","n2c","shc","sh2c","slc","src","nlc","sl2c","sr2c","nl2c",
                   "sp","np","n1p","n2p","shp","sh2p","slp","srp","nlp","sl2p","sr2p","nl2p",
                   "sv","nv","n1v","n2v","shv","sh2v","slv","srv","nlv","sl2v","sr2v","nl2v"]

# Sparse feature sets
#
# Each string specifies a particular feature and has the form:
#   [sn]\d?([hlr]\d?)*[wpdLabAB]
#
# The meaning of each letter is below.  In the following i is a single
# digit integer which is optional if the default value is used:
#
# si: i'th stack word, default i=0 means top
# ni: i'th buffer word, default i=0 means first
# hi: i'th degree head, default i=1 means direct head
# li: i'th leftmost child, default i=1 means the leftmost child
# ri: i'th rightmost child, default i=1 means the rightmost child
# w: word
# p: postag
# d: distance to the right.  e.g. s1d is s0s1 distance, s0d is s0n0
#    distance.  encoding: 1,2,3,4,5-10,10+ (from ZN11)
# L: dependency label (0 is ROOT or NONE)
# a: number of left children.  following ZN11, any number is allowed.
# b: number of right children.  following ZN11, any number is allowed.
# A: set of left dependency labels
# B: set of right dependency labels

# The original zn11 set in sparse format with feature conjunctions
zn11orig = SFeature[
["sw"],                  # STw		StackWord
["sp"],                  # STt		StackTag
["sw","sp"],             # STwt		StackWordTag
["nw"],                  # N0w		NextWord
["np"],                  # N0t		NextTag
["nw","np"],             # N0wt		NextWordTag
["n1w"],                 # N1w		Next+1Word
["n1p"],                 # N1t		Next+1Tag
["n1w","n1p"],           # N1wt		Next+1WordTag
["n2w"],                 # N2w		Next+2Word
["n2p"],                 # N2t		Next+2Tag
["n2w","n2p"],           # N2wt		Next+2WordTag
["shw"],                 # STHw		StackHeadWord
["shp"],                 # STHt		StackHeadTag
["sL"],                  # STi		StackLabel
["sh2w"],                # STHHw		StackHeadHeadWord
["sh2p"],                # STHHt		StackHeadHeadTag
["shL"],                 # STHi		StackLabel ?? StackHeadLabel
["slw"],                 # STLDw		StackLDWord
["slp"],                 # STLDt		StackLDTag
["slL"],                 # STLDi		StackLDLabel
["srw"],                 # STRDw		StackRDWord
["srp"],                 # STRDt		StackRDTag
["srL"],                 # STRDi		StackRDLabel
["nlw"],                 # N0LDw		NextLDWord
["nlp"],                 # N0LDt		NextLDTag
["nlL"],                 # N0LDi		NextLDLabel
["sl2w"],                # STL2Dw	StackL2DWord
["sl2p"],                # STL2Dt	StackL2DTag
["sl2L"],                # STL2Di	StackL2DLabel
["sr2w"],                # STR2Dw	StackR2DWord
["sr2p"],                # STR2Dt	StackR2DTag
["sr2L"],                # STR2Di	StackR2DLabel
["nl2w"],                # N0L2Dw	NextL2DWord
["nl2p"],                # N0L2Dt	NextL2DTag
["nl2L"],                # N0L2Di	NextL2DLabel
			## HTw		HeadStackWord
			## HTt		HeadStackTag
			## HTwt		HeadStackWordTag
["sw","sp","nw","np"],	# STwtN0wt	StackWordTagNextWordTag
["sw","sp","nw"],	# STwtN0w	StackWordTagNextWord
["sw","nw","np"],	# STwN0wt	StackWordNextWordTag
["sp","nw","np"],	# STtN0wt	StackTagNextWordTag
["sw","sp","np"],	# STwtN0t	StackWordTagNextTag
["sw","nw"],             # STwN0w	StackWordNextWord
["sp","np"],             # STtN0t	StackTagNextTag
["np","n1p"],            # N0tN1t	NextTagNext+1Tag
["np","n1p","n2p"],	# N0tN1tN2t	NextTagTrigram
["sp","np","n1p"],	# STtN0tN1t	StackTagNextTagNext+1Tag
["sp","np","nlp"],	# STtN0tN0LDt	StackTagNextTagNextLDTag
["np","nlp","nl2p"],     # N0tN0LDtN0L2Dt	StackTagNextTagNextLDTagNextTagNextL2DTag ?? NextTagNextLDTagNextL2DTag
["shp","sp","np"],       # STHtSTtN0t	StackHeadTagStackTagNextTag
			## HTtHT2tN0t	HeadStackTagHeadStack2TagNextTag
["sh2p","shp","sp"],     # STHHtSTHtSTt	StackHeadHeadTagStackHeadTagStackTag
["sp","slp","np"],       # STtSTLDtN0t	StackTagStackLDTagNextTag
["sp","slp","sl2p"],     # STtSTLDtSTL2Dt	StackTagStackLDTagStackL2DTag
["sp","srp","np"],       # STtSTRDtN0t	StackTagStackRDTagNextTag
["sp","srp","sr2p"],     # STtSTRDtSTR2Dt	StackTagStackRDTagStackR2DTag
["sw","sd"],             # STwd		StackWordDist
["sp","sd"],             # STtd		StackTagDist
["nw","sd"],             # N0wd		NextWordDist
["np","sd"],             # N0td		NextTagDist
["sw","nw","sd"],        # STwN0wd	StackWordNextWordDist
["sp","np","sd"],        # STtN0td	StackTagNextTagDist
["sw","sb"],             # STwra		StackWordRightArity
["sp","sb"],             # STtra		StackTagRightArity
["sw","sa"],             # STwla		StackWordLeftArity
["sp","sa"],             # STtla		StackTagLeftArity
["nw","na"],             # N0wla		NextWordRightArity ?? NextWordLeftArity
["np","na"],             # N0tla		NextTagRightArity  ?? NextTagLeftArity
["sw","sB"],             # STwrp		StackWordRightSetoftags
["sp","sB"],             # STtrp		StackTagRightSetoftags
["sw","sA"],             # STwlp		StackWordLeftSetoftags
["sp","sA"],             # STtlp		StackTagLeftSetoftags
["nw","nA"],             # N0wlp		Next0WordLeftSetoftags ?? NextWordLeftSetoftags
["np","nA"],             # N0tlp		Next0TagLeftSetoftags  ?? NextTagLeftSetoftags
			## STl		StackLemma
			## STc		StackCPOS
			## STf		StackFeats
			## N0l		NextLemma
			## N0c		NextCPOS
			## N0f		NextFeats
			## N1l		Next+1Lemma
			## N1c		Next+1CPOS
			## N1f		Next+1Feats
]

# The reduced zn11 set in sparse format with singleton features
zn11single = SFeature[
["sw"],                  # STw		StackWord
["sp"],                  # STt		StackTag
#["sw","sp"],             # STwt		StackWordTag
["nw"],                  # N0w		NextWord
["np"],                  # N0t		NextTag
#["nw","np"],             # N0wt		NextWordTag
["n1w"],                 # N1w		Next+1Word
["n1p"],                 # N1t		Next+1Tag
#["n1w","n1p"],           # N1wt		Next+1WordTag
["n2w"],                 # N2w		Next+2Word
["n2p"],                 # N2t		Next+2Tag
#["n2w","n2p"],           # N2wt		Next+2WordTag
["shw"],                 # STHw		StackHeadWord
["shp"],                 # STHt		StackHeadTag
["sL"],                  # STi		StackLabel
["sh2w"],                # STHHw		StackHeadHeadWord
["sh2p"],                # STHHt		StackHeadHeadTag
["shL"],                 # STHi		StackLabel ?? StackHeadLabel
["slw"],                 # STLDw		StackLDWord
["slp"],                 # STLDt		StackLDTag
["slL"],                 # STLDi		StackLDLabel
["srw"],                 # STRDw		StackRDWord
["srp"],                 # STRDt		StackRDTag
["srL"],                 # STRDi		StackRDLabel
["nlw"],                 # N0LDw		NextLDWord
["nlp"],                 # N0LDt		NextLDTag
["nlL"],                 # N0LDi		NextLDLabel
["sl2w"],                # STL2Dw	StackL2DWord
["sl2p"],                # STL2Dt	StackL2DTag
["sl2L"],                # STL2Di	StackL2DLabel
["sr2w"],                # STR2Dw	StackR2DWord
["sr2p"],                # STR2Dt	StackR2DTag
["sr2L"],                # STR2Di	StackR2DLabel
["nl2w"],                # N0L2Dw	NextL2DWord
["nl2p"],                # N0L2Dt	NextL2DTag
["nl2L"],                # N0L2Di	NextL2DLabel
			## HTw		HeadStackWord
			## HTt		HeadStackTag
			## HTwt		HeadStackWordTag
#["sw","sp","nw","np"],	# STwtN0wt	StackWordTagNextWordTag
#["sw","sp","nw"],	# STwtN0w	StackWordTagNextWord
#["sw","nw","np"],	# STwN0wt	StackWordNextWordTag
#["sp","nw","np"],	# STtN0wt	StackTagNextWordTag
#["sw","sp","np"],	# STwtN0t	StackWordTagNextTag
#["sw","nw"],             # STwN0w	StackWordNextWord
#["sp","np"],             # STtN0t	StackTagNextTag
#["np","n1p"],            # N0tN1t	NextTagNext+1Tag
#["np","n1p","n2p"],	# N0tN1tN2t	NextTagTrigram
#["sp","np","n1p"],	# STtN0tN1t	StackTagNextTagNext+1Tag
#["sp","np","nlp"],	# STtN0tN0LDt	StackTagNextTagNextLDTag
#["np","nlp","nl2p"],     # N0tN0LDtN0L2Dt	StackTagNextTagNextLDTagNextTagNextL2DTag ?? NextTagNextLDTagNextL2DTag
#["shp","sp","np"],       # STHtSTtN0t	StackHeadTagStackTagNextTag
			## HTtHT2tN0t	HeadStackTagHeadStack2TagNextTag
#["sh2p","shp","sp"],     # STHHtSTHtSTt	StackHeadHeadTagStackHeadTagStackTag
#["sp","slp","np"],       # STtSTLDtN0t	StackTagStackLDTagNextTag
#["sp","slp","sl2p"],     # STtSTLDtSTL2Dt	StackTagStackLDTagStackL2DTag
#["sp","srp","np"],       # STtSTRDtN0t	StackTagStackRDTagNextTag
#["sp","srp","sr2p"],     # STtSTRDtSTR2Dt	StackTagStackRDTagStackR2DTag
["sd"],
#["sw","sd"],             # STwd		StackWordDist
#["sp","sd"],             # STtd		StackTagDist
#["nw","sd"],             # N0wd		NextWordDist
#["np","sd"],             # N0td		NextTagDist
#["sw","nw","sd"],        # STwN0wd	StackWordNextWordDist
#["sp","np","sd"],        # STtN0td	StackTagNextTagDist
["sb"],
#["sw","sb"],             # STwra		StackWordRightArity
#["sp","sb"],             # STtra		StackTagRightArity
["sa"],
#["sw","sa"],             # STwla		StackWordLeftArity
#["sp","sa"],             # STtla		StackTagLeftArity
["na"],
#["nw","na"],             # N0wla		NextWordRightArity ?? NextWordLeftArity
#["np","na"],             # N0tla		NextTagRightArity  ?? NextTagLeftArity
["sB"],
#["sw","sB"],             # STwrp		StackWordRightSetoftags
#["sp","sB"],             # STtrp		StackTagRightSetoftags
["sA"],
#["sw","sA"],             # STwlp		StackWordLeftSetoftags
#["sp","sA"],             # STtlp		StackTagLeftSetoftags
["nA"],
#["nw","nA"],             # N0wlp		Next0WordLeftSetoftags ?? NextWordLeftSetoftags
#["np","nA"],             # N0tlp		Next0TagLeftSetoftags  ?? NextTagLeftSetoftags
			## STl		StackLemma
			## STc		StackCPOS
			## STf		StackFeats
			## N0l		NextLemma
			## N0c		NextCPOS
			## N0f		NextFeats
			## N1l		Next+1Lemma
			## N1c		Next+1CPOS
			## N1f		Next+1Feats
]

# Dense feature sets:

# Each string specifies a particular feature and has the form:
#   [sn]\d?([hlr]\d?)*[vcpdLabAB]
#
# The meaning of each letter is below.  In the following i is a single
# digit integer which is optional if the default value is used:
#
# si: i'th stack word, default i=0 means top
# ni: i'th buffer word, default i=0 means first
# hi: i'th degree head, default i=1 means direct head
# li: i'th leftmost child, default i=1 means the leftmost child
# ri: i'th rightmost child, default i=1 means the rightmost child
# v: word vector
# c: context vector
# p: postag
# d: distance to the right.  e.g. s1d is s0s1 distance, s0d is s0n0
#    distance.  encoding: 1,2,3,4,5-10,10+ (from ZN11)
# L: dependency label (0 is ROOT or NONE)
# a: number of left children.  encoding: 0,1,...,8,9+
# b: number of right children.  encoding: 0,1,...,8,9+
# A: set of left dependency labels
# B: set of right dependency labels

# fs11h54h27 (9144): local best for acl11, ArcHybrid13, start=hybrid27, all=hybrid54, bparser10, shuffled
hybrid25 = split("n0A n0a n0lL n0lc n0lv n0c n0v n1c n1v s0A s0B s0a s0b s0d s0c s0rL s0v s1A s1B s1a s1rL s1rc s1rv s1v s2v")

# fs11e64e27 (9098): local best for acl11, ArcEager13, start=eager27=hybrid27, all=eager64, bparser10, shuffled
eager25 = split("n0A n0a n0l2L n0lL n0lc n0lv n0c n0v n1c n1v n2c n2v s0B s0c s0v s1A s1B s1a s1d s1c s1rL s1rc s1v s2c s2v")

# fs11h54h28 (8997): local best in acl11, ArcEager13 (by mistake), start=hybrid28, all=hybrid54
hybrid27 = split("n0A n0a n0lL n0lc n0lv n0c n0v n1c n1v s0A s0B s0a s0d s0c s0v s1A s1B s1a s1b s1d s1c s1rL s1rc s1rv s1v s2c s2v")
eager27 = hybrid27

# fs11h54h14 (8992): local best for acl11, ArcHybrid13, start=hybrid14, all=hybrid54
hybrid25b = split("n0a n0l2c n0l2v n0lc n0lv n0v n1c n1v n2v s0B s0a s0d s0l2L s0lL s0lc s0c s0rc s0v s1B s1b s1lL s1lc s1c s1r2c s1v")

# fs11e64e20 (8926): local best for acl11, ArcEager13, start=eager20, all=eager64
eager20 = split("n0A n0a n0lL n0c n0v n1c n1v n2c n2v s0B s0L s0b s0d s0hc s0hv s0c s0rL s0v s1c s1v")

# fs11e39e21 (8906): local best for acl11, ArcEager13, start=eager21, all=eager39
eager23 = split("n0A n0lL n0lc n0c n0v n1c n1v n2c n2v s0B s0L s0a s0b s0h2c s0hv s0l2L s0l2c s0l2v s0lc s0c s0r2L s0r2c s0v")

# fs07h54h28 (8797): local best for conll07, ArcHybrid13, start=hybrid28, all=hybrid54
hybrid28b = split("n0A n0a n0l2v n0lL n0lc n0c n0v n1c n1v n2v s0A s0B s0a s0b s0d s0c s0v s1A s1a s1b s1d s1c s1rL s1rc s1rv s1v s2c s2v")

# fs07h54h13 (8791): Local best for conll07, ArcHybrid13, start=tacl13hybrid (aka hybrid13), all=hybrid54
hybrid14 = split("n0lc n0lv n0v n1c n1v s0d s0lc s0c s0rc s0v s1B s1lc s1c s1v")

# fs07e39e39 (8725): local best in conll07, ArcEager13, start=acl11eager, all=acl11eager
eager36 = split("n0A n0a n0l2L n0l2c n0l2v n0lL n0lc n0lv n0c n0v n1c n1v n2c n2v s0A s0B s0L s0b s0d s0h2c s0hL s0hc s0hv s0l2L s0l2c s0l2v s0lL s0lc s0lv s0c s0r2c s0r2v s0rL s0rc s0rv s0v")

# fs07e39e21 (8705): Local best for conll07, ArcEager13, start=eager21, all=eager39
eager20b = split("n0l2L n0lv n0c n0v n1v n2c n2v s0B s0L s0a s0h2c s0hv s0l2L s0l2c s0l2v s0lc s0c s0r2L s0r2c s0v")

# fs07e64e20 (8677): Local best for conll07, ArcEager13, start=eager20, all=eager64
eager21b = split("n0A n0a n0l2L n0lL n0c n0v n1c n1v n2c n2v s0B s0L s0b s0d s0hc s0hv s0c s0rL s0v s1c s1v")

# more complete set than acl11eager: added s1 features
eager64 = split("n0A n0a n0l2L n0l2c n0l2v n0lL n0lc n0lv n0c n0v n1c n1v n2c n2v s0A s0B s0L s0a s0b s0d s0h2c s0h2v s0hL s0hc s0hv s0l2L s0l2c s0l2v s0lL s0lc s0lv s0c s0r2L s0r2c s0r2v s0rL s0rc s0rv s0v s1A s1B s1L s1a s1b s1d s1h2c s1h2v s1hL s1hc s1hv s1l2L s1l2c s1l2v s1lL s1lc s1lv s1c s1r2L s1r2c s1r2v s1rL s1rc s1rv s1v")

# Manually constructed starting point
eager21 = split("n0A n0lL n0c n0v n1c n1v n2c n2v s0B s0L s0a s0h2c s0hv s0l2L s0l2c s0l2v s0lc s0c s0r2L s0r2c s0v")

# Manually constructed starting point
hybrid28 = split("n0A n0a n0lL n0lc n0lv n0c n0v n1c n1v s0A s0B s0a s0b s0d s0c s0v s1A s1B s1a s1b s1d s1c s1rL s1rc s1rv s1v s2c s2v")

# Features that may be relevant for ArcHybrid13
hybrid54 = DFeature[
"n0v", "n0c", "n0A", "n0a",
"s0v", "s0c", "s0A", "s0a", "s0B", "s0b", "s0d",
"s1v", "s1c", "s1A", "s1a", "s1B", "s1b", "s1d",
"n1v", "n1c", "n2v", "n2c", "s2v", "s2c",
"n0lv", "n0l2v", "s0lv", "s0l2v", "s0rv", "s0r2v", "s1lv", "s1l2v", "s1rv", "s1r2v",
"n0lc", "n0l2c", "s0lc", "s0l2c", "s0rc", "s0r2c", "s1lc", "s1l2c", "s1rc", "s1r2c",
"n0lL", "n0l2L", "s0lL", "s0l2L", "s0rL", "s0r2L", "s1lL", "s1l2L", "s1rL", "s1r2L"
]

# Goldberg&Nivre TACL13 ArcHybrid features from: tacl2013dynamicoracles/lefttoright/features:HybridFeatures
# s0,s1,s2: stack word forms: a=-1,-2,-3 b=0 c=4
# w0,w1: buffer word forms: a=0,1 b=0 c=4
# Ts0: tag of s0 (-1 0 -4)
# Trcs1: tag of rightmost child of s1 (-2 1 -4)
# use word half of vector (c=+4) for form, context half of vector (c=-4) for tag
# No need for feature combinations, we have a nonlinear model.

tacl13hybrid = DFeature[
	# (1)
"s0v",	# append("s0_%s" % s0)
"s0c",	# append("Ts0_%s" % Ts0)
        # append("Ts0s0_%s_%s" % (Ts0, s0))

"s1v",	# append("s1_%s" % s1)
"s1c",	# append("Ts1_%s" % Ts1)
     	# append("Ts1s1_%s_%s" % (Ts1, s1))

"n0v",	# append("w0_%s" % w0)
"n0c",	# append("Tw0_%s" % Tw0)
	# append("Tw0w0_%s_%s" % (Tw0, w0))
	# +hybrid
"n1v",	# append("w1_%s" % w1)
"n1c",	# append("Tw1_%s" % Tw1)
	# append("Tw1w1_%s_%s" % (Tw1, w1))

	# (2)
	# append("s0s1_%s_%s" % (s0,s1))
	# append("Ts0Ts1_%s_%s" % (Ts0,Ts1))
	# append("Ts0Tw0_%s_%s" % (Ts0,Tw0))
	# append("s0Ts0Ts1_%s_%s_%s" % (s0,Ts0,Ts1))
	# append("Ts0s1Ts1_%s_%s_%s" % (Ts0,s1,Ts1))
	# append("s0s1Ts1_%s_%s_%s" % (s0,s1,Ts1))
	# append("s0Ts0s1_%s_%s_%s" % (s0,Ts0,s1))
	# append("s0Ts0Ts1_%s_%s_%s" % (s0,Ts0,Ts1))
	# +hybrid
	# append("s0w0_%s_%s" % (s0,w0))
	# append("Ts0Tw0_%s_%s" % (Ts0,Tw0))
	# append("Ts0Tw1_%s_%s" % (Ts0,Tw1))
	# append("s0Ts0Tw0_%s_%s_%s" % (s0,Ts0,Tw0))
	# append("Ts0w0Tw0_%s_%s_%s" % (Ts0,w0,Tw0))
	# append("s0w0Tw0_%s_%s_%s" % (s0,w0,Tw0))
	# append("s0Ts0w0_%s_%s_%s" % (s0,Ts0,w0))
	# append("w0Tw0Tw1_%s_%s_%s" % (w0,Tw0,Tw1)) #

	# (3)
	# append("Ts0Tw0Tw1_%s_%s_%s" % (Ts0,Tw0,Tw1))
	# append("Ts1Ts0Tw0_%s_%s_%s" % (Ts1,Ts0,Tw0))
	# append("s0Tw0Tw1_%s_%s_%s" % (s0,Tw0,Tw1))
	# append("Ts1s0Tw0_%s_%s_%s" % (Ts1,s0,Tw0))

	# (4) rc -1  lc 1
"s1rc",	# append("Ts1Trcs1Tw0_%s_%s_%s" % (Ts1, Trcs1, Tw0)) => Trcs1
	# append("Ts1Trcs1Tw0_%s_%s_%s" % (Ts1, Trcs1, Ts0))
"s1lc",	# append("Ts1Tlcs1Ts0_%s_%s_%s" % (Ts1, Tlcs1, Ts0)) => Tlcs1
"s0lc",	# append("Ts1Ts0Tlcs0_%s_%s_%s" % (Ts1, Ts0, Tlcs0)) => Tlcs0
"s0rc",	# append("Ts1Trcs0Ts0_%s_%s_%s" % (Ts1, Trcs0, Ts0)) => Trcs0
	# append("Ts0Tlcs1s0_%s_%s_%s" % (Ts0, Tlcs1, s0))
	# append("Ts1s0Trcs0_%s_%s_%s" % (Ts1, s0, Trcs0))
	# append("Ts0Tlcs1s0_%s_%s_%s" % (Ts0, Trcs1, s0))
	# append("Ts1s0Trcs0_%s_%s_%s" % (Ts1, s0, Tlcs0))

	# +hybrid
	# append("Ts0Trcs0Tw0_%s_%s_%s" % (Ts0,Trcs0,Tw0))
	# append("Ts0Trcs0Tw0_%s_%s_%s" % (Ts0,Trcs0,Tw1))
	# append("Ts0Tlcs0Tw0_%s_%s_%s" % (Ts0,Tlcs0,Tw0))
"n0lc",	# append("Ts0Tw0Tlcw0_%s_%s_%s" % (Ts0,Tw0,Tlcw0)) => Tlcw0
	# append("Ts0Tlcs0w0_%s_%s_%s" % (Ts0,Tlcs0,w0))
	# append("Ts0Tlcs0w0_%s_%s_%s" % (Ts0,Trcs0,w0))
	# append("Ts0w0Tlcw0_%s_%s_%s" % (Ts0,w0,Tlcw0))

	# append("Ts0Tw0Trcs0Tlcw0_%s_%s_%s_%s" % (Ts0,Tw0,Tlcw0,Trcs0))
	# append("Ts0Ts1Trcs1Tlcs0_%s_%s_%s_%s" % (Ts0,Ts1,Tlcs0,Trcs1))

	# (5)
	# append("Ts2Ts1Ts0_%s_%s_%s" % (Ts2,Ts1,Ts0))
	# append("Ts1Ts0Tw0_%s_%s_%s" % (Ts1,Ts0,Tw0))
	# append("Ts0Tw0Tw1_%s_%s_%s" % (Ts0,Tw0,Tw1))
] # tacl13hybrid

# Variants of tacl13hybrid:

tacl13words = DFeature["s0lv", "s0v", "s0rv", "s1lv", "s1v", "s1rv", "n0lv", "n0v", "n1v"]
tacl13tags  = DFeature["s0lc", "s0c", "s0rc", "s1lc", "s1c", "s1rc", "n0lc", "n0c", "n1c"]
tacl13wordtag = [tacl13words; tacl13tags]

# Renamed:
hybrid13 = tacl13hybrid

# Goldberg&Nivre TACL13 ArcEager features from:
# tacl2013dynamicoracles/lefttoright/features:EagerZhangNivre2011Extractor
# (seems to be the ones used in acl11)
# s0: top of stack (a=-1)
# n0,n1,n2: buffer (a=0,1,2)
# s0h,s0h2: parent and grandparent of s0
# s0l,s0r: left/rightmost child of s0
# s0l2,s0r2: second left/rightmost child of s0
# s0w,s0p,s0wp: form, postag, form-postag for s0
# d: refers to s0-n0 distance, encoded 1,2,3,...,9,10+
# s0vl,s0vr: number of left/right children of s0, flat encoding
# s0L: dependency label for s0 (NA for n0!)
# s0sl,s0sr: set of dependency labels to the left/right of s0

acl11eager = DFeature[
	# # Single Words
	# f("s0wp_%s" % (s0wp))
"s0v",	# f("s0w_%s"  % (s0w))
"s0c",	# f("s0p_%s"  % (s0p))
	# f("n0wp_%s" % (n0wp))
"n0v",	# f("n0w_%s"  % (n0w))
"n0c",	# f("n0p_%s"  % (n0p))
	# f("n1wp_%s" % (n1wp))
"n1v",	# f("n1w_%s"  % (n1w))
"n1c",	# f("n1p_%s"  % (n1p))
	# f("n2wp_%s" % (n2wp))
"n2v",	# f("n2w_%s"  % (n2w))
"n2c",	# f("n2p_%s"  % (n2p))
	#
	# # Pairs
	# f("s0wp,n0wp_%s_%s" % (s0wp, n0wp))
	# f("s0wp,n0w_%s_%s" % (s0wp, n0w))
	# f("s0w,n0wp_%s_%s" % (s0w, n0wp))
	# f("s0wp,n0p_%s_%s" % (s0wp, n0p))
	# f("s0p,n0wp_%s_%s" % (s0p, n0wp))
	# f("s0w,n0w_%s_%s" % (s0w, n0w)) #?
	# f("s0p,n0p_%s_%s" % (s0p, n0p))
	# f("n0p,n1p_%s_%s" % (n0p, n1p))
	#
	# # Tuples
	# f("n0p,n1p,n2p_%s_%s_%s" % (n0p, n1p, n2p))
	# f("s0p,n0p,n1p_%s_%s_%s" % (s0p, n0p, n1p))
	# f("s0hp,s0p,n0p_%s_%s_%s" % (s0hp, s0p, n0p)) => s0hp (handled in unigram section)
	# f("s0p,s0lp,n0p_%s_%s_%s" % (s0p, s0lp, n0p)) => s0lp (handled in unigram section)
	# f("s0p,s0rp,n0p_%s_%s_%s" % (s0p, s0rp, n0p)) => s0rp (handled in unigram section)
	# f("s0p,n0p,n0lp_%s_%s_%s" % (s0p, n0p, n0lp)) => n0lp (handled in unigram section)
	#
	# # Distance
"s0d",	# f("s0wd_%s:%s" % (s0w, d)) => d
	# f("s0pd_%s:%s" % (s0p, d))
	# f("n0wd_%s:%s" % (n0w, d))
	# f("n0pd_%s:%s" % (n0p, d))
	# f("s0w,n0w,d_%s:%s:%s" % (s0w, n0w, d))
	# f("s0p,n0p,d_%s:%s:%s" % (s0p, n0p, d))
	#
	# # Valence
"s0b",	# f("s0wvr_%s:%s" % (s0w, s0vr)) => s0vr
	# f("s0pvr_%s:%s" % (s0p, s0vr))
"s0a",	# f("s0wvl_%s:%s" % (s0w, s0vl)) => s0vl
	# f("s0pvl_%s:%s" % (s0p, s0vl))
"n0a",	# f("n0wvl_%s:%s" % (n0w, n0vl)) => n0vl
	# f("n0pvl_%s:%s" % (n0p, n0vl))
	#
	# # Unigrams
"s0hv",	# f("s0hw_%s" % (s0hw))
"s0hc",	# f("s0hp_%s" % (s0hp))
"s0L",	# f("s0L_%s" % (s0L))
	#
"s0lv",	# f("s0lw_%s" % (s0lw))
"s0lc",	# f("s0lp_%s" % (s0lp))
"s0lL",	# f("s0lL_%s" % (s0lL))
	#
"s0rv",	# f("s0rw_%s" % (s0rw))
"s0rc",	# f("s0rp_%s" % (s0rp))
"s0rL",	# f("s0rL_%s" % (s0rL))
	#
"n0lv",	# f("n0lw_%s" % (n0lw))
"n0lc",	# f("n0lp_%s" % (n0lp))
"n0lL",	# f("n0lL_%s" % (n0lL))
	#
	# # Third-order
	# #do we really need the non-grandparent ones?
"s0h2v",	# f("s0h2w_%s" % (s0h2w))
"s0h2c",	# f("s0h2p_%s" % (s0h2p))
"s0hL",	# f("s0hL_%s"  % (s0hL))
"s0l2v",	# f("s0l2w_%s" % (s0l2w))
"s0l2c",	# f("s0l2p_%s" % (s0l2p))
"s0l2L",	# f("s0l2L_%s" % (s0l2L))
"s0r2v",	# f("s0r2w_%s" % (s0r2w))
"s0r2c",	# f("s0r2p_%s" % (s0r2p))
"s0r2L",	# f("s0r2L_%s" % (s0r2L))
"n0l2v",	# f("n0l2w_%s" % (n0l2w))
"n0l2c",	# f("n0l2p_%s" % (n0l2p))
"n0l2L",	# f("n0l2L_%s" % (n0l2L))
	# f("s0p,s0lp,s0l2p_%s_%s_%s" % (s0p, s0lp, s0l2p))
	# f("s0p,s0rp,s0r2p_%s_%s_%s" % (s0p, s0rp, s0r2p))
	# f("s0p,s0hp,s0h2p_%s_%s_%s" % (s0p, s0hp, s0h2p))
	# f("n0p,n0lp,n0l2p_%s_%s_%s" % (n0p, n0lp, n0l2p))
	#
	# # Labels
"s0B",	# f("s0wsr_%s_%s" % (s0w, s0sr)) => s0sr
	# f("s0psr_%s_%s" % (s0p, s0sr))
"s0A",	# f("s0wsl_%s_%s" % (s0w, s0sl)) => s0sl
	# f("s0psl_%s_%s" % (s0p, s0sl))
"n0A",	# f("n0wsl_%s_%s" % (n0w, n0sl)) => n0sl
	# f("n0psl_%s_%s" % (n0p, n0sl))
] # tacl13eager

# Renamed:
eager39 = acl11eager
# fs11e39e39 (8906): eager39 also local best for acl11, ArcEager13, start=eager39, all=eager39

# Old feature format before we switched to ints:
# f::Features is a nx3 matrix whose rows determine which features to extract
# Each row of f consists of the following three values:
# 1. anchor word: 0:n0, 1:n1, 2:n2, ..., -1:s0, -2:s1, -3:s2, ...
# 2. target word: 0:self, 1:rightmost child, 2:second rightmost, -1:leftmost, -2:second leftmost ...
# 3. feature: One of the features listed below.
#
# 0:wvec (word+context if token, dim depends on encoding)
# +-1: exists/doesn't (one bit)
# +-2: right/left child count (4 bits, == encoding: 0,1,2,3+)
# +3: distance to right (4 bits, == encoding: 1,2,3,4+, root is 4+)
# -3: average of in-between tokens to the right (dim:wvec)
# +-4: word/context half of vector (dim:wvec/2, only valid for token encoding, assumes first half=word, second half=context)
# +-5: right/left child count (4 bits, >= encoding: >=1, >=2, >=3, >=4)
# +-6: right/left child count (4 bits, <= encoding: <=0, <=1, <=2, <=3)
# +7: distance to right, >= encoding (8 bits, >= encoding: >=2, >=3, >=4, >=6, >=8, >=12, >=16, >=20)
# -7: distance to right, <= encoding (8 bits, <= encoding: <=1, <=2, <=3, <=4, <=6, <=8, <=12, <=16)
# +-8: average of in-between word/context vectors to the right (dim:wvec/2)
# +-9: head exists/doesn't (one bit)

# 0.0418668 archybrid_conll07EnglishToken_wikipedia2MUNK-100_rbf376678_1014_cache.mat (trn/tst) (rbf-gamma:0.376747095368119)
fv022b = Int8[-3 0 8;-2 -1 -4;-2 -1 5;-2 0 -9;-2 0 -4;-2 0 4;-2 0 7;-2 1 1;-1 -1 -1;-1 -1 4;-1 0 -9;-1 0 -4;-1 0 4;-1 0 6;-1 1 4;0 -1 -4;0 -1 4;0 0 -4;0 0 -2;0 0 4;1 0 -4;1 0 4]

# 0.0325506937033084 archybrid_conllWSJToken_wikipedia2MUNK-100_rbf333140_cache.mat (rbf-gamma:0.333140404105682)
fv021a = Int8[-3 0 4;-3 1 -2;-2 0 4;-2 0 6;-2 1 -2;-1 -1 -4;-1 -1 2;-1 -1 4;-1 0 -4;-1 0 -1;-1 0 2;-1 0 4;-1 1 2;-1 1 4;0 -1 -4;0 -1 -1;0 -1 4;0 0 -4;0 0 4;1 0 -4;1 0 4]

# 0.0443168273919168 archybrid_conll07EnglishToken_wikipedia2MUNK-100_rbf372064_cache.mat (rbf-gamma:0.372063662109375)
fv019 = Int8[-3 0 8;-2 -1 -4;-2 0 -9;-2 0 -4;-2 0 4;-2 1 1;-1 -1 -1;-1 -1 4;-1 0 -4;-1 0 4;-1 0 6;-1 1 4;0 -1 -4;0 -1 4;0 0 -4;0 0 -2;0 0 4;1 0 -4;1 0 4]

# 0.03492252681764 archybrid_conllWSJToken_wikipedia2MUNK-100_rbf339506_cache.mat (rbf-gamma:0.339506144311523)
fv022a = Int8[-3 0 4;-3 1 -2;-2 0 -4;-2 0 4;-2 0 6;-2 1 -2;-1 -1 -4;-1 -1 2;-1 -1 4;-1 0 -4;-1 0 -1;-1 0 2;-1 0 4;-1 1 2;-1 1 4;0 -1 -4;0 -1 -1;0 -1 4;0 0 -4;0 0 4;1 0 -4;1 0 4]

# 0.0444685231336006 archybrid13_conll07EnglishToken_wikipedia2MUNK-100_rbf3.720637e+05_cache.mat
fv023a = Int8[-3 0 8;-2 -1 -4;-2 -1 -1;-2 0 -9;-2 0 -4;-2 0 4;-2 1 1;-2 1 4;-1 -1 -1;-1 -1 4;-1 0 -4;-1 0 4;-1 0 5;-1 0 6;-1 1 -4;-1 1 4;0 -1 -4;0 -1 4;0 0 -4;0 0 -2;0 0 4;1 0 -4;1 0 4]

# BUGGY: 0.0325637 Best with rbf=0.339506144311523, archybrid_conllWSJToken_wikipedia2MUNK-100_fv130_dump.mat.
fv023 = Int8[-3 0 4;-3 1 -2;-2 -1 -4;-2 0 -4;-2 0 4;-2 0 6;-2 1 -2;-1 -1 -4;-1 -1 2;-1 -1 4;-1 0 -4;-1 0 -1;-1 0 2;-1 0 4;-1 1 2;-1 1 4;0 -1 -4;0 -1 -1;0 -1 4;0 0 -4;0 0 4;1 0 -4;1 0 4]

# BUGGY: 0.0443493 Best with run_featselect_rbf('archybrid13', 'conll07EnglishToken_wikipedia2MUNK-100', 'fv136', 'fv021', 'rbf3721');
fv022 = Int8[-3 0 8;-2 -1 -1;-2 0 -9;-2 0 -4;-2 0 4;-2 1 1;-2 1 4;-1 -1 -1;-1 -1 4;-1 0 -4;-1 0 4;-1 0 6;-1 1 -4;-1 1 4;0 -1 -4;0 -1 4;0 -1 5;0 0 -4;0 0 -2;0 0 4;1 0 -4;1 0 4]

# 0.0445990 Best with rbf372111 archybrid_conll07EnglishToken_wikipedia2MUNK-100_fv136_dump.mat
fv021 = Int8[-3 0 8;-2 -1 -1;-2 0 -9;-2 0 -4;-2 0 4;-2 1 1;-1 -1 -1;-1 -1 4;-1 0 -4;-1 0 4;-1 0 5;-1 0 6;-1 1 -4;-1 1 4;0 -1 -4;0 -1 4;0 0 -4;0 0 -2;0 0 4;1 0 -4;1 0 4]

# Best on run_featselect22('archybrid', 'conllWSJToken_wikipedia2MUNK-100', 'fv130', 'fv018');
fv017a = Int8[-3 0 -4;-3 0 4;-2 0 -4;-2 0 4;-2 1 -2;-1 -1 -4;-1 -1 4;-1 0 -4;-1 0 4;-1 1 -2;-1 1 4;0 -1 -4;0 -1 4;0 0 -4;0 0 4;1 0 -4;1 0 4]

# This one does best on archybrid_conll07EnglishToken_wikipedia2MUNK-100_d5 (degree=5 kernel)
fv018a = Int8[-3 0 8;-2 0 -9;-2 0 -4;-2 0 4;-2 1 1;-1 -1 1;-1 -1 4;-1 0 -4;-1 0 4;-1 1 -4;-1 1 4;0 -1 -4;0 -1 4;0 0 -4;0 0 -2;0 0 4;1 0 -4;1 0 4]

# This one does best on archybrid_conll07EnglishToken_wikipedia2MUNK-100 (degree=3 kernel)
fv015a = Int8[-3 0 8;-2 0 -4;-2 0 4;-2 1 1;-1 -1 4;-1 0 -4;-1 0 4;-1 1 -4;-1 1 4;0 -1 -4;0 -1 4;0 0 -4;0 0 4;1 0 -4;1 0 4]

# This one is a local minimum on archybrid_conll07EnglishToken_wikipedia2MUNK-100 (degree=3 kernel)
fv031a = Int8[-3 0 -8;-3 0 -4;-3 0 -1;-3 0 8;-3 1 -4;-3 1 4;-2 -1 -4;-2 -1 4;-2 0 -8;-2 0 -4;-2 0 4;-2 0 8;-2 1 1;-2 1 2;-1 -1 -4;-1 -1 1;-1 -1 4;-1 0 -8;-1 0 -4;-1 0 2;-1 0 4;-1 0 8;-1 1 -4;-1 1 4;0 -1 -4;0 -1 1;0 -1 4;0 0 -4;0 0 4;1 0 -4;1 0 4]

fv034 = zeros(Int8, 0, 3)
for a=-3:1
  for b=-1:1
    if ((a >= 0) && (b > 0)) continue; end   # no rdeps for buffer words
    if ((a > 0)  && (b < 0)) continue; end   # no ldeps for buffer words other than n0
    if ((a <= -2) && (b < 0)) continue; end  # not interested in ldeps beyond s0
    if ((a <= -3) && (b > 0)) continue; end  # not interested in rdeps beyond s1
    for c=-9:9
      if ((c == 0) || (c == -3)) continue; end  # do not use the word+context combination, keep them separate
      if (abs(c) == 5 || abs(c) == 6) continue; end  # just use = encoding for child count
      if ((abs(c) == 2) && (b != 0)) continue; end # not interested in grand children
      if (abs(c) == 1) continue; end # existence not useful
      if (abs(c) == 9) continue; end # head not useful in archybrid
      if ((c == 3) || (c == -7)) continue; end  # just use >= encoding for distance
      if ((a > 0) && !in(c, [1,-1,4,-4])) continue; end      # no deps/dist/in-between/head for a>0
      if ((a == 0) && (b == 0) && !in(c, [1,-1,4,-4,-2,-5,-6])) continue; end  # no rdeps/dist/in-between/head for a=0
      if ((b != 0) && in(c, [3,-3,7,-7,8,-8,9,-9])) continue; end  # no dist/in-between/head for deps
      fv034 = [fv034; Int8[a b c]]
    end
  end
end

fv039 = Int8[
0 0 4; 		# [-196]    'n0w'
-1 0 4; 	# [-119]    's0w'
1 0 4; 		# [ -73]    'n1w'
-2 0 4; 	# [ -43]    's1w'
0 -1 4; 	# [ -20]    'n0l1w'
0 0 -4; 	# [ -20]    'n0c'
1 0 -4; 	# [ -19]    'n1c'
-1 0 -4; 	# [ -16]    's0c'
-1 -1 4; 	# [ -11]    's0l1w'
-2 0 -4; 	# [ -10]    's1c'
-3 -1 -4;     	# [  -8]    's2l1c'
-1 -1 -4;     	# [  -8]    's0l1c'
0 -1 -4; 	# [  -8]    'n0l1c'
-3 -1 2; 	# [  -7]    's2l1r='
-3 0 8; 	# [  -7]    's2aw'
-2 1 1; 	# [  -7]    's1r1+'
-1 0 -8; 	# [  -7]    's0ac'
-1 0 7; 	# [  -7]    's0d>'
-1 -1 1; 	# [  -6]    's0l1+'
-1 0 8; 	# [  -6]    's0aw'
-3 0 -4; 	# [  -5]    's2c'
-3 0 4; 	# [  -5]    's2w'
-2 -1 -4;     	# [  -5]    's1l1c'
-1 1 4; 	# [  -5]    's0r1w'
-3 0 -8; 	# [  -4]    's2ac'
0 -1 1; 	# [  -4]    'n0l1+'
-3 0 -1; 	# [  -3]    's2-'
-3 0 7; 	# [  -3]    's2d>'
-3 1 4; 	# [  -3]    's2r1w'
-2 -1 4; 	# [  -3]    's1l1w'
-2 0 8; 	# [  -3]    's1aw'
-2 1 2; 	# [  -3]    's1r1r='
-2 -1 -2;     	# [  -2]    's1l1l='
-1 -1 -2;     	# [  -2]    's0l1l='
-2 0 -8; 	# [  -1]    's1ac'
-1 -1 2; 	# [  -1]    's0l1r='
-1 1 -4; 	# [  -1]    's0r1c'
-3 1 -4; 	# [   0]    's2r1c'
-1 0 2; 	# [   0]    's0r='
]

fv012 = Int8[
0 0 4; 		# [-203]    n0w
-1 0 4; 	# [-125]    s0w
-2 0 4; 	# [ -59]    s1w
1 0 4; 		# [ -59]    n1w
0 -1 4; 	# [ -41]    n0l1w
-1 -1 4;	# [ -21]    s0l1w

0 0 -4; 	# [ -17]    n0c
-1 0 -4;	# [ -16]    s0c
-2 0 -4;	# [ -13]    s1c
1 0 -4; 	# [ -15]    n1c
0 -1 -4; 	# [ -11]    n0l1c
-1 -1 -4;	# [ -12]    s0l1c
]

fv017 = Int8[       # 0.0476108, 'archybrid', 'conll07EnglishToken_wikipedia2MUNK-100'
0 0 4; 		# [-203]    n0w
-1 0 4; 	# [-125]    s0w
-2 0 4; 	# [ -59]    s1w
1 0 4; 		# [ -59]    n1w
0 -1 4; 	# [ -41]    n0l1w
-2 1 -2;	# [ -30]    s1r1l=
-1 -1 4;	# [ -21]    s0l1w
-1 1 4; 	# [ -21]    s0r1w
-1 1 -2;	# [ -19]    s0r1l=
0 0 -4; 	# [ -17]    n0c
-1 0 -4;	# [ -16]    s0c
1 0 -4; 	# [ -15]    n1c
-2 0 -4;	# [ -13]    s1c
-3 0 4; 	# [ -12]    s2w
-1 -1 -4;	# [ -12]    s0l1c
-1 1 -4; 	# [ -12]    s0r1c
0 -1 -4; 	# [ -11]    n0l1c
]

fv018 = Int8[       # another version of fv808 that splits token features to word + context
    0 0 4;
    0 0 -4;
    -1 0 4;
    -1 0 -4;
    -2 0 4;
    -2 0 -4;
    1 0 4;
    1 0 -4;
    0 -1 4;
    0 -1 -4;
    -1 1 4;
    -1 1 -4;
    -2 1 -2;
    -1 -1 4;
    -1 -1 -4;
    -1 1 -2;
    -3 0 4;
    -3 0 -4;
]

fv084 = zeros(Int8, 0, 3)
for a=-3:1
  for b=-1:1
    if ((a >= 0) && (b > 0)) continue; end   # no rdeps for buffer words
    if ((a > 0)  && (b < 0)) continue; end   # no ldeps for buffer words other than n0
    for c=-9:9
      if ((c == 0) || (c == -3)) continue; end  # do not use the word+context combination, keep them separate
      if (abs(c) == 5 || abs(c) == 6) continue; end  # just use = encoding for child count
      if ((c == 3) || (c == -7)) continue; end  # just use >= encoding for distance
      if ((a > 0) && !in(c, [1,-1,4,-4])) continue; end      # no deps/dist/in-between/head for a>0
      if ((a == 0) && (b == 0) && !in(c, [1,-1,4,-4,-2,-5,-6])) continue; end  # no rdeps/dist/in-between/head for a=0
      if ((b != 0) && in(c, [3,-3,7,-7,8,-8,9,-9])) continue; end  # no dist/in-between/head for deps
      fv084 = [fv084; Int8[a b c]]
    end
  end
end

# Better initial starting point
fv008w = Int8[
    0  0 4; # n0w
    -1 0 4; # s0w
    1  0 4; # n1w
    -2 0 4; # s1w
    0 -1 4; # n0l1w
    -1 1 4; # s0r1w
    -2 1 4; # s1r1w
    -1 -1 4; # s0l1w
         ]

# Good initial starting point
fv008 = Int8[
    0 0 4;      # n0
    -1 0 4;     # s0
    1 0 4;      # n1
    -2 0 4;     # s1
    0 -1 4;     # n0l1
    -1 1 4;     # s0r1
    0 0 -4;     # n0c
    -1 0 -4;	# s0c
]

# Include the head feature for stack words:
fv136 = zeros(Int8, 0, 3)
for a=-3:2
  for b=-1:1
    if ((a >= 0) && (b > 0)) continue; end   # no rdeps for buffer words
    if ((a > 0)  && (b < 0)) continue; end   # no ldeps for buffer words other than n0
    for c=-9:9
      if ((c == 0) || (c == -3)) continue; end  # do not use the word+context combination, keep them separate
      if ((a > 0) && !in(c, [1,-1,4,-4])) continue; end      # no deps/dist/in-between/head for a>0
      if ((a == 0) && (b == 0) && !in(c, [1,-1,4,-4,-2,-5,-6])) continue; end  # no rdeps/dist/in-between/head for a=0
      if ((b != 0) && in(c, [3,-3,7,-7,8,-8,9,-9])) continue; end  # no dist/in-between/head for deps
      fv136 = [fv136; Int8[a b c]]
    end
  end
end

# All legal features for s2..n2:
fv130 = zeros(Int8, 0, 3)
for a=-3:2
  for b=-1:1
    if ((a >= 0) && (b > 0)) continue; end   # no rdeps for buffer words
    if ((a > 0)  && (b < 0)) continue; end   # no ldeps for a>0
    for c=-8:8
      if ((a > 0) && in(c, [2,-2,3,-3,5,-5,6,-6,7,-7,8,-8])) continue; end      # no deps/dist/in-between for a>0
      if ((a == 0) && (b == 0) && in(c, [2,3,-3,5,6,7,-7,8,-8])) continue; end  # no rdeps/dist/in-between for a=0
      if ((b != 0) && in(c, [3,-3,7,-7,8,-8])) continue; end  # no dist/in-between for deps
      if ((c == 0) || (c == -3)) continue; end  # do not use the word+context combination, keep them separate
      fv130 = [fv130; Int8[a b c]]
    end
  end
end

fv102 = zeros(Int8, 0, 3)
for a=-2:1
  for b=-1:1
    if ((a >= 0) && (b > 0)) continue; end   # no rdeps for buffer words
    if ((a > 0)  && (b < 0)) continue; end   # no ldeps for a>0
    for c=-8:8
      if ((a > 0) && in(c, [2,-2,3,-3,5,-5,6,-6,7,-7,8,-8])) continue; end      # no deps/dist/in-between for a>0
      if ((a == 0) && (b == 0) && in(c, [2,3,-3,5,6,7,-7,8,-8])) continue; end  # no rdeps/dist/in-between for a=0
      if ((b != 0) && in(c, [3,-3,7,-7,8,-8])) continue; end  # no dist/in-between for deps
      fv102 = [fv102; Int8[a b c]]
    end
  end
end

# fv1768 = Int8[
# #n0            n1             n2             s0             s1             s2             s0l1           s0l2           s0r1           s0r2           s1l1           s1l2           s1r1           s1r2           n0l1           n0l2          n0s0 s0s1
# 0  0  0  0  0  1  1  1  1  1  2  2  2  2  2 -1 -1 -1 -1 -1 -2 -2 -2 -2 -2 -3 -3 -3 -3 -3 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -2 -2 -2 -2 -2 -2 -2 -2 -2 -2 -2 -2 -2 -2 -2 -2 -2 -2 -2 -2  0  0  0  0  0  0  0  0  0  0 -1   -2;
# 0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0 -1 -1 -1 -1 -1 -2 -2 -2 -2 -2  1  1  1  1  1  2  2  2  2  2 -1 -1 -1 -1 -1 -2 -2 -2 -2 -2  1  1  1  1  1  2  2  2  2  2 -1 -1 -1 -1 -1 -2 -2 -2 -2 -2  0    0;
# 0 -2  2  1 -1  0 -2  2  1 -1  0 -2  2  1 -1  0 -2  2  1 -1  0 -2  2  1 -1  0 -2  2  1 -1  0 -2  2  1 -1  0 -2  2  1 -1  0 -2  2  1 -1  0 -2  2  1 -1  0 -2  2  1 -1  0 -2  2  1 -1  0 -2  2  1 -1  0 -2  2  1 -1  0 -2  2  1 -1  0 -2  2  1 -1  3    3;
# ]';

fv1768 =
Int8[0 0 0
     0 0 -2
#    0 0 2   # buffer words dont have right children
     0 0 1
     0 0 -1
     1 0 0
#    1 0 -2  # buffer words other than n0 do not have ldeps
#    1 0 2   # buffer words dont have right children
     1 0 1
     1 0 -1
     2 0 0
#    2 0 -2  # buffer words other than n0 do not have ldeps
#    2 0 2   # buffer words dont have right children
     2 0 1
     2 0 -1
     -1 0 0
     -1 0 -2
     -1 0 2
     -1 0 1
     -1 0 -1
     -2 0 0
     -2 0 -2
     -2 0 2
     -2 0 1
     -2 0 -1
     -3 0 0
     -3 0 -2
     -3 0 2
     -3 0 1
     -3 0 -1
     -1 -1 0
     -1 -1 -2
     -1 -1 2
     -1 -1 1
     -1 -1 -1
     -1 -2 0
     -1 -2 -2
     -1 -2 2
     -1 -2 1
     -1 -2 -1
     -1 1 0
     -1 1 -2
     -1 1 2
     -1 1 1
     -1 1 -1
     -1 2 0
     -1 2 -2
     -1 2 2
     -1 2 1
     -1 2 -1
     -2 -1 0
     -2 -1 -2
     -2 -1 2
     -2 -1 1
     -2 -1 -1
     -2 -2 0
     -2 -2 -2
     -2 -2 2
     -2 -2 1
     -2 -2 -1
     -2 1 0
     -2 1 -2
     -2 1 2
     -2 1 1
     -2 1 -1
     -2 2 0
     -2 2 -2
     -2 2 2
     -2 2 1
     -2 2 -1
     0 -1 0
     0 -1 -2
     0 -1 2
     0 -1 1
     0 -1 -1
     0 -2 0
     0 -2 -2
     0 -2 2
     0 -2 1
     0 -2 -1
     -1 0 3
     -2 0 3]

fv804 = Int8[
#n0 s0 s1 n1 n0l1 s0r1 s0l1 s1r1 s0r
  0 -1 -2  1  0   -1   -1   -2   -1;
  0  0  0  0 -1    1   -1    1    0;
  0  0  0  0  0    0    0    0    2;
]';

fv708 = Int8[
#n0 n0l1 n0l2 n1 s0 s0r s0r1 s0s1 s1
  0  0    0    1 -1 -1  -1   -2   -2;
  0 -1   -2    0  0  0   1    0    0;
  0  0    0    0  0  2   0    3    0;
]';

fv808 = Int8[
#n0 s0 s1 n1 n0l1 s0r1 s1r1l s0l1 s0r1l s2
  0 -1 -2  1  0   -1   -2    -1   -1    -3;
  0  0  0  0 -1    1    1    -1    1     0;
  0  0  0  0  0    0   -2     0   -2     0;
]';

end # module Flist


############################################################
# joLinearOperator - subtype operators #####################
############################################################

# joKron
include("joLinearOperatorConstructors/joKron.jl")

# joCoreBlock
include("joLinearOperatorConstructors/joCoreBlock.jl")
# joCoreBlock derivatives: joBlockDiag joDict joStack joBlock
include("joLinearOperatorConstructors/joCoreBlockConstructors.jl")

############################################################
# joLinearOperator - constructor-only operators ############
############################################################


"""
    show(sresult::Simulation)

Print out summary of simulation.
"""
function Base.show(io::IO, sresult::Simulation)
    @printf("===================================================================\n")
    _showtype(sresult)
    @printf("Input parameters: \n")
    @printf("\tMutation rate: %.2f\n", sresult.input.μ)
    _showrates(sresult.input)
    @printf("\tNumber of clonal mutations: %d\n", sresult.input.clonalmutations)
    @printf("\tNumber of subclones: %d\n\n", sresult.input.numclones)
    if sresult.input.numclones > 0
        freqs = subclonefreq(sresult.output)[1]
        for (i, subclone) in enumerate(sresult.output.subclones)
            @printf("Subclone %d \n", i)
            @printf("\tFrequency: %.2f\n", freqs[i])
            @printf("\tNumber of mutations in subclone: %d\n", length(subclone.mutations))
            @printf("\tFitness advantage: %.2f\n", sresult.input.selection[i])
            @printf("\tTime subclone emerges: %.2f\n", subclone.time)
            @printf("\tNumber of divisions: %d\n", subclone.Ndivisions)
            @printf("\tAverage number of divisions per cell: %.2f\n", subclone.avdivisions)
            @printf("\tPopulation size when subclone emerges: %d\n", subclone.N0)
            @printf("\tParent of subclone (0 is host): %d\n\n", subclone.parenttype-1)
        end
    else
        @printf("No clones, tumour growth was neutral\n\n")
    end

end

function Base.show(io::IO, sresult::MultiSimulation)
    @printf("===================================================================\n")
    @printf("Number of simulations = %d\n\n", length(sresult.output))
    @printf("Input parameters: \n")
    _showrates(sresult.input)
    @printf("\tNumber of clonal mutations: %d\n", sresult.input.clonalmutations)
    @printf("\tNumber of subclones: %d\n\n", sresult.input.numclones)

    if sresult.input.numclones > 0
        for i in 1:sresult.input.numclones
            @printf("Subclone %d \n", i)
            @printf("\tFitness advantage: %.2f\n", sresult.input.selection[i])
            @printf("\tEvent time: %.2f\n",
                sresult.input.tevent[i])
            @printf("\tAverage frequency: %.2f (std: %.2f)\n",
                        _get_mean_std(sresult.output, :freq, i)...)
            @printf("\tAverage number of mutations in subclone: %d (std: %.2f)\n",
                        _get_mean_std(sresult.output, :mutations, i)...)
            @printf("\tAverage population size when subclone emerges: %d (std: %.2f)\n",
                        _get_mean_std(sresult.output, :N0, i)...)
            @printf("\tAverage time when subclone emerges: %.2f (std: %.2f)\n",
                        _get_mean_std(sresult.output, :time, i)...)
            @printf("\tDistribution of subclone parents (0 is host): ")
            s = join([@sprintf("%d => %d", parent, freq)
                        for (parent, freq) in parenttype_dist(sresult.output, i)]
                        , ", ")
            @printf("%s\n\n",s)
        end
    else
        @printf("No clones, tumour growth was neutral\n\n")
    end

end

function Base.show(io::IO, population::Population)
    siminput = population.input
    @printf("===================================================================\n")
    @printf("Multilevel branching module simulation\n\n")
    @printf("Input parameters: \n")
    @printf("\tMax age = %.2f\n", siminput.maxtime)
    _showrates(siminput)
    @printf("\tBranching rate = %.4f\n", siminput.branchrate)
    @printf("\tNew module size = %d\n", siminput.branchinitsize)
    @printf("\tMature module size = %d\n", siminput.modulesize)
    @printf("\tNumber of clonal mutations: %d\n", siminput.clonalmutations)
    # @printf("\tNumber of subclones: %d\n\n", siminput.numclones)

    @printf("Population data\n")
    @printf("\tPopulation age = %d\n", age(population))
    @printf("\tNumber of modules = %d\n\n", length(population))

    # @printf("Clonal mutations per module")

    # if siminput.numclones > 0
    #     for i in 1:siminput.numclones
    #         @printf("Subclone %d \n", i)
    #         @printf("\tFitness advantage: %.2f\n", siminput.selection[i])
    #         @printf("\tEvent time: %.2f\n",
    #             siminput.tevent[i])
    #         @printf("\tAverage frequency: %.2f (std: %.2f)\n",
    #                     _get_mean_std(population.output, :freq, i)...)
    #         @printf("\tAverage number of mutations in subclone: %d (std: %.2f)\n",
    #                     _get_mean_std(population.output, :mutations, i)...)
    #         @printf("\tAverage population size when subclone emerges: %d (std: %.2f)\n",
    #                     _get_mean_std(population.output, :N0, i)...)
    #         @printf("\tAverage time when subclone emerges: %.2f (std: %.2f)\n",
    #                     _get_mean_std(population.output, :time, i)...)
    #         @printf("\tDistribution of subclone parents (0 is host): ")
    #         s = join([@sprintf("%d => %d", parent, freq)
    #                     for (parent, freq) in parenttype_dist(population.output, i)]
    #                     , ", ")
    #         @printf("%s\n\n",s)
    #     end
    # else
    #     @printf("No clones, tumour growth was neutral\n\n")
    # end

end

_showtype(sresult::Simulation{BranchingInput}) = @printf("Branching process \n")
_showtype(sresult::Simulation{MoranInput}) = @printf("Moran process \n")
_showtype(sresult::Simulation{BranchingMoranInput}) = @printf("Branching => Moran process\n")
_showtype(sresult::Simulation{MultilevelInput}) = @printf("Branching => Moran proces (multilevel)\n")

function _showrates(siminput::BranchingInput)
    b, d, μ = siminput.b, siminput.d, siminput.μ
    @printf("\tBirth rate of host population: %.2f\n", b)
    @printf("\tDeath rate of host population: %.2f\n", d)
    @printf("\tEffective mutation rate (μ/β): %.2f\n", μ / ((b - d) / b))
end

function _showrates(siminput::MoranInput)
    @printf("\tBirth and death rate of host population: %.2f\n", siminput.bdrate)
end

function _showrates(siminput::Union{BranchingMoranInput, MultilevelInput})
    b, d, μ, bdrate = siminput.b, siminput.d, siminput.μ, siminput.bdrate
    @printf("\tBirth rate of host population (branching): %.2f\n", b)
    @printf("\tDeath rate of host population (branching): %.2f\n", d)
    @printf("\tBirth and death rate of host population (moran): %.2f\n", bdrate)
    @printf("\tMutation rate (μ): %.2f\n", μ)
end

function _get_mean_std(output, subclonefield, i)
    data = [getfield(subclone, subclonefield)[i]
                for simoutput in output
                    for subclone in simoutput.subclones]
    return mean(data), std(data)
end

function parenttype_dist(output, i)
    parenttypes = [subclone.parenttype[i] - 1
                    for simoutput in output
                        for subclone in simoutput.subclones]
    parenttypes = sort(collect(countmap(parenttypes)))
    return parenttypes
end