__precompile__()

"""
This module implements computation of kinship and other identity coefficients.
The kinship matrix is computed recursively by the standard formula.
The delta7 matrix is derived from the kinship matrix for non-inbred pedigrees.
The remaining identity coefficients of Jacquard are computed by simulation.
"""
module MendelKinship
#
# Required OpenMendel packages and modules.
#
using MendelBase
using SnpArrays
#
# Required external modules.
#
using CSV
using DataFrames
using PlotlyJS
using StatsBase

export Kinship

"""
  Kinship(control_file = ""; args...)

This is the wrapper function for the Kinship analysis option. Keyword options
should be specified in the control file.
"""
function Kinship(control_file = ""; args...)
  #
  # Print the logo. Store the initial directory.
  #
  print(" \n \n")
  println("     Welcome to OpenMendel's")
  println("     Kinship analysis option")
  print(" \n \n")
  println("Reading the data.\n")
  initial_directory = pwd()
  #
  # The user specifies the analysis to perform via a set of keywords.
  # Start the keywords at their default values.
  #
  keyword = set_keyword_defaults!(Dict{AbstractString, Any}())
  #
  # Keywords unique to this analysis should be first defined here
  # by setting their default values using the format:
  # keyword["some_keyword_name"] = default_value
  #
  keyword["kinship_output_file"] = ""
  keyword["repetitions"] = 1
  keyword["xlinked_analysis"] = false
  keyword["compare_kinships"] = false
  keyword["maf_threshold"] = 0.01
  keyword["grm_method"] = :MoM # more robust & less sensitive to rare SNPs;
                               # alternatives are :GRM and :Robust
  keyword["deviant_pairs"] = 0
  keyword["kinship_plot"] = ""
  keyword["z_score_plot"] = ""
  keyword["full_pedigree_file"] = ""
  #
  # Process the run-time user-specified keywords that will control the analysis.
  # This will also initialize the random number generator.
  #
  process_keywords!(keyword, control_file, args)
  if keyword["kinship_output_file"] == ""
    keyword["kinship_output_file"] = keyword["output_file"]
  end
  #
  # Check that the correct analysis option was specified.
  #
  lc_analysis_option = lowercase(keyword["analysis_option"])
  if (lc_analysis_option != "" &&
      lc_analysis_option != "kinship")
     throw(ArgumentError(
       "An incorrect analysis option was specified.\n \n"))
  end
  keyword["analysis_option"] = "Kinship"
  #
  # Read the genetic data from the external files named in the keywords.
  #
  (pedigree, person, nuclear_family, locus, snpdata,
    locus_frame, phenotype_frame, person_frame, snp_definition_frame) =
    read_external_data_files(keyword)
  #
  # Check if SNP data were read and names are unique.
  #
  if keyword["compare_kinships"]
    if snpdata.snps == 0
      throw(ArgumentError(
        "\n\nERROR: This analysis requires data from SNP files!\n"))
    elseif snpdata.snps != length(unique(snpdata.snpid))
      throw(ArgumentError(
        " \n\nERROR: There are non-unique names of SNPs.\n"))
    elseif person.people != length(unique(person.name))
      throw(ArgumentError(
        " \n\nERROR: There are non-unique names of individuals.\n"))
    end
  elseif snpdata.snps != 0
      throw(ArgumentError(
        " \n\nERROR: This analysis does not use data from SNP files!\n"))
  end
  #
  # Execute the specified analysis.
  #
  println(" \nAnalyzing the data.\n")
  execution_error = false
  if keyword["compare_kinships"]
    kinship_frame = compare_kinships(pedigree, person, snpdata,
                                     person_frame, keyword)
  else
    kinship_frame = theoretical_coefficients(pedigree, person, keyword)
  end
  # display(kinship_frame)

  if execution_error
    println(" \n \nERROR: Mendel terminated prematurely!\n")
  else
    println(" \n \nMendel's analysis is finished.\n")
  end
  #
  # Finish up by closing, and thus flushing, any output files.
  # Return to the initial directory.
  #
  close(keyword["output_unit"])
  cd(initial_directory)
  return nothing
end # function Kinship

"""
    compare_kinships(genotyped_pedigree, genotyped_person, snpdata,person_frame, keyword)

This function orchestrates the comparison of theoretical
and empirical kinships including Fisher z-scores.
The results are placed in a dataframe.
"""
function compare_kinships(genotyped_pedigree::Pedigree,
  genotyped_person::Person, snpdata::SnpDataStruct,
  person_frame::DataFrame, keyword::Dict{AbstractString, Any})
  #
  # Initialize constants
  #
  genotyped_people = genotyped_person.people
  deviant_pairs = keyword["deviant_pairs"]
  if deviant_pairs == 0
    deviant_pairs = genotyped_people^2
  end
  #
  # Record the order of the individuals in the user data upon entry,
  # before MendelBase permutes them.
  #
  entryorder = convert(Vector{Int64}, person_frame[:, :EntryOrder])
  #
  # If the full pedigree contains people without SNP information,
  # initialize the full pedigree and person including people without SNPs.
  # The boolean vector `contains_snp` (which is not used)
  # records if there are SNPs: contains_snp[i] means person i has SNP data.
  #
  if keyword["full_pedigree_file"] != ""
    full_pedigree, full_person, contains_snp =
      compute_full_person_and_pedigree(keyword, genotyped_person)
  else
    full_pedigree = genotyped_pedigree
    full_person = genotyped_person
  end
  #
  # Initialize more constants.
  #
  full_pedigrees = full_pedigree.pedigrees
  xlinked = keyword["xlinked_analysis"]
  maf_threshold = keyword["maf_threshold"]
  grm_method = Symbol(keyword["grm_method"])
  #
  # People are reordered, and some may not have SNP info.
  # Compute identification maps to match positions of IDs within two vectors.
  # Recall that snpdata.personid has been permuted; thus, we inverse permute.
  #
  personid_in_snpdata = copy(snpdata.personid)
  invpermute!(personid_in_snpdata, entryorder)
  (name_to_id, id_to_name) = correspond(full_person.name, personid_in_snpdata)
  #
  # Compute the genetic relationship matrix and apply Fisher's transform.
  #
  GRM = grm(snpdata.snpmatrix, method=grm_method, minmaf=maf_threshold)
  clamp!(GRM, -0.99999, 0.99999)
  GRM .= atanh.(GRM)
  #
  # For each pedigrees, compute empiric kinship coefficients
  # and subtract theoretical kinship coefficients.
  #
  kinship = Vector{Matrix{Float64}}(undef, full_pedigrees)
  for ped = 1:full_pedigrees
    kinship[ped] = kinship_matrix(full_pedigree, full_person, ped, xlinked)
    q = full_pedigree.pedstart[ped] - 1
    for j = 1:full_pedigree.individuals[ped]
      jj = name_to_id[j + q]
      jj == 0 && continue
      for i = j:full_pedigree.individuals[ped]
        ii = name_to_id[i + q]
        ii == 0 && continue
        GRM[ii, jj] = GRM[ii, jj] - atanh(kinship[ped][i, j])
        GRM[jj, ii] = GRM[ii, jj]
      end
    end
  end
  #
  # Find the indices of the most deviant pairs.
  #
  p = partialsortperm(vec(GRM), 1:deviant_pairs, by = abs, rev = true)
  genotyped_indices = CartesianIndices((genotyped_people, genotyped_people))[p]
  #
  # Enter the most deviant pairs in a dataframe.
  #
  kinship_frame = DataFrame(Pedigree1 = String[], Pedigree2 = String[],
    Person1 = String[], Person2 = String[], theoretical_kinship= Float64[],
    empiric_kinship=Float64[])
  r = 0.0
  for k = 1:deviant_pairs
    #
    # Find the index of the kth largest deviation in the GRM matrix.
    #
    (ii, jj) = (genotyped_indices[k][1], genotyped_indices[k][2])
    #
    # Map this snpid index back to the individual's name.
    #
    (i, j) = (id_to_name[ii], id_to_name[jj])
    j > i && continue
    (pedi, pedj) = (full_person.pedigree[i], full_person.pedigree[j])
    if pedi == pedj
      q = full_pedigree.pedstart[pedi] - 1
      r = GRM[ii, jj] + atanh(kinship[pedi][i - q, j - q])
      push!(kinship_frame,
        [full_pedigree.name[pedi], full_pedigree.name[pedj],
         full_person.name[i], full_person.name[j],
         kinship[pedi][i - q, j - q], tanh(r)])
    else
      push!(kinship_frame,
        [full_pedigree.name[pedi], full_pedigree.name[pedj],
         full_person.name[i], full_person.name[j],
         0.0, tanh(GRM[ii, jj])])
    end
  end
  #
  # Compute the Fisher z-scores.
  #
  kinship_frame = compute_fishers_z(kinship_frame)
  #
  # Create and save plots based on user's requests.
  #
  plot_pair = make_plot_pairs(kinship_frame)
  if keyword["kinship_plot"] != ""
    my_compare_plot = make_compare_plot(kinship_frame, plot_pair)
    # PlotlyBase.savefig(my_compare_plot, keyword["kinship_plot"] * ".pdf")
    PlotlyJS.savefig(my_compare_plot.plot, keyword["kinship_plot"] * ".html")
    println("Kinship plot saved.")
  end
  if keyword["z_score_plot"] != ""
    my_fisher_plot = plot_fisher_z(kinship_frame, plot_pair)
    # PlotlyBase.savefig(my_fisher_plot, keyword["z_score_plot"] * ".pdf")
    PlotlyJS.savefig(my_fisher_plot.plot, keyword["z_score_plot"] * ".html")
    println("Fisher's plot saved.")
  end
  #
  # Write the combined theoretical and empirical kinship frame to a file.
  #
  kinship_table_file = string(keyword["kinship_output_file"])
  CSV.write(kinship_table_file, kinship_frame;
    header = true, delim = keyword["output_field_separator"],
    missingstring = keyword["output_missing_value"])

  return kinship_frame
end # function compare_kinships

"""
    theoretical_coefficients(pedigree::Pedigree, person::Person, keyword::Dict{AbstractString, Any}

This function orchestrates the computation via gene dropping
of the Kinship and Delta7 coefficients deterministically
and Jacquard's 9 identity coefficients stochastically.
The results are placed in a dataframe.
"""
function theoretical_coefficients(pedigree::Pedigree, person::Person,
  keyword::Dict{AbstractString, Any})

  pedigrees = pedigree.pedigrees
  repetitions = keyword["repetitions"]
  xlinked = keyword["xlinked_analysis"]
  #
  # Define dataframes for theoretical Kinship, Delta7, and Jacquard
  # coefficients.
  #
  kinship_dataframe = DataFrame(ped1 = Int[], Pedigree = String[],
    Person1 = String[], Person2 = String[], Kinship = Float64[])
  if !xlinked
    delta_dataframe = DataFrame(ped2 = Int[], Pedigree = String[],
      Person1 = String[], Person2 = String[], Delta7 = Float64[])
  end
  coefficient_dataframe = DataFrame(ped3 = Int[], Pedigree = String[],
    Person1 = String[], Person2 = String[],
    delta1 = Float64[], delta2 = Float64[], delta3 = Float64[],
    delta4 = Float64[], delta5 = Float64[], delta6 = Float64[],
    delta7 = Float64[], delta8 = Float64[], delta9 = Float64[])
  #
  # Loop over all pedigrees.
  #
  for ped = 1:pedigrees
    #
    # Retrieve the various coefficients.
    #
    kinship = kinship_matrix(pedigree, person, ped, xlinked)
    if !xlinked
      delta7 = delta7_matrix(pedigree, person, kinship, ped)
    end
    coefficient = jacquard_coefficients(pedigree, person, ped,
      repetitions, xlinked)
    #
    # Insert the computed coefficients into the appropriate dataframes.
    #
    q = pedigree.pedstart[ped] - 1
    for i = 1:pedigree.individuals[ped]
      for j = i:pedigree.individuals[ped]
        push!(kinship_dataframe, [ped, pedigree.name[ped],
          person.name[i + q], person.name[j + q], kinship[i, j]])
        if !xlinked
          push!(delta_dataframe, [ped, pedigree.name[ped],
            person.name[i + q], person.name[j + q], delta7[i, j]])
        end
        push!(coefficient_dataframe, [ped, pedigree.name[ped],
          person.name[i + q], person.name[j + q],
          coefficient[1, i, j], coefficient[2, i, j], coefficient[3, i, j],
          coefficient[4, i, j], coefficient[5, i, j], coefficient[6, i, j],
          coefficient[7, i, j], coefficient[8, i, j], coefficient[9, i, j]])
      end
    end
  end
  #
  # Join the separate dataframes, and sort the combined dataframe so
  # that relative pairs are lumped by pedigree..
  #
  if xlinked
    combined_dataframe = join(kinship_dataframe, coefficient_dataframe,
      on = [:Pedigree, :Person1, :Person2])
    sort!(combined_dataframe, [:ped1, :Person1, :Person2])
##    deletecols!(combined_dataframe, [:ped1, :ped3])
    names_list = names(combined_dataframe)
    deleteat!(names_list, findall((in)([:ped1, :ped3]), names_list))
    select!(combined_dataframe, names_list)
  else
    temp_dataframe = join(kinship_dataframe, delta_dataframe,
      on = [:Pedigree, :Person1, :Person2])
    combined_dataframe = join(temp_dataframe, coefficient_dataframe,
      on = [:Pedigree, :Person1, :Person2])
    sort!(combined_dataframe, [:ped1, :Person1, :Person2])
##    deletecols!(combined_dataframe, [:ped1, :ped2, :ped3])
    names_list = names(combined_dataframe)
    deleteat!(names_list, findall((in)([:ped1, :ped2, :ped3]), names_list))
    select!(combined_dataframe, names_list)
  end
  #
  # Write the combined coefficient frame to a file and return.
  #
  kinship_table_file = string(keyword["kinship_output_file"])
  CSV.write(kinship_table_file, combined_dataframe;
    header = true, delim = keyword["output_field_separator"],
    missingstring = keyword["output_missing_value"])
  return combined_dataframe
end # function theoretical_coefficients

"""
    kinship_matrix(pedigree::Pedigree, person::Person, ped::Int, xlinked::Bool)

This function recursively computes a kinship matrix for each pedigree.
Children must appear after parents. Each pedigree occurs in a block
demarked by its start and finish and is defined implicitly by
the provided fathers and mothers. Person i is a pedigree founder
if and only if mother[i] = 0. X-linked inheritance is allowed.
"""
function kinship_matrix(pedigree::Pedigree, person::Person,
  ped::Int, xlinked::Bool)
  #
  # Allocate a kinship matrix and an offset.
  #
  pedstart = pedigree.pedstart[ped]
  pedfinish = pedigree.pedfinish[ped]
  kinship = zeros(Float64, pedigree.individuals[ped], pedigree.individuals[ped])
  q = pedstart - 1
  for i = pedstart:pedfinish
    #
    # Initialize the kinship coefficients for founders.
    #
    j = person.mother[i]
    if j == 0
      if xlinked && person.male[i]
        kinship[i - q, i - q] = 1.0
      else
        kinship[i - q, i - q] = 0.5
      end
    else
      #
      # Compute the kinship coefficients of descendants by averaging.
      #
      if xlinked && person.male[i]
        kinship[i - q, i - q] = 1.0
        for m = pedstart:i - 1
          kinship[i - q, m - q] = kinship[j - q, m - q]
          kinship[m - q, i - q] = kinship[i - q, m - q]
        end
      else
        k = person.father[i]
        kinship[i - q, i - q] = 0.5 + 0.5 * kinship[j - q, k - q]
        for m = pedstart:i - 1
          kinship[i - q, m - q] = 0.5 * (kinship[j - q, m - q]
             + kinship[k - q, m - q])
          kinship[m - q, i - q] = kinship[i - q, m - q]
        end
      end
    end
  end
  #
  # Extend the kinship matrix to co-twins.
  #
  cotwin_extension!(kinship, pedigree, person, ped)
  return kinship
end # function kinship_matrix

"""
    delta7_matrix(pedigree::Pedigree, person::Person, kinship::Matrix{Float64}, ped::Int)

This function computes the Delta7 matrix for non-inbred pedigrees.
Children must appear after parents. Each pedigree occurs in a block
and is defined implicitly by the provided fathers and mothers.
Person i is a pedigree founder if and only if mother[i] = 0.
"""
function delta7_matrix(pedigree::Pedigree, person::Person,
  kinship::Matrix{Float64}, ped::Int)

  pedstart = pedigree.pedstart[ped]
  pedfinish = pedigree.pedfinish[ped]
  delta7 = zeros(Float64, pedigree.individuals[ped], pedigree.individuals[ped])
  q = pedstart - 1
  for i = pedstart:pedfinish
    for j = pedstart:i
      if j == i
        delta7[i - q, i - q] = 1.0
      elseif person.mother[i] > 0 && person.mother[j] > 0
        k = person.father[i]
        l = person.mother[i]
        m = person.father[j]
        n = person.mother[j]
        delta7[i - q, j - q] = kinship[k - q, m - q] * kinship[l - q, n - q] +
          kinship[k - q, n - q] * kinship[l - q, m - q]
        delta7[j - q, i - q] = delta7[i - q, j - q]
      end
    end
  end
  #
  # Extend the delta7 matrix to co-twins.
  #
  cotwin_extension!(delta7, pedigree, person, ped)
  return delta7
end # function delta7_matrix

"""
    jacquard_coefficients(pedigree, person, ped::Int, replicates::Int, xlinked::Bool)

This function computes Jacquard's nine condensed identity
coefficients for each pedigree by simulation. Children
must appear after parents. A pedigree is defined implicitly
by giving fathers and mothers. Person i is a pedigree founder
if and only if mother[i] = 0. The more replicates requested,
the more precision attained. X-linked inheritance is possible.
"""
function jacquard_coefficients(pedigree::Pedigree, person::Person,
  ped::Int, replicates::Int, xlinked::Bool)

  pedstart = pedigree.pedstart[ped]
  pedfinish = pedigree.pedfinish[ped]
  ped_size = pedigree.individuals[ped]
  coefficient = zeros(Float64, 9, ped_size, ped_size)
  x = zeros(Float64, ped_size, ped_size)
  source = zeros(Int, 2, ped_size)
  #
  # Loop over all replicates.
  #
  q = pedstart - 1
  for rep = 1:replicates
    f = 0
    for j = pedstart:pedfinish
      if person.mother[j] == 0
        f = f + 1
        source[1, j - q] = f
        if xlinked && person.male[j]
        else
          f = f + 1
        end
        source[2, j - q] = f
      else
        if xlinked && person.male[j]
          u = rand(1:2, 1)
          source[1, j - q] = source[u[1], person.mother[j] - q]
          source[2, j - q] = source[1, j - q]
        else
          u = rand(1:2, 2)
          source[1, j - q] = source[u[1], person.father[j] - q]
          source[2, j - q] = source[u[2], person.mother[j] - q]
        end
      end
    end
    #
    # Loop over all pairs.
    #
    for j = pedstart:pedfinish
      for k = pedstart:pedfinish
        l = identity_state(source[:, j - q], source[:, k - q])
        coefficient[l, j - q, k - q] = coefficient[l, j - q, k - q] + 1.0
      end
    end
  end
  #
  # Extend the coefficients to co-twins.
  #
  for i = 1:9
    x = reshape(coefficient[i, :, :], ped_size, ped_size)
    cotwin_extension!(x, pedigree, person, ped)
    coefficient[i, :, :] = reshape(x, 1, ped_size, ped_size)
  end
  return coefficient / replicates
end # function jacquard_coefficients

"""
    cotwin_extension!(x::Matrix{Float64}, pedigree::Pedigree, person::Person, ped::Int)

This function extends identity coefficients to co-twins.
"""
function cotwin_extension!(x::Matrix{Float64}, pedigree::Pedigree,
  person::Person, ped::Int)

  q = pedigree.pedstart[ped] - 1
  for i = pedigree.pedfinish[ped] + 1:pedigree.twin_finish[ped]
    m = person.primary_twin[i]
    for j = pedigree.pedstart[ped]:pedigree.pedfinish[ped]
      x[i - q, j - q] = x[m - q, j - q]
      x[j - q, i - q] = x[i - q, j - q]
    end
    for j = pedigree.pedfinish[ped] + 1:pedigree.twin_finish[ped]
      n = person.primary_twin[j]
      x[i - q, j - q] = x[m - q, n - q]
    end
  end
end # function cotwin_extension!

"""
    identity_state(source1::Vector{Int}, source2::Vector{Int})

This function assigns one of the 9 condensed identity states
to two sourced genotypes. The integer n is the number of
unique genes among the 4 sampled genes.
"""
function identity_state(source1::Vector{Int}, source2::Vector{Int})

  n = length(union(BitSet(source1), BitSet(source2)))
  if n == 4
    return 9
  elseif n == 3
    if source1[1] == source1[2]
      return 4
    elseif source2[1] == source2[2]
      return 6
    else
      return 8
    end
  elseif n == 2
    if source1[1] == source1[2]
      if source2[1] == source2[2]
        return 2
      else
        return 3
      end
    else
      if source2[1] == source2[2]
        return 5
      else
        return 7
      end
    end
  else
    return 1
  end
end # function identity_state

"""
This function assembles pairs of names and values for printing.
"""
function make_plot_pairs(x::DataFrame)

  plot_pair = Array{String}(undef, size(x, 1))
  for i in 1:length(plot_pair)
    plot_pair[i] = "Person1=" * x[i, 3] * ", " * "Person2=" * x[i, 4]
  end
  return plot_pair
end # function make_plot_pairs

"""
    make_compare_plot(x::DataFrame, plot_pair::Vector{String})

This function composes the comparison plot using PlotlyJS.
"""
function make_compare_plot(x::DataFrame, plot_pair::Vector{String})

  trace1 = scatter(;x=x[!, :theoretical_kinship],
    y=x[!, :empiric_kinship], mode="markers",
    name="empiric kinship", text=plot_pair)
  trace2 = scatter(;x=[1/2, 1/4, 1/8, 1/16, 1/32, 1/64, 1/128, 0.0],
    y=[1/2, 1/4, 1/8, 1/16, 1/32, 1/64, 1/128, 0.0],
    mode="markers", name="marker for midpoint")
  layout = Layout(;title="Compare Empiric and Theoretical Kinships",
    hovermode="closest",
    xaxis=attr(title="Theoretical Kinship",
      showgrid=false, zeroline=false),
    yaxis=attr(title="Empiric Kinship", zeroline=false))
  data = [trace1, trace2]
  plot(data, layout)
end # function make_compare_plot

"""
    compute_fishers_z(x::DataFrame)

This function computes Fisher's atanh transform and standardizes.
"""
function compute_fishers_z(x::DataFrame)

  theoretical_transformed = map(atanh, x[!, :theoretical_kinship])
  empiric_transformed = map(atanh, x[!, :empiric_kinship])
  difference = empiric_transformed - theoretical_transformed
  new_zscore = zscore(difference) #in StatsBase package
  return [x DataFrame(fishers_zscore = new_zscore)]
end # function compute_fishers_z

"""
    plot_fisher_z(x::DataFrame, plot_pair::Vector{String})

This function plots the transformed data using PlotlyJS.
"""
function plot_fisher_z(x::DataFrame, plot_pair::Vector{String})

    trace1 = histogram(x=x[!, :fishers_zscore], text=plot_pair)
    data = [trace1]
    layout = Layout(barmode="overlay",
        title="Z-score plot for Fisher's statistic",
        xaxis=attr(title="Standard Deviations"),
        yaxis=attr(title="Count"))
    plot(data, layout)
end # function plot_fisher_z

"""

"""
function correspond(x::AbstractVector{AbstractString},
  y::AbstractVector{AbstractString})

  (m, n) = (length(x), length(y))
  xperm = sortperm(x)
  yperm = sortperm(y)
  x_to_y = zeros(Int, m)
  y_to_x = zeros(Int, n)
  (i, j) = (1, 1)
  done = false
  while !done
    ii = xperm[i]
    jj = yperm[j]
    if x[ii] == y[jj]
      x_to_y[ii] = jj
      y_to_x[jj] = ii
      (i, j) = (i + 1, j + 1)
    elseif x[ii] < y[jj]
      i = i +1
    elseif y[jj] < x[ii]
      j = j + 1
    end
    done = i > m || j > n
  end
  return (x_to_y, y_to_x)
end # function correspond

# x = ["the", "and", "a", "be", "an"]
# y = ["but", "be", "a", "an", "so"]
# (x_to_y, y_to_x) = correspond(x, y)

"""

"""
function compute_full_person_and_pedigree(keyword :: Dict{AbstractString, Any},
  subset_person :: Person)
#
# Use MendelBase's function to construct the "full" pedigree and person data.
# First, store original key values.
#
  original_pedigree_file = keyword["pedigree_file"]
  snpdefinition_file = keyword["snpdefinition_file"]
  snpdata_file = keyword["snpdata_file"]

  keyword["pedigree_file"] = keyword["full_pedigree_file"]
  keyword["snpdefinition_file"] = "" # Avoid reading in snpdata again.
  keyword["snpdata_file"] = ""
  full_pedigree, full_person, = read_external_data_files(keyword)
  keyword["pedigree_file"] = original_pedigree_file
  keyword["snpdefinition_file"] = snpdefinition_file
  keyword["snpdata_file"] = snpdata_file
#
# Now compute indicator of person i having SNP information.
#
  total_people = full_person.people
  contains_snp = falses(total_people)
  for i in 1:total_people
    person_i = full_person.name[i]
    (person_i in subset_person.name) && (contains_snp[i] = true)
  end

  return full_pedigree, full_person, contains_snp
end # function compute_full_person_and_pedigree

#
# Method to obtain path to this package's data files
# so they can be used in the documentation and testing routines.
# For example, datadir("Control file.txt") will return
# "/path/to/package/data/Control file.txt"
#
datadir(parts...) = joinpath(@__DIR__, "..", "data", parts...)

end # module MendelKinship


module Dionysus
using Flux, FileIO, Pages, Mux

# Build a RNN

rnn = Flux.RNNCell(10, 5)

x = rand(10) # dummy data
h = rand(5)  # initial hidden state

# h, y = rnn2(h, x)

m = Flux.Recur(rnn, h)

seq = [BigFloat(π^i) for i = 1:200]
m.(seq)
seq2 = [BigFloat(e^i) for i = 1:200]

y = m(x)
end # module
msg = hex2bytes("Hello, World!")


_featurepositions(a::Integer, b::Integer) = a:3:b
function _featurepositions(a::AbstractString, b::AbstractString)
    a = Int[ parse(Int,x) for x in split(a,',') ]
    b = Int[ parse(Int,x) for x in split(b,',') ]
    vcat( map((x,y)->x:3:y, a, b)... )
end

function featurepositions(featureTable::DataFrame, featureName)
    row = findfirst(isequal(featureName), featureTable[!,:Feature])
    @assert row!=0
    _featurepositions(featureTable[row,:Start], featureTable[row,:End])
end

function featuresegment(featureTable::DataFrame, featureName)
    row = findfirst(isequal(featureName), featureTable[!,:Feature])
    @assert row!=0
    featureTable[row,:Segment]
end

# returns the most specific (i.e. shortest) feature covering the given position
function featureat(featureTable::DataFrame, position::Integer)
    ind = findall(featureTable[!,:Start].<=position.<=featureTable[!,:End]) # matching features
    isempty(ind) && return ""
    i = argmin( featureTable[ind,:End] - featureTable[ind,:Start] )
    featureTable[ind[i],:Feature] # name of shortest feature
end

function codonposition(featureTable::DataFrame, featureName, position::Integer)
    i = findfirst(isequal(featureName), featureTable[!,:Feature])
    @assert i>0 "Feature $featureName not found in table"
    @assert featureTable[i,:Start]<=position<=featureTable[i,:End]
    p = position-featureTable[i,:Start]
    @assert mod(p,3)==0 "position does does not match the ORF"
    div(p,3)+1
end

function annotatevariants!(swarm::AnnotatedArray, featureTable::DataFrame, reference::AbstractString)
    pos = swarm[:position]
    feature = map(p->featureat(featureTable,p), pos)
    refAA = codon2aa(seq2codons(reference))[pos]
    fpos = map((f,p)->codonposition(featureTable,f,p), feature, pos)
    variantAA = swarm[:aminoacid]

    sz = map(max, size(pos), size(variantAA)) # replace with broadcast_shape?
    # variant in style: feature-refAA-featurePos-variantAA
    variant = broadcast!( (f,r,p,v)->"$f-$r$p$v", Vector{String}(undef,sz), feature, refAA, fpos, variantAA)

    annotate!(swarm,:feature,feature)
    annotate!(swarm,:variant,variant)
end

function variantdescriptions(pos, codons, featureTable::DataFrame, reference::AbstractString)
    pos    = reshape(pos,1,length(pos))
    codons = reshape(codons,length(codons),1)

    feature = map(p->featureat(featureTable,p), pos)
    refAA = codon2aa(seq2codons(reference))[pos]
    fpos = map((f,p)->codonposition(featureTable,f,p), feature, pos)
    variantAA = codon2aa(codons)

    sz = map(max, size(pos), size(variantAA)) # replace with broadcast_shape?
    # variant in style: feature-refAA-featurePos-variantAA
    variants = broadcast!( (f,r,p,v)->"$f-$r$p$v", Vector{String}(undef,sz), feature, refAA, fpos, variantAA)
    variants[:]
end


module Evolutionary
    using Random, LinearAlgebra, Statistics
    using Base: @kwdef
    using UnPack: @unpack
    using NLSolversBase: AbstractObjective, NonDifferentiable, ConstraintBounds,
                         value, value!, nconstraints_x, nconstraints, AbstractConstraints

    import NLSolversBase: NonDifferentiable, f_calls, value, value!
    import Base: show, copy, minimum, summary, identity, getproperty, rand, getindex, length, copyto!, setindex!

    export AbstractStrategy, strategy, mutationwrapper,
           IsotropicStrategy, AnisotropicStrategy, NoStrategy,
           isfeasible, BoxConstraints, apply!, penalty, penalty!,
           PenaltyConstraints, WorstFitnessConstraints, MixedTypePenaltyConstraints,
           # ES mutations
           gaussian, cauchy,
           # GA mutations
           flip, bitinversion, domainrange, inversion, insertion, swap2, scramble, shifting, PM, MIPM,
           # ES recombinations
           average, marriage,
           # GA recombinations
           singlepoint, twopoint, uniform,
           discrete, waverage, intermediate, line, HX, LX, MILX,
           PMX, OX1, CX, OX2, POS,
           # GA selections
           ranklinear, uniformranking, roulette, rouletteinv, sus, susinv, tournament, truncation,
           # DE selections
           random, permutation, randomoffset, best,
           # DE recombinations
           uniformbin, exponential,
           # GP exports
           Terminal, crosstree, mutatetree,
           # Optimization methods
           ES, CMAES, GA, DE, TreeGP,
           # re-export
           NonDifferentiable, value, value!

    # optimize API
    include("api/types.jl")
    include("api/results.jl")
    include("api/utilities.jl")
    include("api/constraints.jl")
    include("api/optimize.jl")

    # ES & GA recombination functions
    include("recombinations.jl")

    # ES & GA mutation functions
    include("mutations.jl")

    # GA selection functions
    include("selections.jl")

    # Evolution Strategy
    include("es.jl")
    include("cmaes.jl")

    # Genetic Algorithms
    include("ga.jl")

    # Differential Evolution
    include("de.jl")

    # Genetic Programming
    include("gp.jl")

    # deprecations
    @deprecate isotropic(recombinant, strategy) gaussian(recombinant, strategy)
    @deprecate anisotropic(recombinant, strategy) gaussian(recombinant, strategy)
    @deprecate isotropicSigma(strategy) gaussian(strategy)
    @deprecate anisotropicSigma(strategy) gaussian(strategy)
    @deprecate averageSigma(strategy) average(strategy)
    @deprecate strategy(kwargs...) IsotropicStrategy(N) false
end


"""
`ReMat` - a scalar random effects matrix

The matrix consists of the grouping factor, `f`, and the transposed dense model
matrix `z`.  The length of `f` must be equal to the number of columns of `z`
"""
abstract AbstractReMat

immutable ReMat <: AbstractReMat
    f::PooledDataVector
    z::Vector
    function ReMat(p::PooledDataVector,z::Vector)
        length(p) == length(z) || throw(DimensionMismatch())
        new(p,z)
    end
end

immutable VectorReMat <: AbstractReMat
    f::PooledDataVector                 # grouping factor
    z::Matrix
    function VectorReMat(p::PooledDataVector,z::Matrix)
        length(p) == size(z,2) || throw(DimensionMismatch())
        size(z,1) > 1 || error("use ReMat instead")
        new(p,z)
    end
end

ReMat(p::PooledDataVector) = ReMat(p,ones(length(p)))

ReMat{T<:Integer}(v::Vector{T}) = ReMat(compact(pool(v)))

Base.eltype(A::AbstractReMat) = eltype(A.z)

vsize(A::AbstractReMat) = isa(A,ReMat) ? 1 : size(A.z,1)

Base.size(A::AbstractReMat) = (length(A.f),vsize(A)*length(A.f.pool))

Base.size(A::AbstractReMat,i::Integer) =
    i < 1 ? throw(BoundsError()) :
    i == 1 ? length(A.f) :
    i == 2 ? vsize(A)*length(A.f.pool) : 1

==(A::AbstractReMat,B::AbstractReMat) = (A.f == B.f) && (A.z == B.z)

# FIXME add a tA boolean argument to combine the code for A_mul_B! and Ac_mul_B!
function Base.A_mul_B!{T}(α::Real,A::AbstractReMat,B::StridedVecOrMat{T},β::Real,R::StridedVecOrMat{T})
    n,q = size(A)
    k = size(B,2)
    size(R,1) == n && size(B,1) == q && size(R,2) == k || throw(DimensionMismatch())
    if β ≠ 1
        scale!(β,R)
    end
    rr = A.f.refs
    zz = A.z
    if isa(A,ReMat)
        for j in 1:k, i in 1:n
            R[i,j] += α * zz[i] * B[rr[i],j]
        end
    else
        l = size(zz,1)
        Bt = reshape(B,(l,div(q,l),k))
        for j in 1:k, i in 1:n
            R[i,j] += α * dot(sub(Bt,:,Int(rr[i]),j),sub(zz,:,i))
        end
    end
    R
end

Base.A_mul_B!{T}(A::AbstractReMat,B::StridedVecOrMat{T},R::StridedVecOrMat{T}) = A_mul_B!(one(T),A,B,zero(T),R)

function Base.Ac_mul_B!{T}(α::Real,A::AbstractReMat,B::StridedVecOrMat{T},β::Real,R::StridedVecOrMat{T})
    n,q = size(A)
    k = size(B,2)
    size(R,1) == q && size(B,1) == n && size(R,2) == k || throw(DimensionMismatch())
    scale!(β,R)
    rr = A.f.refs
    zz = A.z
    if isa(A,ReMat)
        for j in 1:k, i in 1:n
            R[rr[i],j] += α * zz[i] * B[i,j]
        end
    else
        l = size(zz,1)
        rt = reshape(R,(l,div(q,l),k))
        for j in 1:k, i in 1:n
            Base.axpy!(α*B[i,j],sub(zz,:,i),sub(rt,:,Int(rr[i]),j))
        end
    end
    R
end

Base.Ac_mul_B!{T}(R::StridedVecOrMat{T},A::AbstractReMat,B::StridedVecOrMat{T}) = Ac_mul_B!(one(T),A,B,zero(T),R)
function Base.Ac_mul_B(A::AbstractReMat,B::DenseVecOrMat)
    k = size(A,2)
    Ac_mul_B!(Array(eltype(B), isa(B,Vector) ? (k,) : (k, size(B,2))), A, B)
end

function Base.Ac_mul_B(A::ReMat, B::ReMat)
    Az = A.z
    Ar = A.f.refs
    if is(A,B)
        v = zeros(eltype(Az),length(A.f.pool))
        for i in eachindex(Ar)
            v[Ar[i]] += abs2(Az[i])
        end
        return Diagonal(v)
    end
    densify(sparse(convert(Vector{Int32},Ar),convert(Vector{Int32},B.f.refs),Az .* B.z))
end

function Base.Ac_mul_B(A::VectorReMat,B::VectorReMat)
    Az = A.z
    Ar = convert(Vector{Int},A.f.refs)
    if is(A,B)
        l,n = size(Az)
        T = eltype(Az)
        np = length(A.f.pool)
        a = zeros(T,(l,l,np))
        for i in eachindex(Ar)
            Base.LinAlg.BLAS.syr!('L',one(T),sub(Az,:,i),sub(a,:,:,Ar[i]))
        end
        for k in 1:np
            Base.LinAlg.copytri!(sub(a,:,:,k),'L')
        end
        return HBlkDiag(a)
    end
    Bz = B.z
    Br = convert(Vector{Int},B.f.refs)
    (m = length(Ar)) == length(Br) || throw(DimensionMismatch())
    sparse(Ar,Br,[sub(Az,:,i)*sub(Bz,:,i)' for i in 1:m])
end

function Base.Ac_mul_B!{T}(R::DenseVecOrMat{T},A::DenseVecOrMat{T},B::AbstractReMat)
    m = size(A,1)
    n = size(A,2)
    p,q = size(B)
    m == p && size(R,1) == n && size(R,2) == q || throw(DimensionMismatch(""))
    fill!(R,zero(T))
    rr = B.f.refs
    zz = B.z
    if isa(B,ReMat)
        for j in 1:n, i in 1:m
            R[j,rr[i]] += A[i,j] * zz[i]
        end
    else
        l = size(zz,1)
        for j in 1:n, i in 1:m
            Base.LinAlg.axpy!(A[i,j],sub(zz,:,i),sub(R,j,(rr[i]-1)*l + (1:l)))
        end
    end
    R
end

Base.Ac_mul_B(A::DenseVecOrMat,B::AbstractReMat) =
    Ac_mul_B!(Array(eltype(A),(size(A,2),size(B,2))),A,B)


__precompile__()

module ArtBooster

using GR
using XGBoost
using Images

include("dataformats.jl")
include("boosting.jl")
include("rendering.jl")

export boostimage

end # module ArtBooster


using StaticArrays

"""
    clusters::Vector{Cluster}
    data::OrderedDict{FockConfig{N}, OrderedDict{ClusterConfig{N}, Vector{T}}}

This represents an arbitrarily sparse state. E.g., used in TPSCI
"""
struct ClusteredState{T,N,R} <: AbstractState
    clusters::Vector{Cluster}
    data::OrderedDict{FockConfig{N}, OrderedDict{ClusterConfig{N}, MVector{R,T}}}
end
Base.haskey(ts::ClusteredState, i) = return haskey(ts.data,i)
#Base.iterate(ts::ClusteredState, state=1) = iterate(ts.data, state)
#Base.eltype(::Type{ClusteredState{T,N,R}}) where {T,N,R} = OrderedDict{ClusterConfig{N}, MVector{R,T}}

"""
    ClusteredState(clusters; T=Float64, R=1)

Constructor creating an empty vector
# Arguments
- `clusters::Vector{Cluster}`
- `T`:  Type of data for coefficients
- `R`:  Number of roots
# Returns
- `ClusteredState`
"""
function ClusteredState(clusters; T=Float64, R=1)
    N = length(clusters)
    return ClusteredState{T,N,R}(clusters,OrderedDict{FockConfig{N}, OrderedDict{ClusterConfig{N}, MVector{R,T}}}())
end

"""
    function ClusteredState(v::ClusteredState{T,N,R}; T=T, R=R) where {T,N,R}

Constructor creating a `ClusteredState` with the same basis as `v`, but with potentially different `R` and `T`.
Coefficients of new vector are 0.0

# Arguments
- `T`:  Type of data for coefficients
- `R`:  Number of roots
# Returns
- `ClusteredState`
"""
function ClusteredState(v::ClusteredState{TT,NN,RR}; T=TT, R=RR) where {TT,NN,RR}
    out = ClusteredState(v.clusters,T=T,R=R)
    for (fock, configs) in v.data
        add_fockconfig!(out,fock)
        for (config, coeffs) in configs
            out[fock][config] = zeros(T,R)
        end
    end
    return out
end

"""
    ClusteredState(clusters::Vector{Cluster}, fconfig::FockConfig{N}; T=Float64, R=1) where {N}

Constructor using only a single FockConfig. This allows us to turn the CMF state into a ClusteredState.
# Arguments
- `clusters`: vector of clusters types
- `fconfig`: starting FockConfig
- `T`:  Type of data for coefficients
- `R`:  Number of roots
# Returns
- `ClusteredState`
"""
function ClusteredState(clusters::Vector{Cluster}, fconfig::FockConfig{N}; T=Float64, R=1) where {N}
    #={{{=#

    state = ClusteredState(clusters, T=T, R=R)
    add_fockconfig!(state, fconfig)
    conf = ClusterConfig([1 for i in 1:length(clusters)])
    state[fconfig][conf] = zeros(T,R)
    return state
#=}}}=#
end

"""
    add_fockconfig!(s::ClusteredState, fock::FockConfig)
"""
function add_fockconfig!(s::ClusteredState{T,N,R}, fock::FockConfig{N}) where {T<:Number,N,R}
    s.data[fock] = OrderedDict{ClusterConfig{N}, MVector{R,T}}()
    #s.data[fock] = OrderedDict{ClusterConfig{N}, MVector{R,T}}(ClusterConfig([1 for i in 1:N]) => zeros(MVector{R,T}))
end

"""
    getindex(s::ClusteredState, fock::Vector{Tuple{T,T}}) where T<:Integer
"""
#Base.getindex(s::ClusteredState, fock::Vector{Tuple{T,T}}) where T<:Integer = s.data[fock]
@inline Base.getindex(s::ClusteredState, fock) = s.data[fock]
@inline Base.setindex!(s::ClusteredState, a, b) = s.data[b] = a

function Base.size(s::ClusteredState{T,N,R}) where {T,N,R}
    return length(s),R
end
function Base.length(s::ClusteredState)
    l = 0
    for (fock,configs) in s.data
        l += length(keys(configs))
    end
    return l
end
"""
    get_vector(s::ClusteredState; root=1)
"""
function get_vector(s::ClusteredState; root=1)
    v = zeros(length(s))
    idx = 1
    for (fock, configs) in s.data
        for (config, coeff) in configs
            v[idx] = coeff[root]
            idx += 1
        end
    end
    return v
end
"""
    get_vectors(s::ClusteredState)
"""
function get_vectors(s::ClusteredState{T,N,R}) where {T,N,R}
    v = zeros(T,length(s), R)
    idx = 1
    for (fock, configs) in s.data
        for (config, coeff) in configs
            v[idx,:] .= coeff[:]
            idx += 1
        end
    end
    return v
end

"""
    get_vectors!(v, s::ClusteredState)
"""
function get_vectors!(v, s::ClusteredState{T,N,R}) where {T,N,R}
    idx = 1
    for (fock, configs) in s.data
        for (config, coeff) in configs
            v[idx,:] .= coeff[:]
            idx += 1
        end
    end
    return
end

"""
    function set_vector!(ts::ClusteredState{T,N,R}, v::Matrix{T}) where {T,N,R}

Fill the coefficients of `ts` with the values in `v`
"""
function set_vector!(ts::ClusteredState{T,N,R}, v::Matrix{T}) where {T,N,R}

    nbasis = size(v,1)
    nroots = size(v,2)

    length(ts) == nbasis || throw(DimensionMismatch)
    R == nroots || throw(DimensionMismatch)

    idx = 1
    for (fock, tconfigs) in ts.data
        for (tconfig, coeffs) in tconfigs
            #ts[fock][tconfig] = MVector{R}(v[idx,:])
            @views coeffs .= v[idx,:]
            idx += 1
        end
    end
    nbasis == idx-1 || error("huh?", nbasis, " ", idx)
    return
end

#"""
#    function set_vector!(ts::ClusteredState{T,N,R}, v) where {T,N,R}
#
#Fill the coefficients of `ts` with the values in `v`
#"""
#function set_vector!(ts::ClusteredState{T,N,R}, v) where {T,N,R}
#
#    nbasis = size(v,1)
#    nroots = size(v,2)
#
#    length(ts) == nbasis || throw(DimensionMismatch)
#    R == nroots || throw(DimensionMismatch)
#
#    idx = 1
#    for (fock, tconfigs) in ts.data
#        for (tconfig, coeffs) in tconfigs
#            #ts[fock][tconfig] = MVector{R}(v[idx,:])
#            @views coeffs .= v[idx,:]
#            idx += 1
#        end
#    end
#    nbasis == idx-1 || error("huh?", nbasis, " ", idx)
#    return
#end

"""
    Base.display(s::ClusteredState; thresh=1e-3, root=1)

Pretty print
"""
function Base.display(s::ClusteredState; thresh=1e-3, root=1)
    @printf(" --------------------------------------------------\n")
    @printf(" ---------- Fockspaces in state ------: Dim = %5i  \n",length(s))
    @printf(" ----------                root ------:     = %5i  \n",root)
    @printf(" --------------------------------------------------\n")
    @printf(" Printing contributions greater than: %f", thresh)
    @printf("\n")
    @printf(" %-20s%-20s%-20s\n", "Weight", "# Configs", "Fock space(α,β)...")
    @printf(" %-20s%-20s%-20s\n", "-------", "---------", "----------")
    for (fock,configs) in s.data
        prob = 0
        for (config, coeff) in configs
            prob += coeff[root]*coeff[root]
        end
        if prob > thresh
            @printf(" %-20.3f%-20i", prob,length(s.data[fock]))
            for sector in fock
                @printf("(%2i,%-2i)", sector[1],sector[2])
            end
            println()
        end
    end
    print(" --------------------------------------------------\n")
end

"""
    print_configs(s::ClusterState; thresh=1e-3)

Pretty print
"""
function print_configs(s::ClusteredState; thresh=1e-3, root=1)
    #display(keys(s.data))
    idx = 1
    for (fock,configs) in s.data
        length(s.clusters) == length(fock) || throw(Exception)
        length(s.data[fock]) > 0 || continue
        @printf(" Dim %4i fock_space: ",length(s.data[fock]))
        [@printf(" %-2i(%i:%i) ",fii,fi[1],fi[2]) for (fii,fi) in enumerate(fock)]
        println()
        for (config, value) in s.data[fock]
            @printf(" %5i",idx)
            for c in config
                @printf("%3i",c)
            end
            @printf(":%12.8f\n",value[1])
            idx += 1
        end
    end
end

"""
    norm(s::ClusteredState, root)
"""
function LinearAlgebra.norm(s::ClusteredState, root)
    norm = 0
    for (fock,configs) in s.data
        for (config,coeff) in configs
            norm += coeff[root]*coeff[root]
        end
    end
    return sqrt(norm)
end

"""
    norm(s::ClusteredState{T,N,R}) where {T,N,R}
"""
function LinearAlgebra.norm(s::ClusteredState{T,N,R}) where {T,N,R}
    norms = zeros(T,R)
    for (fock,configs) in s.data
        for (config,coeff) in configs
            for r in 1:R
                norms[r] += coeff[r]*coeff[r]
            end
        end
    end
    for r in 1:R
        norms[r] = sqrt(norms[r])
    end
    return norms
end

"""
    normalize!(s::AbstractState)
"""
function normalize!(s::AbstractState)
    scale!(s,1/sqrt(dot(s,s)))
end

"""
    scale!(s::ClusteredState,c)
"""
function scale!(s::ClusteredState{T,N,R},c;root=nothing) where {T,N,R}
    if root == nothing
        for (fock,configs) in s.data
            for (config,coeff) in configs
                s[fock][config] .= coeff.*c
            end
        end
    else
        root <= R || error("root>R")
        for (fock,configs) in s.data
            for (config,coeff) in configs
                s[fock][config][root] = coeff[root]*c
            end
        end
    end
end

"""
    dot(v1::ClusteredState,v2::ClusteredState; r1=1, r2=1)
"""
function dot(v1::ClusteredState{T,N,1},v2::ClusteredState{T,N,1}) where {T,N}
    d = T(0)
    for (fock,configs) in v1.data
        haskey(v2.data, fock) || continue
        for (config,coeff) in configs
            haskey(v2.data[fock], config) || continue
            d += coeff[1] * v2.data[fock][config][1]
        end
    end
    return d
end

"""
    dot(v1::ClusteredState,v2::ClusteredState; r1=1, r2=1)
"""
function dot(v1::ClusteredState{T,N,R}, v2::ClusteredState{T,N,R}, r1, r2) where {T,N,R}
    d = T(0)
    for (fock,configs) in v1.data
        haskey(v2.data, fock) || continue
        for (config,coeff) in configs
            haskey(v2.data[fock], config) || continue
            d += coeff[r1] * v2.data[fock][config][r2]
        end
    end
    return d
end

"""
    dot(v1::ClusteredState,v2::ClusteredState; r1=1, r2=1)
"""
function orth!(v1::ClusteredState{T,N,R}) where {T,N,R}
    d = T(0)
    F = svd(get_vectors(v1))

    set_vector!(v1, F.U*F.Vt)
    return
end

"""
    prune_empty_fock_spaces!(s::ClusteredState)

remove fock_spaces that don't have any configurations
"""
function prune_empty_fock_spaces!(s::ClusteredState)
    keylist = [keys(s.data)...]
    for fock in keylist
        if length(s[fock]) == 0
            delete!(s.data, fock)
        end
    end
#    # I'm not sure why this is necessary
#    idx = 0
#    for (fock,configs) in s.data
#        for (config, coeffs) in s.data[fock]
#            idx += 1
#        end
#    end
    return
end

"""
    zero!(s::ClusteredState)

set all elements to zero
"""
function zero!(s::ClusteredState{T,N,R}) where {T,N,R}
    for (fock,configs) in s.data
        for (config,coeffs) in configs
            s.data[fock][config] = zeros(size(MVector{R,T}))
            #s.data[fock][config] = zeros(MVector{R,T})
        end
    end
end

"""
    function rand!(s::ClusteredState{T,N,R}) where {T,N,R}

set all elements to random values, and orthogonalize
"""
function rand!(s::ClusteredState{T,N,R}) where {T,N,R}
    #={{{=#
    v0 = rand(T,size(s)) .- .5
    set_vector!(s,v0)
    orthonormalize!(s)
end
#=}}}=#

"""
    function orthonormalize!(s::ClusteredState{T,N,R}) where {T,N,R}

orthonormalize
"""
function orthonormalize!(s::ClusteredState{T,N,R}) where {T,N,R}
    #={{{=#
    v0 = get_vectors(s)
    v0[:,1] .= v0[:,1]./norm(v0[:,1])
    for r in 2:R
        #|vr> = |vr> - |v1><v1|vr> - |v2><v2|vr> - ...
        for r0 in 1:r-1
            v0[:,r] .-= v0[:,r0] .* (v0[:,r0]'*v0[:,r])
        end
        v0[:,r] .= v0[:,r]./norm(v0[:,r])
    end
    isapprox(det(v0'*v0), 1.0, atol=1e-14) || @warn "initial guess det(v0'v0) = ", det(v0'v0)
    set_vector!(s,v0)
end
#=}}}=#

"""
    clip!(s::ClusteredState; thresh=1e-5)
"""
function clip!(s::ClusteredState; thresh=1e-5)
#={{{=#
    for (fock,configs) in s.data
        for (config,coeff) in configs
            if all(abs(c) < thresh for c in coeff)
                delete!(s.data[fock], config)
            end
        end
    end
    prune_empty_fock_spaces!(s)
end
#=}}}=#

"""
    add!(s1::ClusteredState, s2::ClusteredState)

Add coeffs in `s2` to `s1`
"""
function add!(s1::ClusteredState, s2::ClusteredState)
    #={{{=#
    for (fock,configs) in s2.data
        if haskey(s1, fock)
            for (config,coeffs) in configs
                if haskey(s1[fock], config)
                    s1[fock][config] .+= s2[fock][config]
                else
                    s1[fock][config] = deepcopy(s2[fock][config])
                end
            end
        else
            s1[fock] = deepcopy(s2[fock])
        end
    end
    #=}}}=#
end

"""
    function extract_roots(v::ClusteredState{T,N,R}, roots)

Extract roots to give new `ClusteredState`
"""
function extract_roots(v::ClusteredState{T,N,R}, roots) where {T,N,R}
    vecs = get_vectors(v)[:,roots]

    out = ClusteredState(v.clusters, T=T, R=length(roots))
    for (fock,configs) in v.data
        add_fockconfig!(out,fock)
        for (config,coeffs) in configs
            out[fock][config] = deepcopy(v[fock][config][roots])
        end
    end

    return out
end


# NOTE in mcmc.jl

using ForwardDiff
using ForwardDiff: value, Dual

CTAG = ForwardDiff.Tag{Val{:custom_tag}, Float64}

#TODO need to pause langevin updates for a moment
"""
    findProposalLaw(xx, tt, P˟, P̃, Ls, Σs; dt=1/5000, timeChange=true,
                    solver::ST=Ralston3())

Initialise the object with proposal law and all the necessary containers needed
for the simulation of the guided proposals
"""
function findProposalLaw(::LangevinUpdt, ::Type{K}, xx, tt, P˟, P̃, Ls, Σs, τ; dt=1/5000,
                         solver::ST=Ralston3(),
                         changePt::ODEChangePt=NoChangePt()) where {K,ST}
    m = length(xx) - 1
    P = Array{ContinuousTimeProcess,1}(undef,m)
    params(Pˣ)
#    P˟_D = clone(Pˣ, )#TODO here is stop point
    for i in m:-1:1
        numPts = Int64(ceil((tt[i+1]-tt[i])/dt))+1
        t = τ(tt[i], tt[i+1]).( range(tt[i], stop=tt[i+1], length=numPts) )
        xx_D = Dual{CT}.(xx[i+1])
        L_D = Dual{CT}.(Ls[i])
        Σ_D = Dual{CT}.(Σs[i])

        P[i] = ( (i==m) ? GuidPropBridge(K, t, P˟, P̃[i], L_D, xx_D, Σ_D;
                                         changePt=changePt, solver=ST()) :
                          GuidPropBridge(K, t, P˟, P̃[i], L_D, xx_D, Σ_D,
                                         P[i+1].H[1], P[i+1].Hν[1], P[i+1].c[1];
                                         changePt=changePt, solver=ST()) )
    end
    P
end

function prepareLangevin(𝓦𝓢::Workspace{ObsScheme}, θ, ::UpdtIdx, y, m, yPr,
                         priors, ::ST, uidx) where {ObsScheme,UpdtIdx,ST}
    idxToUpdt = idx(UpdtIdx())
    function _ll(ϑ)
        XX, WW, P, fpt = 𝓦𝓢.XX, 𝓦𝓢.WW, 𝓦𝓢.P, 𝓦𝓢.fpt
        updateLaws!(P, ϑ)
        solveBackRec!(NoBlocking(), P, ST()) # changes nothing, but needed for ∇
        findPathFromWiener!(XX, y, WW, P, 1:m)

        ll = logpdf(yPr, y)
        ll += pathLogLikhd(ObsScheme(), XX, P, 1:m, fpt)
        ll += lobslikelihood(P[1], y)
        for prior in priors
            ll += logpdf(prior, θ)
        end
        ll
        ϑ[1]
    end
    ϑ = [θ[i] for i in idxToUpdt]
    chunkSize = 1
    result = 𝓦𝓢.result[uidx]
    cfg = ForwardDiff.GradientConfig(_ll, ϑ, ForwardDiff.Chunk{chunkSize}())
    ForwardDiff.gradient!(result, _ll, ϑ, cfg)
    DiffResults.value(result), DiffResults.gradient(result)
end

function postProcessLangevin(𝓦𝓢::Workspace{ObsScheme}, θᵒ, ::UpdtIdx, y, m, yPr,
                         priors, ::ST, uidx) where {ObsScheme,UpdtIdx,ST}
    idxToUpdt = idx(UpdtIdx())
    function _ll(ϑ)
        XXᵒ, WW, Pᵒ, fpt = 𝓦𝓢.XXᵒ, 𝓦𝓢.WW, 𝓦𝓢.Pᵒ, 𝓦𝓢.fpt
        for (i, ui) in enumerate(idxToUpdt)
            θᵒ[ui] = ϑ[i]
        end
        updateLaws!(Pᵒ, θᵒ)
        solveBackRec!(NoBlocking(), Pᵒ, ST()) # changes nothing, but needed for ∇
        findPathFromWiener!(XX, y, WW, P, 1:m)

        yPrᵒ = invStartPt(y, yPr, Pᵒ[1])

        ll = logpdf(yPrᵒ, y)
        ll += pathLogLikhd(ObsScheme(), XXᵒ, Pᵒ, 1:m, fpt)
        ll += lobslikelihood(Pᵒ[1], y)
        for prior in priors
            ll += logpdf(prior, θᵒ)
        end
        ll
        ϑ[2]
    end
    ϑ = [θᵒ[i] for i in idxToUpdt]
    chunkSize = 1
    result = 𝓦𝓢.resultᵒ[uidx]
    cfg = ForwardDiff.GradientConfig(_ll, ϑ, ForwardDiff.Chunk{chunkSize}())
    ForwardDiff.gradient!(result, _ll, ϑ, cfg)

    yPrᵒ = invStartPt(y, yPr, 𝓦𝓢.Pᵒ[1])
    DiffResults.value(result), DiffResults.gradient(result), yPrᵒ
end

function updateParam!(::LangevinUpdt, 𝔅::NoBlocking, tKern, θ,
                      ::UpdtIdx, yPr, 𝓦𝓢::Workspace{ObsScheme}, ll, priors,
                      recomputeODEs; solver::ST=Ralston3(), verbose=false,
                      it=NaN, uidx=NaN) where {ObsScheme <: AbstractObsScheme, ST, UpdtIdx}
    WW, Pᵒ, P, XXᵒ, XX, fpt = 𝓦𝓢.WW, 𝓦𝓢.Pᵒ, 𝓦𝓢.P, 𝓦𝓢.XXᵒ, 𝓦𝓢.XX, 𝓦𝓢.fpt
    m = length(WW)
    y = XX[1].yy[1]
    ll, ∇ll = prepareLangevin(𝓦𝓢, θ, UpdtIdx(), y, m, yPr, priors, ST(), uidx) # TODO pre-allocate ∇ll
    θᵒ = rand(tKern, θ, ∇ll, UpdtIdx())               # sample new parameter
    llᵒ, ∇llᵒ, yPrᵒ = postProcessLangevin(𝓦𝓢, θᵒ, UpdtIdx(), y, m, yPr, priors, ST(), uidx)

    printInfo(verbose, it, ll, llᵒ)

    llr = ( llᵒ - ll + logpdf(tKern, θᵒ, θ, ∇llᵒ, UpdtIdx()) - logpdf(tKern, θ, θᵒ, ∇ll, UpdtIdx()))

    # Accept / reject
    if acceptSample(llr, verbose)
        swap!(XX, XXᵒ, P, Pᵒ, 1:m)
        swap!(𝓦𝓢.resultᵒ, 𝓦𝓢.result, 1:m)
        return llᵒ, true, θᵒ, yPrᵒ
    else
        return ll, false, θ, yPr
    end
end


# MIT license
# Copyright (c) Microsoft Corporation. All rights reserved.
# See LICENSE in the project root for full license information.

#=
Emitters are defined by Pixels and SpatialLayouts. An emitter has a spectrum, and an optical power distribution over the hemisphere.
These are intrinsic physical properties of the emitter.
=#

module Emitters
export generate, visual_size, apply
export right, up, forward
export Spectrum, Directions, Origins, AngularPower, Sources
export pointemitter, collimatedemitter

using ...OpticSim, ..Geometry
using LinearAlgebra
import Unitful.Length
using Unitful.DefaultSymbols

# define some utility functions
right(t::OpticSim.Geometry.Transform{<:Real}) = normalize(Vec3(t[1,1], t[2,1], t[3,1]))
up(t::OpticSim.Geometry.Transform{<:Real}) = normalize(Vec3(t[1,2], t[2,2], t[3,2]))
forward(t::OpticSim.Geometry.Transform{<:Real}) = normalize(Vec3(t[1,3], t[2,3], t[3,3]))
OpticSim.origin(t::OpticSim.Geometry.Transform{<:Real}) = Vec3(t[1,4], t[2,4], t[3,4])

# defining name placeholders to override in nested modules

"""
    generate(???)

[TODO]
"""
function generate end

"""
    visual_size(???)

[TODO]
"""
function visual_size end

"""
    apply(???)

[TODO] Returns ray power
"""
function apply end

include("Spectrum.jl")
include("Directions.jl")
include("Origins.jl")
include("AngularPower.jl")
include("Sources.jl")
include("StandardEmitters.jl")

end # module Emitters

export Emitters


const tri_table = ( (1,9,4,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (1,2,10,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (2,9,4,10,9,2,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (2,3,11,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (1,9,4,2,3,11,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (10,3,11,1,3,10,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (3,9,4,3,11,9,11,10,9,-1,-1,-1,-1,-1,-1),
                    (4,12,3,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (1,12,3,9,12,1,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (2,10,1,3,4,12,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (2,12,3,2,10,12,10,9,12,-1,-1,-1,-1,-1,-1),
                    (4,11,2,12,11,4,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (1,11,2,1,9,11,9,12,11,-1,-1,-1,-1,-1,-1),
                    (4,10,1,4,12,10,12,11,10,-1,-1,-1,-1,-1,-1),
                    (10,9,11,11,9,12,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (5,8,9,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (5,4,1,8,4,5,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (1,2,10,9,5,8,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (5,2,10,5,8,2,8,4,2,-1,-1,-1,-1,-1,-1),
                    (2,3,11,9,5,8,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (4,5,8,4,1,5,2,3,11,-1,-1,-1,-1,-1,-1),
                    (10,3,11,10,1,3,9,5,8,-1,-1,-1,-1,-1,-1),
                    (3,11,10,3,10,8,3,8,4,8,10,5,-1,-1,-1),
                    (9,5,8,4,12,3,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (12,5,8,12,3,5,3,1,5,-1,-1,-1,-1,-1,-1),
                    (10,1,2,9,5,8,3,4,12,-1,-1,-1,-1,-1,-1),
                    (5,8,12,10,5,12,10,12,3,10,3,2,-1,-1,-1),
                    (4,11,2,4,12,11,8,9,5,-1,-1,-1,-1,-1,-1),
                    (2,12,11,2,5,12,2,1,5,8,12,5,-1,-1,-1),
                    (5,8,9,10,1,12,10,12,11,12,1,4,-1,-1,-1),
                    (5,8,12,5,12,10,10,12,11,-1,-1,-1,-1,-1,-1),
                    (10,6,5,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (10,6,5,1,9,4,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (1,6,5,2,6,1,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (9,6,5,9,4,6,4,2,6,-1,-1,-1,-1,-1,-1),
                    (2,3,11,10,6,5,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (4,1,9,2,3,11,5,10,6,-1,-1,-1,-1,-1,-1),
                    (6,3,11,6,5,3,5,1,3,-1,-1,-1,-1,-1,-1),
                    (3,11,6,4,3,6,4,6,5,4,5,9,-1,-1,-1),
                    (10,6,5,3,4,12,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (1,12,3,1,9,12,5,10,6,-1,-1,-1,-1,-1,-1),
                    (1,6,5,1,2,6,3,4,12,-1,-1,-1,-1,-1,-1),
                    (3,2,6,3,6,9,3,9,12,5,9,6,-1,-1,-1),
                    (11,4,12,11,2,4,10,6,5,-1,-1,-1,-1,-1,-1),
                    (5,10,6,1,9,2,9,11,2,9,12,11,-1,-1,-1),
                    (6,5,1,6,1,12,6,12,11,12,1,4,-1,-1,-1),
                    (6,5,9,6,9,11,11,9,12,-1,-1,-1,-1,-1,-1),
                    (10,8,9,6,8,10,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (10,4,1,10,6,4,6,8,4,-1,-1,-1,-1,-1,-1),
                    (1,8,9,1,2,8,2,6,8,-1,-1,-1,-1,-1,-1),
                    (2,6,4,4,6,8,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (10,8,9,10,6,8,11,2,3,-1,-1,-1,-1,-1,-1),
                    (11,2,3,10,6,1,6,4,1,6,8,4,-1,-1,-1),
                    (9,1,3,9,3,6,9,6,8,11,6,3,-1,-1,-1),
                    (3,11,6,3,6,4,4,6,8,-1,-1,-1,-1,-1,-1),
                    (8,10,6,8,9,10,4,12,3,-1,-1,-1,-1,-1,-1),
                    (10,6,8,10,8,3,10,3,1,3,8,12,-1,-1,-1),
                    (3,4,12,1,2,9,2,8,9,2,6,8,-1,-1,-1),
                    (12,3,2,12,2,8,8,2,6,-1,-1,-1,-1,-1,-1),
                    (10,6,9,9,6,8,11,2,4,11,4,12,-1,-1,-1),
                    (6,8,1,6,1,10,8,12,1,2,1,11,12,11,1),
                    (12,11,1,12,1,4,11,6,1,9,1,8,6,8,1),
                    (12,11,6,8,12,6,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (11,7,6,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (1,9,4,6,11,7,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (10,1,2,6,11,7,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (2,9,4,2,10,9,6,11,7,-1,-1,-1,-1,-1,-1),
                    (2,7,6,3,7,2,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (2,7,6,2,3,7,4,1,9,-1,-1,-1,-1,-1,-1),
                    (10,7,6,10,1,7,1,3,7,-1,-1,-1,-1,-1,-1),
                    (6,10,9,6,9,3,6,3,7,4,3,9,-1,-1,-1),
                    (3,4,12,11,7,6,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (12,1,9,12,3,1,11,7,6,-1,-1,-1,-1,-1,-1),
                    (1,2,10,3,4,12,6,11,7,-1,-1,-1,-1,-1,-1),
                    (6,11,7,2,10,3,10,12,3,10,9,12,-1,-1,-1),
                    (7,4,12,7,6,4,6,2,4,-1,-1,-1,-1,-1,-1),
                    (1,9,12,1,12,6,1,6,2,6,12,7,-1,-1,-1),
                    (4,12,7,1,4,7,1,7,6,1,6,10,-1,-1,-1),
                    (7,6,10,7,10,12,12,10,9,-1,-1,-1,-1,-1,-1),
                    (6,11,7,5,8,9,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (5,4,1,5,8,4,7,6,11,-1,-1,-1,-1,-1,-1),
                    (2,10,1,6,11,7,9,5,8,-1,-1,-1,-1,-1,-1),
                    (11,7,6,2,10,8,2,8,4,8,10,5,-1,-1,-1),
                    (7,2,3,7,6,2,5,8,9,-1,-1,-1,-1,-1,-1),
                    (2,3,6,6,3,7,4,1,5,4,5,8,-1,-1,-1),
                    (9,5,8,10,1,6,1,7,6,1,3,7,-1,-1,-1),
                    (8,4,10,8,10,5,4,3,10,6,10,7,3,7,10),
                    (4,12,3,8,9,5,11,7,6,-1,-1,-1,-1,-1,-1),
                    (6,11,7,5,8,3,5,3,1,3,8,12,-1,-1,-1),
                    (1,2,10,5,8,9,3,4,12,6,11,7,-1,-1,-1),
                    (10,3,2,10,12,3,10,5,12,8,12,5,6,11,7),
                    (9,5,8,4,12,6,4,6,2,6,12,7,-1,-1,-1),
                    (6,2,12,6,12,7,2,1,12,8,12,5,1,5,12),
                    (1,6,10,1,7,6,1,4,7,12,7,4,9,5,8),
                    (7,6,10,7,10,12,5,8,10,8,12,10,-1,-1,-1),
                    (11,5,10,7,5,11,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (5,11,7,5,10,11,1,9,4,-1,-1,-1,-1,-1,-1),
                    (11,1,2,11,7,1,7,5,1,-1,-1,-1,-1,-1,-1),
                    (9,4,2,9,2,7,9,7,5,7,2,11,-1,-1,-1),
                    (2,5,10,2,3,5,3,7,5,-1,-1,-1,-1,-1,-1),
                    (4,1,9,2,3,10,3,5,10,3,7,5,-1,-1,-1),
                    (1,3,5,5,3,7,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (9,4,3,9,3,5,5,3,7,-1,-1,-1,-1,-1,-1),
                    (11,5,10,11,7,5,12,3,4,-1,-1,-1,-1,-1,-1),
                    (1,9,3,3,9,12,5,10,11,5,11,7,-1,-1,-1),
                    (4,12,3,1,2,7,1,7,5,7,2,11,-1,-1,-1),
                    (7,5,2,7,2,11,5,9,2,3,2,12,9,12,2),
                    (10,7,5,10,4,7,10,2,4,12,7,4,-1,-1,-1),
                    (9,12,2,9,2,1,12,7,2,10,2,5,7,5,2),
                    (4,12,7,4,7,1,1,7,5,-1,-1,-1,-1,-1,-1),
                    (7,5,9,12,7,9,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (8,11,7,8,9,11,9,10,11,-1,-1,-1,-1,-1,-1),
                    (1,8,4,1,11,8,1,10,11,7,8,11,-1,-1,-1),
                    (11,7,8,2,11,8,2,8,9,2,9,1,-1,-1,-1),
                    (11,7,8,11,8,2,2,8,4,-1,-1,-1,-1,-1,-1),
                    (2,3,7,2,7,9,2,9,10,9,7,8,-1,-1,-1),
                    (3,7,10,3,10,2,7,8,10,1,10,4,8,4,10),
                    (8,9,1,8,1,7,7,1,3,-1,-1,-1,-1,-1,-1),
                    (8,4,3,7,8,3,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (3,4,12,11,7,9,11,9,10,9,7,8,-1,-1,-1),
                    (3,1,8,3,8,12,1,10,8,7,8,11,10,11,8),
                    (2,9,1,2,8,9,2,11,8,7,8,11,3,4,12),
                    (12,3,2,12,2,8,11,7,2,7,8,2,-1,-1,-1),
                    (9,10,7,9,7,8,10,2,7,12,7,4,2,4,7),
                    (1,10,2,12,7,8,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (8,9,1,8,1,7,4,12,1,12,7,1,-1,-1,-1),
                    (8,12,7,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (8,7,12,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (4,1,9,12,8,7,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (1,2,10,12,8,7,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (9,2,10,9,4,2,12,8,7,-1,-1,-1,-1,-1,-1),
                    (11,2,3,7,12,8,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (2,3,11,4,1,9,7,12,8,-1,-1,-1,-1,-1,-1),
                    (3,10,1,3,11,10,7,12,8,-1,-1,-1,-1,-1,-1),
                    (7,12,8,3,11,4,11,9,4,11,10,9,-1,-1,-1),
                    (8,3,4,7,3,8,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (8,1,9,8,7,1,7,3,1,-1,-1,-1,-1,-1,-1),
                    (3,8,7,3,4,8,1,2,10,-1,-1,-1,-1,-1,-1),
                    (2,7,3,2,9,7,2,10,9,9,8,7,-1,-1,-1),
                    (11,8,7,11,2,8,2,4,8,-1,-1,-1,-1,-1,-1),
                    (11,8,7,2,8,11,2,9,8,2,1,9,-1,-1,-1),
                    (1,4,8,1,8,11,1,11,10,7,11,8,-1,-1,-1),
                    (8,7,11,8,11,9,9,11,10,-1,-1,-1,-1,-1,-1),
                    (7,9,5,12,9,7,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (4,7,12,4,1,7,1,5,7,-1,-1,-1,-1,-1,-1),
                    (9,7,12,9,5,7,10,1,2,-1,-1,-1,-1,-1,-1),
                    (10,5,7,10,7,4,10,4,2,12,4,7,-1,-1,-1),
                    (7,9,5,7,12,9,3,11,2,-1,-1,-1,-1,-1,-1),
                    (2,3,11,4,1,12,1,7,12,1,5,7,-1,-1,-1),
                    (5,12,9,5,7,12,1,3,10,3,11,10,-1,-1,-1),
                    (11,10,4,11,4,3,10,5,4,12,4,7,5,7,4),
                    (9,3,4,9,5,3,5,7,3,-1,-1,-1,-1,-1,-1),
                    (1,5,3,5,7,3,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (2,10,1,3,4,5,3,5,7,5,4,9,-1,-1,-1),
                    (2,10,5,2,5,3,3,5,7,-1,-1,-1,-1,-1,-1),
                    (9,2,4,9,7,2,9,5,7,7,11,2,-1,-1,-1),
                    (11,2,1,11,1,7,7,1,5,-1,-1,-1,-1,-1,-1),
                    (5,7,4,5,4,9,7,11,4,1,4,10,11,10,4),
                    (11,10,5,7,11,5,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (5,10,6,8,7,12,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (1,9,4,5,10,6,12,8,7,-1,-1,-1,-1,-1,-1),
                    (6,1,2,6,5,1,8,7,12,-1,-1,-1,-1,-1,-1),
                    (12,8,7,9,4,5,4,6,5,4,2,6,-1,-1,-1),
                    (10,6,5,11,2,3,8,7,12,-1,-1,-1,-1,-1,-1),
                    (7,12,8,2,3,11,1,9,4,5,10,6,-1,-1,-1),
                    (8,7,12,6,5,11,5,3,11,5,1,3,-1,-1,-1),
                    (4,5,9,4,6,5,4,3,6,11,6,3,12,8,7),
                    (8,3,4,8,7,3,6,5,10,-1,-1,-1,-1,-1,-1),
                    (10,6,5,1,9,7,1,7,3,7,9,8,-1,-1,-1),
                    (4,7,3,4,8,7,2,6,1,6,5,1,-1,-1,-1),
                    (7,3,9,7,9,8,3,2,9,5,9,6,2,6,9),
                    (10,6,5,11,2,7,2,8,7,2,4,8,-1,-1,-1),
                    (2,7,11,2,8,7,2,1,8,9,8,1,10,6,5),
                    (5,1,11,5,11,6,1,4,11,7,11,8,4,8,11),
                    (8,7,11,8,11,9,6,5,11,5,9,11,-1,-1,-1),
                    (7,10,6,7,12,10,12,9,10,-1,-1,-1,-1,-1,-1),
                    (4,7,12,1,7,4,1,6,7,1,10,6,-1,-1,-1),
                    (1,12,9,1,6,12,1,2,6,6,7,12,-1,-1,-1),
                    (7,12,4,7,4,6,6,4,2,-1,-1,-1,-1,-1,-1),
                    (2,3,11,10,6,12,10,12,9,12,6,7,-1,-1,-1),
                    (1,12,4,1,7,12,1,10,7,6,7,10,2,3,11),
                    (12,9,6,12,6,7,9,1,6,11,6,3,1,3,6),
                    (7,12,4,7,4,6,3,11,4,11,6,4,-1,-1,-1),
                    (6,9,10,6,3,9,6,7,3,4,9,3,-1,-1,-1),
                    (10,6,7,10,7,1,1,7,3,-1,-1,-1,-1,-1,-1),
                    (2,6,9,2,9,1,6,7,9,4,9,3,7,3,9),
                    (2,6,7,3,2,7,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (2,4,7,2,7,11,4,9,7,6,7,10,9,10,7),
                    (11,2,1,11,1,7,10,6,1,6,7,1,-1,-1,-1),
                    (1,4,9,6,7,11,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (11,6,7,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (12,6,11,8,6,12,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (12,6,11,12,8,6,9,4,1,-1,-1,-1,-1,-1,-1),
                    (6,12,8,6,11,12,2,10,1,-1,-1,-1,-1,-1,-1),
                    (11,8,6,11,12,8,10,9,2,9,4,2,-1,-1,-1),
                    (12,2,3,12,8,2,8,6,2,-1,-1,-1,-1,-1,-1),
                    (1,9,4,2,3,8,2,8,6,8,3,12,-1,-1,-1),
                    (10,8,6,10,3,8,10,1,3,3,12,8,-1,-1,-1),
                    (8,6,3,8,3,12,6,10,3,4,3,9,10,9,3),
                    (3,6,11,3,4,6,4,8,6,-1,-1,-1,-1,-1,-1),
                    (9,3,1,9,6,3,9,8,6,11,3,6,-1,-1,-1),
                    (10,1,2,6,11,4,6,4,8,4,11,3,-1,-1,-1),
                    (10,9,3,10,3,2,9,8,3,11,3,6,8,6,3),
                    (2,4,6,4,8,6,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (1,9,8,1,8,2,2,8,6,-1,-1,-1,-1,-1,-1),
                    (10,1,4,10,4,6,6,4,8,-1,-1,-1,-1,-1,-1),
                    (10,9,8,6,10,8,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (6,9,5,6,11,9,11,12,9,-1,-1,-1,-1,-1,-1),
                    (6,1,5,6,12,1,6,11,12,12,4,1,-1,-1,-1),
                    (1,2,10,9,5,11,9,11,12,11,5,6,-1,-1,-1),
                    (11,12,5,11,5,6,12,4,5,10,5,2,4,2,5),
                    (3,6,2,3,9,6,3,12,9,5,6,9,-1,-1,-1),
                    (1,5,12,1,12,4,5,6,12,3,12,2,6,2,12),
                    (1,3,6,1,6,10,3,12,6,5,6,9,12,9,6),
                    (10,5,6,3,12,4,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (3,6,11,4,6,3,4,5,6,4,9,5,-1,-1,-1),
                    (6,11,3,6,3,5,5,3,1,-1,-1,-1,-1,-1,-1),
                    (4,11,3,4,6,11,4,9,6,5,6,9,1,2,10),
                    (6,11,3,6,3,5,2,10,3,10,5,3,-1,-1,-1),
                    (9,5,6,9,6,4,4,6,2,-1,-1,-1,-1,-1,-1),
                    (1,5,6,2,1,6,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (9,5,6,9,6,4,10,1,6,1,4,6,-1,-1,-1),
                    (10,5,6,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (5,12,8,5,10,12,10,11,12,-1,-1,-1,-1,-1,-1),
                    (1,9,4,5,10,8,10,12,8,10,11,12,-1,-1,-1),
                    (2,11,12,2,12,5,2,5,1,8,5,12,-1,-1,-1),
                    (4,2,5,4,5,9,2,11,5,8,5,12,11,12,5),
                    (5,12,8,10,12,5,10,3,12,10,2,3,-1,-1,-1),
                    (10,8,5,10,12,8,10,2,12,3,12,2,1,9,4),
                    (12,8,5,12,5,3,3,5,1,-1,-1,-1,-1,-1,-1),
                    (12,8,5,12,5,3,9,4,5,4,3,5,-1,-1,-1),
                    (3,10,11,3,8,10,3,4,8,8,5,10,-1,-1,-1),
                    (10,11,8,10,8,5,11,3,8,9,8,1,3,1,8),
                    (4,8,11,4,11,3,8,5,11,2,11,1,5,1,11),
                    (2,11,3,9,8,5,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (5,10,2,5,2,8,8,2,4,-1,-1,-1,-1,-1,-1),
                    (5,10,2,5,2,8,1,9,2,9,8,2,-1,-1,-1),
                    (5,1,4,8,5,4,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (5,9,8,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (10,11,9,11,12,9,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (4,1,10,4,10,12,12,10,11,-1,-1,-1,-1,-1,-1),
                    (1,2,11,1,11,9,9,11,12,-1,-1,-1,-1,-1,-1),
                    (4,2,11,12,4,11,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (2,3,12,2,12,10,10,12,9,-1,-1,-1,-1,-1,-1),
                    (4,1,10,4,10,12,2,3,10,3,12,10,-1,-1,-1),
                    (1,3,12,9,1,12,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (4,3,12,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (3,4,9,3,9,11,11,9,10,-1,-1,-1,-1,-1,-1),
                    (10,11,3,1,10,3,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (3,4,9,3,9,11,1,2,9,2,11,9,-1,-1,-1),
                    (2,11,3,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (2,4,9,10,2,9,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (1,10,2,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1),
                    (1,4,9,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1))

v_table = []
for trimap in tri_table
    a = tuple(filter(x-> x>0, unique!([trimap...]))...)
    b = zeros(UInt8, 12)
    for i in eachindex(a)
        b[i] = UInt8(a[i])
    end
    push!(v_table, tuple(b...))
end

for v in v_table
    println(string(v)*",")
end


using Mimi
using Distributions
using DataFrames
using CSV
using Random

# standard output folder:
global dir_output = "C:/Users/jarmo/Documents/GitHub/PAGEoutput/FixSCC"
global scenario = "NDCs"

function set_default_bools()
    # use annual version
    global use_annual = true
    # use interannual variability
    global use_interannvar = true

    # set default monte carlo parameters to the full set of probability distributions
    global use_only_varMC = false
    global use_no_varMC = false
    # set default interpolation parameters
    global use_linear = false
    global use_logburke = true
    global use_logpopulation = false

    # set only when doing specific variability sensitivity analysis
    global globallyset_use_interannvar = false
end

# set default; interannual global mean standard deviation
global set_gvarsd = 0.11294 # default from https://github.com/jkikstra/climvar
# in case you so desire, set one default; interannual regional mean standard deviation [normally read from rvarsd_regionalvariabilitystandarddeviation.csv]
# global v_multiplier = 1 # introduced for sensitivity analysis for variability; default = 1 -> variability according to empirical results
set_default_bools()

# # Run.
# # with variability
# numMCruns = 20000
# include("getpagefunction.jl")
# include("utils/mctools.jl")
# include("mcs_scc_dist.jl")
#
# get_pre_scc_mcs(numMCruns, 2020, dir_output, scenario)

# Run.
# without variability
global use_interannvar = false
dir_output = "C:/Users/jarmo/Documents/GitHub/PAGEoutput/FixSCC/Novar"

numMCruns = 30000
include("getpagefunction.jl")
include("utils/mctools.jl")
include("mcs_scc_dist.jl")

get_pre_scc_mcs(numMCruns, 2020, dir_output, scenario)


#
# April 2020 Benefit Rates: CAREFUL
#
sys.it.savings_rates   =  [0.0, 20.0, 40.0, 45.0]
sys.it.savings_thresholds   =  [5_000.0, 37_500.0, 150_000.0]
sys.it.savings_basic_rate = 2 # above this counts as higher rate

sys.it.non_savings_rates  =  [19.0,20.0,21.0,41.0,46.0]
sys.it.non_savings_thresholds  =  [2_085, 12_658, 31_931, 150_000.0]
sys.it.non_savings_basic_rate = 2 # above this counts as higher rate

sys.it.dividend_rates  =  [0.0, 7.5,32.5,38.1]
sys.it.dividend_thresholds  =  [2_000.0, 37_500.0, 150_000.0]
sys.it.dividend_basic_rate = 2 # above this counts as higher rate

sys.it.personal_allowance          = 12_500.00
sys.it.personal_allowance_income_limit = 100_000.00
sys.it.personal_allowance_withdrawal_rate = 50.0
sys.it.blind_persons_allowance     = 2_500.00

sys.it.married_couples_allowance   = 9_075.00
sys.it.mca_minimum                 = 3_510.00
sys.it.mca_income_maximum          = 29_600.00
sys.it.mca_credit_rate             = 10.0
sys.it.mca_withdrawal_rate        = 50.0

sys.it.marriage_allowance          = 1_260.00 # check 20/21
sys.it.personal_savings_allowance  = 1_000.00

# FIXME better to have it straight from
# the book with charges per CO2 range
# and the data being an estimate of CO2 per type
merge( sys.it.company_car_charge_by_CO2_emissions,
    Dict([
        Missing_Fuel_Type=>0.1,
        No_Fuel=>0.1,
        Other=>0.1,
        Dont_know=>0.1,
        Petrol=>0.25,
        Diesel=>0.37,
        Hybrid_use_a_combination_of_petrol_and_electricity=>0.16,
        Electric=>0.02,
        LPG=>0.02,
        Biofuel_eg_E85_fuel=>0.02 ]))

sys.it.fuel_imputation  = 24_500.00 # 20/21

#
# pensions
# CHECK THESE FOR UPRATING
sys.it.pension_contrib_basic_amount = 3_600.00
sys.it.pension_contrib_annual_allowance = 40_000.00
sys.it.pension_contrib_annual_minimum = 4_000.00
sys.it.pension_contrib_threshold_income = 240_000.00
sys.it.pension_contrib_withdrawal_rate = 50.0

# sys.it.non_savings_income = NON_SAVINGS_INCOME
# sys.it.all_taxable = ALL_TAXABLE_INCOME
# sys.it.savings_income = SAVINGS_INCOME
# sys.it.dividend_income = DIVIDEND_INCOME

# sys.it.mca_date = MCA_DATE

sys.ni.abolished = false
sys.ni.primary_class_1_rates  = [0.0, 0.0, 12.0, 2.0 ]
sys.ni.primary_class_1_bands  = [120.0, 183.0, 962.0, 9999999999999.9] # the '-1' here is because json can't write inf
sys.ni.secondary_class_1_rates  = [0.0, 13.8, 13.8 ] # keep 2 so
sys.ni.secondary_class_1_bands  = [170.0, 967.0, 99999999999999.9 ]
sys.ni.state_pension_age = 66; # fixme move
sys.ni.class_2_threshold = 6_475.0;
sys.ni.class_2_rate = 3.05;
sys.ni.class_4_rates  = [0.0, 9.0, 2.0 ]
sys.ni.class_4_bands  = [9_500.0, 50_270.0, 99999999999999.9 ]
# sys.ni.class_1_income = IncludedItems([WAGES],[PENSION_CONTRIBUTIONS_EMPLOYER])
# sys.ni.class_4_income = [SELF_EMPLOYMENT_INCOME]

# sys.uc.

sys.lmt.abolished = false

## FIXME we can't turn off pension credit individually here..

sys.lmt.premia.family = 17.60
sys.lmt.premia.family_lone_parent = 22.20
sys.lmt.premia.carer_single = 37.50
sys.lmt.premia.carer_couple = 2*37.50
sys.lmt.premia.disabled_child = 65.52
sys.lmt.premia.disability_single = 34.95
sys.lmt.premia.disability_couple = 49.80
sys.lmt.premia.enhanced_disability_child = 26.60
sys.lmt.premia.enhanced_disability_single = 17.10
sys.lmt.premia.enhanced_disability_couple = 24.50
sys.lmt.premia.severe_disability_single = 66.95
sys.lmt.premia.severe_disability_couple = 133.90
sys.lmt.premia.pensioner_is = 148.40

sys.lmt.allowances.age_18_24 = 58.90
sys.lmt.allowances.age_25_and_over = 74.35
sys.lmt.allowances.age_18_and_in_work_activity = 74.35
sys.lmt.allowances.over_pension_age = 187.75 #
sys.lmt.allowances.lone_parent = 74.70
sys.lmt.allowances.lone_parent_over_pension_age = 191.15
sys.lmt.allowances.couple_both_under_18 = 74.35
sys.lmt.allowances.couple_both_over_18 = 74.35
sys.lmt.allowances.couple_over_pension_age =  280.85
sys.lmt.allowances.couple_one_over_18_high =  116.80
sys.lmt.allowances.couple_one_over_18_med = 74.35
sys.lmt.allowances.pa_couple_one_over_18_low = 58.90
sys.lmt.allowances.child = 68.27
sys.lmt.allowances.pc_mig_single = 173.75
sys.lmt.allowances.pc_mig_couple = 265.20

# sys.lmt.income_rules.
sys.lmt.income_rules.permitted_work = 140.00
sys.lmt.income_rules.lone_parent_hb = 25.0
sys.lmt.income_rules.high = 20.0
sys.lmt.income_rules.low_couple = 10.0
sys.lmt.income_rules.low_single = 5.0
sys.lmt.income_rules.hb_additional = 17.10
sys.lmt.income_rules.childcare_max_1 = 175.0
sys.lmt.income_rules.childcare_max_2 = 300.0
sys.lmt.income_rules.incomes     = LEGACY_MT_INCOME
sys.lmt.income_rules.hb_incomes  = LEGACY_HB_INCOME
sys.lmt.income_rules.pc_incomes  = LEGACY_PC_INCOME
sys.lmt.income_rules.sc_incomes  = LEGACY_SAVINGS_CREDIT_INCOME
sys.lmt.income_rules.capital_min = 6_000.0
sys.lmt.income_rules.capital_max = 16_000.0
sys.lmt.income_rules.pc_capital_min = 10_000.0
sys.lmt.income_rules.pc_capital_max = 99999999999999.9
sys.lmt.income_rules.pensioner_capital_min = 10_000.0
sys.lmt.income_rules.pensioner_capital_max = 16_000.0

sys.lmt.income_rules.capital_tariff = 250
sys.lmt.income_rules.pensioner_tariff = 500
# FIXME why do we need a seperate copy of HoursLimits here?

sys.lmt.hours_limits.lower = 16
sys.lmt.hours_limits.med = 24
sys.lmt.hours_limits.higher = 30

sys.lmt.savings_credit.abolished = false
sys.lmt.savings_credit.withdrawal_rate = 60.0
sys.lmt.savings_credit.threshold_single = 150.47
sys.lmt.savings_credit.threshold_couple = 239.17
sys.lmt.savings_credit.max_single = 13.97
sys.lmt.savings_credit.max_couple = 15.62
sys.lmt.savings_credit.available_till = Date( 2016, 04, 06 )

sys.lmt.child_tax_credit.abolished = false
sys.lmt.child_tax_credit.family = 545.0
sys.lmt.child_tax_credit.child  = 2_830.0
sys.lmt.child_tax_credit.disability = 3_415
sys.lmt.child_tax_credit.severe_disability = 4825
sys.lmt.child_tax_credit.threshold = 16_385.0

sys.lmt.working_tax_credit.basic = 3_040 # with the 10pw uplift
sys.lmt.working_tax_credit.lone_parent = 2_045
sys.lmt.working_tax_credit.couple  = 2_045
sys.lmt.working_tax_credit.hours_ge_30 = 825
sys.lmt.working_tax_credit.disability = 3_220
sys.lmt.working_tax_credit.severe_disability = 1_390
sys.lmt.working_tax_credit.age_50_plus  = 1_365.00  # not used?
sys.lmt.working_tax_credit.age_50_plus_30_hrs = 2_030.00 # not used?
sys.lmt.working_tax_credit.childcare_max_2_plus_children = 300.0 # pw
sys.lmt.working_tax_credit.childcare_max_1_child  = 175.0
sys.lmt.working_tax_credit.childcare_proportion = 70.0
sys.lmt.working_tax_credit.taper = 41.0
sys.lmt.working_tax_credit.threshold = 6_530.0
sys.lmt.working_tax_credit.non_earnings_minima = 300.0 # FIXME check

sys.lmt.hb.taper = 65.0
sys.lmt.hb.ndd_deductions = [15.85,36.45,50.05,81.90,93.25,102.35]
sys.lmt.hb.ndd_incomes = [149.0,217.0,283.0,377.0,469.0,99999999999999.9]

sys.lmt.ctr.taper = 20.0
sys.lmt.ctr.ndd_deductions = []
sys.lmt.ctr.ndd_incomes = []

sys.uc.abolished = false
sys.uc.threshold = 2_500.0 ## NOT USED
sys.uc.age_18_24 = 342.72
sys.uc.age_25_and_over = 409.89

sys.uc.couple_both_under_25 = 488.59
sys.uc.couple_oldest_25_plus = 594.04

sys.uc.first_child  = 281.25
sys.uc.subsequent_child = 235.83
sys.uc.disabled_child_lower = 128.25
sys.uc.disabled_child_higher = 400.29
sys.uc.limited_capcacity_for_work_activity = 341.92
sys.uc.carer = 163.73

sys.uc.ndd = 75.15

sys.uc.childcare_max_2_plus_children  = 1_108.04 # pm
sys.uc.childcare_max_1_child  = 646.35
sys.uc.childcare_proportion = 85.0 # pct

sys.uc.minimum_income_floor_hours = 35*WEEKS_PER_MONTH

sys.uc.work_allowance_w_housing = 292.0
sys.uc.work_allowance_no_housing = 512.0
sys.uc.other_income = UC_OTHER_INCOME
# sys.uc.earned_income :: IncludedItems = UC_EARNED_INCOME
sys.uc.capital_min = 6_000.0
sys.uc.capital_max = 16_000.0
# £1 *per week* ≆ 4.35 pm FIXME make 4.35 WEEKS_PER_MONTH?
sys.uc.capital_tariff = 250.0/4.35
sys.uc.taper = 63.0
sys.uc.ctr_taper = 20.0

sys.age_limits.state_pension_ages = pension_ages()
sys.age_limits.savings_credit_to_new_state_pension = Date( 2016, 04, 06 )

sys.hours_limits.lower = 16
sys.hours_limits.med = 24
sys.hours_limits.higher = 30

sys.child_limits.max_children = 2
# TILL APRIL 22
sys.minwage.ages = [16,18,21,23]
sys.minwage.wage_per_hour = [4.62, 6.56, 8.36, 8.91]
sys.minwage.apprentice_rate = 4.30

sys.hr.maximum_rooms = 4
sys.hr.rooms_rent_reduction = [14.0,25.0]
sys.hr.single_room_age = 35
#
# These are unchanged in 3 years; see:
# https://www.gov.scot/publications/local-housing-allowance-rates-2021-2022/
#
sys.hr.brmas = loadBRMAs( 4, T, DEFAULT_BRMA_2021 )

sys.nmt_bens.attendance_allowance.abolished = false
sys.nmt_bens.attendance_allowance.higher = 89.60
sys.nmt_bens.attendance_allowance.lower = 60.00

sys.nmt_bens.child_benefit.abolished = false
sys.nmt_bens.child_benefit.first_child = 21.05
sys.nmt_bens.child_benefit.other_children = 13.95
sys.nmt_bens.child_benefit.high_income_thresh = 50_000.0
sys.nmt_bens.child_benefit.withdrawal = 1/100
sys.nmt_bens.child_benefit.guardians_allowance = 17.90

sys.nmt_bens.dla.abolished = false
sys.nmt_bens.dla.care_high = 89.15
sys.nmt_bens.dla.care_middle  = 59.70
sys.nmt_bens.dla.care_low  = 23.60
sys.nmt_bens.dla.mob_high  = 62.25
sys.nmt_bens.dla.mob_low  = 23.60

sys.nmt_bens.carers.abolished = false
sys.nmt_bens.carers.allowance =  67.25
sys.nmt_bens.carers.scottish_supplement = 231.40
sys.nmt_bens.carers.hours :: Int = 35
sys.nmt_bens.carers.gainful_employment_min = 128.00

sys.nmt_bens.pip.abolished = false
sys.nmt_bens.pip.dl_standard = 59.70
sys.nmt_bens.pip.dl_enhanced = 89.15
sys.nmt_bens.pip.mobility_standard = 23.60
sys.nmt_bens.pip.mobility_enhanced = 62.25

sys.nmt_bens.esa.abolished = false
sys.nmt_bens.esa.assessment_u25 = 58.90
sys.nmt_bens.esa.assessment_25p = 74.35
sys.nmt_bens.esa.main           = 74.35
sys.nmt_bens.esa.work           = 29.55
sys.nmt_bens.esa.support        = 39.20

sys.nmt_bens.jsa.abolished = false
sys.nmt_bens.jsa.u25 = 58.90
sys.nmt_bens.jsa.o24 = 74.35

sys.nmt_bens.pensions.abolished = false
sys.nmt_bens.pensions.new_state_pension = 175.20
# pension_start_date = Date( 2016, 04, 06 )
sys.nmt_bens.pensions.cat_a     = 134.25
sys.nmt_bens.pensions.cat_b     = 134.25
sys.nmt_bens.pensions.cat_b_survivor = 80.45
sys.nmt_bens.pensions.cat_d     = 80.45

sys.nmt_bens.bereavement.abolished = false
# higher effectively just means 'with children';
sys.nmt_bens.bereavement.lump_sum_higher = 3_500
sys.nmt_bens.bereavement.lump_sum_lower  = 2_500
sys.nmt_bens.bereavement.higher = 350
sys.nmt_bens.bereavement.lower  = 100

sys.nmt_bens.widows_pension.abolished = false
sys.nmt_bens.widows_pension.industrial_higher = 134.25
sys.nmt_bens.widows_pension.industrial_lower = 40.28
sys.nmt_bens.widows_pension.standard_rate =  121.95
sys.nmt_bens.widows_pension.parent =  121.95
sys.nmt_bens.widows_pension.ages = collect(54:-1:45)
sys.nmt_bens.widows_pension.age_amounts = [113.41,104.88,96.34,87.80,79.27,70.73,62.19,53.66,45.12,36.59]

#
# young carer grant
sys.nmt_bens.maternity.abolished = false
sys.nmt_bens.maternity.rate = 151.20

sys.nmt_bens.smp = 151.97 ## 90% of earn cpag 21/2 812
# = XX

sys.bencap.abolished = false
sys.bencap.outside_london_single = 257.69
sys.bencap.outside_london_couple = 384.62
# not really needed, but anyway ..
sys.bencap.inside_london_single = 296.35
sys.bencap.inside_london_couple  = 442.31
sys.bencap.uc_incomes_limit  = 617

sys.scottish_child_payment.amount = 10.0
sys.scottish_child_payment.maximum_age = 5

sys.ubi.abolished = true
sys.ubi.adult_amount = 4_800.0
sys.ubi.child_amount= 3_000.0
sys.ubi.universal_pension = 8_780.0
sys.ubi.adult_age = 17
sys.ubi.retirement_age = 66

weeklyise!( sys )


module UTFDetect

utf8_lead_1(b::UInt8) = leading_ones(b) == 0
utf8_lead_2(b::UInt8) = leading_ones(b) == 2
utf8_lead_3(b::UInt8) = leading_ones(b) == 3
utf8_lead_4(b::UInt8) = leading_ones(b) == 4
utf8_cont(b::UInt8)   = leading_ones(b) == 1

utf16_lone(b::UInt8)  = (b >> 3) != 0b11011
utf16_lead(b::UInt8)  = (b >> 2) == 0b110110
utf16_trail(b::UInt8) = (b >> 2) == 0b110111

utf32_byte_4(b::UInt8) = b == 0x00
utf32_byte_3(b::UInt8) = b <= 0x10

false_p(b::UInt8) = false
true_p(b::UInt8)  = true

const predicates = [
    utf8_lead_1, utf8_lead_2, utf8_lead_3, utf8_lead_4, utf8_cont,
    utf16_lone, utf16_lead, utf16_trail,
    utf32_byte_4, utf32_byte_3,
    false_p, true_p,
]

p_mask(byte::UInt8) =
    sum(pred(byte) << (i-1) for (i, pred) in enumerate(predicates))
p_mask(p::Vector{<:Function}) =
    sum((pred in p) << (i-1) for (i, pred) in enumerate(predicates))

p_masks = sort!(unique(p_mask(byte) for byte = 0x0:0xff))
# 10 total predicate masks < 4 bits
@assert length(p_masks) == 10

# construct a predicate lookup table indexing into p_masks
p_table = UInt8[findfirst(==(p_mask(UInt8(b))), p_masks)-1 for b=0x00:0xff]
@assert all(p_mask(b) == p_masks[p_table[b+1]+1] for b = 0x00:0xff)

# UTF-8 state: counter of remaining trailing bytes from 3 down to 0 (bits: 2)
# UTF-16 state: hi/lo alternator, trail indicator (bits: 3, one shared with UTF-32)
# UTF-32 state: byte counter for 0-3 (bits: 2, one shared with UTF-16)
# total state: 6 bits (3 data-dependent, 3 data-independent)

# preds: value from p_table, index into p_masks
# represents a possible set of true predicates

function ⋲(p::Function, preds::UInt8)
    mask = p_masks[preds+1]
    for (i, p′) in enumerate(predicates)
        p′ == p && return !iszero(mask & (1 << (i-1)))
    end
    error("predicate not found: $p")
end

# u8_next: u8_trail must be 0 at start of a character
# set to the number of trailing continuation characters
# decremented after each continuation character

function u8_next(preds::UInt8, u8_trail::UInt8)::UInt8
    utf8_cont   ⋲ preds ? max(0, u8_trail-1) :
    utf8_lead_4 ⋲ preds ? 3 :
    utf8_lead_3 ⋲ preds ? 2 :
    utf8_lead_2 ⋲ preds ? 1 : 0
end

function u8_valid(u8_trail::UInt8)
    u8_trail == 0 ?
        p_mask([utf8_lead_1, utf8_lead_2, utf8_lead_3, utf8_lead_4]) :
        p_mask([utf8_cont])
end

# u16_next: only called on high bytes, for little-endian & big-endian
# there's a separate state bit for each one that's passed through as-is
# for bytes that aren't the high byte for that endianness

function u16_next(preds::UInt8)::UInt8
    utf16_lead ⋲ preds
end

function u16_valid(u16_trail::UInt8, count::UInt8, be::Bool)
    hi = be ⊻ isodd(count) # count is zero-based
    hi && u16_trail == 0 ? p_mask([utf16_lone, utf16_lead]) :
    hi && u16_trail == 1 ? p_mask([utf16_trail]) :
                           p_mask([true_p])
end

# u32_next: doesn't exist since UTF-32 has no data-dependent state

function u32_valid(count::UInt8, be::Bool)
    (count & 3) == (be ? 0 : 3) ? p_mask([utf32_byte_4]) :
    (count & 3) == (be ? 1 : 2) ? p_mask([utf32_byte_3]) :
                                  p_mask([true_p])
end

# build the transition table

t_table = zeros(UInt8, 2^4*2^2*10)
indices = Int[]
for u8_trail = 0b00:0b11, u16l_trail = 0b0:0b1, u16b_trail = 0b0:0b1
    state = u8_trail | (u16l_trail << 2) | (u16b_trail << 3)
    for count = 0b00:0b11
        mask_u8   = u8_valid(u8_trail)
        mask_u16l = u16_valid(u16l_trail, count, false)
        mask_u16b = u16_valid(u16b_trail, count, true)
        mask_u32l = u32_valid(count, false)
        mask_u32b = u32_valid(count, true)
        for preds in 0x0:0x9
            index = state | (count << 4) | (UInt16(preds) << 6)
            # compute the valid mask
            mask  = p_masks[preds+1]
            u8    = UInt8(!iszero(mask & mask_u8))
            u16l  = UInt8(!iszero(mask & mask_u16l))
            u16b  = UInt8(!iszero(mask & mask_u16b))
            u32l  = UInt8(!iszero(mask & mask_u32l))
            u32b  = UInt8(!iszero(mask & mask_u32b))
            valid = u8 | (u16l << 1) | (u16b << 2) | (u32l << 3) | (u32b << 4)
            # compute the next state
            u8_trail′ = u8_next(preds, u8_trail)
            if iseven(count)
                u16l_trail′ = u16l_trail
                u16b_trail′ = u16_next(preds)
            else
                u16l_trail′ = u16_next(preds)
                u16b_trail′ = u16b_trail
            end
            state′ = u8_trail′ | (u16l_trail′ << 2) | (u16b_trail′ << 3)
            # populate the table
            t_table[index + 1] = valid | (state′ << 5)
            push!(indices, index)
        end
    end
end
@assert sort!(indices) == 0:length(t_table)-1

const table = Tuple([p_table; t_table])

function detect(bytes::Union{AbstractVector{UInt8}, IO})
    valid = 0b11111
    state = 0b000
    count = 0b00
    @inbounds for byte in bytes
        preds = table[byte + 1]
        index = state | (count << 4) | (UInt16(preds) << 6)
        state = table[index + 1 + 256]
        valid &= state
        state >>= 5
        count = (count + 0x1) & 0x3
    end
    return valid, state, count
end

end # module


#Written By Johnathan Bizzano
module Initializer
    using ..Utils

    @Fun(Func{StorageType}, cons_and_bounds::StorageType, idx::Int)

    export RNG, None

    None() = MemoryWrapper(sett -> Func{sett.storageType}(idx -> 0))
    RNG() = MemoryWrapper(sett -> Func{sett.storageType}(idx -> rand(range)))
end


abstract type AbstractEnv end

"""
    Supply Chain Simulation Environment Constructor

# Fields:
- `network::MetaDiGraph`: Supply chain network directed graph.
- `markets::Vector`: Vector of market nodes where demand occurs.
- `producers::Vector`: Vector of producer nodes where material transformation occurs.
- `distributors::Vector`: Vector of distribution centers (excludes market nodes).
- `echelons::Dict`: Dictionary with Vector of nodes downstream of each node in the network (including that node).
- `materials::Vector`: Vector with the names of all materials in the system.
- `products::Vector`: Vector with the names of all final products in the system.
- `bill_of_materials::Array`: Square matrix with BOM (rows = inputs, cols = ouputs); indices follow materials list; row with positive value is a co-product, row with negative value is a input (reactant).
- `inv_on_hand::DataFrame`: Timeseries with on hand inventories @ each node.
- `inv_level::DataFrame`: Timeseries with inventory level @ each node (on-hand minus backlog, if backlogging is allowed)
- `inv_pipeline::DataFrame`: Timeseries with pipeline inventories on each arc.
- `inv_position::DataFrame`: Timeseries with inventory positions @ each node (inventory level + placed replenishments).
- `ech_position::DataFrame`: Timeseries with echelon inventory position @ each node.
- `replenishments::DataFrame`: Timeseries with replenishment orders placed on each arc.
- `shipments::DataFrame`: Temp table with active shipments and time to arrival on each arc.
- `production::DataFrame`: Temp table with active material production commited to an arc and time to ship.
- `demand::DataFrame`: Timeseries with realization of demand at each market, and amounts sold, unfulfilled demand, and backlog.
- `profit::DataFrame`: Timeseries with profit @ each node.
- `reward::Float64`: Final reward in the system (used for RL)
- `period::Int`: Current period in the simulation.
- `num_periods::Int`: Number of periods in the simulation.
- `discount::Float64`: Time discount factor (i.e. interest rate).
- `options::Dict`: Simulation options
  - `backlog::Bool`: Indicator if backlogging is allowed.
  - `reallocate::Bool`: Indicator if unfulfilled requests should be reallocated to alternate suppliers.
  - `evaluate_profit::Bool`: Indicator if the profit should be evaluated at each node
  - `capacitated_inventory::Bool`: Indicator if inventory limits should be enforced
- `seed::Int`: Random seed.
"""
mutable struct SupplyChainEnv <: AbstractEnv
    network::MetaDiGraph
    markets::Vector
    producers::Vector
    distributors::Vector
    echelons::Dict
    materials::Vector
    products::Vector
    bill_of_materials::Array
    inv_on_hand::DataFrame
    inv_level::DataFrame
    inv_pipeline::DataFrame
    inv_position::DataFrame
    ech_position::DataFrame
    replenishments::DataFrame
    shipments::DataFrame
    production::DataFrame
    demand::DataFrame
    profit::DataFrame
    reward::Float64
    period::Int
    num_periods::Int
    discount::Float64
    options::Dict
    seed::Int
end

"""
    SupplyChainEnv(network::MetaDiGraph, num_periods::Int;
                        discount::Float64=0.0, backlog::Bool=false,
                        reallocate::Bool=false, evaluate_profit::Bool=true,
                        capacitated_inventory::Bool=true, seed::Int=0)

Create a `SupplyChainEnv` from a directed graph with metadata (`MetaDiGraph`).
"""
function SupplyChainEnv(network::MetaDiGraph, num_periods::Int;
                        discount::Float64=0.0, backlog::Bool=false,
                        reallocate::Bool=false, evaluate_profit::Bool=true,
                        capacitated_inventory::Bool=true, seed::Int=0)
    #copy network (avoids issues when changing say num_periods after the Env was already created)
    net = copy(network)
    #get main nodes
    nodes = vertices(net)
    #get nodes in each echelon
    echelons = Dict(n => identify_echelons(net, n) for n in nodes)
    #get main edges
    arcs = [(e.src,e.dst) for e in edges(net)]
    #get end distributors, producers, and distribution centers
    market_keys = [:demand_distribution, :demand_frequency, :sales_price, :demand_penalty, :demand_sequence] #keys to identify a market
    plant_keys = [:production_cost, :production_time, :production_capacity] #keys to identify a plant (producer)
    mrkts = [n for n in nodes if !isempty(intersect(market_keys, keys(net.vprops[n])))]
    plants = [n for n in nodes if !isempty(intersect(plant_keys, keys(net.vprops[n])))]
    sinks = [n for n in nodes if isempty(outneighbors(net, n))]
    dcs = setdiff(nodes, plants, sinks)
    #get materials
    mats = get_prop(net, :materials)
    prods = union(
        [
            intersect([:demand_distribution, :demand_sequence], keys(net.vprops[n]))[1] |> #find which of the keys is used in the market node (demand_distribution or demand_sequence)
            param -> findall(i -> i isa Sampleable || !iszero(i), get_prop(net,n,param))
            for n in mrkts
        ]...
    )
    #check inputs
    check_inputs(net, nodes, arcs, mrkts, plants, mats, num_periods)
    #get bill of materials
    bom = get_prop(net, :bill_of_materials)
    #get material conversion to products
    _, conversion_dict = material_conversion(net)
    set_prop!(net, :conversion_dictionary, conversion_dict)
    #create logging dataframes
    inv_on_hand = DataFrame(:period => Int[], :node => Int[], :material => [], :level => Float64[], :discarded => [])
    for n in nodes, p in mats
        init_inv = get_prop(net, n, :initial_inventory)
        push!(inv_on_hand, (0, n, p, init_inv[p], 0))
    end
    inv_pipeline = DataFrame(:period => zeros(Int, length(mats)*length(arcs)),
                             :arc => repeat(arcs, inner = length(mats)),
                             :material => repeat(mats, outer = length(arcs)),
                             :level => zeros(length(mats)*length(arcs)))
    inv_position = select(inv_on_hand, [:period, :node, :material, :level])
    inv_level = copy(inv_position)
    ech_position = DataFrame(:period => Int[], :node => Int[], :material => [], :level => Float64[])
    for n in nodes, p in mats
        if get_prop(net, n, :inventory_capacity)[p] > 0 #only update if that node can store that material
            ech_pos = sum(filter([:node, :material] => (i1, i2) -> i1 in echelons[n] && i2 == p, inv_position).level)
            push!(ech_position, (0, n, p, ech_pos))
        end
    end
    replenishments = DataFrame(:period => Int[],#zeros(Int, length(mats)*length(arcs)),
                               :arc => Tuple[],#repeat(arcs, inner = length(mats)),
                               :material => Any[],#repeat(mats, outer = length(arcs)),
                               :requested => Float64[],#total amount requested
                               :accepted => Float64[],#zeros(length(mats)*length(arcs)),
                               :lead => Float64[],#zeros(Int, length(mats)*length(arcs)),
                               :unfulfilled => Float64[],#zeros(length(mats)*length(arcs)),
                               :reallocated => Any[])#Any[missing for i in 1:length(mats)*length(arcs)])
    shipments = DataFrame(:arc => [],
                          :material => [],
                          :amount => Float64[],
                          :lead => Float64[])
    production = DataFrame(:arc => [],
                           :material => [],
                           :amount => Float64[],
                           :lead => Float64[])
    demand = DataFrame(:period => Int[],#zeros(Int, length(mats)*length(mrkts)),
                       :node => Int[],#repeat(mrkts, inner = length(mats)),
                       :material => Any[],#repeat(mats, outer = length(mrkts)),
                       :demand => Float64[],#zeros(length(mats)*length(mrkts)),
                       :sold => Float64[],#zeros(length(mats)*length(mrkts)),
                       :unfulfilled => Float64[])#zeros(length(mats)*length(mrkts)))
    profit = DataFrame(:period => Int[],#zeros(Int, length(nodes)),
                       :value => Float64[],#zeros(length(nodes)),
                       :node => Int[])#nodes)
    reward = 0
    period = 0
    num_periods = num_periods
    options = Dict(:backlog => backlog, :reallocate => reallocate,
                   :evaluate_profit => evaluate_profit,
                   :capacitated_inventory => capacitated_inventory)
    env = SupplyChainEnv(
        net, mrkts, plants, dcs, echelons, mats, prods, bom,
        inv_on_hand, inv_level, inv_pipeline, inv_position, ech_position,
        replenishments, shipments, production, demand,
        profit, reward, period, num_periods, discount, options, seed
    )

    Random.seed!(env)

    return env
end

"""
    reset!(env::SupplyChainEnv)

Reset a `SupplyChainEnv` (empty all logging dataframes and set simulation time to 0).
"""
function reset!(env::SupplyChainEnv)
    env.period = 0
    env.reward = 0
    filter!(i -> i.period == 0, env.inv_on_hand)
    filter!(i -> i.period == 0, env.inv_pipeline)
    filter!(i -> i.period == 0, env.inv_position)
    filter!(i -> i.period == 0, env.ech_position)
    filter!(i -> i.period == 0, env.replenishments)
    filter!(i -> i.period == 0, env.demand)
    filter!(i -> i.period == 0, env.profit)
    env.shipments = DataFrame(:arc => [],
                          :material => [],
                          :amount => Float64[],
                          :lead => Int[])
    env.production = DataFrame(:arc => [],
                           :material => [],
                           :amount => Float64[],
                           :lead => Int[])

    Random.seed!(env)
end

"""
    is_terminated(env::SupplyChainEnv)

Check if a simulation has terminated (i.e., has reached the maximum number of periods).
"""
is_terminated(env::SupplyChainEnv) = env.period == env.num_periods

"Set the random seed for a simulation."
Random.seed!(env::SupplyChainEnv) = Random.seed!(env.seed)


function TetgenIO(
    points::Vector{Point{3,T}};
    pointattributes = SVector{0,T}[],
    pointmtrs = SVector{0,T}[],
    pointmarkers = Cint[],
    tetrahedrons = SimplexFace{4,Cint}[],
    tetrahedronattributes = T[],
    tetrahedronvolumes = T[],
    neighbors = Cint[],
    facets = JLFacet{T,Vector{Cint}}[],
    facetmarkers = Cint[],
    holes = Point{3,T}[],
    regions = Region{T}[],
    facetconstraints = FacetConstraint{inttype(T),T}[],
    segmentconstraints = SegmentationConstraint{inttype(T),T}[],
    trifaces = TriangleFace{Cint}[],
    trifacemarkers = Cint[],
    edges = LineFace{Cint}[],
    edgemarkers = Cint[],
) where T
    TetgenIO(
        points,
        pointattributes,
        pointmtrs,
        pointmarkers,
        tetrahedrons,
        tetrahedronattributes,
        tetrahedronvolumes,
        neighbors,
        facets,
        facetmarkers,
        holes,
        regions,
        facetconstraints,
        segmentconstraints,
        trifaces,
        trifacemarkers,
        edges,
        edgemarkers,
    )

end