#reml.jl
 Base.@propagate_inbounds function subutri!(a, b)
    s = size(a,1)
    if s == 1 return a[1,1] -= b[1,1] end
    @simd for m = 1:s
        @simd for n = m:s
            a[m,n] -= b[m,n]
        end
    end
    a
end

function fillzeroutri!(a::AbstractArray{T})  where T
    tr = UpperTriangular(a)
    fill!(tr, zero(T))
end

################################################################################
#                     REML without provided β
################################################################################
function reml_sweep_β(lmm, θ::Vector{T}; syrkblas::Bool = false) where T <: Number
    data = LMMDataViews(lmm)
    reml_sweep_β(lmm, data, θ; syrkblas = syrkblas)
end
function reml_sweep_β(lmm, data::AbstractLMMDataBlocks, θ::Vector{T}; syrkblas::Bool = false) where T <: Number
    n             = length(lmm.covstr.vcovblock)
    N             = length(lmm.data.yv)
    c             = (N - lmm.rankx)*log(2π)
    #---------------------------------------------------------------------------
    V⁻¹           = Vector{AbstractArray{T}}(undef, n)
    θ₁            = zero(T)
    θ₂            = zeros(T, lmm.rankx, lmm.rankx)
    θ₃            = zero(T)
    βm            = zeros(T, lmm.rankx)
    β             = Vector{T}(undef, lmm.rankx)
    #---------------------------------------------------------------------------
    logdetθ₂      = zero(T)
    θs₂           = Symmetric(θ₂)
    #θ₂ut          = UpperTriangular(θ₂)
    #akk = Vector{T}(undef, lmm.covstr.maxn + lmm.rankx) #temp for sweep
    noerror       = true

        ncore     = min(Polyester.num_cores(), n)
        accθ₁     = zeros(T, ncore)
        accθ₂     = Vector{Matrix{T}}(undef, ncore)
        accβm     = Vector{Vector{T}}(undef, ncore)
        erroracc  = trues(ncore)
        d, r = divrem(n, Polyester.num_cores())
        Base.Threads.@threads for t = 1:ncore

        #@batch per=core for i = 1:n #@fastmath
            # Vp - matrix for sweep operation
            # [V  X
            #  X' 0]
            offset = min(t-1, r) + (t-1)*d
            accθ₂[t] = zeros(T, lmm.rankx, lmm.rankx)
            accβm[t] = zeros(T, lmm.rankx)

            for j ∈ 1:d+(t ≤ r)
                i =  offset + j

                q    = length(lmm.covstr.vcovblock[i])
                qswm = q + lmm.rankx
                Vp   = Matrix{T}(undef, qswm, qswm)
                V    = view(Vp, 1:q, 1:q)
                Vx   = view(Vp, 1:q, q+1:qswm)
                Vc   = view(Vp, q+1:qswm, q+1:qswm)
                fillzeroutri!(V)
                copyto!(Vx, data.xv[i])
                fillzeroutri!(Vc)
            #-------------------------------------------------------------------
            # Make V matrix
                vmatrix!(V, θ, lmm, i)
            #-----------------------------------------------------------------------
            #swr  = sweepb!(view(akk, 1:qswm), Vp, 1:q; logdet = true, syrkblas = syrkblas)
                swm, swr, ne  = sweepb!(Vector{T}(undef, qswm), Vp, 1:q; logdet = true, syrkblas = syrkblas)
                V⁻¹[i] = V
            #-----------------------------------------------------------------------
                if ne == false erroracc[t] = false end
                accθ₁[t] += swr
                subutri!(accθ₂[t], Vc)
                mulαtβinc!(accβm[t], Vx, data.yv[i])
            end
            #-----------------------------------------------------------------------
        end
        θ₁ = sum(accθ₁)
        map(x->θ₂ .+= x, accθ₂) # make upper triangular use only!
        map(x->βm .+= x, accβm)
        noerror = all(erroracc)
        # Cholesky decomposition for matrix inverse θs₂ - Summetric(θ₂); C = θ₂⁻¹
        cθs₂ = cholesky(θs₂)
        # β calculation
        mul!(β, inv(cθs₂), βm)
        # θ₃
        @inbounds @simd for i = 1:n
            θ₃ += mulθ₃(data.yv[i], data.xv[i], β, V⁻¹[i])
        end
        # final θ₂
        logdetθ₂ = logdet(cθs₂)

    return   θ₁ + logdetθ₂ + θ₃ + c, β, θs₂, θ₃, noerror #REML, β, iC, θ₃, errors
end
################################################################################
#                     REML with provided β
################################################################################
function reml_sweep_β(lmm, θ::Vector{T}, β::Vector) where T <: Number
    data = LMMDataViews(lmm)
    reml_sweep_β(lmm, data, θ, β)
end
function reml_sweep_β(lmm, data::AbstractLMMDataBlocks, θ::Vector{T}, β::Vector) where T <: Number
    n             = length(lmm.covstr.vcovblock)
    N             = length(lmm.data.yv)
    c             = (N - lmm.rankx)*log(2π)
    #---------------------------------------------------------------------------
    θ₁            = zero(T)
    θ₂            = zeros(T, lmm.rankx, lmm.rankx)
    θ₃            = zero(T)
    #---------------------------------------------------------------------------
    logdetθ₂      = zero(T)
    akk           = zeros(T, lmm.covstr.maxn + lmm.rankx) #temp for sweep
    #Vm            = Matrix{T}(undef, lmm.covstr.maxn + lmm.rankx, lmm.covstr.maxn + lmm.rankx) #!!
    θs₂           = Symmetric(θ₂)
    noerror       = true

        l = Base.Threads.SpinLock()
        Base.Threads.@threads for i = 1:n
            q    = length(lmm.covstr.vcovblock[i])
            qswm = q + lmm.rankx
            Vp   = Matrix{T}(undef, qswm, qswm)
            V    = view(Vp, 1:q, 1:q)
            Vx   = view(Vp, 1:q, q+1:qswm)
            Vc   = view(Vp, q+1:qswm, q+1:qswm)
            #Vp   = view(Vm, 1:q + lmm.rankx, 1:q + lmm.rankx)
            #V    = view(Vm, 1:q, 1:q)
            #Vx   = view(Vm, 1:q, q+1:q+lmm.rankx)
            #Vc   = view(Vm, q + 1:qswm, q + 1:qswm)
            fillzeroutri!(V)
            copyto!(Vx, data.xv[i])
            fillzeroutri!(Vc)
            vmatrix!(V, θ, lmm, i)
            #-----------------------------------------------------------------------
            swm, swr, ne = sweepb!(Vector{T}(undef, qswm), Vp, 1:q; logdet = true)
            #-----------------------------------------------------------------------
            θ₃t = mulθ₃(data.yv[i], data.xv[i], β, V)
            lock(l) do
                if ne == false noerror = false end
                θ₁  += swr
                subutri!(θ₂, view(Vp, q + 1:qswm, q + 1:qswm))
                θ₃  += θ₃t
            end
        end
        logdetθ₂ = logdet(θs₂)

    return   θ₁ + logdetθ₂ + θ₃ + c, θs₂, θ₃, noerror #REML, iC, θ₃
end
################################################################################
#                     REML AI-like / scoring part
################################################################################
function sweep_ai(lmm, data::AbstractLMMDataBlocks, θ::Vector{T}, β::Vector) where T <: Number
    n             = length(lmm.covstr.vcovblock)
    θ₃            = zero(T)
    #akk           = zeros(T, lmm.covstr.maxn + lmm.rankx) #temp for sweep
    #Vm            = Matrix{T}(undef, lmm.covstr.maxn + lmm.rankx, lmm.covstr.maxn + lmm.rankx) #!!
    l = Base.Threads.SpinLock()
    @inbounds Base.Threads.@threads for i = 1:n
        q    = length(lmm.covstr.vcovblock[i])
        qswm = q + lmm.rankx
        Vp   = Matrix{T}(undef, qswm, qswm)
        V    = view(Vp, 1:q, 1:q)
        Vx   = view(Vp, 1:q, q+1:qswm)
        Vc   = view(Vp, q+1:qswm, q+1:qswm)
        #Vp   = view(Vm, 1:q + lmm.rankx, 1:q + lmm.rankx)
        #V    = view(Vm, 1:q, 1:q)
        #Vx   = view(Vm, 1:q, q+1:q+lmm.rankx)
        #Vc   = view(Vm, q + 1:qswm, q + 1:qswm)
        fillzeroutri!(V)
        copyto!(Vx, data.xv[i])
        fillzeroutri!(Vc)
        vmatrix!(V, θ, lmm, i)
        sweepb!(Vector{T}(undef, qswm), Vp, 1:q)
        θ₃t = mulθ₃(data.yv[i], data.xv[i], β, V)
        lock(l) do
            θ₃  += θ₃t
        end
    end
    return  θ₃
end
function sweep_score(lmm, data::AbstractLMMDataBlocks, θ::Vector{T}, β::Vector) where T <: Number
    n             = length(lmm.covstr.vcovblock)
    θ₁            = zero(T)
    θ₃            = zero(T)
    #---------------------------------------------------------------------------
    #akk           = zeros(T, lmm.covstr.maxn + lmm.rankx) #temp for sweep
    #Vm            = Matrix{T}(undef, lmm.covstr.maxn + lmm.rankx, lmm.covstr.maxn + lmm.rankx) #!!
    l = Base.Threads.SpinLock()
    @inbounds Base.Threads.@threads for i = 1:n
        q    = length(lmm.covstr.vcovblock[i])
        qswm = q + lmm.rankx
        Vp   = Matrix{T}(undef, qswm, qswm)
        V    = view(Vp, 1:q, 1:q)
        Vx   = view(Vp, 1:q, q+1:qswm)
        Vc   = view(Vp, q+1:qswm, q+1:qswm)

        #Vp   = view(Vm, 1:q + lmm.rankx, 1:q + lmm.rankx)
        #V    = view(Vm, 1:q, 1:q)
        #Vx   = view(Vm, 1:q, q+1:q+lmm.rankx)
        #Vc   = view(Vm, q + 1:qswm, q + 1:qswm)

        fillzeroutri!(V)
        copyto!(Vx, data.xv[i])
        fillzeroutri!(Vc)
        vmatrix!(V, θ, lmm, i)
        #-----------------------------------------------------------------------
        swr  = sweepb!(Vector{T}(undef, qswm), Vp, 1:q; logdet = true)

        θ₃t  = mulθ₃(data.yv[i], data.xv[i], β, V)
        lock(l) do
            θ₁  += swr[2]
            θ₃  += θ₃t
        end
    end

    return   -θ₁ + θ₃
end

################################################################################
#                     variance-covariance matrix of β
################################################################################
function sweep_β_cov(lmm, θ::Vector{T}, β::Vector) where T <: Number
    n             = length(lmm.covstr.vcovblock)
    θ₂            = zeros(T, lmm.rankx, lmm.rankx)
    akk           = zeros(T, lmm.covstr.maxn + lmm.rankx) #temp for sweep
    Vm            = Matrix{T}(undef, lmm.covstr.maxn + lmm.rankx, lmm.covstr.maxn + lmm.rankx) #!!
    @inbounds @simd for i = 1:n
        q    = length(lmm.covstr.vcovblock[i])
        qswm = q + lmm.rankx
        Vp   = view(Vm, 1:q + lmm.rankx, 1:q + lmm.rankx)
        V    = view(Vm, 1:q, 1:q)
        fillzeroutri!(V)
        Vx   = view(Vm, 1:q, q+1:q+lmm.rankx)
        copyto!(Vx,  view(lmm.data.xv,  lmm.covstr.vcovblock[i],:))
        Vc   = view(Vm, q + 1:qswm, q + 1:qswm)
        fillzeroutri!(Vc)
        vmatrix!(V, θ, lmm, i)
        #-----------------------------------------------------------------------
        sweepb!(fill!(view(akk, 1:qswm), zero(T)), Vp, 1:q)
        subutri!(θ₂, view(Vp, q + 1:qswm, q + 1:qswm))
    end
    return Symmetric(θ₂)
end


# function mpi_collapse_solu()# (comms_ctx, horizontal_mesh, quad, z_max, z_elem, z_stretch, Y) # TODO: it's using the global scope here
if is_distributed # replace sol.u on the root processor with the global sol.u

    if ClimaComms.iamroot(comms_ctx)
        global_h_space = make_horizontal_space(horizontal_mesh, quad, nothing)
        global_center_space, global_face_space = make_hybrid_spaces(global_h_space, z_max, z_elem, z_stretch)
        global_Y_c_type = Fields.Field{typeof(Fields.field_values(Y.c)), typeof(global_center_space)}
        global_Y_f_type = Fields.Field{typeof(Fields.field_values(Y.f)), typeof(global_face_space)}
        global_Y_type = Fields.FieldVector{FT, NamedTuple{(:c, :f), Tuple{global_Y_c_type, global_Y_f_type}}}
        global_Y_slab_type =
            Fields.Field{typeof(Fields.field_values(slab_sim.integrator.u.T_sfc)), typeof(global_h_space)}
        global_sol_u_atmos = similar(sol_atm.u, global_Y_type)
        global_sol_u_slab = similar(sol_slab.u, global_Y_slab_type)
    end
    for i in 1:length(sol_atm.u)
        global_Y_c = DataLayouts.gather(comms_ctx, Fields.field_values(sol_atm.u[i].c))
        global_Y_f = DataLayouts.gather(comms_ctx, Fields.field_values(sol_atm.u[i].f))
        global_Y_slab = DataLayouts.gather(comms_ctx, Fields.field_values(sol_slab.u[i].T_sfc))
        if ClimaComms.iamroot(comms_ctx)
            global_sol_u_atmos[i] = Fields.FieldVector(
                c = Fields.Field(global_Y_c, global_center_space),
                f = Fields.Field(global_Y_f, global_face_space),
            )
            global_sol_u_slab[i] = Fields.Field(global_Y_slab, global_h_space)
        end
    end
    if ClimaComms.iamroot(comms_ctx)
        solu_atm = global_sol_u_atmos
        solu_slab = global_sol_u_slab
        return solu_atm, solu_slab
    end
else
    solu_atm = sol_atm.u
    h_space = make_horizontal_space(horizontal_mesh, quad, nothing) #TODO move this to the beginning (once same the instance error sorted)
    solu_slab = Fields.FieldVector(T_sfc = [Fields.Field(Fields.field_values(u.T_sfc), h_space) for u in sol_slab.u])
end


using RigidBodyDynamics
using RigidBodyDynamics: Bounds
using MeshCat
using MeshCatMechanisms
using LCPSim
using JuMP, Gurobi
using Base.Test

@testset "rimless wheel" begin
    urdf = joinpath(@__DIR__, "urdf", "rimless_wheel.urdf")
    mechanism = parse_urdf(Float64, urdf)
    floating_base = findjoint(mechanism, "floating_base")
    position_bounds(floating_base) .= Bounds(-100, 100)
    velocity_bounds(floating_base) .= Bounds(-100, 100)

    vis = Visualizer()
    if !haskey(ENV, "CI")
        open(vis)
        wait(vis)
    end
    mvis = MechanismVisualizer(mechanism, URDFVisuals(urdf), vis[:robot])

    x0 = MechanismState(mechanism)
    set_configuration!(x0, [-1.5, 1.8, -0.05])
    env = LCPSim.parse_contacts(mechanism, urdf);
    controller = LCPSim.passive_controller()
    Δt = 0.005
    N = 1700
    lcp_solver = GurobiSolver(Gurobi.Env(); OutputFlag=0)

    results = LCPSim.simulate(x0, controller, env, Δt, N, lcp_solver)

    for r in results
        set_configuration!(mvis, configuration(r.state))
        sleep(Δt)
    end
end


## ExoplanetsSysSim/examples/hsu_etal_2018/calc_sum_stats_pr.jl
## 2017 Keir A. Ashby
#  Calculates summary statistics that can be used to generate period-radius distribution plots
#  Based primarily off of calc_summary_stats_obs_binned_rates from christiansen_func.jl

function calc_sum_stats_pr(cat_obs::KeplerObsCatalog, param::SimParam, trueobs_cat::Bool = false, cat_phys::KeplerPhysicalCatalog=KeplerPhysicalCatalog([]))
  ssd = Dict{ASCIIString,Any}()
  cache = Dict{ASCIIString,Any}()

  if !trueobs_cat
    num_targets = get_int(param,"num_targets_sim_pass_one")
    ssd["num targets"] = num_targets
  else
    ssd["num targets"] = get_int(param,"num_kepler_targets")
  end

  max_tranets_in_sys = get_int(param,"max_tranets_in_sys")    # Demo that simulation parameters can specify how to evalute models, too
  @assert max_tranets_in_sys >= 1
  idx_tranets = find(x::KeplerTargetObs-> length(x.obs) > 0, cat_obs.target)::Array{Int64,1}             # Find indices of systems with at least 1 tranet = potentially detectable transiting planet
# Count total number of tranets and compile indices for N-tranet systems
  num_tranets = 0
  idx_n_tranets = Vector{Int64}[ Int64[] for m = 1:max_tranets_in_sys]
  for n in 1:max_tranets_in_sys-1
    idx_n_tranets[n] = find(x::KeplerTargetObs-> length(x.obs) == n, cat_obs.target[idx_tranets] )
    num_tranets += n*length(idx_n_tranets[n])
  end
  idx_n_tranets[max_tranets_in_sys] = find(x::KeplerTargetObs-> length(x.obs) >= max_tranets_in_sys, cat_obs.target[idx_tranets] )

  num_tranets += max_tranets_in_sys*length(idx_n_tranets[max_tranets_in_sys])  # WARNING: this means we need to ignore planets w/ indices > max_tranets_in_sys
  if ( length( find(x::KeplerTargetObs-> length(x.obs) > max_tranets_in_sys, cat_obs.target[idx_tranets] ) ) > 0)   # Make sure max_tranets_in_sys is at least big enough for observed systems
    warn("Observational data has more transiting planets in one systems than max_tranets_in_sys allows.")
  end
  num_tranets  = convert(Int64,num_tranets)            # TODO OPT: Figure out why isn't this already an Int.  I may be doing something that prevents some optimizations

  num_sys_tranets = zeros(max_tranets_in_sys)                           # Since observed data, don't need to calculate probabilities.
  for n in 1:max_tranets_in_sys                                         # Make histogram of N-tranet systems
    num_sys_tranets[n] = length(idx_n_tranets[n])
  end
  ssd["num_sys_tranets"] = num_sys_tranets
  ssd["planets detected"] = num_tranets

    period_obs_list = zeros(num_tranets)
    weight_list = zeros(num_tranets)
    radius_obs_list = zeros(num_tranets)

  if trueobs_cat==true
    mult_true   = zeros(max_tranets_in_sys)

    n = 1    # tranet id
    for i in idx_tranets
      for j in 1:num_planets(cat_obs.target[i])
	if length(cat_obs.target[i].obs)==0 continue end
        period_obs_list[n] = cat_obs.target[i].obs[j].period
        weight_list[n] = 1.0
        radius_obs_list[n] = sqrt(cat_obs.target[i].obs[j].depth)*cat_obs.target[i].star.radius
        num_planets_in_sys = length(cat_obs.target[i].obs)
        mult_true[num_planets_in_sys]+=1
        n = n+1
      end
    end

    ssd["period_list"] = period_obs_list
    ssd["radius_list"] = radius_obs_list
    ssd["multiplicity_true"] = mult_true

  else  #Here we'll calculate periods and radii for the physical and observed catalogs as well as their planet multiplicities.
    period_phys_list = Float64[] # zeros(length(idx_phys))      #Holds the periods for every planet in the physical catalog
    radius_phys_list = Float64[] # zeros(length(idx_phys))	#Holds the radii for every planet in the physical catalog
    mult_phys        = zeros(max_tranets_in_sys) 		#Holds the number of systems for each variation of planet multiplicity in the physical catalog
    mult_obs         = zeros(max_tranets_in_sys) 		#Holds the number of systems for each variation of planet multiplicity in the observed catalog
    a = 1    # tranet id
    b = 1    # tranet id
    for i in 1:length(cat_phys.target) 				#get from physical catalog
       for s in 1:length(cat_phys.target[i].sys)
         if length(cat_phys.target[i].sys)==0 continue end
	 for p in 1:length(cat_phys.target[i].sys[s].planet)
	   if length(cat_phys.target[i].sys[s].planet)==0 continue end
	   #period_phys_list[a] = cat_phys.target[i].sys[s].orbit[p].P
           #radius_phys_list[a] = cat_phys.target[i].sys[s].planet[p].radius
	   push!(period_phys_list,cat_phys.target[i].sys[s].orbit[p].P)
           push!(radius_phys_list,cat_phys.target[i].sys[s].planet[p].radius)
	   num_planets_in_sys = length(cat_phys.target[i].sys[s].planet)
           mult_phys[num_planets_in_sys]+=1
           a+=1
         end
	end
    end
    for j in idx_tranets
      for h in 1:num_planets(cat_obs.target[j])
	if length(cat_obs.target[j].obs)==0 continue end
        period_obs_list[b] = cat_obs.target[j].obs[h].period
	weight_list[b] = 1.0
        radius_obs_list[b] = sqrt(cat_obs.target[j].obs[h].depth*cat_obs.target[j].star.radius)
        num_planets_in_sys = length(cat_obs.target[j].obs)
        mult_obs[num_planets_in_sys]+=1
        b+=1
      end
    end

    ssd["period_phys_list"]   = period_phys_list
    ssd["radius_phys_list"]   = radius_phys_list
    ssd["period_obs_list"]    = period_obs_list
    ssd["radius_obs_list"]    = radius_obs_list
    ssd["max_tranets_in_sys"] = max_tranets_in_sys
    ssd["multiplicity_phys"]  = mult_phys
    ssd["multiplicity_obs"]   = mult_obs
  end

  limitP::Array{Float64,1} = get_any(param, "p_lim_arr", Array{Float64,1})
  limitRp::Array{Float64,1} = get_any(param, "r_lim_arr", Array{Float64,1})

  np_bin = zeros((length(limitP)-1) * (length(limitRp)-1))
  np_bin_idx = 1
  for i in 1:(length(limitP)-1)
    P_match = find(x -> ((x > limitP[i]) && (x < limitP[i+1])), period_obs_list)
    for j in 1:(length(limitRp)-1)
      R_match = find(x -> ((x > limitRp[j]) && (x < limitRp[j+1])), radius_obs_list)

      bin_match = intersect(P_match, R_match)

      np_bin[np_bin_idx] = sum(weight_list[bin_match])
      np_bin_idx += 1
    end
  end

  #ssd["planets detected"] = sum(np_bin)
  ssd["planets table"] = np_bin

  return CatalogSummaryStatistics(ssd, cache)
end


# Advent of code 2021, day 7

posvec = [
    1101,
    1,
    29,
    67,
    1102,
    0,
    1,
    65,
    1008,
    65,
    35,
    66,
    1005,
    66,
    28,
    1,
    67,
    65,
    20,
    4,
    0,
    1001,
    65,
    1,
    65,
    1106,
    0,
    8,
    99,
    35,
    67,
    101,
    99,
    105,
    32,
    110,
    39,
    101,
    115,
    116,
    32,
    112,
    97,
    115,
    32,
    117,
    110,
    101,
    32,
    105,
    110,
    116,
    99,
    111,
    100,
    101,
    32,
    112,
    114,
    111,
    103,
    114,
    97,
    109,
    10,
    775,
    180,
    473,
    1346,
    1189,
    1553,
    196,
    427,
    788,
    25,
    159,
    204,
    851,
    24,
    1404,
    91,
    308,
    1096,
    747,
    278,
    1185,
    36,
    752,
    57,
    54,
    583,
    570,
    260,
    735,
    1192,
    72,
    552,
    103,
    693,
    383,
    202,
    78,
    1050,
    453,
    116,
    333,
    182,
    765,
    60,
    372,
    201,
    291,
    1642,
    49,
    80,
    918,
    98,
    1443,
    36,
    577,
    696,
    1289,
    56,
    220,
    56,
    51,
    550,
    1666,
    651,
    652,
    1508,
    70,
    40,
    1473,
    57,
    1065,
    32,
    6,
    537,
    1053,
    315,
    711,
    163,
    476,
    1006,
    3,
    1092,
    18,
    1304,
    237,
    358,
    457,
    888,
    36,
    639,
    39,
    1051,
    723,
    590,
    1242,
    210,
    1217,
    473,
    488,
    1554,
    729,
    776,
    307,
    375,
    243,
    186,
    436,
    94,
    451,
    1230,
    495,
    861,
    480,
    28,
    323,
    529,
    92,
    65,
    43,
    564,
    143,
    183,
    81,
    965,
    82,
    168,
    303,
    331,
    99,
    921,
    583,
    1349,
    182,
    353,
    626,
    150,
    475,
    1388,
    381,
    539,
    1190,
    664,
    923,
    1579,
    564,
    31,
    186,
    171,
    415,
    69,
    82,
    621,
    579,
    636,
    787,
    154,
    384,
    463,
    124,
    213,
    270,
    318,
    16,
    21,
    429,
    1285,
    1052,
    755,
    248,
    67,
    1021,
    20,
    165,
    789,
    7,
    456,
    18,
    1432,
    1379,
    3,
    108,
    96,
    40,
    42,
    1148,
    665,
    526,
    392,
    616,
    405,
    633,
    399,
    152,
    388,
    9,
    1078,
    159,
    454,
    945,
    330,
    2,
    455,
    288,
    288,
    72,
    313,
    827,
    521,
    939,
    186,
    680,
    253,
    386,
    917,
    317,
    346,
    1897,
    520,
    662,
    558,
    919,
    31,
    1141,
    1025,
    29,
    80,
    601,
    1001,
    199,
    128,
    1721,
    1221,
    367,
    29,
    722,
    186,
    344,
    136,
    415,
    718,
    122,
    7,
    879,
    195,
    1430,
    250,
    65,
    391,
    1296,
    154,
    39,
    647,
    861,
    175,
    18,
    448,
    131,
    568,
    322,
    20,
    290,
    828,
    844,
    135,
    622,
    409,
    236,
    40,
    341,
    767,
    31,
    917,
    5,
    335,
    27,
    205,
    63,
    18,
    1262,
    615,
    1819,
    291,
    1263,
    179,
    206,
    686,
    102,
    18,
    483,
    525,
    331,
    303,
    932,
    664,
    337,
    774,
    1080,
    22,
    177,
    891,
    780,
    523,
    362,
    38,
    219,
    281,
    72,
    861,
    48,
    1382,
    1497,
    249,
    343,
    404,
    292,
    1252,
    1,
    532,
    370,
    421,
    155,
    1728,
    110,
    1778,
    804,
    944,
    142,
    1508,
    155,
    967,
    1201,
    1066,
    706,
    613,
    1222,
    40,
    1000,
    558,
    616,
    693,
    378,
    124,
    35,
    91,
    514,
    1445,
    7,
    280,
    775,
    744,
    421,
    236,
    597,
    143,
    380,
    91,
    1564,
    111,
    1359,
    711,
    453,
    73,
    247,
    427,
    119,
    182,
    508,
    598,
    514,
    543,
    297,
    182,
    397,
    24,
    1317,
    107,
    766,
    428,
    877,
    580,
    1135,
    13,
    266,
    568,
    369,
    570,
    5,
    214,
    1222,
    150,
    225,
    93,
    1168,
    7,
    793,
    346,
    17,
    70,
    127,
    734,
    1428,
    1513,
    274,
    409,
    1291,
    498,
    958,
    535,
    37,
    268,
    994,
    165,
    662,
    59,
    125,
    267,
    557,
    28,
    259,
    77,
    1226,
    588,
    499,
    105,
    15,
    238,
    272,
    272,
    100,
    329,
    642,
    68,
    356,
    782,
    90,
    674,
    608,
    431,
    1,
    442,
    835,
    3,
    780,
    51,
    394,
    146,
    71,
    231,
    582,
    81,
    595,
    921,
    913,
    398,
    831,
    1107,
    491,
    801,
    371,
    407,
    746,
    1337,
    196,
    7,
    86,
    427,
    72,
    217,
    3,
    98,
    717,
    1268,
    991,
    187,
    103,
    256,
    616,
    172,
    125,
    831,
    1380,
    935,
    281,
    1534,
    435,
    868,
    1291,
    51,
    894,
    91,
    527,
    443,
    868,
    1300,
    72,
    1108,
    259,
    641,
    381,
    1103,
    580,
    422,
    310,
    953,
    97,
    8,
    198,
    1249,
    1069,
    342,
    953,
    464,
    66,
    648,
    683,
    184,
    702,
    1488,
    440,
    389,
    408,
    0,
    47,
    1023,
    45,
    63,
    999,
    131,
    73,
    135,
    256,
    1586,
    798,
    61,
    43,
    708,
    138,
    500,
    952,
    10,
    170,
    1287,
    956,
    1454,
    886,
    1117,
    405,
    1064,
    252,
    1218,
    334,
    316,
    1116,
    485,
    63,
    336,
    1218,
    528,
    309,
    86,
    833,
    168,
    955,
    45,
    993,
    841,
    480,
    336,
    512,
    835,
    296,
    285,
    340,
    81,
    1291,
    61,
    51,
    5,
    284,
    1114,
    120,
    794,
    1444,
    889,
    28,
    0,
    327,
    134,
    71,
    1040,
    230,
    48,
    49,
    837,
    1191,
    558,
    3,
    26,
    760,
    1038,
    104,
    320,
    87,
    464,
    270,
    115,
    357,
    277,
    285,
    488,
    1649,
    477,
    972,
    423,
    524,
    657,
    20,
    395,
    958,
    528,
    13,
    125,
    391,
    90,
    334,
    314,
    1022,
    1147,
    200,
    1357,
    1092,
    559,
    610,
    193,
    296,
    60,
    188,
    0,
    2,
    277,
    540,
    365,
    79,
    278,
    199,
    1327,
    573,
    615,
    95,
    677,
    285,
    143,
    182,
    226,
    12,
    661,
    492,
    189,
    526,
    75,
    1358,
    923,
    228,
    59,
    417,
    535,
    544,
    270,
    1040,
    197,
    2,
    216,
    1217,
    372,
    1034,
    84,
    16,
    725,
    1798,
    352,
    147,
    290,
    528,
    1091,
    105,
    476,
    725,
    137,
    474,
    75,
    1313,
    644,
    92,
    43,
    286,
    47,
    118,
    11,
    865,
    1316,
    1464,
    1189,
    673,
    67,
    612,
    183,
    379,
    309,
    464,
    207,
    31,
    64,
    1375,
    34,
    413,
    618,
    131,
    1459,
    178,
    179,
    81,
    245,
    316,
    223,
    230,
    697,
    337,
    977,
    188,
    1335,
    811,
    163,
    592,
    181,
    93,
    108,
    865,
    112,
    20,
    497,
    986,
    1124,
    73,
    128,
    96,
    107,
    1288,
    179,
    229,
    145,
    1293,
    1224,
    1308,
    748,
    768,
    143,
    38,
    33,
    1842,
    64,
    45,
    1209,
    984,
    269,
    371,
    1451,
    876,
    1372,
    65,
    275,
    173,
    1569,
    298,
    187,
    91,
    522,
    133,
    39,
    709,
    878,
    2,
    123,
    195,
    1435,
    1569,
    482,
    1047,
    322,
    382,
    796,
    38,
    903,
    24,
    950,
    387,
    510,
    460,
    570,
    499,
    545,
    561,
    158,
    383,
    213,
    978,
    1329,
    380,
    938,
    280,
    267,
    762,
    841,
    713,
    111,
    357,
    71,
    19,
    121,
    581,
    91,
    177,
    869,
    1138,
    173,
    14,
    145,
    155,
    21,
    353,
    340,
    1145,
    113,
    594,
    685,
    91,
    781,
    558,
    500,
    10,
    33,
    300,
    270,
    457,
    675,
    850,
    64,
    49,
    81,
    311,
    906,
    404,
    207,
    176,
    309,
    45,
    855,
    16,
    9,
    881,
    428,
    194,
    300,
    329,
    715,
    985,
    559,
    656,
    66,
    184,
    1529,
    8,
    1131,
    610,
    78,
    522,
    338,
    492,
    1378,
    47,
    163,
    448,
    111,
    700,
    3,
    19,
    796,
    876,
    224,
    212,
    51,
    524,
    273,
    597,
    980,
    0,
    10,
    205,
    8,
    985,
    38,
    876,
    6,
    91,
    435,
    1273,
    38,
    147,
    214,
    362,
    1,
    95,
    87,
    724,
    1126,
    807,
    378,
    105,
    89,
    276,
    1076,
    107,
    552,
    1082,
    32,
    896,
    202,
    177,
    946,
    753,
    1106,
    464,
    72,
    61,
    225,
    55,
]

movementcosts = [sum([abs(n - p) for p in posvec]) for n in 1:maximum(posvec)]
movementcosts2 = [sum([sum(1:abs(n - p)) for p in posvec]) for n in 1:maximum(posvec)]
println("Part 1: ", first(findmin(movementcosts)))
println("Part2: ", first(findmin(movementcosts2)))


using Dates: Day, Hour

warsaw = first(compile("Europe/Warsaw", tzdata["europe"]))

normal = DateTime(2015, 1, 1, 0)   # a 24 hour day in warsaw
spring = DateTime(2015, 3, 29, 0)  # a 23 hour day in warsaw
fall = DateTime(2015, 10, 25, 0)   # a 25 hour day in warsaw

# Unary plus
@test +ZonedDateTime(normal, warsaw) == ZonedDateTime(normal, warsaw)

# Period arithmetic
@test ZonedDateTime(normal, warsaw) + Day(1) == ZonedDateTime(normal + Day(1), warsaw)
@test ZonedDateTime(spring, warsaw) + Day(1) == ZonedDateTime(spring + Day(1), warsaw)
@test ZonedDateTime(fall, warsaw) + Day(1) == ZonedDateTime(fall + Day(1), warsaw)

@test ZonedDateTime(normal, warsaw) + Hour(24) == ZonedDateTime(normal + Hour(24), warsaw)
@test ZonedDateTime(spring, warsaw) + Hour(24) == ZonedDateTime(spring + Hour(25), warsaw)
@test ZonedDateTime(fall, warsaw) + Hour(24) == ZonedDateTime(fall + Hour(23), warsaw)

# Do the same calculations but backwards over the transitions.
@test ZonedDateTime(normal + Day(1), warsaw) - Day(1) == ZonedDateTime(normal, warsaw)
@test ZonedDateTime(spring + Day(1), warsaw) - Day(1) == ZonedDateTime(spring, warsaw)
@test ZonedDateTime(fall + Day(1), warsaw) - Day(1) == ZonedDateTime(fall, warsaw)

@test ZonedDateTime(normal + Day(1), warsaw) - Hour(24) == ZonedDateTime(normal, warsaw)
@test ZonedDateTime(spring + Day(1), warsaw) - Hour(23) == ZonedDateTime(spring, warsaw)
@test ZonedDateTime(fall + Day(1), warsaw) - Hour(25) == ZonedDateTime(fall, warsaw)

# Ensure that arithmetic around transitions works.
@test ZonedDateTime(spring, warsaw) + Hour(1) == ZonedDateTime(spring + Hour(1), warsaw)
@test ZonedDateTime(spring, warsaw) + Hour(2) == ZonedDateTime(spring + Hour(3), warsaw)
@test ZonedDateTime(fall, warsaw) + Hour(2) == ZonedDateTime(fall + Hour(2), warsaw, 1)
@test ZonedDateTime(fall, warsaw) + Hour(3) == ZonedDateTime(fall + Hour(2), warsaw, 2)

# Non-Associativity
explicit_hour_day = (ZonedDateTime(spring, warsaw) + Hour(24)) + Day(1)
explicit_day_hour = (ZonedDateTime(spring, warsaw) + Day(1)) + Hour(24)
implicit_hour_day = ZonedDateTime(spring, warsaw) + Hour(24) + Day(1)
implicit_day_hour = ZonedDateTime(spring, warsaw) + Day(1) + Hour(24)

@test explicit_hour_day == ZonedDateTime(2015, 3, 31, 1, warsaw)
@test explicit_day_hour == ZonedDateTime(2015, 3, 31, 0, warsaw)
@test implicit_hour_day == ZonedDateTime(2015, 3, 31, 0, warsaw)
@test implicit_day_hour == ZonedDateTime(2015, 3, 31, 0, warsaw)

# CompoundPeriod canonicalization interacting with period arithmetic. Since `spring_zdt` is
# a 23 hour day this means adding `Day(1)` and `Hour(23)` are equivalent.
spring_zdt = ZonedDateTime(spring, warsaw)
@test spring_zdt + Day(1) + Minute(1) == spring_zdt + Hour(23) + Minute(1)

# When canonicalization happens automatically `Hour(24) + Minute(1)` is converted into
# `Day(1) + Minute(1)`. Fixed in `JuliaLang/julia#19268`
@test spring_zdt + Hour(23) + Minute(1) < spring_zdt + Hour(24) + Minute(1)

# Arithmetic with a StepRange should always work even when the start/stop lands on
# ambiguous or non-existent DateTimes.
ambiguous = DateTime(2015, 10, 25, 2)   # Ambiguous hour in Warsaw
nonexistent = DateTime(2014, 3, 30, 2)  # Non-existent hour in Warsaw

range = ZonedDateTime(ambiguous - Day(1), warsaw):Hour(1):ZonedDateTime(ambiguous - Day(1) + Hour(1), warsaw)
@test range .+ Day(1) == ZonedDateTime(ambiguous, warsaw, 1):Hour(1):ZonedDateTime(ambiguous + Hour(1), warsaw)

range = ZonedDateTime(ambiguous - Day(1) - Hour(1), warsaw):Hour(1):ZonedDateTime(ambiguous - Day(1), warsaw)
@test range .+ Day(1) == ZonedDateTime(ambiguous - Hour(1), warsaw, 1):Hour(1):ZonedDateTime(ambiguous, warsaw, 2)

range = ZonedDateTime(nonexistent - Day(1), warsaw):Hour(1):ZonedDateTime(nonexistent - Day(1) + Hour(1), warsaw)
@test range .+ Day(1) == ZonedDateTime(nonexistent + Hour(1), warsaw):Hour(1):ZonedDateTime(nonexistent + Hour(1), warsaw)

range = ZonedDateTime(nonexistent - Day(1) - Hour(1), warsaw):Hour(1):ZonedDateTime(nonexistent - Day(1), warsaw)
@test range .+ Day(1) == ZonedDateTime(nonexistent - Hour(1), warsaw):Hour(1):ZonedDateTime(nonexistent - Hour(1), warsaw)


### A Pluto.jl notebook ###
# v0.19.0

using Markdown
using InteractiveUtils

# ╔═╡ 7a3506e2-6287-4755-a715-cd1de9dffc4c
using Pkg, DrWatson

# ╔═╡ 4087e639-e1b9-4034-bb09-051ae0cc4e3d
begin
	using RegressionAndOtherStories
end

# ╔═╡ 27fd555f-2a13-4106-bcb5-6644e41dfb29
md" ##### As this is a good example of real life data-mangling, this file is in notebook format!."

# ╔═╡ d5f05d8b-416e-4648-89d8-afa937f30e77
md"
!!! note

Unfortunately this notebook is not complete. Translating R to Julia can be tricky!!!
"

# ╔═╡ ad1df579-dd25-4be3-9194-62019182e9c2
html"""
<style>
	main {
		margin: 0 auto;
		max-width: 2000px;
    	padding-left: max(160px, 10%);
    	padding-right: max(160px, 10%);
	}
</style>
"""

# ╔═╡ a8a9aee2-f2fd-4176-9a45-570672d0eb94
begin
	pew_pre_raw = CSV.read(ros_datadir("Pew", "pew.csv"), DataFrame; missingstring="NA", pool=false)
	#pew_pre_raw = CSV.read(ros_datadir("Pew", "pew2.csv"), DataFrame; missingstring="NA", pool=false)
	pew_pre = pew_pre_raw[:, [:survey, :regicert,  :party, :state, :heat2, :heat4, :income2, :party4,
		:date, :weight, :voter_weight2, :pid, :ideology, :inc]]
end

# ╔═╡ 8ef8d7c5-219d-42e7-85a2-6dee7d237edb
N = size(pew_pre, 1)

# ╔═╡ 4859f90d-ccdf-4995-be0c-85faa9c02978
names(pew_pre)

# ╔═╡ 424f704a-0d61-45fa-a88f-3ad851b81f53
heat2_vals = unique(pew_pre.heat2)

# ╔═╡ 663a35fc-5c55-459c-be4a-b6be96d04eaf
pew_pre.heat2[1:10]

# ╔═╡ e05b1923-3137-47df-b49f-839cfae0bd2e
heat4_vals = unique(pew_pre.heat4)

# ╔═╡ 6383993d-ce7c-401d-a6a2-f87e19f85dd7
let
	res = Int[]
	ok = findall(x -> !ismissing(x), pew_pre.heat4)
	for heat4_val in heat4_vals[2:end]
		append!(res, [length(findall(x -> x == heat4_val, pew_pre.heat4[ok]))])
	end
	res
end

# ╔═╡ cff39c41-c8b6-49b3-9e5e-eeabf141a61f
rvote::Vector{Union{Missing, Int}} = repeat([missing]; inner=N);

# ╔═╡ b78a4c9a-452e-4a27-8cb3-63154d48d965
begin
	which_question = [!ismissing(pew_pre.heat2[i]) ? 2 : !ismissing(pew_pre.heat4[i]) ? 4 : 0 for i in 1:N]
end

# ╔═╡ 91a5866c-6723-46bd-b806-c2c9059c1244
length(which_question)

# ╔═╡ e8e32ea1-35f1-48fa-b5e4-bb5f2e845e10
findfirst(x -> x == 4, which_question)

# ╔═╡ 478230d4-f210-4df8-98e2-fdeaa88d3c7a
findall(x -> x == 4, which_question)

# ╔═╡ 0f27ef85-a4ce-4978-aac7-a1804f723ee0
length(findall(x -> x == 4, which_question))

# ╔═╡ 127ec62d-406e-40ca-93de-c32562e2a29f
mean(which_question)

# ╔═╡ bad9cf89-cc3a-4cb2-b139-ff8e1052818e
for i in 1:N
	if which_question[i] == 2
		rvote[i] = pew_pre.heat2[i] == "rep/lean rep" ? 1 : pew_pre.heat2[i] == "dem/lean dem" ? 0 : missing
	elseif which_question[i] == 4
		rvote[i] = pew_pre.heat4[i] == "rep/lean rep" ? 1 : pew_pre.heat4[i] == "dem/lean dem" ? 0 : missing
	end
end

# ╔═╡ 75e4d07d-90ff-402a-8918-4f0e68bbcf5d
pew_pre.rvote = rvote

# ╔═╡ 1040c140-5d2a-4352-a7f0-901f3a2b275a
(question = mean(which_question), rvote = mean(filter(!ismissing, rvote)))

# ╔═╡ d6ad38c9-5bc9-4d59-a1d8-b19ad76634dc
unique(pew_pre.regicert)

# ╔═╡ 9ce44159-004f-424a-8f8c-99e41b2dd622
registered = [!ismissing(pew_pre.regicert[i]) && pew_pre.regicert[i] == "absolutely certain" ? 1 : 0 for i in 1:N]

# ╔═╡ fbd7dfb4-1cd7-4aad-ad4b-b89e675376e7
mean(registered)

# ╔═╡ 836e78ed-98d9-4ff6-adda-e7cfa3080bcf
begin
	early = Vector(pew_pre.date) .> 90008
	late = Vector(pew_pre.date) .> 90008
	month = Int.(floor.(pew_pre.date ./ 10000))
	day = Int.(floor.(pew_pre.date ./ 100) .- 100month)
	day_numeric = month .* 31 .+ day
	poll_id = [
		month[i] == 6 ? 1 :
			month[i] == 7 && day[i] < 28 ? 2 :
				month[i] == 7 && day[i] >= 28 || month[i] == 8 ? 3 :
					month[i] == 9 && day[i] < 16 ? 4 :
						month[i] == 9 && day[i] >= 16 ? 5 :
							month[i] == 10 && day[i] < 14 ? 6 :
								month[i] == 10 && day[i]>=14 && day[i] < 20 ? 7 :
									month[i] == 10 && day[i] >= 20 && day[i] < 27 ? 8 : 9
										for i in 1:N]
	max_poll_id = maximum(poll_id)
end

# ╔═╡ db88ef04-e742-4336-b6c8-73eb5055e655
# R: 266804, 542365, 138996
[sum(month), sum(day), sum(poll_id)]

# ╔═╡ 745fb351-97c9-490c-b242-91937aefb895
let
	state_abr = ["AL", "AK", "AZ", "AR", "CA", "CO", "CT", "DE", "DC", "FL", "GA", "HI", "ID", "IL", "IN", "IA", "KS", "KY",
		"LA", "ME", "MD", "MA", "MI", "MN", "MS", "MO", "MT", "NE", "NV", "NH", "NJ", "NM", "NY", "NC", "ND", "OH", "OK",
		"OR", "PA", "RI", "SC", "SD", "TN", "TX", "UT", "VT", "VA", "WA", "WV", "WI", "WY"];
	sn = unique(Vector{String}(sort(pew_pre.state)))
	state_names = vcat(sn[1], "alaska", sn[2:7], "dc", sn[8:46], sn[48:50])
	votes08 = [60,39, 62,36, 54,45, 58,39, 40,59, 46,53, 39,60, 37,62, 7,93, 49,51, 53,46, 25,74, 61,36, 38,61,
		49,50, 45,54, 57,42, 58,41, 59,40, 40,58, 39,60, 36,62, 42,56, 44,54, 57,43, 49,49, 50,47, 57,41, 41,57,
		44,55, 42,57, 42,57, 37,62, 49,50, 53,45, 47,51, 66,34, 42,56, 44,55, 35,63, 54,45, 54,44, 57,42, 55,44,
		62,35, 32,67, 48,52, 43,56, 56,43, 43,56, 65,33]
	obama08 = votes08[2:2:102]
	mccain08 = votes08[1:2:101]
	ovote_actual = obama08 ./ (obama08 .+ mccain08)
	global state_df = DataFrame(row=1:51, state=state_names, state_abbr=state_abr, obama08=obama08, mccain08=mccain08,
		actual=ovote_actual)
	state_df
end

# ╔═╡ 07fd84bd-7778-4ea1-ab87-190f4693c46d
begin
	pop_weight0 = Vector(pew_pre.weight)
	voter_weight0 = [ismissing(rvote[i]) || registered[i] == 0 ? missing : pop_weight0[i] for i in 1:N]

	pop_weight = repeat([missing], N)
	pop_weight = convert(Vector{Union{Missing, Float64}}, pop_weight)
	voter_weight = repeat([missing], N)
	voter_weight = convert(Vector{Union{Missing, Float64}}, voter_weight)
	for i in 1:max_poll_id
		ok = findall( x -> x == i, poll_id)
		pop_weight[ok] = pop_weight0[ok] / mean(pop_weight0[ok])
		voter_weight[ok] = voter_weight0[ok] / mean(filter(!ismissing, voter_weight0[ok]))
	end
end

# ╔═╡ 3c2f895a-f1d3-4c80-a9d6-1cb4ab3a083c
# [1] 1.326923 0.822000 0.493000 0.492000 2.000000 1.800000 1.384615 0.492000 1.081633 1.777778 0.571000 1.458000
pew_pre.pop_weight0 = pop_weight0

# ╔═╡ faf2626a-ac8a-4e74-b93c-8ccc8d8a4b69
# R: [1] 0.5936557  NA 0.5352379 0.5341522 0.8947854 0.6640835 0.6194668 0.5341522 0.4839145 0.6558850
pew_pre.voter_weight = voter_weight

# ╔═╡ 5344134a-9e47-4cf1-83a8-017f43736fa0
begin
	dems = Int[]; reps = Int[]
	votes = setdiff(1:N, findall(ismissing, rvote))
	for st in state_df.state
		state = findall(x -> x == st, pew_pre.state)
		if length(state) > 10
			valid = intersect(state, votes)
			append!(dems, [sum(rvote[valid] .== 0)])
			append!(reps, [sum(rvote[valid] .== 1)])
		else
			append!(dems, [0])
			append!(reps, [0])
		end
	end
	state_df.dems=dems
	state_df.reps=reps
	state_df
end

# ╔═╡ 354b330f-c86e-4f29-a2a7-2efc82f8f46a
pew_pre[:, [:state, :rvote, :voter_weight2]]

# ╔═╡ 02ab023b-199c-4134-9dc8-387c7963cdb3
let
	pew_df = pew_pre[completecases(pew_pre, [:rvote, :voter_weight]), [:state, :rvote, :voter_weight]]
end

# ╔═╡ 78426ef1-922f-4277-af1f-e3a1ef61ed6e
md"
!!! note

In below DataFrame I'm surprised `inc` does not always correspond with `income2`.
"

# ╔═╡ bace2090-e331-4a04-ab5e-6f68b6c6ce27
pew_pre[:, [:state, :inc, :income2]]

# ╔═╡ 235cb6da-5cc7-456a-8874-c1b1923cf956
sort(unique(pew_pre.inc))

# ╔═╡ e58d4b6f-820e-4d69-8919-682fbf9051fb
inc_vals = [5,15,25,35,45,62.5,87.5,125,200]

# ╔═╡ d3978285-5e7e-44c5-b06b-9a46fe6b62e0
max_inc = 9

# ╔═╡ d35a817a-1fe9-4ccc-9484-7b272985c194
sort(unique(pew_pre.pid))

# ╔═╡ cd1810bf-11ed-46b1-b5a3-39054202e740
sort(unique(filter(x -> !ismissing(x), pew_pre.pid)))

# ╔═╡ e6cf6020-0953-4e6d-9174-f21928a742da
pid_labels = ["Democrat", "Lean Dem.", "Independent", "Lean Rep.", "Republican"]

# ╔═╡ 4310a3bb-74f4-4746-a4df-a2f3528de248
max_pid = 5

# ╔═╡ 922cbd2d-2ef0-4d0b-a9d5-c3759774068f
sort(unique(pew_pre.ideology))

# ╔═╡ 21cd4f93-cd3f-4813-a292-d1a6aef39db9
ideology_labels = ["Very liberal", "Liberal", "Moderate", "Conservative", "Very conservative"]

# ╔═╡ 9a173de0-b44b-4ec7-ba86-f34a8f0fdf0e
max_ideo = 5

# ╔═╡ 2b6be5ad-411a-4e82-a713-e3a8e6140314
begin
	incprop = zeros(Union{Missing, Float64}, max_pid+1, max_inc)
	incprop[max_pid + 1, :] = ones(max_inc)
	vc = pew_pre[completecases(pew_pre, [:pid, :inc]), [:pid, :inc, :pop_weight0]]

	for i in 2:max_pid
		for j in 1:max_inc
			vcs = vc[vc.pid .< i .&& vc.inc .== j, :]
			incprop[i, j] = mean(vcs.pid, Weights(vcs.pop_weight0))
		end
	end
	incprop

end

# ╔═╡ d9c1c0a5-44fe-472f-a93c-9a3d38ef1a84
begin
	pid_incprob = CSV.read(ros_datadir("Pew", "pid_incprop.csv"), DataFrame; missingstring="NA", pool=false)
end

# ╔═╡ 272c6581-962e-461b-b99d-75c3ca9629c1
begin
	ideo_incprob = CSV.read(ros_datadir("Pew", "ideo_incprop.csv"), DataFrame; missingstring="NA", pool=false)
end

# ╔═╡ 7d9ce69d-6f62-4a99-91a2-2d9bab97eda1
let
	party_incprob_df = CSV.read(ros_datadir("Pew", "party_incprop.csv"), DataFrame; missingstring="NA", pool=false)
	party_incprob = reshape(Array(party_incprob_df)[:, 2:end], :, 3, 9)
	party_incprob[:, :, 9]
end

# ╔═╡ 00000000-0000-0000-0000-000000000001
PLUTO_PROJECT_TOML_CONTENTS = """
[deps]
DrWatson = "634d3b9d-ee7a-5ddf-bec9-22491ea816e1"
Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
RegressionAndOtherStories = "21324389-b050-441a-ba7b-9a837781bda0"

[compat]
DrWatson = "~2.9.1"
RegressionAndOtherStories = "~0.1.2"
"""

# ╔═╡ 00000000-0000-0000-0000-000000000002
PLUTO_MANIFEST_TOML_CONTENTS = """
# This file is machine-generated - editing it directly is not advised

julia_version = "1.9.0-DEV"
manifest_format = "2.0"
project_hash = "c1af439e4b3a1abbf7795826d31176bd2310c62c"

[[deps.ArgTools]]
uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f"
version = "1.1.1"

[[deps.Artifacts]]
uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"

[[deps.Base64]]
uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"

[[deps.CSV]]
deps = ["CodecZlib", "Dates", "FilePathsBase", "InlineStrings", "Mmap", "Parsers", "PooledArrays", "SentinelArrays", "Tables", "Unicode", "WeakRefStrings"]
git-tree-sha1 = "873fb188a4b9d76549b81465b1f75c82aaf59238"
uuid = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
version = "0.10.4"

[[deps.CategoricalArrays]]
deps = ["DataAPI", "Future", "Missings", "Printf", "Requires", "Statistics", "Unicode"]
git-tree-sha1 = "109664d3a6f2202b1225478335ea8fea3cd8706b"
uuid = "324d7699-5711-5eae-9e2f-1d82baa6b597"
version = "0.10.5"

[[deps.ChainRulesCore]]
deps = ["Compat", "LinearAlgebra", "SparseArrays"]
git-tree-sha1 = "9950387274246d08af38f6eef8cb5480862a435f"
uuid = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
version = "1.14.0"

[[deps.ChangesOfVariables]]
deps = ["ChainRulesCore", "LinearAlgebra", "Test"]
git-tree-sha1 = "bf98fa45a0a4cee295de98d4c1462be26345b9a1"
uuid = "9e997f8a-9a97-42d5-a9f1-ce6bfc15e2c0"
version = "0.1.2"

[[deps.CodecZlib]]
deps = ["TranscodingStreams", "Zlib_jll"]
git-tree-sha1 = "ded953804d019afa9a3f98981d99b33e3db7b6da"
uuid = "944b1d66-785c-5afd-91f1-9de20f533193"
version = "0.7.0"

[[deps.Combinatorics]]
git-tree-sha1 = "08c8b6831dc00bfea825826be0bc8336fc369860"
uuid = "861a8166-3701-5b0c-9a16-15d98fcdc6aa"
version = "1.0.2"

[[deps.Compat]]
deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "SHA", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"]
git-tree-sha1 = "96b0bc6c52df76506efc8a441c6cf1adcb1babc4"
uuid = "34da2185-b29b-5c13-b0c7-acf172513d20"
version = "3.42.0"

[[deps.CompilerSupportLibraries_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae"
version = "0.5.2+0"

[[deps.Crayons]]
git-tree-sha1 = "249fe38abf76d48563e2f4556bebd215aa317e15"
uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f"
version = "4.1.1"

[[deps.DataAPI]]
git-tree-sha1 = "cc70b17275652eb47bc9e5f81635981f13cea5c8"
uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a"
version = "1.9.0"

[[deps.DataFrames]]
deps = ["Compat", "DataAPI", "Future", "InvertedIndices", "IteratorInterfaceExtensions", "LinearAlgebra", "Markdown", "Missings", "PooledArrays", "PrettyTables", "Printf", "REPL", "Reexport", "SortingAlgorithms", "Statistics", "TableTraits", "Tables", "Unicode"]
git-tree-sha1 = "ae02104e835f219b8930c7664b8012c93475c340"
uuid = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
version = "1.3.2"

[[deps.DataStructures]]
deps = ["Compat", "InteractiveUtils", "OrderedCollections"]
git-tree-sha1 = "3daef5523dd2e769dad2365274f760ff5f282c7d"
uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
version = "0.18.11"

[[deps.DataValueInterfaces]]
git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6"
uuid = "e2d170a0-9d28-54be-80f0-106bbe20a464"
version = "1.0.0"

[[deps.Dates]]
deps = ["Printf"]
uuid = "ade2ca70-3891-5945-98fb-dc099432e06a"

[[deps.DelimitedFiles]]
deps = ["Mmap"]
uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab"

[[deps.DensityInterface]]
deps = ["InverseFunctions", "Test"]
git-tree-sha1 = "80c3e8639e3353e5d2912fb3a1916b8455e2494b"
uuid = "b429d917-457f-4dbc-8f4c-0cc954292b1d"
version = "0.4.0"

[[deps.Distributed]]
deps = ["Random", "Serialization", "Sockets"]
uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b"

[[deps.Distributions]]
deps = ["ChainRulesCore", "DensityInterface", "FillArrays", "LinearAlgebra", "PDMats", "Printf", "QuadGK", "Random", "SparseArrays", "SpecialFunctions", "Statistics", "StatsBase", "StatsFuns", "Test"]
git-tree-sha1 = "5a4168170ede913a2cd679e53c2123cb4b889795"
uuid = "31c24e10-a181-5473-b8eb-7969acd0382f"
version = "0.25.53"

[[deps.DocStringExtensions]]
deps = ["LibGit2"]
git-tree-sha1 = "b19534d1895d702889b219c382a6e18010797f0b"
uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
version = "0.8.6"

[[deps.Downloads]]
deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"]
uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
version = "1.6.0"

[[deps.DrWatson]]
deps = ["Dates", "FileIO", "JLD2", "LibGit2", "MacroTools", "Pkg", "Random", "Requires", "Scratch", "UnPack"]
git-tree-sha1 = "67e9001646db6e45006643bf37716ecd831d37d2"
uuid = "634d3b9d-ee7a-5ddf-bec9-22491ea816e1"
version = "2.9.1"

[[deps.FileIO]]
deps = ["Pkg", "Requires", "UUIDs"]
git-tree-sha1 = "80ced645013a5dbdc52cf70329399c35ce007fae"
uuid = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549"
version = "1.13.0"

[[deps.FilePathsBase]]
deps = ["Compat", "Dates", "Mmap", "Printf", "Test", "UUIDs"]
git-tree-sha1 = "129b104185df66e408edd6625d480b7f9e9823a0"
uuid = "48062228-2e41-5def-b9a4-89aafe57970f"
version = "0.9.18"

[[deps.FileWatching]]
uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee"

[[deps.FillArrays]]
deps = ["LinearAlgebra", "Random", "SparseArrays", "Statistics"]
git-tree-sha1 = "246621d23d1f43e3b9c368bf3b72b2331a27c286"
uuid = "1a297f60-69ca-5386-bcde-b61e274b549b"
version = "0.13.2"

[[deps.Formatting]]
deps = ["Printf"]
git-tree-sha1 = "8339d61043228fdd3eb658d86c926cb282ae72a8"
uuid = "59287772-0a20-5a39-b81b-1366585eb4c0"
version = "0.4.2"

[[deps.Future]]
deps = ["Random"]
uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820"

[[deps.InlineStrings]]
deps = ["Parsers"]
git-tree-sha1 = "61feba885fac3a407465726d0c330b3055df897f"
uuid = "842dd82b-1e85-43dc-bf29-5d0ee9dffc48"
version = "1.1.2"

[[deps.InteractiveUtils]]
deps = ["Markdown"]
uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"

[[deps.InverseFunctions]]
deps = ["Test"]
git-tree-sha1 = "91b5dcf362c5add98049e6c29ee756910b03051d"
uuid = "3587e190-3f89-42d0-90ee-14403ec27112"
version = "0.1.3"

[[deps.InvertedIndices]]
git-tree-sha1 = "bee5f1ef5bf65df56bdd2e40447590b272a5471f"
uuid = "41ab1584-1d38-5bbf-9106-f11c6c58b48f"
version = "1.1.0"

[[deps.IrrationalConstants]]
git-tree-sha1 = "7fd44fd4ff43fc60815f8e764c0f352b83c49151"
uuid = "92d709cd-6900-40b7-9082-c6be49f344b6"
version = "0.1.1"

[[deps.IteratorInterfaceExtensions]]
git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856"
uuid = "82899510-4779-5014-852e-03e436cf321d"
version = "1.0.0"

[[deps.JLD2]]
deps = ["FileIO", "MacroTools", "Mmap", "OrderedCollections", "Pkg", "Printf", "Reexport", "TranscodingStreams", "UUIDs"]
git-tree-sha1 = "81b9477b49402b47fbe7f7ae0b252077f53e4a08"
uuid = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
version = "0.4.22"

[[deps.JLLWrappers]]
deps = ["Preferences"]
git-tree-sha1 = "abc9885a7ca2052a736a600f7fa66209f96506e1"
uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210"
version = "1.4.1"

[[deps.LaTeXStrings]]
git-tree-sha1 = "f2355693d6778a178ade15952b7ac47a4ff97996"
uuid = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f"
version = "1.3.0"

[[deps.LibCURL]]
deps = ["LibCURL_jll", "MozillaCACerts_jll"]
uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21"
version = "0.6.3"

[[deps.LibCURL_jll]]
deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"]
uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0"
version = "7.81.0+0"

[[deps.LibGit2]]
deps = ["Base64", "NetworkOptions", "Printf", "SHA"]
uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"

[[deps.LibSSH2_jll]]
deps = ["Artifacts", "Libdl", "MbedTLS_jll"]
uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8"
version = "1.10.2+0"

[[deps.Libdl]]
uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"

[[deps.LinearAlgebra]]
deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"]
uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"

[[deps.LogExpFunctions]]
deps = ["ChainRulesCore", "ChangesOfVariables", "DocStringExtensions", "InverseFunctions", "IrrationalConstants", "LinearAlgebra"]
git-tree-sha1 = "a7e100b068a6cbead98b9f4e5c8b488934b7aea0"
uuid = "2ab3a3ac-af41-5b50-aa03-7779005ae688"
version = "0.3.11"

[[deps.Logging]]
uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"

[[deps.MacroTools]]
deps = ["Markdown", "Random"]
git-tree-sha1 = "3d3e902b31198a27340d0bf00d6ac452866021cf"
uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
version = "0.5.9"

[[deps.Markdown]]
deps = ["Base64"]
uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"

[[deps.MbedTLS_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1"
version = "2.28.0+0"

[[deps.Missings]]
deps = ["DataAPI"]
git-tree-sha1 = "bf210ce90b6c9eed32d25dbcae1ebc565df2687f"
uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28"
version = "1.0.2"

[[deps.Mmap]]
uuid = "a63ad114-7e13-5084-954f-fe012c677804"

[[deps.MozillaCACerts_jll]]
uuid = "14a3606d-f60d-562e-9121-12d972cd8159"
version = "2022.2.1"

[[deps.NamedArrays]]
deps = ["Combinatorics", "DataStructures", "DelimitedFiles", "InvertedIndices", "LinearAlgebra", "Random", "Requires", "SparseArrays", "Statistics"]
git-tree-sha1 = "2fd5787125d1a93fbe30961bd841707b8a80d75b"
uuid = "86f7a689-2022-50b4-a561-43c23ac3c673"
version = "0.9.6"

[[deps.NamedTupleTools]]
git-tree-sha1 = "befc30261949849408ac945a1ebb9fa5ec5e1fd5"
uuid = "d9ec5142-1e00-5aa0-9d6a-321866360f50"
version = "0.14.0"

[[deps.NetworkOptions]]
uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908"
version = "1.2.0"

[[deps.OpenBLAS_jll]]
deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"]
uuid = "4536629a-c528-5b80-bd46-f80d51c5b363"
version = "0.3.20+0"

[[deps.OpenLibm_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "05823500-19ac-5b8b-9628-191a04bc5112"
version = "0.8.1+0"

[[deps.OpenSpecFun_jll]]
deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "13652491f6856acfd2db29360e1bbcd4565d04f1"
uuid = "efe28fd5-8261-553b-a9e1-b2916fc3738e"
version = "0.5.5+0"

[[deps.OrderedCollections]]
git-tree-sha1 = "85f8e6578bf1f9ee0d11e7bb1b1456435479d47c"
uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
version = "1.4.1"

[[deps.PDMats]]
deps = ["LinearAlgebra", "SparseArrays", "SuiteSparse"]
git-tree-sha1 = "e8185b83b9fc56eb6456200e873ce598ebc7f262"
uuid = "90014a1f-27ba-587c-ab20-58faa44d9150"
version = "0.11.7"

[[deps.Parsers]]
deps = ["Dates"]
git-tree-sha1 = "621f4f3b4977325b9128d5fae7a8b4829a0c2222"
uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
version = "2.2.4"

[[deps.Pkg]]
deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"]
uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
version = "1.8.0"

[[deps.PooledArrays]]
deps = ["DataAPI", "Future"]
git-tree-sha1 = "28ef6c7ce353f0b35d0df0d5930e0d072c1f5b9b"
uuid = "2dfb63ee-cc39-5dd5-95bd-886bf059d720"
version = "1.4.1"

[[deps.Preferences]]
deps = ["TOML"]
git-tree-sha1 = "d3538e7f8a790dc8903519090857ef8e1283eecd"
uuid = "21216c6a-2e73-6563-6e65-726566657250"
version = "1.2.5"

[[deps.PrettyTables]]
deps = ["Crayons", "Formatting", "Markdown", "Reexport", "Tables"]
git-tree-sha1 = "dfb54c4e414caa595a1f2ed759b160f5a3ddcba5"
uuid = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d"
version = "1.3.1"

[[deps.Printf]]
deps = ["Unicode"]
uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7"

[[deps.QuadGK]]
deps = ["DataStructures", "LinearAlgebra"]
git-tree-sha1 = "78aadffb3efd2155af139781b8a8df1ef279ea39"
uuid = "1fd47b50-473d-5c70-9696-f719f8f3bcdc"
version = "2.4.2"

[[deps.REPL]]
deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"]
uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"

[[deps.Random]]
deps = ["SHA", "Serialization"]
uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"

[[deps.Reexport]]
git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b"
uuid = "189a3867-3050-52da-a836-e630ba90ab69"
version = "1.2.2"

[[deps.RegressionAndOtherStories]]
deps = ["CSV", "CategoricalArrays", "DataFrames", "DataStructures", "Dates", "DelimitedFiles", "Distributions", "LaTeXStrings", "LinearAlgebra", "NamedArrays", "NamedTupleTools", "Reexport", "Statistics", "StatsBase", "Unicode"]
git-tree-sha1 = "6cbfc20f44776778474198647fefe075f83fab6b"
uuid = "21324389-b050-441a-ba7b-9a837781bda0"
version = "0.1.2"

[[deps.Requires]]
deps = ["UUIDs"]
git-tree-sha1 = "838a3a4188e2ded87a4f9f184b4b0d78a1e91cb7"
uuid = "ae029012-a4dd-5104-9daa-d747884805df"
version = "1.3.0"

[[deps.Rmath]]
deps = ["Random", "Rmath_jll"]
git-tree-sha1 = "bf3188feca147ce108c76ad82c2792c57abe7b1f"
uuid = "79098fc4-a85e-5d69-aa6a-4863f24498fa"
version = "0.7.0"

[[deps.Rmath_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "68db32dff12bb6127bac73c209881191bf0efbb7"
uuid = "f50d1b31-88e8-58de-be2c-1cc44531875f"
version = "0.3.0+0"

[[deps.SHA]]
uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
version = "0.7.0"

[[deps.Scratch]]
deps = ["Dates"]
git-tree-sha1 = "0b4b7f1393cff97c33891da2a0bf69c6ed241fda"
uuid = "6c6a2e73-6563-6170-7368-637461726353"
version = "1.1.0"

[[deps.SentinelArrays]]
deps = ["Dates", "Random"]
git-tree-sha1 = "6a2f7d70512d205ca8c7ee31bfa9f142fe74310c"
uuid = "91c51154-3ec4-41a3-a24f-3f23e20d615c"
version = "1.3.12"

[[deps.Serialization]]
uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"

[[deps.SharedArrays]]
deps = ["Distributed", "Mmap", "Random", "Serialization"]
uuid = "1a1011a3-84de-559e-8e89-a11a2f7dc383"

[[deps.Sockets]]
uuid = "6462fe0b-24de-5631-8697-dd941f90decc"

[[deps.SortingAlgorithms]]
deps = ["DataStructures"]
git-tree-sha1 = "b3363d7460f7d098ca0912c69b082f75625d7508"
uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c"
version = "1.0.1"

[[deps.SparseArrays]]
deps = ["LinearAlgebra", "Random"]
uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"

[[deps.SpecialFunctions]]
deps = ["ChainRulesCore", "IrrationalConstants", "LogExpFunctions", "OpenLibm_jll", "OpenSpecFun_jll"]
git-tree-sha1 = "5ba658aeecaaf96923dce0da9e703bd1fe7666f9"
uuid = "276daf66-3868-5448-9aa4-cd146d93841b"
version = "2.1.4"

[[deps.Statistics]]
deps = ["LinearAlgebra", "SparseArrays"]
uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"

[[deps.StatsAPI]]
deps = ["LinearAlgebra"]
git-tree-sha1 = "c3d8ba7f3fa0625b062b82853a7d5229cb728b6b"
uuid = "82ae8749-77ed-4fe6-ae5f-f523153014b0"
version = "1.2.1"

[[deps.StatsBase]]
deps = ["DataAPI", "DataStructures", "LinearAlgebra", "LogExpFunctions", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics", "StatsAPI"]
git-tree-sha1 = "8977b17906b0a1cc74ab2e3a05faa16cf08a8291"
uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
version = "0.33.16"

[[deps.StatsFuns]]
deps = ["ChainRulesCore", "InverseFunctions", "IrrationalConstants", "LogExpFunctions", "Reexport", "Rmath", "SpecialFunctions"]
git-tree-sha1 = "5950925ff997ed6fb3e985dcce8eb1ba42a0bbe7"
uuid = "4c63d2b9-4356-54db-8cca-17b64c39e42c"
version = "0.9.18"

[[deps.SuiteSparse]]
deps = ["Libdl", "LinearAlgebra", "Serialization", "SparseArrays"]
uuid = "4607b0f0-06f3-5cda-b6b1-a6196a1729e9"

[[deps.TOML]]
deps = ["Dates"]
uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
version = "1.0.0"

[[deps.TableTraits]]
deps = ["IteratorInterfaceExtensions"]
git-tree-sha1 = "c06b2f539df1c6efa794486abfb6ed2022561a39"
uuid = "3783bdb8-4a98-5b6b-af9a-565f29a5fe9c"
version = "1.0.1"

[[deps.Tables]]
deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "LinearAlgebra", "OrderedCollections", "TableTraits", "Test"]
git-tree-sha1 = "5ce79ce186cc678bbb5c5681ca3379d1ddae11a1"
uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
version = "1.7.0"

[[deps.Tar]]
deps = ["ArgTools", "SHA"]
uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
version = "1.10.0"

[[deps.Test]]
deps = ["InteractiveUtils", "Logging", "Random", "Serialization"]
uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

[[deps.TranscodingStreams]]
deps = ["Random", "Test"]
git-tree-sha1 = "216b95ea110b5972db65aa90f88d8d89dcb8851c"
uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa"
version = "0.9.6"

[[deps.UUIDs]]
deps = ["Random", "SHA"]
uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"

[[deps.UnPack]]
git-tree-sha1 = "387c1f73762231e86e0c9c5443ce3b4a0a9a0c2b"
uuid = "3a884ed6-31ef-47d7-9d2a-63182c4928ed"
version = "1.0.2"

[[deps.Unicode]]
uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"

[[deps.WeakRefStrings]]
deps = ["DataAPI", "InlineStrings", "Parsers"]
git-tree-sha1 = "b1be2855ed9ed8eac54e5caff2afcdb442d52c23"
uuid = "ea10d353-3f73-51f8-a26c-33c1cb351aa5"
version = "1.4.2"

[[deps.Zlib_jll]]
deps = ["Libdl"]
uuid = "83775a58-1f1d-513f-b197-d71354ab007a"
version = "1.2.12+3"

[[deps.libblastrampoline_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "8e850b90-86db-534c-a0d3-1478176c7d93"
version = "5.1.0+0"

[[deps.nghttp2_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d"
version = "1.41.0+1"

[[deps.p7zip_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0"
version = "16.2.1+1"
"""

# ╔═╡ Cell order:
# ╟─27fd555f-2a13-4106-bcb5-6644e41dfb29
# ╟─d5f05d8b-416e-4648-89d8-afa937f30e77
# ╠═ad1df579-dd25-4be3-9194-62019182e9c2
# ╠═7a3506e2-6287-4755-a715-cd1de9dffc4c
# ╠═4087e639-e1b9-4034-bb09-051ae0cc4e3d
# ╠═a8a9aee2-f2fd-4176-9a45-570672d0eb94
# ╠═8ef8d7c5-219d-42e7-85a2-6dee7d237edb
# ╠═4859f90d-ccdf-4995-be0c-85faa9c02978
# ╠═424f704a-0d61-45fa-a88f-3ad851b81f53
# ╠═663a35fc-5c55-459c-be4a-b6be96d04eaf
# ╠═e05b1923-3137-47df-b49f-839cfae0bd2e
# ╠═6383993d-ce7c-401d-a6a2-f87e19f85dd7
# ╠═cff39c41-c8b6-49b3-9e5e-eeabf141a61f
# ╠═b78a4c9a-452e-4a27-8cb3-63154d48d965
# ╠═91a5866c-6723-46bd-b806-c2c9059c1244
# ╠═e8e32ea1-35f1-48fa-b5e4-bb5f2e845e10
# ╠═478230d4-f210-4df8-98e2-fdeaa88d3c7a
# ╠═0f27ef85-a4ce-4978-aac7-a1804f723ee0
# ╠═127ec62d-406e-40ca-93de-c32562e2a29f
# ╠═bad9cf89-cc3a-4cb2-b139-ff8e1052818e
# ╠═75e4d07d-90ff-402a-8918-4f0e68bbcf5d
# ╠═1040c140-5d2a-4352-a7f0-901f3a2b275a
# ╠═d6ad38c9-5bc9-4d59-a1d8-b19ad76634dc
# ╠═9ce44159-004f-424a-8f8c-99e41b2dd622
# ╠═fbd7dfb4-1cd7-4aad-ad4b-b89e675376e7
# ╠═836e78ed-98d9-4ff6-adda-e7cfa3080bcf
# ╠═db88ef04-e742-4336-b6c8-73eb5055e655
# ╠═745fb351-97c9-490c-b242-91937aefb895
# ╠═07fd84bd-7778-4ea1-ab87-190f4693c46d
# ╠═3c2f895a-f1d3-4c80-a9d6-1cb4ab3a083c
# ╠═faf2626a-ac8a-4e74-b93c-8ccc8d8a4b69
# ╠═5344134a-9e47-4cf1-83a8-017f43736fa0
# ╠═354b330f-c86e-4f29-a2a7-2efc82f8f46a
# ╠═02ab023b-199c-4134-9dc8-387c7963cdb3
# ╟─78426ef1-922f-4277-af1f-e3a1ef61ed6e
# ╠═bace2090-e331-4a04-ab5e-6f68b6c6ce27
# ╠═235cb6da-5cc7-456a-8874-c1b1923cf956
# ╠═e58d4b6f-820e-4d69-8919-682fbf9051fb
# ╠═d3978285-5e7e-44c5-b06b-9a46fe6b62e0
# ╠═d35a817a-1fe9-4ccc-9484-7b272985c194
# ╠═cd1810bf-11ed-46b1-b5a3-39054202e740
# ╠═e6cf6020-0953-4e6d-9174-f21928a742da
# ╠═4310a3bb-74f4-4746-a4df-a2f3528de248
# ╠═922cbd2d-2ef0-4d0b-a9d5-c3759774068f
# ╠═21cd4f93-cd3f-4813-a292-d1a6aef39db9
# ╠═9a173de0-b44b-4ec7-ba86-f34a8f0fdf0e
# ╠═2b6be5ad-411a-4e82-a713-e3a8e6140314
# ╠═d9c1c0a5-44fe-472f-a93c-9a3d38ef1a84
# ╠═272c6581-962e-461b-b99d-75c3ca9629c1
# ╠═7d9ce69d-6f62-4a99-91a2-2d9bab97eda1
# ╟─00000000-0000-0000-0000-000000000001
# ╟─00000000-0000-0000-0000-000000000002


#=
An instance of `ILMSystem` holds the base cache, any problem-specific extra cache,
and containers for physical parameters, boundary condition data/functions, forcing data/functions,
a timestep function (which returns the timestep) and motion data/functions.

If the system needs to be regenerated because of surface motion, only the two
caches will need to be regenerated.
=#

Base.length(sys::ILMSystem) = length(sys.base_cache)

"""
    construct_system(prob::AbstractILMProblem) -> ILMSystem

Return a system of type of type `ILMSystem` from the given problem
instance `prob`.
"""
@inline construct_system(prob::AbstractILMProblem) = __init(prob)

"""
    update_system(sysold::ILMSystem,body::Body/BodyList)

From an existing system `sysold`, return a new system based on
the Body or BodyList `body`.
"""
function update_system(sysold::ILMSystem,body::Union{Body,BodyList})
    prob = regenerate_problem(sysold,body)
    return __init(prob)
end

"""
    update_system!(sys::ILMSystem,u,sysold,t)

From an existing system `sysold` at time `t`, return a new system `sys`
in place, based on the solution vector `u`, whose body state
information will be used to replace the body information and
subsequent operators in `sysold`.
"""
function update_system!(sys::ILMSystem,u,sysold::ILMSystem,t)
    @unpack base_cache, motions = sysold
    @unpack bl = base_cache
    x = aux_state(u)
    bodies = deepcopy(bl)
    update_body!(bodies,x,motions)
    sysnew = update_system(sysold,bodies)
    sys.base_cache = sysnew.base_cache
    sys.extra_cache = sysnew.extra_cache
    return sys
end

"""
    __init(prob::AbstractILMProblem)

Initialize `ILMSystem` with the given problem `prob` specification.
Depending on the type of problem, this sets up a base cache of scalar or
vector type, as well as an optional extra cache
"""
function __init(prob::AbstractILMProblem{DT,ST,DTP}) where {DT,ST,DTP}
    @unpack g, bodies, phys_params, bc, forcing, timestep_func, motions = prob

    base_cache = _construct_base_cache(bodies,g,DT,ST,DTP,prob,Val(length(bodies)))

    extra_cache = prob_cache(prob,base_cache)

    return ILMSystem{_static_surfaces(motions),typeof(prob),length(base_cache),typeof(phys_params),typeof(bc),typeof(forcing),
                    typeof(timestep_func),typeof(motions),typeof(base_cache),typeof(extra_cache)}(
              phys_params,bc,forcing,timestep_func,motions,base_cache,extra_cache)

end

@inline _construct_base_cache(bodies,g,DT,ST,DTP,::PT,::Val{0}) where {PT <: AbstractScalarILMProblem} =
              SurfaceScalarCache(g,ddftype=DT,scaling=ST,dtype=DTP)

@inline _construct_base_cache(bodies,g,DT,ST,DTP,::PT,::Val{N}) where {PT <: AbstractScalarILMProblem,N} =
              SurfaceScalarCache(bodies,g,ddftype=DT,scaling=ST,dtype=DTP)

@inline _construct_base_cache(bodies,g,DT,ST,DTP,::PT,::Val{0}) where {PT <: AbstractVectorILMProblem} =
              SurfaceVectorCache(g,ddftype=DT,scaling=ST,dtype=DTP)

@inline _construct_base_cache(bodies,g,DT,ST,DTP,::PT,::Val{N}) where {PT <: AbstractVectorILMProblem,N} =
              SurfaceVectorCache(bodies,g,ddftype=DT,scaling=ST,dtype=DTP)

_static_surfaces(::Nothing) = true
_static_surfaces(::Any) = false

# Extend surface_velocity! and allow for null motions
function RigidBodyTools.surface_velocity!(u::AbstractVector,v::AbstractVector,sys::ILMSystem,t)
  @unpack base_cache, motions = sys
  @unpack bl = base_cache
  fill!(u,0.0)
  fill!(v,0.0)
  if !isnothing(motions)
    surface_velocity!(u,v,bl,motions,t)
  end
end

RigidBodyTools.surface_velocity!(u::VectorData,sys::ILMSystem,t) = surface_velocity!(u.u,u.v,sys,t)

# Create the basic solve function, to be extended
function solve(prob::AbstractILMProblem,sys::ILMSystem) end

## Extend functions on `BasicILMCache` type to `ILMSystem`
for f in [:zeros_surface,:zeros_surfacescalar,
          :zeros_grid,:zeros_gridcurl,:zeros_griddiv,:zeros_gridgrad,:zeros_gridgradcurl,
          :similar_surface,:similar_surfacescalar,
          :similar_grid,:similar_gridcurl,:similar_griddiv,:similar_gridgrad,:similar_gridgradcurl,
          :ones_surface,:ones_surfacescalar,
          :ones_grid,:ones_gridgrad,:ones_gridcurl,:ones_griddiv,:ones_gridgradcurl,
          :x_grid,:y_grid,:x_gridcurl,:y_gridcurl,:x_griddiv,:y_griddiv,
          :normals,:areas,:points,:arcs,
          :create_nRTRn,:create_GLinvD,:create_CLinvCT,:create_CL2invCT,
          :create_RTLinvR,:create_GLinvD_symm,
          :create_GLinvD_cross,:create_surface_filter]
   @eval $f(sys::ILMSystem) = $f(sys.base_cache)
end

for f in [:regularize!, :interpolate!, :regularize_normal!,
          :normal_interpolate!,
          :regularize_normal_cross!,:normal_cross_interpolate!,
          :regularize_normal_dot!,:normal_dot_interpolate!,
          :regularize_normal_symm!,:normal_interpolate_symm!,
          :surface_curl!,:surface_divergence!,:surface_grad!,
          :surface_divergence_symm!,:surface_grad_symm!,
          :surface_curl_cross!,:surface_divergence_cross!,:surface_grad_cross!]
  @eval $f(a,b,sys::ILMSystem) = $f(a,b,sys.base_cache)
end

for f in [:norm,:integrate]
   @eval $f(a,sys::ILMSystem) = $f(a,sys.base_cache)
   @eval $f(a,sys::ILMSystem,i::Int) = $f(a,sys.base_cache,i)
end

for f in [:view]
  @eval $f(a::PointData,sys::ILMSystem,i::Int) = $f(a,sys.base_cache,i)
end

for f in [:dot,:copyto!]
   @eval $f(a,b,sys::ILMSystem) = $f(a,b,sys.base_cache)
   @eval $f(a,b,sys::ILMSystem,i::Int) = $f(a,b,sys.base_cache,i)
end

for f in [:RegularizationMatrix,:InterpolationMatrix]
  @eval CartesianGrids.$f(sys::ILMSystem,args...) = $f(sys.base_cache,args...)
end


@testset "$TEST $G" begin

f, fio = mktemp()

g = G(10,20)
gpath = joinpath(testdir,"testdata","gtest.gt.gz")
writegraph(gpath, g)
h = readgraph(gpath, :gt, G, compressed=true)
@test h == g
h = readgraph(gpath, G)
@test h == g
if G <: ANetwork
    vprop!(g, "w", rand(nv(g)))
    writenetwork(gpath, g)
    h = readnetwork(gpath, :gt, G, compressed=true)
    test_networks_eq(h, g)
    h = readnetwork(gpath, G)
    test_networks_eq(h, g)
end
rm(gpath)

g = DG(10,20)
gpath = joinpath(testdir,"testdata","gtest.gt.gz")
writegraph(gpath, g)
h = readgraph(gpath, :gt, G, compressed=true)
@test h == g
h = readgraph(gpath, G)
@test h == g
if G <: ANetwork
    vprop!(g, "w", rand(nv(g)))
    eprop!(g, "w", EdgeMap(g, rand(ne(g)))) #TODO

    writenetwork(gpath, g)
    h = readnetwork(gpath, :gt, G, compressed=true)
    test_networks_eq(h, g)
    h = readnetwork(gpath, G)
    test_networks_eq(h, g)
end
rm(gpath)

g = readgraph(:lesmis, G)
@test typeof(g) == G
@test nv(g) == 77
@test ne(g) == 254

writegraph(f, g, :gt)
h = readgraph(f, :gt, G)
@test g == h

g = readgraph(:serengetifoodweb, G)
@test typeof(g) == DG
@test nv(g) == 161
@test ne(g) == 592

writegraph(f, g, :gt)
h = readgraph(f, :gt, G)
@test g == h

g = G(10,0)
writegraph(f, g, :gt)
ga = readgraph(f, :gt, G)
@test g == ga

g = G(10,20)
writegraph(f, g, :gt)
ga = readgraph(f, :gt, G)
@test g == ga

g = DG(10,20)
writegraph(f, g, :gt)
ga = readgraph(f, :gt, G)
@test g == ga

if G <: ANetwork ##############################

g = readnetwork(:karate, G)
@test sort(gprop_names(g)) == ["description","readme"]
@test sort(vprop_names(g)) == ["pos"]
@test sort(eprop_names(g)) == []

writenetwork(f, g, :gt)
h = readnetwork(f, :gt, G)
@test h == g
@test h.props == g.props

g = readnetwork(:lesmis)
@test sort(gprop_names(g)) == ["description","readme"]
@test sort(vprop_names(g)) == ["label","pos"]
@test sort(eprop_names(g)) == ["value"]

writenetwork(f, g, :gt)
h = readnetwork(f, :gt, G)
@test h == g
# @test h.props == g.props #dataset graphs' edges index can change
writenetwork(f, h, :gt)
h2 = readnetwork(f, :gt, G)
@test h == h2
@test h.props == h2.props

end #if network ####################

isfile(f) && rm(f)
 end #testset


function solve_tsp(tsp::TSP, parameters=AlgorithmParameters(); verbose=true, use_dist_mtx=false)
    n = tsp.dimension
    x = tsp.nodes[:, 1]
    y = tsp.nodes[:, 2]
    serv_time = zeros(size(x))
    dem = ones(size(x))
    dem[1] = 0.0

    vehicleCapacity = n
    n_vehicles = 1

    isRoundingInteger = true
    isDurationConstraint = false
    duration_limit = C_DBL_MAX

    if use_dist_mtx
        c_dist_mtx = Matrix(tsp.weights')
        for i in 1:n
            c_dist_mtx[i, i] = 0.0
        end
        # need to input dist_mtx' instead of dist_mtx
        # Julia: column-first indexing
        # C: row-first indexing
        return c_api_solve_cvrp_dist_mtx(
            n, x, y, c_dist_mtx, serv_time, dem, vehicleCapacity, duration_limit, isDurationConstraint,
            n_vehicles, parameters, verbose
        )
    else
        return c_api_solve_cvrp(
            n, x, y, serv_time, dem,
            vehicleCapacity, duration_limit, isRoundingInteger, isDurationConstraint, n_vehicles, parameters, verbose
        )
    end
end

function solve_tsp(dist_mtx::Matrix, parameters=AlgorithmParameters(); verbose=true, x_coordinates=Float64[], y_coordinates=Float64[])
    n = size(dist_mtx, 1)

    serv_time = zeros(n)
    dem = ones(n)
    dem[1] = 0.0

    vehicleCapacity = n
    n_vehicles = 1

    c_dist_mtx = Matrix(dist_mtx')
    for i in 1:n
        c_dist_mtx[i, i] = 0.0
    end

    duration_limit = C_DBL_MAX
    isDurationConstraint = false

    # need to input dist_mtx' instead of dist_mtx
    # Julia: column-first indexing
    # C: row-first indexing
    return c_api_solve_cvrp_dist_mtx(
        n, x_coordinates, y_coordinates, c_dist_mtx, serv_time, dem,
        vehicleCapacity, duration_limit, isDurationConstraint,
        n_vehicles, parameters, verbose
    )
end
function solve_tsp(x::Vector, y:: Vector, parameters=AlgorithmParameters(); verbose=true)
    n = length(x)
    serv_time = zeros(n)
    dem = ones(n)
    dem[1] = 0.0
    vehicleCapacity = n
    n_vehicles = 1

    isRoundingInteger = true
    duration_limit = C_DBL_MAX
    isDurationConstraint = false

    return c_api_solve_cvrp(
        n, x, y, serv_time, dem,
        vehicleCapacity, duration_limit, isRoundingInteger, isDurationConstraint,
        n_vehicles, parameters, verbose
    )
end


# Implements invariants that combine other invariants
# - `AllInvariant` passes only if all child invariants are true
# - `AnyInvariant` passes only if any one child invariant is true
# - `SequenceInvariant` runs multiple invariants in an order and fails
#    early if one fails

abstract type InvariantList <: AbstractInvariant end

title(invs::InvariantList) = invs.title
description(invs::InvariantList) = invs.description

AbstractTrees.children(invs::InvariantList) = invs.invariants

# ## AllInvariant

struct AllInvariant{I<:AbstractInvariant} <: InvariantList
    invariants::Vector{I}
    title::String
    description::Union{Nothing, String}
    shortcircuit::Bool
end

AllInvariant(
    invariants, title::String;
    description = nothing,
    shortcircuit = true,
    kwargs...
) = invariant(AllInvariant(invariants, title, description, shortcircuit); kwargs...)

# TODO: check for errors thrown and check invariants individually
# TODO: remove SequenceInvariant and add `shortcircuit` kwarg to `AllInvariant`

function satisfies(invs::AllInvariant, input)
    results = []
    keepchecking = true
    for inv in invs.invariants
        if !keepchecking
            push!(results, missing)
            continue
        else
            res = try
                satisfies(inv, input)
            catch e
                "Unexpected error while checking invariant: $e"
            end
            push!(results, res)
            if !isnothing(res) && invs.shortcircuit
                keepchecking = false
            end
        end

    end
    return all(isnothing, results) ? nothing : results
end

function errormessage(io::IO, invs::AllInvariant, msgs)
    __combinator_errormessage(io, invs, msgs, map(__getmarker, msgs),
        faint("All invariants listed below should be satisfied:\n\n "))
end

__getmarker(::Nothing) = PASS
__getmarker(::Missing) = UNKNOWN
__getmarker(_) = FAIL

# ## AnyInvariant

struct AnyInvariant{I<:AbstractInvariant} <: InvariantList
    invariants::Vector{I}
    title::String
    description::Union{Nothing, String}
end

AnyInvariant(
    invariants, title::String;
    description = nothing,
    kwargs...
) = invariant(AnyInvariant(invariants, title, description); kwargs...)

function satisfies(invs::AnyInvariant, input)
    results = []

    for inv in invs.invariants
        res = satisfies(inv, input)
        push!(results, res)
        if isnothing(res)
            return nothing
        end
    end
    return results
end

function errormessage(io::IO, invs::AnyInvariant, msgs)
    __combinator_errormessage(io, invs, msgs, map(m -> isnothing(m) ? PASS : FAIL, msgs),
        faint("At least one of the invariants listed below should be satisfied:\n\n"))
end

# ## Helpers for printing children

const PASS = "\e[32m✔\e[0m\e[2m"
const FAIL = "\e[31m⨯\e[0m"
const UNKNOWN = "\e[33m?\e[0m"

function __combinator_errormessage(io::IO, invs, msgs, markers, msg)
    showdescription(io, invs)
    print(io, msg)
    for (inv, message, marker) in zip(invs.invariants, msgs, markers)
        __errormessage_child(io, inv; marker, message)
    end
end

function __errormessage_child(
        io,
        inv;
        marker = "o",
        message=nothing,
        indent = "    "
        )
    print(io, marker, " ")
    showtitle(io, inv)
    print(io, "\e[0m")
    println(io)

    if !(isnothing(message) || ismissing(message))
        ioi = WrapIO(io; indent)
        errormessage(ioi, inv, message)
    end
end

@testset "Combinators" begin
    i = exampleinvariant()
    @test invariant([i, i], "test") isa AllInvariant
    @test invariant([i, i], "test", :all) isa AllInvariant
    @test invariant([i, i], "test", :any) isa AnyInvariant
end


using CUDAnative

function kernel_vadd(a, b, c)
    i = (blockIdx().x-1) * blockDim().x + threadIdx().x
    c[i] = a[i] + b[i]

    return nothing
end

Base.code_llvm(stdout, kernel_vadd, Tuple{CuDeviceArray{Float32,1,AS.Global},
                                          CuDeviceArray{Float32,1,AS.Global},
                                          CuDeviceArray{Float32,1,AS.Global}})

using TinyCassette
struct Ctx end

Base.code_llvm(stdout, TinyCassette.execute,
               Tuple{Ctx, typeof(kernel_vadd),
                     CuDeviceArray{Float32,1,AS.Global},
                     CuDeviceArray{Float32,1,AS.Global},
                     CuDeviceArray{Float32,1,AS.Global}})


# Run after including ../src/CGP.jl
#  or uncomment the following line
include("../src/CGP.jl")

# Note that the parameters are set in the file ../src/SetParams.jl
# The default functions are defined in ../src/Func.jl
# The type  MyInt is defined in CGP.jl

#using Base.Test
using Printf
using Main.CGP

funcs = default_funcs()
const context = construct_contexts(Main.CGP.numinputs)[Main.CGP.numinputs]

# Generate random circuits and count the logic functions that are computed.
# Similar to results of Raman and Wagner 2011 to generate their Figure 2b.
#num_iterations = 50
num_iterations = 50000
println("num_iterations: ",num_iterations)
counts = create_count_function_hash( p.numinputs, p.numoutputs )
for _ = 1:num_iterations
    c = random_chromosome(p, funcs)
    #print_chromosome( c )
    # Executing on these inputs tests all possible bit combinations for inputs
    outputs = execute_chromosome(c, context)
  #=
    print("outputs: ",outputs)
    if length(outputs) > 1
      println("   hamming: ",hamming(outputs[1],outputs[2]))
    else
      println()
    end
  =#
    concat_outputs = concatenate_outputs(p.numinputs, outputs)
    #print("concat:  ")
    #Printf.@printf("0x%2x\n",concat_outputs)
    increment_count_function_hash(concat_outputs, counts )
    outputs
end

#if num_iterations <= 50
  Main.CGP.print_count_function_hash(counts)
#end
#Main.CGP.print_count_function_summary(counts)


using OpSel
using Test

@testset "OpSel.jl" begin
    # Write your own tests here.
    OpSel.testUnequal(2045);
    OpSel.testEqual(200,50);
end