"""
This function creates the MarketClearingProblem struct for CEM.
"""
function create_cem_mkt_clr_problem(investor_dir::String,
                                    market_names::Vector{Symbol},
                                    expected_portfolio::Vector{<: Project{<: BuildPhase}},
                                    zones::Vector{String},
                                    lines::Vector{ZonalLine},
                                    peak_load::Float64,
                                    rep_hour_weight::Vector{Float64},
                                    average_capital_cost_multiplier::Float64,
                                    scenario::Scenario,
                                    iteration_year::Int64,
                                    yearly_horizon::Int64)

    num_invperiods = yearly_horizon

    #Get updated load growth belief
    load_growth = AxisArrays.AxisArray(zeros(length(zones), num_invperiods), zones, collect(1:num_invperiods))
    for (idx, zone) in enumerate(zones)
        load_growth[idx, :] = get_parameter_values(scenario)[iteration_year]["load_$(zone)", iteration_year:iteration_year + num_invperiods - 1]
    end

    reserve_up_market_bool = false
    reserve_down_market_bool = false
    capacity_market_bool = false
    rec_market_bool = false

    # Find if the investor participates in the following markets:
    if in(:ReserveUp, market_names)
        reserve_up_market_bool = true
    end

    if in(:ReserveDown, market_names)
        reserve_down_market_bool = true
    end

    if in(:Capacity, market_names)
        capacity_market_bool = true
    end

    if in(:REC, market_names)
        rec_market_bool = true
    end

    # Gather markets data-------------------------------------------------------------------------------
    load_data = read_data(joinpath(investor_dir, "timeseries_data_files", "Load", "load_$(iteration_year - 1).csv"))
    reserve_up_demand_data = read_data(joinpath(investor_dir, "timeseries_data_files", "Reserves", "reserve_up_$(iteration_year - 1).csv"))
    reserve_down_demand_data = read_data(joinpath(investor_dir, "timeseries_data_files", "Reserves", "reserve_down_$(iteration_year - 1).csv"))

    num_hours = DataFrames.nrow(load_data)

    energy_mkt_params = read_data(joinpath(investor_dir, "markets_data", "energy_mkt_param.csv"))
    price_cap_energy = AxisArrays.AxisArray(energy_mkt_params.price_cap * 1.0, zones)

    reserve_up_mkt_params = read_data(joinpath(investor_dir, "markets_data", "reserve_up_mkt_param.csv"))
    reserve_up_num_points = AxisArrays.AxisArray(zeros(Int64,length(zones)), zones)
    reserve_up_break_points = AxisArrays.AxisArray([Float64[] for z in zones], zones)
    reserve_up_price_points = AxisArrays.AxisArray([Float64[] for z in zones], zones)

    reserve_down_mkt_params = read_data(joinpath(investor_dir, "markets_data", "reserve_down_mkt_param.csv"))
    price_cap_reservedown = AxisArrays.AxisArray(reserve_down_mkt_params.price_cap * 1.0, zones)
    perc_req_reserve_down = AxisArrays.AxisArray(reserve_down_mkt_params.perc_req, zones)

    zonal_load = AxisArrays.AxisArray(zeros(length(zones), num_hours), zones, (1:num_hours))
    zonal_reserve_up = AxisArrays.AxisArray(zeros(length(zones), num_hours), zones, (1:num_hours))
    zonal_reserve_down = AxisArrays.AxisArray(zeros(length(zones), num_hours), zones, (1:num_hours))

    for (zone_num, zone) in enumerate(zones)
        idx = findfirst(x -> x == "$(zone)", reserve_up_mkt_params.zones)
        if reserve_up_mkt_params.ORDC[idx]
            reserve_up_num_points[zone] = reserve_up_mkt_params.ORDC_points[idx]
            for point in 1:reserve_up_num_points[zone]
                push!(reserve_up_break_points[zone], reserve_up_mkt_params[idx, Symbol("ORDC_x$(point)")] * reserve_up_market_bool)
                push!(reserve_up_price_points[zone], reserve_up_mkt_params[idx, Symbol("ORDC_y$(point)")] * reserve_up_market_bool)
            end
        else
            reserve_up_break_points[zone] = [0.0, 1.0, 1.0, 10.0] * reserve_up_market_bool
            reserve_up_price_points[zone] = [reserve_up_mkt_params.price_cap[idx], reserve_up_mkt_params.price_cap[idx], 0.0, 0.0] * reserve_up_market_bool
        end

        price_cap_reservedown[zone] = reserve_down_mkt_params.price_cap[idx]

        for h in 1:num_hours
            zonal_load[zone, h] = load_data[:, Symbol(zone_num)][h]
            zonal_reserve_up[zone, h] = reserve_up_demand_data[:, Symbol(zone_num)][h] * reserve_up_market_bool
            zonal_reserve_down[zone, h] = reserve_down_demand_data[:, Symbol(zone_num)][h] * reserve_down_market_bool
        end
    end

    energy_annual_increment = AxisArrays.AxisArray(ones(length(zones), num_invperiods), zones, collect(1:num_invperiods))
    reserveup_annual_increment = AxisArrays.AxisArray(ones(length(zones), num_invperiods), zones, collect(1:num_invperiods))
    reservedown_annual_increment = AxisArrays.AxisArray(ones(length(zones), num_invperiods), zones, collect(1:num_invperiods))

    for z in zones
        for p in 1:num_invperiods
            for i in 1:p
                energy_annual_increment[z, p] = energy_annual_increment[z, p] * (1 + load_growth[z, i])
                reserveup_annual_increment[z, p] = energy_annual_increment[z, p] * (1 + load_growth[z, i])
                reservedown_annual_increment[z, p] = energy_annual_increment[z, p] * (1 + load_growth[z, i])
            end
        end
    end

    capacity_mkt_param_file = joinpath(investor_dir, "markets_data", "capacity_mkt_param.csv")
    capacity_annual_increment = load_growth

    REC_mkt_params = read_data(joinpath(investor_dir, "markets_data", "REC_mkt_param.csv"))
    price_cap_rec = REC_mkt_params.price_cap[1]
    rec_req = REC_mkt_params.rec_req[1] * rec_market_bool
    rec_annual_increment = REC_mkt_params.annual_increment[1] * rec_market_bool

    capacity_markets = Vector{CapacityMarket}(undef, num_invperiods)
    energy_markets = Vector{EnergyMarket}(undef, num_invperiods)
    reserve_up_markets = Vector{ReserveUpMarket}(undef, num_invperiods)
    reserve_down_markets = Vector{ReserveDownMarket}(undef, num_invperiods)

    rec_markets = Vector{RECMarket}(undef, num_invperiods)

    average_capacity_growth = Statistics.mean(capacity_annual_increment)

    # Create market products data for the horizon
    for p in 1:num_invperiods
        system_peak_load = (1 + average_capacity_growth) ^ (p) * peak_load
        capacity_markets[p] = create_capacity_demand_curve(capacity_mkt_param_file, system_peak_load, capacity_market_bool)

        energy_markets[p] = EnergyMarket(AxisArrays.AxisArray(zonal_load .* energy_annual_increment[:, p], zones, (1:num_hours)),
                                     price_cap_energy)
        reserve_up_markets[p] = ReserveUpMarket(reserve_up_break_points,
                                                reserve_up_price_points,
                                                AxisArrays.AxisArray(zonal_reserve_up .* reserveup_annual_increment[:, p], zones, (1:num_hours)))
        reserve_down_markets[p] = ReserveDownMarket(AxisArrays.AxisArray(zonal_reserve_down .* reservedown_annual_increment[:, p], zones, (1:num_hours)),
                                                  price_cap_reservedown)
        rec_markets[p] = RECMarket(min(rec_req + rec_annual_increment * (p + iteration_year - 1), 1),
                             price_cap_rec)
    end

    max_peak_loads = AxisArrays.AxisArray([maximum([maximum(market.demand[z, :]) for market in energy_markets]) for z in zones], zones)

    markets = MarketCollection.(capacity_markets,
                                energy_markets,
                                reserve_up_markets,
                                reserve_down_markets,
                                rec_markets)
    #-----------------------------------------------------------------------------------------------------------------------------
    availability_df = read_data(joinpath(investor_dir, "timeseries_data_files", "Availability", "DAY_AHEAD_availability.csv"))

    invested_portfolio = find_active_invested_projects(expected_portfolio)
    option_portfolio = find_option_projects(expected_portfolio)

    cem_projects = MarketProject[]

    for project in invested_portfolio
        tech = get_tech(project)
        zone = get_zone(tech)
        cem_project = create_market_project(project,
                                          price_cap_energy[zone],
                                          maximum(reserve_up_break_points[zone]),
                                          price_cap_reservedown[zone],
                                          max_peak_loads,
                                          iteration_year,
                                          num_hours,
                                          num_invperiods,
                                          availability_df)

        if !isnothing(cem_project)
            push!(cem_projects, cem_project)
        end
    end

    aggregated_options = MarketProject[]
    for option in option_portfolio
        tech = get_tech(option)
        zone = get_zone(tech)
        similar_option = filter(p -> (get_type(tech) == p.tech_type && zone == p.zone), aggregated_options)

        if length(similar_option) < 1
            aggregated_option = create_market_project(option,
                                          price_cap_energy[zone],
                                          maximum(reserve_up_break_points[zone]),
                                          price_cap_reservedown[zone],
                                          max_peak_loads,
                                          iteration_year,
                                          num_hours,
                                          num_invperiods,
                                          availability_df)
            push!(aggregated_options, aggregated_option)
        else
            investment_cost = get_investment_cost(get_finance_data(option))[iteration_year:iteration_year + num_invperiods - 1]
            for (i, cost) in enumerate(similar_option[1].expansion_cost)
                if investment_cost[i] > cost
                    similar_option[1].expansion_cost[i] = investment_cost[i]
                end
            end

            discount_rate = get_discount_rate(get_finance_data(option))
            if discount_rate > similar_option[1].discount_rate
                similar_option[1].discount_rate = discount_rate
            end

            owner = get_ownedby(get_finance_data(option))
            if !in(owner, similar_option[1].ownedby)
               similar_option[1].base_cost_units = similar_option[1].base_cost_units + 1
               push!(similar_option[1].ownedby, owner)
            end
        end
    end

    append!(cem_projects, aggregated_options)

    system = MarketClearingProblem(zones, lines, average_capital_cost_multiplier, markets, cem_projects, rep_hour_weight)

    return system
end


if R123_USE_AESNI
    import RandomNumbers: split_uint
    key = rand(UInt128)
    r = ARS1x(key)
    r1 = ARS4x(split_uint(key, UInt32))
    @test seed_type(r) == UInt128
    @test seed_type(r1) == NTuple{4, UInt32}
    @test copyto!(copy(r), r) == r
    @test copyto!(copy(r1), r1) == r1
    @test r.x == rand(r1, UInt128)
    @test rand(r, UInt128) == rand(r1, UInt128)
    set_counter!(r, 0)
    set_counter!(r1, 1)
    @test rand(r, Tuple{UInt128})[1] == rand(r1, UInt128)
end


# Compare benchmarks
using GPUBenchmarks, BenchmarkTools
db = GPUBenchmarks.get_database()
using GPUBenchmarks: name, timestamp

ts = last(sort(unique(timestamp.(db))))

newbenchs = filter(x-> x.timestamp == ts, db)

unique(GPUBenchmarks.device.(db))

oldbenchs = filter(x-> x.device == "opencl", db)
old_newest_ts = last(sort(unique(timestamp.(oldbenchs))))
oldbenchs = filter(x-> x.timestamp == old_newest_ts, oldbenchs)

sort(unique(GPUArrays.name.(db))

for elem in unique(GPUBenchmarks.name.(newbenchs))
    A = filter(x-> x.name == elem, oldbenchs)
    B = filter(x-> x.name == elem, oldbenchs)
    for (a, b) in zip(A, B)
        x = judge(minimum(a.benchmark), minimum(b.benchmark))
        if x.time != :invariant
            println(name)
            @show x
        end
    end
end


function _ensure_avar(model)
    if (model.avar_cache != nothing)
        return model.avar_cache
    end
    model.avar_cache = reshop_avar()
    return model.avar_cache
end

function _set_rhs(model::Optimizer, eidx, fn_cst, set::LS)
    # Add bound to constraint.
    if isa(set, MOI.LessThan{Float64})
        val = set.upper
    elseif isa(set, MOI.GreaterThan{Float64})
        val = set.lower
    elseif isa(set, MOI.EqualTo{Float64})
        val = set.value
    end

    # Set the rhs
    reshop_set_rhs(model.ctx, eidx, val - fn_cst)
    reshop_set_equtype(model.ctx, eidx, set_to_reshop[typeof(set)])

end

function _set_rhs(model::Optimizer, eidx, fn_cst, set::VLS)
    # Add bound to constraint.

    # Set the rhs
    reshop_set_rhs(model.ctx, eidx, - fn_cst)
    reshop_set_equtype(model.ctx, eidx, set_to_reshop[typeof(set)])

end

function get_status(ctx::Ptr{context})
    return (solver_stat[rhp_get_solvestat(ctx)],
            model_stat[rhp_get_modelstat(ctx)])
end

function is_valid_index(idx::Cint)
    return idx >= 0
end

function is_constraint_active(ctx::Ptr{context}, ci::MOI.ConstraintIndex{MOI.SingleVariable, MOI.EqualTo{Float64}})
     return true
end

function is_constraint_active(ctx::Ptr{context}, ci::MOI.ConstraintIndex{MOI.SingleVariable, MOI.LessThan{Float64}})
     return ctx_getvarbstat(ctx, ci.value-1) == RHP_BASIS_STATUS_UPPER
end

function is_constraint_active(ctx::Ptr{context}, ci::MOI.ConstraintIndex{MOI.SingleVariable, MOI.GreaterThan{Float64}})
     return ctx_getvarbstat(ctx, ci.value-1) == RHP_BASIS_STATUS_LOWER
end

function is_constraint_active(ctx::Ptr{context}, ci::MOI.ConstraintIndex{MOI.SingleVariable, MOI.Interval{Float64}})
     bstat = ctx_getvarbstat(ctx, ci.value-1)
     return bstat == RHP_BASIS_STATUS_LOWER || bstat == RHP_BASIS_STATUS_UPPER
end

function is_constraint_active(ctx::Ptr{context}, ci::MOI.ConstraintIndex{<:SF, MOI.EqualTo{Float64}})
     return true
end

function is_constraint_active(ctx::Ptr{context}, ci::MOI.ConstraintIndex{<:SF, MOI.LessThan{Float64}})
     return ctx_getequbstat(ctx, ci.value-1) == RHP_BASIS_STATUS_UPPER
end

function is_constraint_active(ctx::Ptr{context}, ci::MOI.ConstraintIndex{<:SF, MOI.GreaterThan{Float64}})
     return ctx_getequbstat(ctx, ci.value-1) == RHP_BASIS_STATUS_LOWER
end

function is_constraint_active(ctx::Ptr{context}, ci::MOI.ConstraintIndex{<:SF, MOI.Interval{Float64}})
     bstat = ctx_getequbstat(ctx, ci.value-1)
     return bstat == RHP_BASIS_STATUS_LOWER || bstat == RHP_BASIS_STATUS_UPPER
end

function reset_var_bnd(ctx::Ptr{context}, ci::MOI.ConstraintIndex{MOI.SingleVariable, MOI.Interval{Float64}})
     ctx
end

# Utils for MathOptInterface
#
##################################################
# Import legacy from LinQuadOptInterface to ease the integration
# of ReSHOP quadratic and linear facilities.
##################################################
# URL: https://github.com/JuliaOpt/LinQuadOptInterface.jl
#
# LICENSE:
# MIT License
# Copyright (c) 2017 Oscar Dowson, Joaquim Dias Garcia and contributors
##################################################

function reduce_duplicates(rows::Vector{T}, cols::Vector{T}, vals::Vector{S}) where T where S
    @assert length(rows) == length(cols) == length(vals)
    for i in 1:length(rows)
        if rows[i] > cols[i]
            tmp = rows[i]
            rows[i] = cols[i]
            cols[i] = tmp
        end
    end
    findnz(dropzeros!(sparse(rows, cols, vals)))
end

"""
    canonical_quadratic_reduction(func::ScalarQuadraticFunction)

Reduce a ScalarQuadraticFunction into three arrays, returned in the following
order:
 1. a vector of quadratic row indices
 2. a vector of quadratic column indices
 3. a vector of quadratic coefficients

Warning: we assume in this function that all variables are correctly
ordered, that is no deletion or swap has occured.
"""
function canonical_quadratic_reduction(func::MOI.ScalarQuadraticFunction)
    quad_columns_1, quad_columns_2, quad_coefficients = (
        Int32[term.variable_index_1.value for term in func.quadratic_terms],
        Int32[term.variable_index_2.value for term in func.quadratic_terms],
        [term.coefficient for term in func.quadratic_terms]
    )
    # MOI stores the diagonal element with a coefficient of 2.
    # This is most likely very smart for derivative, but it is not a panacee
    for i in 1:length(quad_coefficients)
        @inbounds if quad_columns_1[i] == quad_columns_2[i]
            quad_coefficients[i] *= .5
        end
    end
    r, c, v = reduce_duplicates(quad_columns_1, quad_columns_2, quad_coefficients)
    return (r .- 1., c .- 1., v)
end

"""
    canonical_linear_reduction(func::Quad)

Reduce a ScalarQuadraticFunction into two arrays, returned in the following
order:
 1. a vector of linear column indices
 2. a vector of linear coefficients

Warning: we assume in this function that all variables are correctly
ordered, that is no deletion or swap has occured.
"""
function canonical_linear_reduction(func::MOI.ScalarQuadraticFunction)
    f = MOIU.canonical(func)
    affine_columns = Int32[term.variable_index.value - 1 for term in f.affine_terms]
    affine_coefficients = [term.coefficient for term in f.affine_terms]
    return affine_columns, affine_coefficients
end

function canonical_linear_reduction(func::MOI.ScalarAffineFunction)
    f = MOIU.canonical(func)
    affine_columns = Int32[term.variable_index.value - 1 for term in f.terms]
    affine_coefficients = [term.coefficient for term in f.terms]
    return affine_columns, affine_coefficients
end

function canonical_vector_affine_reduction(func::MOI.VectorAffineFunction)
    index_cols = Int32[]
    index_vars = Int32[]
    coefs = Float64[]

    for t in func.terms
        push!(index_cols, t.output_index - 1)
        push!(index_vars, t.scalar_term.variable_index.value - 1)
        push!(coefs, t.scalar_term.coefficient)
    end
    return index_cols, index_vars, coefs
end


using DSGE, Test, CSV, DataFrames
using ModelConstructors, Nullables, Dates, OrderedCollections

prob_vecs = [[1., 0.], [.5, .5], [0., 1.]]
Hbar_vec = [14, 10, 6]

m = Model1002("ss59"; custom_settings = Dict{Symbol, Setting}(:flexible_ait_policy_change =>
                                                              Setting(:flexible_ait_policy_change,
                                                                      false),
                                                              :add_pgap => Setting(:add_pgap, false),
                                                              :add_ygap => Setting(:add_ygap, false),
                                                              :add_altpolicy_pgap => Setting(:add_altpolicy_pgap, true),
                                                              :add_altpolicy_ygap => Setting(:add_altpolicy_ygap, true))) # Set to false unless you want to re-generate any saved output
m <= Setting(:flexible_ait_policy_change, false) # Set to false unless you want to re-generate any saved output
m <= Setting(:date_forecast_start, Date(2020, 6, 30))
m <= Setting(:date_conditional_end, Date(2020, 6, 30))
m <= Setting(:cond_full_names, [:obs_gdp, :obs_corepce, :obs_spread, # Have to add anticipated rates to conditional data
                                   :obs_nominalrate, :obs_longrate,
                                   :obs_nominalrate1, :obs_nominalrate2, :obs_nominalrate3,
                                   :obs_nominalrate4, :obs_nominalrate5, :obs_nominalrate6])
m <= Setting(:cond_semi_names, [:obs_spread,
                                   :obs_nominalrate, :obs_longrate,
                                   :obs_nominalrate1, :obs_nominalrate2, :obs_nominalrate3,
                                   :obs_nominalrate4, :obs_nominalrate5, :obs_nominalrate6])

output_vars = [:histobs, :forecastobs, :histpseudo, :forecastpseudo]

df_full = CSV.read(joinpath(dirname(@__FILE__), "../reference/uncertain_altpolicy_data.csv"), DataFrame)
modal_params = map(x -> x.value, m.parameters)

hist_rule = DSGE.taylor_rule()
setup_regime_switching_inds!(m; cond_type = :full) # Reset the regime dates b/c using full conditional forecast

# Set up parameters
for j in 5:30 # Just add a bunch
    ModelConstructors.set_regime_val!(m[:σ_g], j, m[:σ_g].value)
    ModelConstructors.set_regime_val!(m[:σ_b], j, m[:σ_b].value)
    ModelConstructors.set_regime_val!(m[:σ_μ], j, m[:σ_μ].value)
    ModelConstructors.set_regime_val!(m[:σ_ztil], j, m[:σ_ztil].value)
    ModelConstructors.set_regime_val!(m[:σ_λ_f], j, m[:σ_λ_f].value)
    ModelConstructors.set_regime_val!(m[:σ_λ_w], j, m[:σ_λ_w].value)
    ModelConstructors.set_regime_val!(m[:σ_r_m], j, m[:σ_r_m].value)
    ModelConstructors.set_regime_val!(m[:σ_σ_ω], j, m[:σ_σ_ω].value)
    ModelConstructors.set_regime_val!(m[:σ_μ_e], j, m[:σ_μ_e].value)
    ModelConstructors.set_regime_val!(m[:σ_γ], j, m[:σ_γ].value)
    ModelConstructors.set_regime_val!(m[:σ_π_star], j, m[:σ_π_star].value)
    ModelConstructors.set_regime_val!(m[:σ_lr], j, m[:σ_lr].value)
    ModelConstructors.set_regime_val!(m[:σ_z_p], j, m[:σ_z_p].value)
    ModelConstructors.set_regime_val!(m[:σ_tfp], j, m[:σ_tfp].value)
    ModelConstructors.set_regime_val!(m[:σ_gdpdef], j, m[:σ_gdpdef].value)
    ModelConstructors.set_regime_val!(m[:σ_corepce], j, m[:σ_corepce].value)
    ModelConstructors.set_regime_val!(m[:σ_gdp], j, m[:σ_gdp].value)
    ModelConstructors.set_regime_val!(m[:σ_gdi], j, m[:σ_gdi].value)

    for l = 1:DSGE.n_mon_anticipated_shocks(m)
        ModelConstructors.set_regime_val!(m[Symbol("σ_r_m$(l)")], j, m[Symbol("σ_r_m$(l)")].value)
    end

    ModelConstructors.set_regime_val!(m[:σ_φ], j, 0.)
    ModelConstructors.set_regime_val!(m[:σ_biidc], j, 0.)
    ModelConstructors.set_regime_val!(m[:σ_ziid], j, 0.)
    if haskey(m.keys, :σ_φ_prop)
        ModelConstructors.set_regime_val!(m[:σ_φ_prop], j, 0.)
    end
    if haskey(m.keys, :σ_biidc_prop)
        ModelConstructors.set_regime_val!(m[:σ_biidc_prop], j, 0.)
    end
    if haskey(m.keys, :σ_ziid_prop)
        ModelConstructors.set_regime_val!(m[:σ_ziid_prop], j, 0.)
    end
end

# Impose the historical policy as the permanent and "alternative" rule
T_unc = Dict()
R_unc = Dict()
C_unc = Dict()
for j in 1:length(prob_vecs)
    T_unc[j], R_unc[j], C_unc[j] = DSGE.gensys_uncertain_altpol(m, prob_vecs[j], DSGE.AltPolicy[hist_rule])
    T_unc[j], R_unc[j], C_unc[j] = DSGE.augment_states(m, T_unc[j], R_unc[j], C_unc[j])
end

# Some setup
tempZLB_regimes = Dict()
m <= Setting(:alternative_policies, AltPolicy[hist_rule])

autoTs = Dict() # automatically calculated, these are what we are testing
autoRs = Dict()
autoCs = Dict()
auto_fcasts = Dict()

manTs = Dict()  # manually calculated, these are the "truth"
manRs = Dict()
manCs = Dict()
fcasts = Dict()

# Calculate baseline forecast to calculate the states in 2020-Q3
baseline_fcasts = DSGE.forecast_one_draw(m, :mode, :full, vcat(:forecaststates, output_vars), modal_params,
                                         df_full; regime_switching = true, n_regimes =
                                         get_setting(m, :n_regimes))

# Now calculate everything else
for j in 1:length(prob_vecs)
    # Set up initial state
    s₀ = baseline_fcasts[:forecaststates][:, 1]

    # Set up temporary ZLB regimes
    tempZLB_regimes[j] = 4:(4 + Hbar_vec[j] - 1)
    new_regime_dates = Dict{Int, Date}()
    new_regime_dates[1] = date_presample_start(m)
    for (regind, date) in zip(2:(maximum(tempZLB_regimes[j]) + 1), # This won't actually go to 3000, but this makes sure we have
                              Date(2020, 3, 31):Dates.Month(3):Date(3000,3,31)) # enough regimes to fill up hist_regime_dates properly
        new_regime_dates[regind] = date
    end

    m <= Setting(:regime_dates, new_regime_dates)
    setup_regime_switching_inds!(m; cond_type = :full)

    m <= Setting(:replace_eqcond, true)
    m <= Setting(:gensys2, true)
    m <= Setting(:gensys2_separate_cond_regimes, true) # required b/c not using an alternative policy technically
    replace_eqcond = Dict{Int, DSGE.EqcondEntry}()     # Which rule to replace with in which periods
    for regind in tempZLB_regimes[j]
        replace_eqcond[regind] = DSGE.EqcondEntry(DSGE.zlb_rule(), prob_vecs[j]) # Temp ZLB rule in this regimes
    end
    replace_eqcond[tempZLB_regimes[j][end]+1] = DSGE.EqcondEntry(AltPolicy(:historical, eqcond, solve), prob_vecs[j]) # Will set the "lift-off" regime directly
    m <= Setting(:regime_eqcond_info, replace_eqcond)
    m <= Setting(:temporary_altpolicy_names, [:zlb_rule])
    fcasts[j] = deepcopy(baseline_fcasts)

    # Set up matrices for regime-switching. Note that we have
    # plus 1 for lift-off b/c historical and conditional forecast period are accounted for by fact
    # tempZLB_regimes calculated from forecastobs (which includes conditional forecast periods)
    Γ0s = Vector{Matrix{Float64}}(undef, maximum(tempZLB_regimes[j]) + 1)
    Γ1s = Vector{Matrix{Float64}}(undef, maximum(tempZLB_regimes[j]) + 1)
    Cs = Vector{Vector{Float64}}(undef, maximum(tempZLB_regimes[j]) + 1)
    Ψs = Vector{Matrix{Float64}}(undef, maximum(tempZLB_regimes[j]) + 1)
    Πs = Vector{Matrix{Float64}}(undef, maximum(tempZLB_regimes[j]) + 1)

    TTTs_uzlb = Vector{Matrix{Float64}}(undef, maximum(tempZLB_regimes[j]) + 1) # uncertain ZLB
    RRRs_uzlb = Vector{Matrix{Float64}}(undef, maximum(tempZLB_regimes[j]) + 1)
    CCCs_uzlb = Vector{Vector{Float64}}(undef, maximum(tempZLB_regimes[j]) + 1)

    # Automatic calculation
    m <= Setting(:uncertain_altpolicy, true)
    m <= Setting(:uncertain_temporary_altpolicy, true)
    autoTs[j], autoRs[j], autoCs[j] = solve(m; regime_switching = true,
                                            gensys_regimes = [1:get_setting(m, :n_hist_regimes) + 1],
                                            gensys2_regimes =
                                            [(get_setting(m, :n_hist_regimes) + 1):get_setting(m, :n_regimes)],
                                            regimes = collect(1:get_setting(m, :n_regimes)))
    auto_fcasts[j] = DSGE.forecast_one_draw(m, :mode, :full, output_vars, modal_params,
                                            df_full; regime_switching = true, n_regimes =
                                            get_setting(m, :n_regimes))
    m <= Setting(:uncertain_altpolicy, false) # Need to turn these off afterward
    m <= Setting(:uncertain_temporary_altpolicy, false)

    # Onto the manual calculation

    # Only need to fill up the matrices for the forecast regimes for this part
    for fcast_reg in 3:length(TTTs_uzlb)
        Γ0s[fcast_reg], Γ1s[fcast_reg], Cs[fcast_reg], Ψs[fcast_reg], Πs[fcast_reg] =
        DSGE.zlb_rule().eqcond(m, fcast_reg)
    end
    TTT_gensys_final, RRR_gensys_final, CCC_gensys_final = solve(m)
    TTT_gensys_final = TTT_gensys_final[1:n_states(m), 1:n_states(m)]
    RRR_gensys_final = RRR_gensys_final[1:n_states(m), :]
    CCC_gensys_final = CCC_gensys_final[1:n_states(m)]

    # Calculate temp ZLB regimes under assumption smooth AIT-GDP occurs forever after ZLB ends
    gensys2_regimes = (tempZLB_regimes[j][1] - 1):(tempZLB_regimes[j][end] + 1)
    Tcal, Rcal, Ccal = DSGE.gensys2(m, Γ0s[gensys2_regimes], Γ1s[gensys2_regimes],
                                    Cs[gensys2_regimes], Ψs[gensys2_regimes], Πs[gensys2_regimes],
                                    TTT_gensys_final, RRR_gensys_final, CCC_gensys_final, Hbar_vec[j] + 1)
    Tcal[end] = TTT_gensys_final
    Rcal[end] = RRR_gensys_final
    Ccal[end] = CCC_gensys_final

    Th, Rh, Ch = hist_rule.solve(m)
    TTTs_uzlb[1] = Th
    TTTs_uzlb[2] = Th
    TTTs_uzlb[3] = Th
    CCCs_uzlb[1] = Ch
    CCCs_uzlb[2] = Ch
    CCCs_uzlb[3] = Ch

    # Prep for calculation of new transition matrices
    Γ0_til, Γ1_til, Γ2_til, C_til, Ψ_til = DSGE.gensys_to_predictable_form(Γ0s[4], Γ1s[4], Cs[4], Ψs[4], Πs[4])

    # Get new transition matrices for "uncertain" ZLB
    # Note we pass in Tcal[2:end] since we want the period t + 1 ZLB matrix for period t.
    # TTTs_uzlb[inreg] is the same size as Tcal though b/c TTTs_uzbl[inreg[end]] is just Tcal[end], etc.
    inreg = gensys2_regimes[2:end]
    TTTs_uzlb[inreg], RRRs_uzlb[inreg], CCCs_uzlb[inreg] =
        DSGE.gensys2_uncertain_altpol(prob_vecs[j], Th[1:n_states(m), 1:n_states(m)],
                                      Ch[1:n_states(m)], Tcal[2:end], Rcal[2:end], Ccal[2:end],
                                      Γ0_til, Γ1_til, Γ2_til, C_til, Ψ_til)
    for li in inreg
        TTTs_uzlb[li], RRRs_uzlb[li], CCCs_uzlb[li] = DSGE.augment_states(m, TTTs_uzlb[li], RRRs_uzlb[li], CCCs_uzlb[li])
    end

    # Now update the last regime matrices to feature "uncertainty" about the alt policy, too, post ZLB
    TTTs_uzlb[end] = T_unc[j]
    RRRs_uzlb[end] = R_unc[j]
    CCCs_uzlb[end] = C_unc[j]

    manTs[j] = TTTs_uzlb
    manRs[j] = RRRs_uzlb
    manCs[j] = CCCs_uzlb
end

@testset "Check automatic calculation of uncertain ZLB with an uncertain alternative policy." begin
    obs_inds = vcat(1:9, 12:13) # Only consider non-anticipating variables b/c still handling how to calculate measurement equations for them
    for j in 1:length(prob_vecs)
        for i in 4:length(manTs[j])
            @test @test_matrix_approx_eq autoTs[j][i] manTs[j][i]
        end
    end
end

nothing


function eval_f_wrapper(ips::InteriorPointSolver, x::Vector{Float64})
    nlp = ips.nlp
    cnt = ips.cnt
    @trace(ips.logger,"Evaluating objective.")
    cnt.eval_function_time += @elapsed obj_val = (get_minimize(nlp) ? 1. : -1.) * obj(nlp,view(x,1:get_nvar(nlp)))
    cnt.obj_cnt+=1
    cnt.obj_cnt==1 && (is_valid(obj_val) || throw(InvalidNumberException()))
    return obj_val*ips.obj_scale[]
end

function eval_grad_f_wrapper!(ips::InteriorPointSolver, f::Vector{Float64},x::Vector{Float64})
    nlp = ips.nlp
    cnt = ips.cnt
    @trace(ips.logger,"Evaluating objective gradient.")
    cnt.eval_function_time += @elapsed grad!(nlp,view(x,1:get_nvar(nlp)),view(f,1:get_nvar(nlp)))
    f.*=ips.obj_scale[] * (get_minimize(nlp) ? 1. : -1.)
    cnt.obj_grad_cnt+=1
    cnt.obj_grad_cnt==1 && (is_valid(f)  || throw(InvalidNumberException()))
    return f
end

function eval_cons_wrapper!(ips::InteriorPointSolver, c::Vector{Float64},x::Vector{Float64})
    nlp = ips.nlp
    cnt = ips.cnt
    @trace(ips.logger, "Evaluating constraints.")
    cnt.eval_function_time += @elapsed cons!(nlp,view(x,1:get_nvar(nlp)),view(c,1:get_ncon(nlp)))
    view(c,ips.ind_ineq).-=view(x,get_nvar(nlp)+1:ips.n)
    c.-=ips.rhs
    c.*=ips.con_scale
    cnt.con_cnt+=1
    cnt.con_cnt==2 && (is_valid(c) || throw(InvalidNumberException()))
    return c
end

function eval_jac_wrapper!(ipp::InteriorPointSolver, kkt::AbstractKKTSystem, x::Vector{Float64})
    nlp = ipp.nlp
    cnt = ipp.cnt
    ns = length(ipp.ind_ineq)
    @trace(ipp.logger, "Evaluating constraint Jacobian.")
    jac = get_jacobian(kkt)
    cnt.eval_function_time += @elapsed jac_coord!(nlp,view(x,1:get_nvar(nlp)),view(jac,1:get_nnzj(nlp)))
    compress_jacobian!(kkt)
    cnt.con_jac_cnt+=1
    cnt.con_jac_cnt==1 && (is_valid(jac) || throw(InvalidNumberException()))
    @trace(ipp.logger,"Constraint jacobian evaluation started.")
    return jac
end

function eval_lag_hess_wrapper!(ipp::InteriorPointSolver, kkt::AbstractKKTSystem, x::Vector{Float64},l::Vector{Float64};is_resto=false)
    nlp = ipp.nlp
    cnt = ipp.cnt
    @trace(ipp.logger,"Evaluating Lagrangian Hessian.")
    dual(ipp._w1) .= l.*ipp.con_scale
    hess = get_hessian(kkt)
    cnt.eval_function_time += @elapsed hess_coord!(
        nlp, view(x,1:get_nvar(nlp)), dual(ipp._w1), view(hess, 1:get_nnzh(nlp));
        obj_weight = (get_minimize(nlp) ? 1. : -1.) * (is_resto ? 0.0 : ipp.obj_scale[]))
    compress_hessian!(kkt)
    cnt.lag_hess_cnt+=1
    cnt.lag_hess_cnt==1 && (is_valid(hess) || throw(InvalidNumberException()))
    return hess
end

function eval_jac_wrapper!(ipp::InteriorPointSolver, kkt::AbstractDenseKKTSystem, x::Vector{Float64})
    nlp = ipp.nlp
    cnt = ipp.cnt
    ns = length(ipp.ind_ineq)
    @trace(ipp.logger, "Evaluating constraint Jacobian.")
    jac = get_jacobian(kkt)
    cnt.eval_function_time += @elapsed jac_dense!(nlp,view(x,1:get_nvar(nlp)),jac)
    compress_jacobian!(kkt)
    cnt.con_jac_cnt+=1
    cnt.con_jac_cnt==1 && (is_valid(jac) || throw(InvalidNumberException()))
    @trace(ipp.logger,"Constraint jacobian evaluation started.")
    return jac
end

function eval_lag_hess_wrapper!(ipp::InteriorPointSolver, kkt::AbstractDenseKKTSystem, x::Vector{Float64},l::Vector{Float64};is_resto=false)
    nlp = ipp.nlp
    cnt = ipp.cnt
    @trace(ipp.logger,"Evaluating Lagrangian Hessian.")
    dual(ipp._w1) .= l.*ipp.con_scale
    hess = get_hessian(kkt)
    cnt.eval_function_time += @elapsed hess_dense!(
        nlp, view(x,1:get_nvar(nlp)), dual(ipp._w1), hess;
        obj_weight = (get_minimize(nlp) ? 1. : -1.) * (is_resto ? 0.0 : ipp.obj_scale[]))
    compress_hessian!(kkt)
    cnt.lag_hess_cnt+=1
    cnt.lag_hess_cnt==1 && (is_valid(hess) || throw(InvalidNumberException()))
    return hess
end


module Vars

jets = mapreduce(vcat, 0:5) do i
    ["Jet_Px[$i]", "Jet_Py[$i]", "Jet_Pz[$i]", "Jet_E[$i]"]
end
tight_lepton = "TightLepton_" .* ["Px", "Py", "Pz", "E"] .* "[0]"
jet_csv = string.("Jet_CSV[", 0:5, "]")

scalar_features = [
    "Evt_CSV_avg_tagged",
    "Evt_CSV_avg",
    "Evt_M_minDrLepTag",
    "Evt_Pt_JetsAverage",
    "Evt_Pt_minDrTaggedJets",
    "Evt_Deta_TaggedJetsAverage",
    "Evt_Deta_JetsAverage",
    #"N_Jets",
    "Reco_JABDT_ttbar_Jet_CSV_whaddau2",
    "Reco_ttbar_toplep_m",
    "Reco_tHq_bestJABDToutput",
    "Reco_JABDT_tHq_abs_ljet_eta",
    "Reco_JABDT_tHq_Jet_CSV_hdau1",
    "Reco_JABDT_tHW_Jet_CSV_btop",
    #"memDBp",
]

all_features = [jets; tight_lepton; jet_csv; scalar_features]

weights = [
    "Weight_XS",
    #"Weight_btagSF",
    "Weight_CSV",
    "Weight_GEN_nom",
    "lumiWeight",
]

classes = [:ttH, :ttbb, :ttcc, :ttlf]

names_lola_out = [
    mapreduce(vcat, 0:5) do i
        ["Jet_m²[$i]"; "Jet_p_T[$i]"; "Jet_weighted_E[$i]"; string.("Jet_weighted_d", 1:4, "[$i]")]
    end
    ["Lepton_m²"; "Lepton_p_T"; "Lepton_weighted_E"; string.("Lepton_weighted_d", 1:4)]
]

tier1 = [
    "Evt_CSV_avg_tagged",
    "Evt_CSV_avg",
    "Evt_M_minDrLepTag",
    "Evt_Pt_JetsAverage",
    "Evt_Pt_minDrTaggedJets",
    "Evt_Deta_TaggedJetsAverage",
]
tier2 = [
    "Evt_Deta_JetsAverage",
]
tier3 = [
    "Reco_JABDT_ttbar_Jet_CSV_whaddau2",
    "Reco_ttbar_toplep_m",
    "Reco_tHq_bestJABDToutput",
    "Reco_JABDT_tHq_abs_ljet_eta",
    "Reco_JABDT_tHq_Jet_CSV_hdau1",
    "Reco_JABDT_tHW_Jet_CSV_btop",
]

end


function runModels( root::String, modelsDict::Dict, dfd::DataFrame=NA)
   res = OrderedDict()
   q = Symbol[]
   cnt=0
   for (k,v) in OrderedDict(m=>modelsDict[m][:raneff] for m in [:ipen,:idolocc,:iocc])
        for r in v push!(q,k); push!(q,r); cnt=cnt+1 end
   end
   x=reshape(q, (2,cnt))
   ml = permutedims(x, [2, 1])
   m=:empty
   for i in 1:length(ml[:,1])
       if m != ml[i,:][1]
           gr=Symbol("glm_"*string(ml[i,:][1]))
           isna(dfd) ? ( @swarm gr runGlm(root, ml[i,:][1]) ) : ( res[gr] = runGlm(dfd,modelsDict[ml[i,:][1]]) )
           m=ml[i,:][1]
       end
       r=Symbol("glmm_"*string(ml[i,:][1])*"_"*string(ml[i,:][2]))
       isna(dfd) ? (@swarm r runGlmm(root, ml[i,:][1], ml[i,:][2]) ) : (res[r] = runGlmm(dfd,modelsDict[m], [ml[i,:][2]], true) )
       covg=Symbol("covglm_"*string(ml[i,:][1])*"_"*string(ml[i,:][2])   )
       isna(dfd) ? ( @swarm covg runGlm(root, ml[i,:][1], ml[i,:][2]) ) : ( res[covg] = runGlm(dfd,modelsDict[m], ml[i,:][2])  )
   end
   if isna(dfd)
       sleep(20)
       while !iscompletew() println("No!"); sleep(5); end
       println("DONE")
   end
   return res
end

function runModels(root::String, modelsDict::Dict, )
    dfd = read_dfd(root) #dfd = readtable(mod_fname,header=true);
    poolit!(dfd,modelsDict[:factors])
    res = OrderedDict()

    q = Symbol[]
    cnt=0
    for (k,v) in OrderedDict(m=>modelsDict[m][:raneff] for m in [:ipen,:idolocc,:iocc])
         for r in v push!(q,k); push!(q,r); cnt=cnt+1 end
    end
    x=reshape(q, (2,cnt))
    ml = permutedims(x, [2, 1])
    m=:empty
    for i in 1:length(ml[:,1])
        if m != ml[i,:][1]
            gr=Symbol("glm_"*string(ml[i,:][1]))
            res[gr] = runGlm(dfd,modelsDict[ml[i,:][1]]) #res[gr] = runGlm(mod_fname, jmod_fname, ml[i,:][1])
            m=ml[i,:][1]
        end
        r=Symbol("glmm_"*string(ml[i,:][1])*"_"*string(ml[i,:][2]))
        res[r] = runGlmm(dfd,modelsDict[m], [ml[i,:][2]], true)   #res[r] = runGlmm(mod_fname, jmod_fname, ml[i,:][1], ml[i,:][2])
        covg=Symbol("covglm_"*string(ml[i,:][1])*"_"*string(ml[i,:][2]))
        res[covg] = runGlm(dfd,modelsDict[m], ml[i,:][2])   #res[covg] = runGlm(mod_fname, jmod_fname, ml[i,:][1], ml[i,:][2])
    end
    return res
end

function genDHHMeans(dfx::DataFrame)
    println("TOTAL DOLHH")
    o=dfx[(dfx[:modelType].=="GLM")&(dfx[:model].=="occ")&(dfx[:parameter].=="group"),:adj_mean_score0][1]
    y=dfx[(dfx[:modelType].=="GLM")&(dfx[:model].=="dolocc")&(dfx[:parameter].=="group"),:adj_mean_score0][1]
    p=dfx[(dfx[:modelType].=="GLM")&(dfx[:model].=="pen")&(dfx[:parameter].=="group"),:adj_mean_score0][1]
    adj_mean_score0=o*y*p
    println(".... 0 : $adj_mean_score0 ($o, $y,$p) ...")
    o=dfx[(dfx[:modelType].=="GLM")&(dfx[:model].=="occ")&(dfx[:parameter].=="group"),:adj_mean_score1][1]
    y=dfx[(dfx[:modelType].=="GLM")&(dfx[:model].=="dolocc")&(dfx[:parameter].=="group"),:adj_mean_score1][1]
    p=dfx[(dfx[:modelType].=="GLM")&(dfx[:model].=="pen")&(dfx[:parameter].=="group"),:adj_mean_score1][1]
    adj_mean_score1=o*y*p
    println(".... 1 : $adj_mean_score1 ($o, $y,$p) ...")
    unadj_avg_cntrl_hh_pre = mean(dfd[ (dfd[:group] .== 0), :prd_1_net_pr_pre] )
    unadj_avg_expsd_hh_pre = mean(dfd[ (dfd[:group] .== 1), :prd_1_net_pr_pre] )
    unadj_avg_cntrl_hh_pst = mean(dfd[ (dfd[:group] .== 0), :prd_1_net_pr_pos] )
    unadj_avg_expsd_hh_pst = mean(dfd[ (dfd[:group] .== 1), :prd_1_net_pr_pos] )
    pre=["group",0.0,0.0,0.0,0.0,"dolhh","ranef","GLM"]
    vars=[0.0,0.0,adj_mean_score0,adj_mean_score1,unadj_avg_expsd_hh_pre,unadj_avg_expsd_hh_pst,unadj_avg_cntrl_hh_pre,unadj_avg_cntrl_hh_pst]
    cis = zeros(10)  # cis and pvals [0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0]
    pvals=[0.0,0.0]
    cnts=convert(Array{Int64},collect(values(getCnts(dfd))))
    push!(dfx, filler(dfx,vcat(pre,vars,cis,cnts)))
    for ranef in unique(dfx[(dfx[:modelType].=="GLMM"),:ranef])
        for level in unique(dfx[(dfx[:modelType].=="GLMM")&(dfx[:ranef].==ranef),:parameter])
#            println(ranef," ~~ ",level)
            o=dfx[(dfx[:modelType].=="GLMM")&(dfx[:model].=="occ")&(dfx[:ranef].==ranef)&(dfx[:parameter].==level),:adj_mean_score0][1]
            y=dfx[(dfx[:modelType].=="GLMM")&(dfx[:model].=="dolocc")&(dfx[:ranef].==ranef)&(dfx[:parameter].==level),:adj_mean_score0][1]
            p=dfx[(dfx[:modelType].=="GLMM")&(dfx[:model].=="pen")&(dfx[:ranef].==ranef)&(dfx[:parameter].==level),:adj_mean_score0][1]
            adj_mean_score0=o*y*p
            println(".... 1 : $adj_mean_score1 ($o, $y,$p) ...")
            o=dfx[(dfx[:modelType].=="GLMM")&(dfx[:model].=="occ")&(dfx[:ranef].==ranef)&(dfx[:parameter].==level),:adj_mean_score1][1]
            y=dfx[(dfx[:modelType].=="GLMM")&(dfx[:model].=="dolocc")&(dfx[:ranef].==ranef)&(dfx[:parameter].==level),:adj_mean_score1][1]
            p=dfx[(dfx[:modelType].=="GLMM")&(dfx[:model].=="pen")&(dfx[:ranef].==ranef)&(dfx[:parameter].==level),:adj_mean_score1][1]
            adj_mean_score1=o*y*p
            println(".... 0 : ",adj_mean_score0," ... 1 :",adj_mean_score1)
            unadj_avg_expsd_hh_pre = mean(dfd[ (dfd[:group].==1) & (dfd[Symbol(ranef)].==level), :prd_1_net_pr_pre] )
            unadj_avg_expsd_hh_pst = mean(dfd[ (dfd[:group].==1) & (dfd[Symbol(ranef)].==level), :prd_1_net_pr_pos] )
            unadj_avg_cntrl_hh_pre = mean(dfd[ (dfd[:group].==0) & (dfd[Symbol(ranef)].==level), :prd_1_net_pr_pre] )
            unadj_avg_cntrl_hh_pst = mean(dfd[ (dfd[:group].==0) & (dfd[Symbol(ranef)].==level), :prd_1_net_pr_pos] )
            exposed=true # temp hack - all exposed
            if exposed
                unadj_avg_cntrl_hh_pre = dfx[(dfx[:modelType].=="GLM")&(dfx[:model].=="dolhh") ,:unadj_avg_cntrl_hh_pre][1]
                unadj_avg_cntrl_hh_pst = dfx[(dfx[:modelType].=="GLM")&(dfx[:model].=="dolhh") ,:unadj_avg_cntrl_hh_pst][1]
            end
            pre=[level,0.0,0.0,0.0,0.0,"dolhh",ranef,"GLMM"]
            vars=[0.0,0.0,adj_mean_score0,adj_mean_score1,unadj_avg_expsd_hh_pre,unadj_avg_expsd_hh_pst,unadj_avg_cntrl_hh_pre,unadj_avg_cntrl_hh_pst ]
            cis=zeros(10) #[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0]
            cnts=convert(Array{Int64},collect(values(getCnts(dfd,ranef,level))))
            push!(dfx, filler(dfx,vcat(pre,vars,cis,cnts)))
        end
    end
end
genDHHMeans(dfx)

ZDict = Dict("onetail_80_pct_intrvl" => 0.84 ,"onetail_90_pct_intrvl" => 1.28, "twotail_80_pct_intrvl" => 1.28, "twotail_90_pct_intrvl" => 1.65)

v_ttl=2

function CIs_O_LB(iDict::OrderedDict, zscore::Float64, iAccuracy::Float64=0.000000001)
    M = get(iDict, :M, NA)
    Mt = get(iDict, :Mt, NA)
    Mc = get(iDict, :Mc, NA)
    N = get(iDict, :N, NA)
    Nt = get(iDict, :Mt, NA)
    Nc = get(iDict, :Nc, NA)
    B1 = get(iDict, :B1, NA)
    B2 = get(iDict, :B2, NA)
    B3 = get(iDict, :B3, NA)
    SE1 = get(iDict, :SE1, NA)
    SE2 = get(iDict, :SE2, NA)
    SE3 = get(iDict, :SE3, NA)
    o_SE0 = get(iDict, :o_SE0, 0)
    y_SE0 = get(iDict, :y_SE0, 0)
    p_SE0 = get(iDict, :p_SE0, 0)
SEsq=sqrt(SE1^2+SE2^2+SE3^2+o_SE0^2+y_SE0^2+p_SE0^2)
    o_B0 = get(iDict, :o_B0, 0)
    y_B0 = get(iDict, :y_B0, 0)
    p_B0 = get(iDict, :p_B0, 0)

    o_mean_score0 = get(iDict, :o_mean_score0, NA)
    o_mean_score1 = get(iDict, :o_mean_score1, NA)
    y_mean_score0 = get(iDict, :y_mean_score0, NA)
    y_mean_score1 = get(iDict, :y_mean_score1, NA)
    p_mean_score0 = get(iDict, :p_mean_score0, NA)
    p_mean_score1 =get(iDict, :p_mean_score1, NA)
Bsum=(B1+B2+B3)-(o_B0+y_B0+p_B0)

    ztot = Bsum-(zscore*SEsq)
    ######CONFIDENCE INTERVAL - LB ########
    m=nothing
    m = Model(solver=NLoptSolver(algorithm=:LD_MMA, maxtime=v_ttl))
    @variable(m, Bocc <= (B1-o_B0))
    @variable(m, Bdolocc <= (B2-y_B0))
    @variable(m, Bpen <= (B3-p_B0))
    @NLobjective(m, Min, ((((p_mean_score1*(Nt/N))+(p_mean_score0*exp(Bpen)*(Nc/N)))*
                                  ((o_mean_score1*(Mt/M))+(o_mean_score0*exp(Bocc)*(Mc/M)))*
                                  ((y_mean_score1*(Mt/M))+(y_mean_score0*exp(Bdolocc)*(Mc/M)))
                                  )
                                  -(((p_mean_score1*(Nt/N)*exp(-Bpen))+(p_mean_score0*(Nc/N)))*
                                    ((o_mean_score1*(Mt/M)*exp(-Bocc))+(o_mean_score0*(Mc/M)))*
                                    ((y_mean_score1*(Mt/M)*exp(-Bdolocc))+(y_mean_score0*(Mc/M)))
                                    )
                                  )
                               )
    @constraint(m, (0.000000000<= (((Bocc+Bpen+Bdolocc)-ztot))<= iAccuracy))
    status = solve(m)
    mval=getobjectivevalue(m)
    println(mval)
    mval_out=mval/(o_mean_score0*y_mean_score0*p_mean_score0)
    return status == :Optimal ? mval_out : -Inf
end
CIs_O_LB(od, 0.84, 0.000000001)

onetail_80 FAILED : DataFrameRow (row 112)
modelType                 GLMM
model                     dolhh
ranef                     ad_type
parameter                 Desktop
adj_dod_effct             10.02936692284249
*****onetail_80_pct_intrvl_lb  10.955272420022512
onetail_80_pct_intrvl_ub  20.010743178509298

LB onetail_80 (ad_type~Desktop):=  - 0.000000001, DataStructures.
od = OrderedDict(:M=>108092,:Mt=>10810,:Mc=>106004,:N=>594930,:Nc=>584120,:B1=>-0.000998217,:B2=>0.0583124,:B3=>0.066694,:SE1=>0.0201122,:SE2=>0.0193296,:SE3=>0.0301203,:o_SE0=>0,:y_SE0=>0,:p_SE0=>0,:o_B0=>0,:y_B0=>0,:p_B0=>0,:o_mean_score0=>1.24344,:o_mean_score1=>1.2422,:y_mean_score0=>3.3212,:y_mean_score1=>3.52063,:p_mean_score0=>0.181431,:p_mean_score1=>0.193944,:metakey=>"ad_type~Desktop",:Bsum=>0.124008,:onetail_pval=>0.998738,:twotail_pval=>0.997476,:twotail_90_pct_intrvl_lb=>0.0678382,:twotail_90_pct_intrvl_ub=>0.245724,:twotail_80_pct_intrvl_lb=>0.0867394,:twotail_80_pct_intrvl_ub=>0.224721)
0.08208352266101582
            : Confidence Interval : 0.10955272420022512

mDict=collectModels(dfx,"GLMM","ad_type","Desktop")
calcPValue_Opt(mDict)
dfx[:adj_dod_effct] = ((dfx[:adj_mean_score1] .- dfx[:adj_mean_score0]) ./ dfx[:adj_mean_score0] ) *100

dfx[(dfx[:modelType].=="GLMM")&(dfx[:ranef].=="ad_type")&(dfx[:parameter].=="Desktop"),:]

            adjctl = md[:o_mean_score0] * md[:y_mean_score0] * md[:p_mean_score0]
            adjexp = md[:o_mean_score1] * md[:y_mean_score1] * md[:p_mean_score1]
            println(k," ~~ ",adjctl,"~",adjexp) #,md)
            sdf[sdf[:key].==k,:adj_mean_cntrl_grp] = adjctl
            sdf[sdf[:key].==k,:adj_mean_expsd_grp] = adjexp
            sdf[sdf[:key].==k,:adj_dod_effct] = ((adjexp - adjctl) ./ adjctl ) * 100

function collectModels(dfx::DataFrame,modelType::String,ranef::String="",level::String="")
    mDict = OrderedDict()
    if modelType=="GLM"
        o=dfx[(dfx[:modelType].=="GLM")&(dfx[:model].=="occ")&(dfx[:parameter].=="group"),:]
        y=dfx[(dfx[:modelType].=="GLM")&(dfx[:model].=="dolocc")&(dfx[:parameter].=="group"),:]
        p=dfx[(dfx[:modelType].=="GLM")&(dfx[:model].=="pen")&(dfx[:parameter].=="group"),:]
    else
        o=dfx[(dfx[:modelType].==modelType)&(dfx[:model].=="occ")&(dfx[:ranef].==ranef)&(dfx[:parameter].==level),:]
        y=dfx[(dfx[:modelType].==modelType)&(dfx[:model].=="dolocc")&(dfx[:ranef].==ranef)&(dfx[:parameter].==level),:]
        p=dfx[(dfx[:modelType].==modelType)&(dfx[:model].=="pen")&(dfx[:ranef].==ranef)&(dfx[:parameter].==level),:]
    end
    mDict[:M]=p[:M][1]
    mDict[:Mt]=p[:Mt][1]
    mDict[:Mc]=p[:Mc][1]
    mDict[:N]=p[:N][1]
    mDict[:Mt]=p[:Nt][1]
    mDict[:Nc]=p[:Nc][1]
    if modelType=="GLM"
        mDict[:B1]=o[:coef][1]
        mDict[:B2]=y[:coef][1]
        mDict[:B3]=p[:coef][1] #fx[:adj_dod_effct] = ((dfx[:adj_mean_score1] .- dfx[:adj_mean_score0]) ./ dfx[:adj_mean_score0] ) *100
        mDict[:SE1]=o[:stderr][1]
        mDict[:SE2]=y[:stderr][1]
        mDict[:SE3]=p[:stderr][1] #fx[:adj_dod_effct] = ((dfx[:adj_mean_score1] .- dfx[:adj_mean_score0]) ./ dfx[:adj_mean_score0] ) *100S
    else
        mDict[:B1]=o[:adj_coef][1]
        mDict[:B2]=y[:adj_coef][1]
        mDict[:B3]=p[:adj_coef][1] #fx[:adj_dod_effct] = ((dfx[:adj_mean_score1] .- dfx[:adj_mean_score0]) ./ dfx[:adj_mean_score0] ) *100
        mDict[:SE1]=o[:adj_stderr][1]
        mDict[:SE2]=y[:adj_stderr][1]
        mDict[:SE3]=p[:adj_stderr][1] #fx[:adj_dod_effct] = ((dfx[:adj_mean_score1] .- dfx[:adj_mean_score0]) ./ dfx[:adj_mean_score0] ) *100S
    end
    mDict[:o_SE0]=0
    mDict[:y_SE0]=0
    mDict[:p_SE0]=0
    mDict[:o_B0]=0
    mDict[:y_B0]=0
    mDict[:p_B0]=0
    mDict[:o_mean_score0]=o[:adj_mean_score0][1]
    mDict[:o_mean_score1]=o[:adj_mean_score1][1]
    mDict[:y_mean_score0]=y[:adj_mean_score0][1]
    mDict[:y_mean_score1]=y[:adj_mean_score1][1]
    mDict[:p_mean_score0]=p[:adj_mean_score0][1]
    mDict[:p_mean_score1]=p[:adj_mean_score1][1]
    return mDict
end

function ConfidenceIntervals(dfx::DataFrame)
   # mDict=collectModels(dfx,"GLM")
#    println("Running CI for : Total")
#    mDict[:metakey] = "Total Campaign"
#    calcPValue_Opt(mDict)
#    CIs_O(mDict)
#    dfx[(dfx[:model].=="dolhh")&(dfx[:modelType].=="GLM")&(dfx[:parameter].=="group"),:onetail_pval] = mDict[:onetail_pval]*100
#    dfx[(dfx[:model].=="dolhh")&(dfx[:modelType].=="GLM")&(dfx[:parameter].=="group"),:twotail_pval] = mDict[:twotail_pval]*100
#    for k in keys(ZDict)
#        dfx[(dfx[:model].=="dolhh")&(dfx[:modelType].=="GLM")&(dfx[:parameter].=="group"),Symbol(k*"_lb")] = mDict[Symbol(k*"_lb")]*100
#        dfx[(dfx[:model].=="dolhh")&(dfx[:modelType].=="GLM")&(dfx[:parameter].=="group"),Symbol(k*"_ub")] = mDict[Symbol(k*"_ub")]*100
#    end
    for ranef in ["ad_type"] #unique(dfx[(dfx[:modelType].=="GLMM")&(dfx[:ranef].=="ad_type")&(dfx[:parameter].=="Desktop"),:ranef])
        for level in ["Desktop"] #unique(dfx[(dfx[:modelType].=="GLMM")&(dfx[:ranef].==ranef),:parameter])
            mDict=collectModels(dfx,"GLMM",ranef,level)
            println("Running CI for : $ranef : $level")
            mDict[:metakey] = ranef*"~"*level
            calcPValue_Opt(mDict)
            CIs_O(mDict)
            dfx[(dfx[:model].=="dolhh")&(dfx[:ranef].==ranef)&(dfx[:parameter].==level),:onetail_pval] = mDict[:onetail_pval]*100
            dfx[(dfx[:model].=="dolhh")&(dfx[:ranef].==ranef)&(dfx[:parameter].==level),:twotail_pval] = mDict[:twotail_pval]*100
            for k in keys(ZDict)
                dfx[(dfx[:model].=="dolhh")&(dfx[:ranef].==ranef)&(dfx[:parameter].==level),Symbol(k*"_lb")] = mDict[Symbol(k*"_lb")]*100
                dfx[(dfx[:model].=="dolhh")&(dfx[:ranef].==ranef)&(dfx[:parameter].==level),Symbol(k*"_ub")] = mDict[Symbol(k*"_ub")]*100
            end
        end
    end
end
ConfidenceIntervals(dfx)

# ---------------------------------------------------------

#Generate GLM Formula
#function genFmula(iv::Array{Symbol},isLastOffset::Bool=false)  # Y=1st, if isLastOffset:: offset last
#    isLastOffset ?  genFmula(iv[1],iv,last(iv)) : genFmula(iv[1],iv)
#end
#function genFmula(y::Symbol, iv::Array{Symbol},offset::Symbol=:Missing)
#    xvars=setdiff(iv,[y,offset])
#    eval(parse( string(y)*" ~ 1"* reduce(*, [ " + "*  string(c) for c in xvars ] )  ))
#end
#function genFmula(y::Symbol, iv::Array{Symbol})
#    xvars=setdiff(iv,[y])
#    eval(parse( string(y)*" ~ 1"* reduce(*, [ " + "*  string(c) for c in xvars ] )  ))
#end
function genF(y::Symbol, iv::Array{Symbol},ranef::Array{Symbol}=Symbol[])
        vars=setdiff(iv,vcat([y],ranef))
        f = string(y)*" ~ 1"
        f = length(vars) > 0 ?  f * reduce(*, [ " + "*  string(c) for c in unique(vars) ] ) : f
        f = length(ranef) > 0 ?  f * reduce(*, [ " + "*  "(1 | "*string(c)*")" for c in unique(ranef) ] )  : f
        eval(parse(f))
end

# remove single level vars
function FS_singleLevel(dfd::DataFrame,vars::Array{Symbol}=Symbol[])
    if length(vars)==0  vars=names(dfd)  end
    return [c for c in filter(x -> length(unique( dfd[x] ))==1, vars)]
end

function raneffect(m::MixedModels.GeneralizedLinearMixedModel)
    DataFrame(parameter=levels(m.LMM.trms[1].f), coef=vec(ranef(m, named=true)'[1]), stderr = sqrt.(vec(condVar(m)[1])) )  # stderrX =vec(condVar(m)[1])
end

isnumeric{T<:Number}(::AbstractArray{T}) = true
isnumeric(::Any) = false
isfloat{T<:Float64}(::AbstractArray{T}) = true
isfloat(::Any) = false
isint{T<:Int64}(::AbstractArray{T}) = true
isint(::Any) = false
isString{T<:String}(::AbstractArray{T}) = true
isString(::Any) = false
isBool{T<:Bool}(::AbstractArray{T}) = true
isBool(::Any) = false
isSymbol{T<:Symbol}(::AbstractArray{T}) = true
isSymbol(::Any) = false

Base.lowercase(df::DataFrame) = names!(df, map(x->Symbol(lowercase(string(x))),names(df)))
Base.lowercase(da::DataArray{String}) = [lowercase(x) for x in da]

function filler(dfx::DataFrame, rowIn::Array{Any})
    row=filler(dfx)
    for i in 1:length(rowIn)
        row[i] = rowIn[i]
    end
    return row
end

function filler(dfx::DataFrame, cols::Vector{Symbol}=Symbol[])
    a = Any[]
    if length(cols)==0 cols=names(dfx) end
    for (n, v) in eachcol(dfx)
        if n ∈ cols && isfloat(v)
            push!(a, 0.0)
        elseif n ∈ cols && isint(v)
            push!(a, 0)
        elseif n ∈ cols && isString(v)
            push!(a, "")
        elseif n ∈ cols && isBool(v)
            push!(a, false)
        end
    end
    return a
end

function cov2cor(m)
    d = Diagonal([inv(sqrt(m[i, i])) for i in 1:size(m, 2)])
    A_mul_B!(d, m * d)
end

function vif2(m)
    v = vcov(m)
    assign = m.mm.assign
    nms = coefnames(m.mf)
    if (ipos = findfirst(nms, "(Intercept)")) > 0
        inds = deleteat!(collect(eachindex(nms)), ipos)
        v = view(v, inds, inds)
        assign = view(assign, inds)
    else
        warning("No intercept: vifs may not be sensible.")
    end
    terms = convert(Vector{Symbol}, m.mf.terms.terms)
    n_terms = length(terms)
    if n_terms < 2
        error("model contains fewer than 2 terms")
    end
    R = cov2cor(v)
    detR = det(Symmetric(R))
    result = NamedArray(Array(eltype(R), (length(terms), 3)))
    setnames!(result, string.(terms), 1)
    setnames!(result, ["GVIF", "Df", "GVIF^(1/(2*Df))"], 2)
    inds = collect(eachindex(terms))
    ainds = collect(eachindex(assign))
    for i in inds
        subs = find(x -> x == i, assign)
        notsubs = setdiff(ainds, subs)
        result[i, 1] = det(Symmetric(R[notsubs, notsubs])) / detR
        if (result[i, 2] = length(subs)) > 1
            result[i, 1] *= det(Symmetric(R[subs, subs]))
        end
    end
    if all(result[:, 2] .== 1)
        return result[:, 1:1]
    end
    result
end

function vifDF2(m::RegressionModel)
    sdf=coefDF(m,true)
    v=vif2(m)
    vdf=DataFrame(parameter=convert(Array{Symbol},names(v)[1]), vif=convert(Array{Float64}, v)[:,1]   )
    #vdf[:parameter] = convert(Array{Symbol}, vdf[:variable])
    join(sdf,vdf[[:parameter,:vif]], on = :parameter, kind=:outer)
end

""" Doug
    profileθ(m::GeneralizedLinearMixedModel)

Profile the deviance as a function of θ for models with a single, scalar r.e. term
"""
function profileθ{T}(m::GeneralizedLinearMixedModel{T})
    θ = getθ(m)
    isa(θ, Vector{T}) && length(θ) == 1 ||
        throw(ArgumentError("m must have a single scalar r.e. term"))
    th = zeros(T, 1)
    δ = zeros(T, 1)
    fill!(θ, 0)
    decreasing = false
    basedev = pirls!(setθ!(m, θ), true)
    for t in 0.001:0.001:2.0
        θ[1] = t
        dd = pirls!(setθ!(m, θ), true) - basedev
        sgn = sign(dd - δ[end])
        push!(δ, dd)
        push!(th, t)
        dd ≤ 4.0 || break
        decreasing && sgn > 0 && break
        decreasing = sgn < 0
    end
    th, δ
end

function gethostworkers()
    d = Dict()
    @sync @async for (idx, pid) in enumerate(workers())
        d[pid] = remotecall_fetch(getipaddr,pid)
    end
    iout=Dict()
    for ip in [string(ip) for ip in unique(values(d))]
        a=Int64[]
        for (key, value) in d
            if string(value) == ip
                push!(a,key)
            end
        end
        iout[ip] = sort(a)
    end
    l = get(iout,string(getipaddr()),[])
    iout[string(getipaddr())] = convert(Array{Int64},vcat([1],l))
    return iout
end

function gethostModelDistribution(raneff::Array{Symbol})
    function allocateCore(a::Array{Int64})
        proc = a[1]
        ao = length(setdiff(a,proc)) > 0 ?  setdiff(a,proc) : a
        return proc, ao
    end
    nodehosts = gethostworkers()
    hosts = [k for k in keys(nodehosts)]
    hostcnt = length(hosts)
    #raneff = [:creative, :publisher1, :placement]
    h=1
    m=:pen
    penprocs = Dict()
    for r in raneff
        proc, nodehosts[hosts[h]] = allocateCore(nodehosts[hosts[h]])
        penprocs[r] = Dict(:proc=>proc, :host=>hosts[h])
        println(r,"  :  ", hosts[h], "  ",proc)
        h = h == hostcnt ? h=1 : h+1
    end

    static_nodehosts = deepcopy(nodehosts)

    println("Pen : ",penprocs )
    println(nodehosts)
end

function checksingularity(form::Formula, data::DataFrame, tolerance = 1.e-8)
    mf = ModelFrame(form, data)
    mm = ModelMatrix(mf)
    qrf = qrfact!(mm.m, Val{true})
    vals = abs.(diag(qrf[:R]))
    firstbad = findfirst(x -> x < min(tolerance, 0.5) * vals[1], vals)
    if firstbad == 0
        return Symbol[]
    end
    mf.terms.terms[view(mm.assign[qrf[:p]], firstbad:length(vals))]
end

function coefDF(m::RegressionModel, paramsAsSymbols::Bool=false)
    vout = DataFrame( #vars=vcat([:intercept],m.mf.terms.terms)  #g.model.mm.assign # m.mm.assign
               #vars=vcat([:intercept],[m.mf.terms.terms[x] for x in m.mm.assign[2:end]])
               parameter=DataFrames.coefnames(m.mf)
             , coef=GLM.coef(m)
             , stderr=stderr(m)
             , zval=GLM.coef(m)./stderr(m)
             , pval= ccdf(FDist(1, dof_residual(m)), abs2(GLM.coef(m)./stderr(m))  )
             )
    if paramsAsSymbols == true
        vout[:parameter] = map(x->Symbol(replace(replace(x,"group: 1","group"),"(Intercept)","Intercept")), vout[:parameter])
    end
    return vout
end

"""
Relevel DF for GLM
"""
function relevel{T,I}(v::PooledDataVector{T,I}, lev::T)
    pool = v.pool
    i = convert(I, findfirst(pool, lev))
    i ≠ 0 || throw(ArgumentError("\"$lev\" is not in v.pool"))
    if i == 1
        return copy(v)
    end
    orig = convert(Vector{I}, 1:length(pool))
    perm = append!([i], deleteat!(orig, i))
    PooledDataArray(DataArrays.RefArray(invperm(perm)[v.refs]), v.pool[perm])
end
#dfd[ranef] = relevel(dfd[ranef], "none")

#GLM REsiduals
function xResiduals(g::DataFrames.DataFrameRegressionModel)
    resp = g.model.rr
    #sign(resp.y - resp.mu) .* sqrt(resp.devresid)    # Original
    #sign(resp.y - resp.mu) .* sqrt(complex(resp.devresid))
    sign(resp.y - resp.mu) .* sqrt(((resp.devresid)+1)-1)
    #sign(resp.y - resp.mu) .* sqrt(abs(resp.devresid))
end

function unpool(dfd::DataFrame)
    v_factors = Symbol[]
    for c in names(dfd)
        if typeof(dfd[c]) in [ PooledDataArray{String,UInt8,1} ]
            println("         Converting : ", c )
            push!(v_factors,c)
            dfd[c] = Array(dfd[c])
        end
    end
    return v_factors
end

function poolit!(dfd::DataFrame, vars::Array{Symbol}=Symbol[])
    if length(vars) == 0   vars=names(dfd) end
    for (i, c) in enumerate(dfd.columns)
        if typeof(c) == DataVector{String}
            if dfd.colindex.names[i] in vars
                #println("[",i,"]  : ",dfd.colindex.names[i])
                #unique(dfd[i])
                dfd[i] = pool(dfd[i])
            end
        end
        #typeof(c) == DataVector{String} && println("[",i,"]  : ",dfd.colindex.names[i])
    end
end

function getColswithType(dtype::String, dfd::DataFrame, vars::Array{Symbol}=Symbol[])
    clst=Symbol[]
    typLst = Dict("num" => [DataVector{Float64},DataVector{Int64}], "str" => [DataVector{String}] )
    if length(vars) == 0   vars=names(dfd) end
    if dtype in ["num","str"]
        for (i, c) in enumerate(dfd.columns)
            if dfd.colindex.names[i] in vars
                    if typeof(c) in typLst[dtype]
                #if typeof(c) in [ DataVector{Float64}, DataVector{Int64} ] # = = DataVector{String}
                    #println("[",i,"]  : ",dfd.colindex.names[i])
                    push!(clst,dfd.colindex.names[i])
                end
             end
         end
    else
        println("ERROR: getColswithType : Invalid dtype!")
    end
    return clst
end

function corDFD(dfd::DataFrame, vars::Array{Symbol}=Symbol[])
    if length(vars) == 0
        vars=names(dfd)
    #else
    #    vars=convert(Array{Symbol},vars)
    end
    cols=getColswithType("num", dfd, convert(Array{Symbol},vars) )
    carr= cor(Array(dfd[cols]))
    tcarr = triu(carr,1)
    d = DataFrame(tcarr)
    names!(d,cols)
    d[:vars] = cols
    return stack(d,cols)
end
#function corDFD(dfd::DataFrame, vars::Array{Any}=[])
#    varsS=Symbol[]
#    if length(vars) > 0 varsS=convert(Array{Symbol},vars) end
#    corrDF(dfd,varsS)
#end

# ***************************************************************************************************************************
#export JULIA_NUM_THREADS=4
ENV["JULIA_PKGDIR"] = "/mapr/mapr04p/analytics0001/analytic_users/jpkg"
for i in ["10.106.128.212","10.106.128.213","10.106.128.214","10.106.128.215","10.106.128.216","10.106.128.217",
          "10.106.128.218","10.106.128.219","10.106.128.220","10.106.128.221"]
             if i != string(getipaddr()) println(i); addprocs([(i, 3)]) end
end #addprocs(1)
@everywhere insert!(Base.LOAD_CACHE_PATH, 1, "/mapr/mapr04p/analytics0001/analytic_users/jpkg/lib/v0.5")
@everywhere pop!(Base.LOAD_CACHE_PATH)
using DataStructures, DataArrays , DataFrames, StatsFuns, GLM , Distributions, MixedModels, StatsBase, JSON, StatLib, NamedArrays, JuMP, NLopt, NLsolve
@everywhere using DataStructures, DataArrays , DataFrames, StatsFuns, GLM , Distributions, MixedModels, StatsBase, JSON, StatLib, NamedArrays, JuMP, NLopt, NLsolve

root = !isdefined(:root) ? pwd() : root
# ************ CUSTOMIZE *********************
#root="/mnt/resource/analytics/models/Jenny-o"
#root="/mnt/resource/analytics/models/NatChoice"
#root="/mnt/resource/analytics/models/rev"
#root="/mnt/resource/analytics/models/ALL#10"
#root="/mnt/resource/analytics/models/ALL#11"
#root="/mnt/resource/analytics/models/CDW#6"
#root="/mnt/resource/analytics/models/HormelChili#8"
#root="/mnt/resource/analytics/models/Rev#8"
#root="/mnt/resource/analytics/models/Jenny-o"
#root="/mnt/resource/analytics/models/hunts"
#root="/mapr/mapr04p/analytics0001/analytic_users/jmdl/rev"
#root="/mapr/mapr04p/analytics0001/analytic_users/jmdl/NatChoice"
#root="/mapr/mapr04p/analytics0001/analytic_users/jmdl/ALL10"

# ************ END CUSTOMIZE *********************

#jmod_fname = root*"/dfd_model.json"
#mod_fname = root*"/dfd_model.csv"
modelsDict = read_modelsDict(root) #modelsDict = readModels(jmod_fname)

@everywhere function runGlm(root::String, modelname::Symbol, ranef::Symbol=:empty)
    println("Loding data : ")
    dfd = read_dfd(root) #dfd = readtable(mod_fname,header=true);
    modelsDict = read_modelsDict(root)#modelsDict = readModels(jmod_fname)
    poolit!(dfd,modelsDict[:factors])
    m=modelsDict[modelname]
    return runGlm(dfd, m, ranef)
end

@everywhere function runGlm(dfd::DataFrame,m::Dict, ranef::Symbol=:empty)
    if ranef==:empty
        f=genF(m[:y_var],m[:finalvars])
        cols = convert(Array{Symbol},vcat(m[:finalvars], [m[:y_var], m[:logvar]]))
        println("Ok - Running ",m[:modelName]," -  Glm on proc ",myid()," with : ",f)
    else
        f=genF(m[:y_var],setdiff(vcat(m[:finalvars],ranef),[:group]))
        dfd[ranef] = relevel(dfd[ranef], "none")
        cols = convert(Array{Symbol},vcat(m[:finalvars], [m[:y_var], m[:logvar]], ranef ))
        println("Ok - Running ",m[:modelName]," - ",ranef," Glm on proc ",myid()," with : ",f)
    end
    if m[:Buyer_Pos_P1_is1]
        return glm(f, dfd[(dfd[:buyer_pos_p1].==1),cols] , m[:dist], m[:lnk])
    else
        return return glm(f, dfd[cols] , m[:dist], m[:lnk])
    end
end

@everywhere function runGlmm(root::String, modelname::Symbol, ranef::Symbol=:empty, runθ::Bool=true)
    println("Loding data : Glmm : ",modelname," ~~~ ",ranef)
    dfd = read_dfd(root)   #dfd = readtable(mod_fname,header=true);
    modelsDict = read_modelsDict(root) #modelsDict = readModels(jmod_fname)
    poolit!(dfd,modelsDict[:factors])
    m=modelsDict[modelname]
    if ranef==:empty
        v_out=Dict()
        for r in m[:raneff]
            v_out[r] = runGlmm(dfd, m, [r], runθ)
        end
        return v_out
    else
        return runGlmm(dfd, m, [ranef], runθ)
    end
end

@everywhere function runGlmm(dfd::DataFrame,m::Dict, ranef::Array{Symbol}=[:empty], runθ::Bool=true)
    ranef = ranef==[:empty] ? m[:raneff]  : ranef
    for r in ranef
        dfd[r] = relevel(dfd[r], "none")
    end
    f=genF(m[:y_var],setdiff(m[:finalvars],[:group]),ranef)
    cols = convert(Array{Symbol},vcat(m[:finalvars], [m[:y_var], m[:logvar]], ranef ))
    println("Ok - Running ",m[:modelName]," - ",ranef," Glmm on proc ",myid()," with : ",f)
    if m[:Buyer_Pos_P1_is1]
        #gmm1 = fit!(glmm(f, dfd[(dfd[:buyer_pos_p1].==1),cols] , m[:dist], m[:lnk]), false, 0)   #gmm1 = fit!(glmm(f, dfd[(dfd[:buyer_pos_p1].==1),cols] , m[:dist], m[:lnk]), nAGQ=0)
        gmm1 = fit!(glmm(f, dfd[(dfd[:buyer_pos_p1].==1),cols] , m[:dist], m[:lnk]), verbose=false, fast=true)
        if runθ if sum(raneffect(gmm1)[:coef]) == 0 profileθ(gmm1) end  end
        return gmm1
    else
        #gmm1 = fit!(glmm(f, dfd[cols], m[:dist], m[:lnk]), false, 0)   #gmm1 = fit!(glmm(f, dfd[cols], m[:dist], m[:lnk]), nAGQ=0)
        gmm1 = fit!(glmm(f, dfd[cols] , m[:dist], m[:lnk]), verbose=false, fast=true)
        if runθ if sum(raneffect(gmm1)[:coef]) == 0 profileθ(gmm1) end end
        return gmm1
    end
end

"""
SWARMING - hold on
"""
const SHELF = OrderedDict()
freew() = setdiff(workers(),[v[:worker] for (k,v) in SHELF])
hasfreew() = length(freew()) > 0 ? true : false
nextFreew() = length(freew()) > 0 ? freew()[1]   : NA
#statusW() =  [ string(k)*"   status: "*string(v[:status]) for (k,v) in values(SHELF)]
#statusw() =  for (k,v) in SHELF println(string(k)*"   status: "*string(v[:status])*"   "*string(v[:worker]) ) end
notcompletew() = filter((k,v)-> v[:status]!=:complete ,SHELF)
iscompletew() = length(notcompletew()) == 0 ? true : false
keysw() = collect(keys(SHELF))
getw(k::Symbol) = SHELF[k][:results]
function takew(k::Symbol)
    vo = deepcopy(SHELF[k][:results])
    delete!(SHELF, k)
    return vo
end

type statusW function statusW()  return new() end  end
function Base.show(io::IO, statusw::statusW)  for (k,v) in SHELF println(string(k)*"   status: "*string(v[:status])*"   "*string(v[:worker]) ) end end
statusw=statusW()

function swarm(k::Symbol, thunk)
    #println("SWARMING  : ",k)
    #println("SWARM DUMP : ",typeof(thunk)==Expr)
    SHELF[k] = Dict{Symbol,Any}(:worker =>0,:expr => thunk, :status =>:ready, :channel => Channel(1))
    t = SHELF[k]
    while !hasfreew() sleep(15) end   # println("waiting for free worker");
    t[:worker] = nextFreew()
    wid = t[:worker]
    if isna(wid)
        return false
    else
        #remotecall_fetch(runGlmm, wid, mod_fname, jmod_fname, w[:model], w[:renef] )
        t[:status] = :running
        #splice!(expr.args, 1:1,[:remotecall_fetch,expr.args[1],wid])
        #println("Starting model")
        println("SWARMING  : ",k, " on ", wid)
        @async put!(t[:channel], fetch(Base.sync_add(remotecall(thunk, wid)))  )
        while isready(t[:channel])==false sleep(15) end  #println("NOT Ready : ",t[:worker] );
        t[:results] = take!(t[:channel])
        t[:status] = :complete
        t[:worker] = 0
        println("SWARM Task Completed : ",k)
        close(t[:channel])
        return true
    end
end

macro swarm(p, expr)
    expr = Base.localize_vars(esc(:(()->($expr))), false)
    ex = :( @async swarm($(esc(p)), $expr))
end

# ******************************************

function runModels( root::String, modelsDict::Dict)
   @swarm :glm_ipen runGlm(root, :ipen)
   sleep(25)
   @swarm :glm_iocc runGlm(root, :iocc)
   sleep(25)
   @swarm :glm_idolocc runGlm(root, :idolocc)
   q = Symbol[]
   cnt=0
   for (k,v) in OrderedDict(m=>modelsDict[m][:raneff] for m in [:ipen,:idolocc,:iocc])
        for r in v push!(q,k); push!(q,r); cnt=cnt+1 end
   end
   x=reshape(q, (2,cnt))
   ml = permutedims(x, [2, 1])
   m=:empty
   for i in 1:length(ml[:,1])
       r=Symbol("glmm_"*string(ml[i,:][1])*"_"*string(ml[i,:][2]))
       @swarm r runGlmm(root, ml[i,:][1], ml[i,:][2])
       covg=Symbol("covglm_"*string(ml[i,:][1])*"_"*string(ml[i,:][2])   )
       @swarm covg runGlm(root, ml[i,:][1], ml[i,:][2])
   end
   sleep(20)
   while !iscompletew() println("Not Complete yet!"); sleep(5); end
   println("DONE!DONE!DONE!")
end
runModels(root,modelsDict)

function consolidateResults(modelsDict::Dict,dx::OrderedDict=OrderedDict())
  sdf = DataFrame(parameter=String[], coef=Int64[], stderr=Int64[], zval=Int64[], pval=Int64[], model=String[], ranef=String[], modelType=String[])
    for m in [:iocc, :idolocc, :ipen]
        g = length(dx) > 0 ? dx[Symbol("glm_"*string(m))] : getw( Symbol("glm_"*string(m)) )
        #g = takew(  Symbol("glm_"*string(m))    )
        dfx = coefDF(g,false)
        dfx[:model] = replace(string(m),"i","")
        dfx[:ranef] = "none"
        dfx[:modelType] = "GLM"
        sdf = vcat(sdf,dfx)
        grp = sdf[(sdf[:modelType].=="GLM")&(sdf[:model].==replace(string(m),"i",""))&(sdf[:parameter].=="group"),:coef][1]
        err = sdf[(sdf[:modelType].=="GLM")&(sdf[:model].==replace(string(m),"i",""))&(sdf[:parameter].=="group"),:stderr][1]
        for r in  modelsDict[m][:raneff]
            k="_"*string(m)*"_"*string(r)
            g = length(dx) > 0 ?  dx[Symbol("glmm"*k)]  : getw(Symbol("glmm"*k))
            dfx = raneffect(g)
            dfx[:zval]=0.0
            dfx[:pval]=0.0
            dfx[:model] = replace(string(m),"i","")
            dfx[:ranef]=string(r)
            dfx[:modelType]="GLMM"
            sdf = vcat(sdf,dfx)

            g = length(dx) > 0 ?  dx[Symbol("covglm"*k)]    : getw(Symbol("covglm"*k))
            dfx = coefDF(g,false)
            dfx[:model] = replace(string(m),"i","")
                dfx[:ranef] = string(r)
            dfx[:modelType] = "RanCov"
            dfx = dfx[findin(dfx[:parameter],filter(x->contains(x,string(r)) ,Array(dfx[:parameter]))),:]
            #dfx[:coef]=dfx[:coef]-grp
            #dfx[:stderr]=sqrt(dfx[:stderr].^2+err^2)
            dfx[:parameter] = map(x-> replace(x,string(r)*": ",""),dfx[:parameter])
            sdf = vcat(sdf,dfx)
        end
    end
    return sdf
end
dfx = consolidateResults(modelsDict)
save_dfx(root,dfx)      #writetable(root*"/campaign.csv", dfx)

# ***************************************************************************************************************************
# ***************************************************************************************************************************
# ***************************************************************************************************************************
# ***************************************************************************************************************************
# ***************************************************************************************************************************

#mv(root*"/dfx.csv",root*"/dfx.csv_prev")
#writetable(root*"/dfx_ready.csv", dfx)
#root="/mapr/mapr04p/analytics0001/analytic_users/jmdl/rev"
#modelsDict = read_modelsDict(root)
#modelsDict[:occ]=modelsDict[:iocc]; delete!(modelsDict,:iocc)
#modelsDict[:dolocc]=modelsDict[:idolocc]; delete!(modelsDict,:idolocc)
#modelsDict[:pen]=modelsDict[:ipen]; delete!(modelsDict,:ipen)
##dfx = read_dfx(root)

#export JULIA_NUM_THREADS=4
ENV["JULIA_PKGDIR"] = "/mapr/mapr04p/analytics0001/analytic_users/jpkg"
for i in ["10.106.128.212","10.106.128.213","10.106.128.214","10.106.128.215","10.106.128.216","10.106.128.217",
          "10.106.128.218","10.106.128.219","10.106.128.220","10.106.128.221"]
             if i != string(getipaddr()) println(i); addprocs([(i, 3)]) end
end #addprocs(1)
@everywhere insert!(Base.LOAD_CACHE_PATH, 1, "/mapr/mapr04p/analytics0001/analytic_users/jpkg/lib/v0.5")
@everywhere pop!(Base.LOAD_CACHE_PATH)
using DataStructures, DataArrays , DataFrames, StatsFuns, GLM , Distributions, MixedModels, StatsBase, JSON, StatLib, NamedArrays, JuMP, NLopt, NLsolve
@everywhere using DataStructures, DataArrays , DataFrames, StatsFuns, GLM , Distributions, MixedModels, StatsBase, JSON, StatLib, NamedArrays, JuMP, NLopt, NLsolve

"""
SWARMING - hold on
"""
const SHELF = OrderedDict()
freew() = setdiff(workers(),[v[:worker] for (k,v) in SHELF])
hasfreew() = length(freew()) > 0 ? true : false
nextFreew() = length(freew()) > 0 ? freew()[1]   : NA
#statusW() =  [ string(k)*"   status: "*string(v[:status]) for (k,v) in values(SHELF)]
#statusw() =  for (k,v) in SHELF println(string(k)*"   status: "*string(v[:status])*"   "*string(v[:worker]) ) end
notcompletew() = filter((k,v)-> v[:status]!=:complete ,SHELF)
iscompletew() = length(notcompletew()) == 0 ? true : false
keysw() = collect(keys(SHELF))
getw(k::Symbol) = SHELF[k][:results]
function takew(k::Symbol)
    vo = deepcopy(SHELF[k][:results])
    delete!(SHELF, k)
    return vo
end
            # NOTE : create function rmW() # remove value from SHELF
            #                        clearW() #remove all values from SHELF
            #                        waitW()  # while !iscompletew() println("Not Complete yet!"); sleep(5); end

type statusW function statusW()  return new() end  end
function Base.show(io::IO, statusw::statusW)  for (k,v) in SHELF println(string(k)*"   status: "*string(v[:status])*"   "*string(v[:worker]) ) end end
statusw=statusW()

function swarm(k::Symbol, thunk)
    #println("SWARMING  : ",k)
    #println("SWARM DUMP : ",typeof(thunk)==Expr)
    SHELF[k] = Dict{Symbol,Any}(:worker =>0,:expr => thunk, :status =>:ready, :channel => Channel(1))
    t = SHELF[k]
    while !hasfreew() sleep(15) end   # println("waiting for free worker");
    t[:worker] = nextFreew()
    wid = t[:worker]
    if isna(wid)
        return false
    else
        #remotecall_fetch(runGlmm, wid, mod_fname, jmod_fname, w[:model], w[:renef] )
        t[:status] = :running
        #splice!(expr.args, 1:1,[:remotecall_fetch,expr.args[1],wid])
        #println("Starting model")
        println("SWARMING  : ",k)
        @async put!(t[:channel], fetch(Base.sync_add(remotecall(thunk, wid)))  )
        while isready(t[:channel])==false sleep(15) end  #println("NOT Ready : ",t[:worker] );
        t[:results] = take!(t[:channel])
        t[:status] = :complete
        t[:worker] = 0
        println("SWARM Task Completed : ",k)
        close(t[:channel])
        return true
    end
end
macro swarm(p, expr)
    expr = Base.localize_vars(esc(:(()->($expr))), false)
    ex = :( @async swarm($(esc(p)), $expr))
end

@everywhere function MP_collectModels(dfx::DataFrame, modelType::String,ranef::String="",level::String="")
    mDict = OrderedDict()
    if modelType=="GLM"
        o=dfx[(dfx[:modelType].=="GLM")&(dfx[:model].=="occ")&(dfx[:parameter].=="group"),:]
        y=dfx[(dfx[:modelType].=="GLM")&(dfx[:model].=="dolocc")&(dfx[:parameter].=="group"),:]
        p=dfx[(dfx[:modelType].=="GLM")&(dfx[:model].=="pen")&(dfx[:parameter].=="group"),:]
    else
        o=dfx[(dfx[:modelType].==modelType)&(dfx[:model].=="occ")&(dfx[:ranef].==ranef)&(dfx[:parameter].==level),:]
        y=dfx[(dfx[:modelType].==modelType)&(dfx[:model].=="dolocc")&(dfx[:ranef].==ranef)&(dfx[:parameter].==level),:]
        p=dfx[(dfx[:modelType].==modelType)&(dfx[:model].=="pen")&(dfx[:ranef].==ranef)&(dfx[:parameter].==level),:]
    end
    mDict[:M]=p[:M][1]
    mDict[:Mt]=p[:Mt][1]
    mDict[:Mc]=p[:Mc][1]
    mDict[:N]=p[:N][1]
    mDict[:Nt]=p[:Nt][1]
    mDict[:Nc]=p[:Nc][1]
    if modelType=="GLM"
        mDict[:B1]=o[:coef][1]
        mDict[:B2]=y[:coef][1]
        mDict[:B3]=p[:coef][1] #fx[:adj_dod_effct] = ((dfx[:adj_mean_score1] .- dfx[:adj_mean_score0]) ./ dfx[:adj_mean_score0] ) *100
        mDict[:SE1]=o[:stderr][1]
        mDict[:SE2]=y[:stderr][1]
        mDict[:SE3]=p[:stderr][1] #fx[:adj_dod_effct] = ((dfx[:adj_mean_score1] .- dfx[:adj_mean_score0]) ./ dfx[:adj_mean_score0] ) *100S
    else
        mDict[:B1]=o[:adj_coef][1]
        mDict[:B2]=y[:adj_coef][1]
        mDict[:B3]=p[:adj_coef][1] #fx[:adj_dod_effct] = ((dfx[:adj_mean_score1] .- dfx[:adj_mean_score0]) ./ dfx[:adj_mean_score0] ) *100
        mDict[:SE1]=o[:adj_stderr][1]
        mDict[:SE2]=y[:adj_stderr][1]
        mDict[:SE3]=p[:adj_stderr][1] #fx[:adj_dod_effct] = ((dfx[:adj_mean_score1] .- dfx[:adj_mean_score0]) ./ dfx[:adj_mean_score0] ) *100S
    end
    mDict[:o_SE0]=0
    mDict[:y_SE0]=0
    mDict[:p_SE0]=0
    mDict[:o_B0]=0
    mDict[:y_B0]=0
    mDict[:p_B0]=0
    mDict[:o_mean_score0]=o[:adj_mean_score0][1]
    mDict[:o_mean_score1]=o[:adj_mean_score1][1]
    mDict[:y_mean_score0]=y[:adj_mean_score0][1]
    mDict[:y_mean_score1]=y[:adj_mean_score1][1]
    mDict[:p_mean_score0]=p[:adj_mean_score0][1]
    mDict[:p_mean_score1]=p[:adj_mean_score1][1]
    println("Running CI for : $ranef : $level")
    mDict[:metakey] = ranef*"~"*level
    calcPValue_Opt(mDict)
    CIs_O(mDict)
    return mDict
end

function MP_ConfidenceIntervals(dfx::DataFrame)
                @swarm :total MP_collectModels(dfx, "GLM")
    for ranef in unique(dfx[(dfx[:modelType].=="GLMM"),:ranef])
        for level in unique(dfx[(dfx[:modelType].=="GLMM")&(dfx[:ranef].==ranef),:parameter])
                k = Symbol(ranef*"~"*level)
                @swarm k  MP_collectModels(dfx, "GLMM",ranef,level)
        end
    end
    sleep(20)
    while !iscompletew() println("Not Complete yet!"); sleep(5); end
    mDict = takew(:total)
    dfx[(dfx[:model].=="dolhh")&(dfx[:modelType].=="GLM")&(dfx[:parameter].=="group"),:onetail_pval] = mDict[:onetail_pval]*100
    dfx[(dfx[:model].=="dolhh")&(dfx[:modelType].=="GLM")&(dfx[:parameter].=="group"),:twotail_pval] = mDict[:twotail_pval]*100
    dfx[(dfx[:model].=="dolhh")&(dfx[:modelType].=="GLM")&(dfx[:parameter].=="group"),:pval] = mDict[:pval]
    for k in keys(ZDict)
        dfx[(dfx[:model].=="dolhh")&(dfx[:modelType].=="GLM")&(dfx[:parameter].=="group"),Symbol(k*"_lb")] = mDict[Symbol(k*"_lb")]*100
        dfx[(dfx[:model].=="dolhh")&(dfx[:modelType].=="GLM")&(dfx[:parameter].=="group"),Symbol(k*"_ub")] = mDict[Symbol(k*"_ub")]*100
    end
    for ranef in unique(dfx[(dfx[:modelType].=="GLMM"),:ranef])
        for level in unique(dfx[(dfx[:modelType].=="GLMM")&(dfx[:ranef].==ranef),:parameter])
                mDict = takew( Symbol(ranef*"~"*level) )
                dfx[(dfx[:model].=="dolhh")&(dfx[:ranef].==ranef)&(dfx[:parameter].==level),:pval] = mDict[:pval]
                for k in keys(ZDict)
                    dfx[(dfx[:model].=="dolhh")&(dfx[:ranef].==ranef)&(dfx[:parameter].==level),Symbol(k*"_lb")] = mDict[Symbol(k*"_lb")]*100
                    dfx[(dfx[:model].=="dolhh")&(dfx[:ranef].==ranef)&(dfx[:parameter].==level),Symbol(k*"_ub")] = mDict[Symbol(k*"_ub")]*100
                end
        end
    end
end

MP_ConfidenceIntervals(dfx)

# ***************************************************************************************************************************
# ***************************************************************************************************************************
# ***************************************************************************************************************************
# ***************************************************************************************************************************
# ***************************************************************************************************************************

# *******************************************
function runModels(root::String, modelsDict::Dict)
    dfd = read_dfd(root) #dfd = readtable(mod_fname,header=true);
    poolit!(dfd,modelsDict[:factors])
    res = OrderedDict()
    q = Symbol[]
    cnt=0
    for (k,v) in OrderedDict(m=>modelsDict[m][:raneff] for m in [:ipen,:idolocc,:iocc])
         for r in v push!(q,k); push!(q,r); cnt=cnt+1 end
    end
    x=reshape(q, (2,cnt))
    ml = permutedims(x, [2, 1])
    m=:empty
    for i in 1:length(ml[:,1])
        if m != ml[i,:][1]
            gr=Symbol("glm_"*string(ml[i,:][1]))
            res[gr] = runGlm(dfd,modelsDict[ml[i,:][1]]) #res[gr] = runGlm(mod_fname, jmod_fname, ml[i,:][1])
            m=ml[i,:][1]
        end
        r=Symbol("glmm_"*string(ml[i,:][1])*"_"*string(ml[i,:][2]))
        res[r] = runGlmm(dfd,modelsDict[m], [ml[i,:][2]], true)   #res[r] = runGlmm(mod_fname, jmod_fname, ml[i,:][1], ml[i,:][2])
        covg=Symbol("covglm_"*string(ml[i,:][1])*"_"*string(ml[i,:][2]))
        res[covg] = runGlm(dfd,modelsDict[m], ml[i,:][2])   #res[covg] = runGlm(mod_fname, jmod_fname, ml[i,:][1], ml[i,:][2])
    end
    return res
end
#dx = runModels(root,modelsDict)

function consolidateResults(modelsDict::Dict,dx::OrderedDict)
  sdf = DataFrame(parameter=String[], coef=Int64[], stderr=Int64[], zval=Int64[], pval=Int64[], model=Symbol[], ranef=String[], modelType=String[])
    for m in [:iocc, :idolocc, :ipen]
        g = dx[Symbol("glm_"*string(m))]
        dfx = coefDF(g,false)
        dfx[:model] = replace(string(m),"i","")
        dfx[:ranef] = "none"
        dfx[:modelType] = "GLM"
        sdf = vcat(sdf,dfx)
        grp = sdf[(sdf[:modelType].=="GLM")&(sdf[:model].==replace(string(m),"i",""))&(sdf[:parameter].=="group"),:coef][1]
        err = sdf[(sdf[:modelType].=="GLM")&(sdf[:model].==replace(string(m),"i",""))&(sdf[:parameter].=="group"),:stderr][1]
        for r in  modelsDict[m][:raneff]
            k="_"*string(m)*"_"*string(r)
            g = dx[Symbol("glmm"*k)]
            dfx = raneffect(g)
            dfx[:zval]=0.0
            dfx[:pval]=0.0
            dfx[:model] = replace(string(m),"i","")
            dfx[:ranef]=string(r)
            dfx[:modelType]="GLMM"
            sdf = vcat(sdf,dfx)
            g = dx[Symbol("covglm"*k)]
            dfx = coefDF(g,false)
            dfx[:model] = replace(string(m),"i","")
                dfx[:ranef] = string(r)
            dfx[:modelType] = "RanCov"
            dfx = dfx[findin(dfx[:parameter],filter(x->contains(x,string(r)) ,Array(dfx[:parameter]))),:]
            dfx[:parameter] = map(x-> replace(x,string(r)*": ",""),dfx[:parameter])
            sdf = vcat(sdf,dfx)
        end
    end
    return sdf
end
#dx = runModels(root,modelsDict)
#dfx = consolidateResults(modelsDict, dx)
#save_dfx(root,dfx)   #writetable(root*"/campaign.csv", dfx)

#dx = runModels(root,modelsDict)
#dfx = consolidateResults(modelsDict, dx)
#save_dfx(root,dfx)   #writetable(root*"/campaign.csv", dfx)

 # ***************************************************************************************************************************

#function distributeDataset()
#    #jmod_fname = root*"/dfd_model.json"
#    #mod_fname
#    #ppp="/mnt/resource/analytics/models/"; run(`ls $ppp`)
#    ips = ["10.63.36.22","10.63.36.23"]
#    for ip in ips
#            cmd=`scp $root/dfd_model.* $ip:$root/`
#            println(cmd)
#            run(cmd)
#        #run(`ssh $ip mkdir $root`)
#        #run(`scp $root/dfd_model.* $ip:$root/`)
#    end
#end
#addprocs([("iriadmin@10.63.36.22", 1), ("iriadmin@10.63.36.23", 1)])
#addprocs(2)

function cmparr(a1::Array{Symbol},a2::Array{Symbol},ignore::Array{Symbol}=Symbol[])
    a1=setdiff(a1,ignore)
    a2=setdiff(a2,ignore)
    a1_only=setdiff(a1,a2)
    a2_only=setdiff(a2,a1)
    l1=length(a1_only)
    l2=length(a2_only)
    println(l1,"--",l2)
    l1>l2 ? DataFrame(a1_only=a1_only,a2_only=vcat(a2_only,fill(:x, l1-l2))) : DataFrame(DataFrame(a1_only=vcat(a1_only,fill(:x, l2-l1)),a2_only=a2_only))
end

function cmparr(a1::Array{Any},a2::Array{Symbol},ignore::Array{Symbol}=Symbol[])
    cmparr(convert(Array{Symbol},a1), a2)
end


module MdvspCg

using GLPK

include("instance.jl")

export Instance,
    pullout_cost,
    pullin_cost,
    deadhead_cost,
    read_huisman_instance

include("main.jl")

export build_main_problem,
    add_column!

include("subproblem.jl")

export optimize!

function gencol()
    inst = read_huisman_instance("../mdvsp-colgen/instances/m4n500s0.inp")
    #inst = read_huisman_instance("../mdvsp-colgen/instances/m8n1500s4.inp")

    main = build_main_problem(inst)
    sp = map(1:inst.num_depots) do k
        Subproblem(inst, k)
    end

    timings = open("timings.csv", "w")
    write(timings, "event,iter,time,bytes,gctime\n")
    function tm_append(msg, iter, tm)
        write(timings, "$(msg),$(iter),$(tm.time),$(tm.bytes),$(tm.gctime)\n")
    end

    # Configures the optimizer
    set_optimizer(main, GLPK.Optimizer)

    for iter in 1:10000
        tm = @timed optimize!(main)
        tm_append("solve_main", iter, tm)

        println("Iter: $iter  Main obj: $(objective_value(main))")

        depot_duals = dual.(main[:depot_cap])
        task_duals = dual.(main[:task_asg])
        newcols = Vector{Any}(undef, inst.num_depots)

        tm = @timed begin
            Threads.@threads for k in 1:inst.num_depots
                newcols[k] = optimize!(sp[k], depot_duals[k], task_duals)
            end
        end
        tm_append("solve_pricing", iter, tm)

        converged = true
        tm = @timed begin
            for k in 1:inst.num_depots
                if !isempty(newcols[k].path)
                    converged = false
                    add_column!(main, k, newcols[k].cost, newcols[k].path)
                end
            end
        end
        tm_append("add_columns", iter, tm)

        if converged
            if !main[:asg_relax]
                println("** COLGEN CONVERGED! **")
                break
            else
                println("*** CHANGING RELAXATION STATUS ***")
                tm = @timed set_main_asg_equals!(main)
                tm_append("set_equals", iter, tm)
            end
        end

        # write_to_file(main, "main.$iter.lp")
    end

    write_to_file(main, "main.final.lp")
    close(timings)

    return main
end

end # module