compat w/ivreg2

compat w/ivreg2

// --------------------------------------------------------------------------
// FixedEffects main class
// --------------------------------------------------------------------------

mata:

class FixedEffects
{
    // Factors
    `Integer'               G                   // Number of sets of FEs
    `Integer'               N                   // number of obs
    `Factors'               factors
    `Vector'                sample
    `Varlist'               absvars
    `Varlist'               ivars
    `Varlist'               cvars
    `Boolean'               has_intercept
    `RowVector'             intercepts
    `RowVector'             num_slopes
    `Integer'               num_singletons
    `Boolean'               save_any_fe
    `Boolean'               save_all_fe
    `Varlist'               targets

    // Optimization options
    `Real'                  tolerance
    `Integer'               maxiter
    `String'                transform           // Kaczmarz Cimmino Symmetric_kaczmarz (k c s)
    `String'                acceleration        // Acceleration method. None/No/Empty is none\
    `Integer'               accel_start         // Iteration where we start to accelerate // set it at 6? 2?3?
    `string'                slope_method
    `Boolean'               prune               // Whether to recursively prune degree-1 edges
    `Boolean'               abort               // Raise error if convergence failed?
    `Integer'               accel_freq          // Specific to Aitken's acceleration

    // Optimization objects
    `BipartiteGraph'        bg                  // Used when pruning 1-core vertices
    `Vector'                pruned_weight       // temp. weight for the factors that were pruned
    `Integer'               prune_g1            // Factor 1/2 in the bipartite subgraph that gets pruned
    `Integer'               prune_g2            // Factor 2/2 in the bipartite subgraph that gets pruned

    // Misc
    `Integer'               verbose
    `Boolean'               timeit

    // Weight-specific
    `Boolean'               has_weights
    `Variabe'               weight              // unsorted weight
    `String'                weight_var          // Weighting variable
    `String'                weight_type         // Weight type (pw, fw, etc)

    // Absorbed degrees-of-freedom computations
    `Integer'               G_extended          // Number of intercepts plus slopes
    `Integer'               df_a_redundant      // e(mobility)
    `Integer'               df_a_initial
    `Integer'               df_a                // df_a_inital - df_a_redundant
    `Vector'                doflist_M
    `Vector'                doflist_K
    `Vector'                doflist_M_is_exact
    `Vector'                doflist_M_is_nested
    `Vector'                is_slope
    `Integer'               df_a_nested // Redundant due to bein nested; used for: r2_a r2_a_within rmse

    // VCE and cluster variables
    `String'                vcetype
    `Integer'               num_clusters
    `Varlist'               clustervars
    `Varlist'               base_clustervars
    `String'                vceextra

    // Regression-specific
    `String'                original_varlist    // y x1 x2 (x3 x4 = z1 z2 z3)
    `String'                varlist             // y x1 x2 x3 x4 z1 z2 z3
    `String'                original_depvar
    `String'                original_indepvars
    `String'                original_endogvars
    `String'                original_instruments
    `String'                depvar              // y
    `String'                indepvars           // x1 x2
    `String'                endogvars           // x3 x4
    `String'                instruments         // z1 z2 z3

    `Boolean'               drop_singletons
    `String'                absorb              // contents of absorb()
    `String'                select_if           // If condition
    `String'                select_in           // In condition
    `String'                model               // ols, iv
    `String'                summarize_stats
    `Boolean'               summarize_quietly
    `StringRowVector'       dofadjustments // firstpair pairwise cluster continuous
    `Varname'               groupvar
    `String'                residuals
    `String'                diopts

    // Output
    `String'                cmdline
    `String'                subcmd
    `String'                title
    `Boolean'               converged
    `Integer'               iteration_count // e(ic)
    `Varlist'               extended_absvars
    `Integer'               N_hdfe
    `String'                notes
    `String'                equation_d
    `Integer'               df_r
    `Integer'               df_m
    `Integer'               N_clust
    `Integer'               N_clust_list
    `Real'                  rss
    `Real'                  rmse
    `Real'                  F
    `Real'                  tss
    `Real'                  tss_within
    `Real'                  sumweights
    `Real'                  r2
    `Real'                  r2_within
    `Real'                  r2_a
    `Real'                  r2_a_within
    `Real'                  ll
    `Real'                  ll_0

    // Methods
    `Void'                  new()
    `Matrix'                partial_out()
    `Void'                  _partial_out()
    `Variables'             project_one_fe()
    `Void'                  prune_1core()
    `Void'                  _expand_1core()
    `Void'                  estimate_dof()
    `Void'                  save_touse()
    `Void'                  save_variable()
    `Void'                  post_footnote()
    `Void'                  post()
}

// Set default value of properties
`Void' FixedEffects::new()
{
    num_singletons = .
    sample = J(0, 1, .)
    weight = 1 // set to 1 so cross(x, S.weight, y)==cross(x, y)

    verbose = 0

    // Optimization defaults
    slope_method = "invsym"
    maxiter = 1e4
    tolerance = 1e-8
    transform = "symmetric_kaczmarz"
    acceleration = "conjugate_gradient"
    accel_start = 6

    prune = 1
    converged = 0
    abort = 1

    // Specific to Aitken:
    accel_freq = 3
}

`Variables' FixedEffects::partial_out(`Anything' data, | `Boolean' save_tss)
{
    // -data- is either a varlist or a matrix
    `Variables'             y
    `Varlist'               vars
    `RowVector'             idx
    `Integer'               i

    if (eltype(data) == "string") {
        vars = tokens(invtokens(data))
        if (verbose > 0) printf("\n{txt} ## Partialling out %g variables: {res}%s{txt}\n\n", cols(vars), invtokens(vars))
        if (verbose > 0) printf("{txt}    - Loading variables into Mata\n")
        y = st_data(sample, invtokens(vars))

        if (cols(y) < cols(vars)) {
            printf("{err}(some columns were dropped due to a bug/quirk in st_data() (%g cols created instead of %g for %s); running slower workaround)\n", cols(y), cols(vars), invtokens(vars))
            assert(0) // when does this happen?
        }
        else if (cols(y) > cols(vars)) {
            printf("{err}(some empty columns were added due to a bug/quirk in {bf:st_data()}; %g cols created instead of %g for {it:%s}; running slower workaround)\n", cols(y), cols(vars), invtokens(vars))
            y = J(rows(y), 0, .)
            for (i=1; i<=cols(vars); i++) {
                y = y, st_data(sample, vars[i])
            }
        }

        assert_msg(cols(y)==cols(vars), "st_data() constructed more columns than expected")

        idx = `selectindex'(colmax(abs(colminmax(y))) :== 0)
        if (cols(idx) & verbose>-1) {
            printf("{err}WARNING: after demeaning, some variables are just zeros: %s\n", invtokens(vars[idx]))
        }

        _partial_out(y, save_tss)
    }
    else {
        if (verbose > 0) printf("\n{txt} ## Partialling out %g variables\n\n", cols(data))
        _partial_out(y=data, save_tss)
    }
    //if (verbose==0) printf("\n")
    if (verbose==0) printf("{txt}(converged in %s iterations)\n", strofreal(iteration_count))
    return(y)
}

`Void' FixedEffects::_partial_out(`Variables' y, | `Boolean' save_tss)
{
    `RowVector'             stdevs, needs_zeroing
    `FunctionP'             funct_transform, func_accel // transform
    `Real'                y_mean

    if (args()<2 | save_tss==.) save_tss = 0

    // Solver Warnings
    if (transform=="kaczmarz" & acceleration=="conjugate_gradient") {
        printf("{err}(WARNING: convergence is {bf:unlikely} with transform=kaczmarz and accel=CG)\n")
    }

    // Load transform pointer
    if (transform=="cimmino") funct_transform = &transform_cimmino()
    if (transform=="kaczmarz") funct_transform = &transform_kaczmarz()
    if (transform=="symmetric_kaczmarz") funct_transform = &transform_sym_kaczmarz()
    if (transform=="random_kaczmarz") funct_transform = &transform_rand_kaczmarz()

    // Pointer to acceleration routine
    if (acceleration=="test") func_accel = &accelerate_test()
    if (acceleration=="none") func_accel = &accelerate_none()
    if (acceleration=="conjugate_gradient") func_accel = &accelerate_cg()
    if (acceleration=="steepest_descent") func_accel = &accelerate_sd()
    if (acceleration=="aitken") func_accel = &accelerate_aitken()
    if (acceleration=="hybrid") func_accel = &accelerate_hybrid()

    // Shortcut for trivial case (1 FE)
    if (G==1) func_accel = &accelerate_none()

    // Compute TSS of depvar
    if (save_tss & tss==.) {
        if (has_intercept) {
            y_mean = mean(y[., 1], weight)
            tss = crossdev(y[., 1], y_mean, weight, y[., 1], y_mean) // Sum of w[i] * (y[i]-y_mean) ^ 2
        }
        else {
            tss = cross(y[., 1], weight, y[., 1]) // Sum of w[i] * y[i] ^ 2
        }
        if (weight_type=="aweight" | weight_type=="pweight") tss = tss * rows(y) / sum(weight)
    }

    // Standardize variables
    if (verbose > 0) printf("{txt}    - Standardizing variables\n")
    stdevs = reghdfe_standardize(y)

    // Solve
    if (verbose>0) printf("{txt}    - Running solver (acceleration={res}%s{txt}, transform={res}%s{txt} tol={res}%-1.0e{txt})\n", acceleration, transform, tolerance)
    if (verbose==1) printf("{txt}    - Iterating:")
    if (verbose>1) printf("{txt}      ")
    converged = 0
    iteration_count = 0
    y = (*func_accel)(this, y, funct_transform) :* stdevs
    // converged gets updated by check_convergence()

    if (prune) {
        _expand_1core(y)
    }

    // Standardizing makes it hard to detect values that are perfectly collinear with the absvars
    // in which case they should be 0.00 but they end up as 1e-16
    // EG: reghdfe price ibn.foreign , absorb(foreign)

    // This will edit to zero entire columns where *ALL* values are very close to zero
    needs_zeroing = colmax(abs(colminmax(y))) :< 1e-8 // chose something relatively close to epsilon() ~ 1e-16
    needs_zeroing = `selectindex'(needs_zeroing)
    if (cols(needs_zeroing)) {
        y[., needs_zeroing] = J(rows(y), cols(needs_zeroing), 0)
    }

    if (weight==1) weight = J(rows(y), 1, .) // bugbug remove this!??!
}

`Variables' FixedEffects::project_one_fe(`Variables' y, `Integer' g)
{
    `Factor'                f

    // Cons+K+W, Cons+K, K+W, K, Cons+W, Cons = 6 variants

    f = factors[g]

    if (num_slopes[g]==0) {
        // bugbug see if we can remove the .]
        return(panelmean(f.sort(y), f)[f.levels, .])
    }
    else if (intercepts[g]) {
        return(panelsolve_invsym(f.sort(y), f, intercepts[g]))
    }
    else {
        return(panelsolve_invsym(f.sort(y), f, intercepts[g]))
    }
}

`Void' FixedEffects::estimate_dof()
{
    `Boolean'               has_int
    `Integer'               g, h                        // index FEs (1..G)
    `Integer'               num_intercepts              // Number of absvars with an intercept term
    `Integer'               i_cluster, i_intercept, j_intercept
    `Integer'               i                           // index 1..G_extended
    `Integer'               j
    `Integer'               bg_verbose                  // verbose level when calling BipartiteGraph()
    `Integer'               m                           // Mobility groups between a specific pair of FEs
    `RowVector'             SubGs
    `RowVector'             offsets, idx, zeros, results
    `Matrix'                tmp
    `Variables'             data
    `DataCol'               cluster_data
    `String'                absvar, clustervar
    `Factor'                F
    `BipartiteGraph'        BG
    `Integer'               pair_count

    if (verbose > 0) printf("\n{txt} ## Estimating degrees-of-freedom absorbed by the fixed effects\n\n")

    // Count all FE intercepts and slopes
    SubGs = intercepts + num_slopes
    G_extended = sum(SubGs)
    num_intercepts = sum(intercepts)
    offsets = runningsum(SubGs) - SubGs :+ 1 // start of each FE within the extended list
    idx = `selectindex'(intercepts) // Select all FEs with intercepts
    if (verbose > 0) printf("{txt}    - there are %f fixed intercepts and slopes in the %f absvars\n", G_extended, G)

    // Initialize result vectors and scalars
    doflist_M_is_exact = J(1, G_extended, 0)
    doflist_M_is_nested = J(1, G_extended, 0)
    df_a_nested = 0

    // (1) M will hold the redundant coefs for each extended absvar (G_extended, not G)
    doflist_M = J(1, G_extended, 0)
    assert(0 <= num_clusters & num_clusters <= 10)
    if (num_clusters > 0 & anyof(dofadjustments, "clusters")) {

        // (2) (Intercept-Only) Look for absvars that are clustervars
        for (i_intercept=1; i_intercept<=length(idx); i_intercept++) {
            g = idx[i_intercept]
            i = offsets[g]
            absvar = invtokens(tokens(ivars[g]), "#")
            if (anyof(clustervars, absvar)) {
                doflist_M[i] = factors[g].num_levels
                df_a_nested = df_a_nested + doflist_M[i]
                doflist_M_is_exact[i] = doflist_M_is_nested[i] = 1
                idx[i_intercept] = 0
                if (verbose > 0) printf("{txt} - categorical variable {res}%s{txt} is also a cluster variable, so it doesn't reduce DoF\n", absvar)
            }
        }
        idx = select(idx, idx)

        // (3) (Intercept-Only) Look for absvars that are nested within a clustervar
        for (i_cluster=1; i_cluster<= num_clusters; i_cluster++) {
            cluster_data = .
            if (!length(idx)) break // no more absvars to process
            for (i_intercept=1; i_intercept<=length(idx); i_intercept++) {

                g = idx[i_intercept]
                i = offsets[g]
                absvar = invtokens(tokens(ivars[g]), "#")
                clustervar = clustervars[i_cluster]
                if (doflist_M_is_exact[i]) continue // nothing to do

                if (cluster_data == .) {
                    if (strpos(clustervar, "#")) {
                        clustervar = base_clustervars[i_cluster]
                        F = factor(clustervar, sample, ., "", 0, 0, ., 0)
                        cluster_data = F.levels
                        F = Factor() // clear
                    }
                    else {
                        cluster_data = __fload_data(clustervar, sample)
                    }
                }

                if (factors[g].nested_within(cluster_data)) {
                    doflist_M[i] = factors[g].num_levels
                    doflist_M_is_exact[i] = doflist_M_is_nested[i] = 1
                    df_a_nested = df_a_nested + doflist_M[i]
                    idx[i_intercept] = 0
                    if (verbose > 0) printf("{txt} - categorical variable {res}%s{txt} is nested within a cluster variable, so it doesn't reduce DoF\n", absvar)
                }
            }
            idx = select(idx, idx)
        }
        cluster_data = . // save memory
    } // end of the two cluster checks (absvar is clustervar; absvar is nested within clustervar)

    // (4) (Intercept-Only) Every intercept but the first has at least one redundant coef.
    if (length(idx) > 1) {
        if (verbose > 0) printf("{txt}    - there is at least one redundant coef. for every set of FE intercepts after the first one\n")
        doflist_M[offsets[idx[2..num_intercepts]]] = J(1, num_intercepts-1, 1) // Set DoF loss of all intercepts but the first one to 1
    }

    // (5) (Intercept-only) Mobility group algorithm
    // Excluding those already solved, the first absvar is exact

    if (length(idx)) {
        i = idx[1]
        doflist_M_is_exact[i] = 1
    }

    // Compute number of dijsoint subgraphs / mobility groups for each pair of remaining FEs
    if (anyof(dofadjustments, "firstpair") | anyof(dofadjustments, "pairwise")) {
        BG = BipartiteGraph()
        bg_verbose = max(( verbose - 1 , 0 ))
        pair_count = 0

        for (i_intercept=1; i_intercept<=length(idx)-1; i_intercept++) {
            for (j_intercept=i_intercept+1; j_intercept<=length(idx); j_intercept++) {
                g = idx[i_intercept]
                h = idx[j_intercept]
                i = offsets[h]
                BG.init(&factors[g], &factors[h], bg_verbose)
                m = BG.init_zigzag()
                ++pair_count
                if (verbose > 0) printf("{txt}    - mobility groups between FE intercepts #%f and #%f: {res}%f\n", g, h, m)
                doflist_M[i] = max(( doflist_M[i] , m ))
                if (j_intercept==2) doflist_M_is_exact[i] = 1
                if (pair_count & anyof(dofadjustments, "firstpair")) break
            }
            if (pair_count & anyof(dofadjustments, "firstpair")) break
        }
        BG = BipartiteGraph() // clear
    }
    // TODO: add group3hdfe

    // (6) See if cvars are zero (w/out intercept) or just constant (w/intercept)
    if (anyof(dofadjustments, "continuous")) {
        for (i=g=1; g<=G; g++) {
            // If model has intercept, redundant cvars are those that are CONSTANT
            // Without intercept, a cvar has to be zero within a FE for it to be redundant
            // Since S.fes[g].x are already demeaned IF they have intercept, we don't have to worry about the two cases
            has_int = intercepts[g]
            if (has_int) i++
            if (!num_slopes[g]) continue

            data = asarray(factors[g].extra, "x")
            assert(num_slopes[g]==cols(data))
            results = J(1, cols(data), 0)
            // float(1.1) - 1 == 2.384e-08 , so let's pick something bigger, 1e-6
            zeros = J(1, cols(data), 1e-6)
            // This can be speed up by moving the -if- outside the -for-
            for (j = 1; j <= factors[g].num_levels; j++) {
                tmp = colminmax(panelsubmatrix(data, j, factors[g].info))
                if (has_int) {
                    results = results + ((tmp[2, .] - tmp[1, .]) :<= zeros)
                }
                else {
                    results = results + (colsum(abs(tmp)) :<= zeros)
                }
            }
            data = .
            if (sum(results)) {
                if (has_int  & verbose) printf("{txt}    - the slopes in the FE #%f are constant for {res}%f{txt} levels, which don't reduce DoF\n", g, sum(results))
                if (!has_int & verbose) printf("{txt}    - the slopes in the FE #%f are zero for {res}%f{txt} levels, which don't reduce DoF\n", g, sum(results))
                doflist_M[i..i+num_slopes[g]-1] = results
            }
            i = i + num_slopes[g]
        }
    }

    // Store results (besides doflist_..., etc.)
    doflist_K = J(1, G_extended, .)
    for (g=1; g<=G; g++) {
        i = offsets[g]
        j = g==G ? G_extended : offsets[g+1]
        doflist_K[i..j] = J(1, j-i+1, factors[g].num_levels)
    }
    df_a_initial = sum(doflist_K)
    df_a_redundant = sum(doflist_M)
    df_a = df_a_initial - df_a_redundant
}

`Void' FixedEffects::prune_1core()
{
    // Note that we can't prune degree-2 nodes, to keep the graph bipartite
    `Integer'               i, j, g
    `Vector'                subgraph_id

    `Vector'                idx
    `RowVector'             i_prune

    assert(G==2) // bugbug remove?

    // Abort if the user set HDFE.prune = 0
    if (!prune) return

    // Pick two factors, and check if we really benefit from pruning
    prune = 0
    i_prune = J(1, 2, 0)
    for (g=i=1; g<=2; g++) {
        //if (intercepts[g] & !num_slopes[g] & factors[g].num_levels>100) {
        if (intercepts[g] & !num_slopes[g]) {
            i_prune[i++] = g // increments at the end
            if (i > 2) { // success!
                prune = 1
                break
            }
        }
    }

    if (!prune) return

    // for speed, the factor with more levels goes first
    i = i_prune[1]
    j = i_prune[2]
    //if (factors[i].num_levels < factors[j].num_levels) swap(i, j) // bugbug uncomment it!
    prune_g1 = i
    prune_g2 = j

    bg = BipartiteGraph()
    bg.init(&factors[prune_g1], &factors[prune_g2], verbose)
    bg.init_zigzag(1) // 1 => save subgraphs into bg.subgraph_id
    bg.compute_cores()
    bg.prune_1core(weight)
}

// bugbug fix or remove this fn altogether
`Void' FixedEffects::_expand_1core(`Variables' y)
{
    y = bg.expand_1core(y)
}

`Void' FixedEffects::save_touse(`Varname' touse, | `Boolean' replace)
{
    `Integer'               idx
    `Vector'                mask

    if (verbose > 0) printf("\n{txt} ## Saving e(sample)\n")
    mask = J(st_nobs(), 1, 0)
    mask[sample] = J(rows(sample), 1, 1)

    if (args()<2 | replace==.) {
        // Generate
        idx = st_addvar("byte", touse)
        st_store(., idx, mask)
    }
    else {
        // Replace
        st_store(., touse, mask)
    }
}

`Void' FixedEffects::save_variable(`Varname' varname,
                                   `Variable' data,
                                 | `String' varlabel)
{
    `Integer'               idx
    `Vector'                mask
    idx = st_addvar("double", varname)
    st_store(sample, idx, data)
    if (args()>=3 & varlabel!="") st_varlabel(idx, varlabel)

}

`Void' FixedEffects::post_footnote()
{
    `Matrix'                table
    `StringVector'          rowstripe
    `StringRowVector'       colstripe
    `String'                text

    assert(!missing(G))
    st_numscalar("e(N_hdfe)", G)
    st_numscalar("e(N_hdfe_extended)", G_extended)
    st_numscalar("e(df_a)", df_a)
    st_numscalar("e(df_a_initial)", df_a_initial)
    st_numscalar("e(df_a_redundant)", df_a_redundant)
    st_numscalar("e(df_a_nested)", df_a_nested)

    st_global("e(absvars)", invtokens(absvars))
    text = invtokens(extended_absvars)
    text = subinstr(text, "1.", "")
    st_global("e(extended_absvars)", text)

    // Absorbed degrees-of-freedom table
    table = (doflist_K \ doflist_M \ (doflist_K-doflist_M) \ !doflist_M_is_exact \ doflist_M_is_nested)'
    rowstripe = extended_absvars'
    rowstripe = J(rows(table), 1, "") , extended_absvars' // add equation col
    colstripe = "Categories" \ "Redundant" \ "Num Coefs" \ "Exact?" \ "Nested?" // colstripe cannot have dots on Stata 12 or earlier
    colstripe = J(cols(table), 1, "") , colstripe // add equation col
    st_matrix("e(dof_table)", table)
    st_matrixrowstripe("e(dof_table)", rowstripe)
    st_matrixcolstripe("e(dof_table)", colstripe)

    st_numscalar("e(ic)", iteration_count)
    st_numscalar("e(drop_singletons)", drop_singletons)
    st_numscalar("e(num_singletons)", num_singletons)
}

`Void' FixedEffects::post(`Options' options)
{
    `String'        text
    `Integer'       i

    post_footnote()

    // ---- constants -------------------------------------------------------

    st_global("e(predict)", "reghdfe_p")
    st_global("e(estat_cmd)", "reghdfe_estat")
    st_global("e(footnote)", "reghdfe_footnote")
    st_global("e(marginsok)", "")
    st_numscalar("e(df_m)", df_m)

    assert(title != "")
    text = sprintf("HDFE %s", title)
    st_global("e(title)", text)

    text = sprintf("Absorbing %g HDFE %s", G, plural(G, "group"))
    st_global("e(title2)", text)

    st_global("e(model)", model)
    st_global("e(cmdline)", cmdline)

    st_numscalar("e(tss)", tss)
    st_numscalar("e(tss_within)", tss_within)
    st_numscalar("e(rss)", rss)
    st_numscalar("e(mss)", tss - rss)
    st_numscalar("e(rmse)", rmse)
    st_numscalar("e(F)", F)

    st_numscalar("e(ll)", ll)
    st_numscalar("e(ll_0)", ll_0)

    st_numscalar("e(r2)", r2)
    st_numscalar("e(r2_within)", r2_within)
    st_numscalar("e(r2_a)", r2_a)
    st_numscalar("e(r2_a_within)", r2_a_within)

    if (!missing(N_clust)) {
        st_numscalar("e(N_clust)", N_clust)
        for (i=1; i<=num_clusters; i++) {
            text = sprintf("e(N_clust%g)", i)
            st_numscalar(text, N_clust_list[i])
        }
        text = "Statistics robust to heteroskedasticity"
        st_global("e(title3)", text)
    }

    if (!missing(sumweights)) st_numscalar("e(sumweights)", sumweights)

    // ---- .options properties ---------------------------------------------

    st_global("e(depvar)", depvar)
    st_global("e(indepvars)", indepvars)
    st_global("e(endogvars)", endogvars)
    st_global("e(instruments)", instruments)

    if (!missing(N_clust)) {
        st_numscalar("e(N_clustervars)", num_clusters)
        st_global("e(clustvar)", invtokens(clustervars))
        for (i=1; i<=num_clusters; i++) {
            text = sprintf("e(clustvar%g)", i)
            st_global(text, clustervars[i])
        }
    }

    if (residuals != "") {
        st_global("e(resid)", residuals)
    }

    // Stata uses e(vcetype) for the SE column headers
    // In the default option, leave it empty.
    // In the cluster and robust options, set it as "Robust"
    text = strproper(vcetype)
    if (text=="Cluster") text = "Robust"
    if (text=="Unadjusted") text = ""
    assert(anyof( ("", "Robust", "Jackknife", "Bootstrap") , text))
    if (text!="") st_global("e(vcetype)", text)

    text = vcetype
    if (text=="unadjusted") text = "ols"
    st_global("e(vce)", text)

    // Weights
    if (weight_type != "") {
        st_global("e(wexp)", "= " + weight_var)
        st_global("e(wtype)", weight_type)
    }
}

end

// --------------------------------------------------------------------------
// FixedEffects main class
// --------------------------------------------------------------------------

mata:

class FixedEffects
{
    // Factors
    `Integer'               G                   // Number of sets of FEs
    `Integer'               N                   // number of obs
    `Factors'               factors
    `Vector'                sample
    `Varlist'               absvars
    `Varlist'               ivars
    `Varlist'               cvars
    `Boolean'               has_intercept
    `RowVector'             intercepts
    `RowVector'             num_slopes
    `Integer'               num_singletons
    `Boolean'               save_any_fe
    `Boolean'               save_all_fe
    `Varlist'               targets

    // Optimization options
    `Real'                  tolerance
    `Integer'               maxiter
    `String'                transform           // Kaczmarz Cimmino Symmetric_kaczmarz (k c s)
    `String'                acceleration        // Acceleration method. None/No/Empty is none\
    `Integer'               accel_start         // Iteration where we start to accelerate // set it at 6? 2?3?
    `string'                slope_method
    `Boolean'               prune               // Whether to recursively prune degree-1 edges
    `Boolean'               abort               // Raise error if convergence failed?
    `Integer'               accel_freq          // Specific to Aitken's acceleration

    // Optimization objects
    `BipartiteGraph'        bg                  // Used when pruning 1-core vertices
    `Vector'                pruned_weight       // temp. weight for the factors that were pruned
    `Integer'               prune_g1            // Factor 1/2 in the bipartite subgraph that gets pruned
    `Integer'               prune_g2            // Factor 2/2 in the bipartite subgraph that gets pruned

    // Misc
    `Integer'               verbose
    `Boolean'               timeit

    // Weight-specific
    `Boolean'               has_weights
    `Variabe'               weight              // unsorted weight
    `String'                weight_var          // Weighting variable
    `String'                weight_type         // Weight type (pw, fw, etc)

    // Absorbed degrees-of-freedom computations
    `Integer'               G_extended          // Number of intercepts plus slopes
    `Integer'               df_a_redundant      // e(mobility)
    `Integer'               df_a_initial
    `Integer'               df_a                // df_a_inital - df_a_redundant
    `Vector'                doflist_M
    `Vector'                doflist_K
    `Vector'                doflist_M_is_exact
    `Vector'                doflist_M_is_nested
    `Vector'                is_slope
    `Integer'               df_a_nested // Redundant due to bein nested; used for: r2_a r2_a_within rmse

    // VCE and cluster variables
    `String'                vcetype
    `Integer'               num_clusters
    `Varlist'               clustervars
    `Varlist'               base_clustervars
    `String'                vceextra

    // Regression-specific
    `String'                original_varlist    // y x1 x2 (x3 x4 = z1 z2 z3)
    `String'                varlist             // y x1 x2 x3 x4 z1 z2 z3
    `String'                original_depvar
    `String'                original_indepvars
    `String'                original_endogvars
    `String'                original_instruments
    `String'                depvar              // y
    `String'                indepvars           // x1 x2
    `String'                endogvars           // x3 x4
    `String'                instruments         // z1 z2 z3

    `Boolean'               drop_singletons
    `String'                absorb              // contents of absorb()
    `String'                select_if           // If condition
    `String'                select_in           // In condition
    `String'                model               // ols, iv
    `String'                summarize_stats
    `Boolean'               summarize_quietly
    `StringRowVector'       dofadjustments // firstpair pairwise cluster continuous
    `Varname'               groupvar
    `String'                residuals
    `String'                diopts

    // Output
    `String'                cmdline
    `String'                subcmd
    `String'                title
    `Boolean'               converged
    `Integer'               iteration_count // e(ic)
    `Varlist'               extended_absvars
    `Integer'               N_hdfe
    `String'                notes
    `String'                equation_d
    `Integer'               df_r
    `Integer'               df_m
    `Integer'               N_clust
    `Integer'               N_clust_list
    `Real'                  rss
    `Real'                  rmse
    `Real'                  F
    `Real'                  tss
    `Real'                  tss_within
    `Real'                  sumweights
    `Real'                  r2
    `Real'                  r2_within
    `Real'                  r2_a
    `Real'                  r2_a_within
    `Real'                  ll
    `Real'                  ll_0

    // Methods
    `Void'                  new()
    `Matrix'                partial_out()
    `Void'                  _partial_out()
    `Variables'             project_one_fe()
    `Void'                  prune_1core()
    `Void'                  _expand_1core()
    `Void'                  estimate_dof()
    `Void'                  save_touse()
    `Void'                  save_variable()
    `Void'                  post_footnote()
    `Void'                  post()
}

// Set default value of properties
`Void' FixedEffects::new()
{
    num_singletons = .
    sample = J(0, 1, .)
    weight = 1 // set to 1 so cross(x, S.weight, y)==cross(x, y)

    verbose = 0

    // Optimization defaults
    slope_method = "invsym"
    maxiter = 1e4
    tolerance = 1e-8
    transform = "symmetric_kaczmarz"
    acceleration = "conjugate_gradient"
    accel_start = 6

    prune = 1
    converged = 0
    abort = 1

    // Specific to Aitken:
    accel_freq = 3
}

`Variables' FixedEffects::partial_out(`Anything' data, | `Boolean' save_tss)
{
    // -data- is either a varlist or a matrix
    `Variables'             y
    `Varlist'               vars
    `RowVector'             idx
    `Integer'               i

    if (eltype(data) == "string") {
        vars = tokens(invtokens(data))
        if (verbose > 0) printf("\n{txt} ## Partialling out %g variables: {res}%s{txt}\n\n", cols(vars), invtokens(vars))
        if (verbose > 0) printf("{txt}    - Loading variables into Mata\n")
        y = st_data(sample, invtokens(vars))

        if (cols(y) < cols(vars)) {
            printf("{err}(some columns were dropped due to a bug/quirk in st_data() (%g cols created instead of %g for %s); running slower workaround)\n", cols(y), cols(vars), invtokens(vars))
            assert(0) // when does this happen?
        }
        else if (cols(y) > cols(vars)) {
            printf("{err}(some empty columns were added due to a bug/quirk in {bf:st_data()}; %g cols created instead of %g for {it:%s}; running slower workaround)\n", cols(y), cols(vars), invtokens(vars))
            y = J(rows(y), 0, .)
            for (i=1; i<=cols(vars); i++) {
                y = y, st_data(sample, vars[i])
            }
        }

        assert_msg(cols(y)==cols(vars), "st_data() constructed more columns than expected")

        idx = `selectindex'(colmax(abs(colminmax(y))) :== 0)
        if (cols(idx) & verbose>-1) {
            printf("{err}WARNING: after demeaning, some variables are just zeros: %s\n", invtokens(vars[idx]))
        }

        _partial_out(y, save_tss)
    }
    else {
        if (verbose > 0) printf("\n{txt} ## Partialling out %g variables\n\n", cols(data))
        _partial_out(y=data, save_tss)
    }
    //if (verbose==0) printf("\n")
    if (verbose==0) printf("{txt}(converged in %s iterations)\n", strofreal(iteration_count))
    return(y)
}

`Void' FixedEffects::_partial_out(`Variables' y, | `Boolean' save_tss)
{
    `RowVector'             stdevs, needs_zeroing
    `FunctionP'             funct_transform, func_accel // transform
    `Real'                y_mean

    if (args()<2 | save_tss==.) save_tss = 0

    // Solver Warnings
    if (transform=="kaczmarz" & acceleration=="conjugate_gradient") {
        printf("{err}(WARNING: convergence is {bf:unlikely} with transform=kaczmarz and accel=CG)\n")
    }

    // Load transform pointer
    if (transform=="cimmino") funct_transform = &transform_cimmino()
    if (transform=="kaczmarz") funct_transform = &transform_kaczmarz()
    if (transform=="symmetric_kaczmarz") funct_transform = &transform_sym_kaczmarz()
    if (transform=="random_kaczmarz") funct_transform = &transform_rand_kaczmarz()

    // Pointer to acceleration routine
    if (acceleration=="test") func_accel = &accelerate_test()
    if (acceleration=="none") func_accel = &accelerate_none()
    if (acceleration=="conjugate_gradient") func_accel = &accelerate_cg()
    if (acceleration=="steepest_descent") func_accel = &accelerate_sd()
    if (acceleration=="aitken") func_accel = &accelerate_aitken()
    if (acceleration=="hybrid") func_accel = &accelerate_hybrid()

    // Shortcut for trivial case (1 FE)
    if (G==1) func_accel = &accelerate_none()

    // Compute TSS of depvar
    if (save_tss & tss==.) {
        if (has_intercept) {
            y_mean = mean(y[., 1], weight)
            tss = crossdev(y[., 1], y_mean, weight, y[., 1], y_mean) // Sum of w[i] * (y[i]-y_mean) ^ 2
        }
        else {
            tss = cross(y[., 1], weight, y[., 1]) // Sum of w[i] * y[i] ^ 2
        }
        if (weight_type=="aweight" | weight_type=="pweight") tss = tss * rows(y) / sum(weight)
    }

    // Standardize variables
    if (verbose > 0) printf("{txt}    - Standardizing variables\n")
    stdevs = reghdfe_standardize(y)

    // Solve
    if (verbose>0) printf("{txt}    - Running solver (acceleration={res}%s{txt}, transform={res}%s{txt} tol={res}%-1.0e{txt})\n", acceleration, transform, tolerance)
    if (verbose==1) printf("{txt}    - Iterating:")
    if (verbose>1) printf("{txt}      ")
    converged = 0
    iteration_count = 0
    y = (*func_accel)(this, y, funct_transform) :* stdevs
    // converged gets updated by check_convergence()

    if (prune) {
        _expand_1core(y)
    }

    // Standardizing makes it hard to detect values that are perfectly collinear with the absvars
    // in which case they should be 0.00 but they end up as 1e-16
    // EG: reghdfe price ibn.foreign , absorb(foreign)

    // This will edit to zero entire columns where *ALL* values are very close to zero
    needs_zeroing = colmax(abs(colminmax(y))) :< 1e-8 // chose something relatively close to epsilon() ~ 1e-16
    needs_zeroing = `selectindex'(needs_zeroing)
    if (cols(needs_zeroing)) {
        y[., needs_zeroing] = J(rows(y), cols(needs_zeroing), 0)
    }

    if (weight==1) weight = J(rows(y), 1, .) // bugbug remove this!??!
}

`Variables' FixedEffects::project_one_fe(`Variables' y, `Integer' g)
{
    `Factor'                f

    // Cons+K+W, Cons+K, K+W, K, Cons+W, Cons = 6 variants

    f = factors[g]

    if (num_slopes[g]==0) {
        // bugbug see if we can remove the .]
        return(panelmean(f.sort(y), f)[f.levels, .])
    }
    else if (intercepts[g]) {
        return(panelsolve_invsym(f.sort(y), f, intercepts[g]))
    }
    else {
        return(panelsolve_invsym(f.sort(y), f, intercepts[g]))
    }
}

`Void' FixedEffects::estimate_dof()
{
    `Boolean'               has_int
    `Integer'               g, h                        // index FEs (1..G)
    `Integer'               num_intercepts              // Number of absvars with an intercept term
    `Integer'               i_cluster, i_intercept, j_intercept
    `Integer'               i                           // index 1..G_extended
    `Integer'               j
    `Integer'               bg_verbose                  // verbose level when calling BipartiteGraph()
    `Integer'               m                           // Mobility groups between a specific pair of FEs
    `RowVector'             SubGs
    `RowVector'             offsets, idx, zeros, results
    `Matrix'                tmp
    `Variables'             data
    `DataCol'               cluster_data
    `String'                absvar, clustervar
    `Factor'                F
    `BipartiteGraph'        BG
    `Integer'               pair_count

    if (verbose > 0) printf("\n{txt} ## Estimating degrees-of-freedom absorbed by the fixed effects\n\n")

    // Count all FE intercepts and slopes
    SubGs = intercepts + num_slopes
    G_extended = sum(SubGs)
    num_intercepts = sum(intercepts)
    offsets = runningsum(SubGs) - SubGs :+ 1 // start of each FE within the extended list
    idx = `selectindex'(intercepts) // Select all FEs with intercepts
    if (verbose > 0) printf("{txt}    - there are %f fixed intercepts and slopes in the %f absvars\n", G_extended, G)

    // Initialize result vectors and scalars
    doflist_M_is_exact = J(1, G_extended, 0)
    doflist_M_is_nested = J(1, G_extended, 0)
    df_a_nested = 0

    // (1) M will hold the redundant coefs for each extended absvar (G_extended, not G)
    doflist_M = J(1, G_extended, 0)
    assert(0 <= num_clusters & num_clusters <= 10)
    if (num_clusters > 0 & anyof(dofadjustments, "clusters")) {

        // (2) (Intercept-Only) Look for absvars that are clustervars
        for (i_intercept=1; i_intercept<=length(idx); i_intercept++) {
            g = idx[i_intercept]
            i = offsets[g]
            absvar = invtokens(tokens(ivars[g]), "#")
            if (anyof(clustervars, absvar)) {
                doflist_M[i] = factors[g].num_levels
                df_a_nested = df_a_nested + doflist_M[i]
                doflist_M_is_exact[i] = doflist_M_is_nested[i] = 1
                idx[i_intercept] = 0
                if (verbose > 0) printf("{txt} - categorical variable {res}%s{txt} is also a cluster variable, so it doesn't reduce DoF\n", absvar)
            }
        }
        idx = select(idx, idx)

        // (3) (Intercept-Only) Look for absvars that are nested within a clustervar
        for (i_cluster=1; i_cluster<= num_clusters; i_cluster++) {
            cluster_data = .
            if (!length(idx)) break // no more absvars to process
            for (i_intercept=1; i_intercept<=length(idx); i_intercept++) {

                g = idx[i_intercept]
                i = offsets[g]
                absvar = invtokens(tokens(ivars[g]), "#")
                clustervar = clustervars[i_cluster]
                if (doflist_M_is_exact[i]) continue // nothing to do

                if (cluster_data == .) {
                    if (strpos(clustervar, "#")) {
                        clustervar = base_clustervars[i_cluster]
                        F = factor(clustervar, sample, ., "", 0, 0, ., 0)
                        cluster_data = F.levels
                        F = Factor() // clear
                    }
                    else {
                        cluster_data = __fload_data(clustervar, sample)
                    }
                }

                if (factors[g].nested_within(cluster_data)) {
                    doflist_M[i] = factors[g].num_levels
                    doflist_M_is_exact[i] = doflist_M_is_nested[i] = 1
                    df_a_nested = df_a_nested + doflist_M[i]
                    idx[i_intercept] = 0
                    if (verbose > 0) printf("{txt} - categorical variable {res}%s{txt} is nested within a cluster variable, so it doesn't reduce DoF\n", absvar)
                }
            }
            idx = select(idx, idx)
        }
        cluster_data = . // save memory
    } // end of the two cluster checks (absvar is clustervar; absvar is nested within clustervar)

    // (4) (Intercept-Only) Every intercept but the first has at least one redundant coef.
    if (length(idx) > 1) {
        if (verbose > 0) printf("{txt}    - there is at least one redundant coef. for every set of FE intercepts after the first one\n")
        doflist_M[offsets[idx[2..num_intercepts]]] = J(1, num_intercepts-1, 1) // Set DoF loss of all intercepts but the first one to 1
    }

    // (5) (Intercept-only) Mobility group algorithm
    // Excluding those already solved, the first absvar is exact

    if (length(idx)) {
        i = idx[1]
        doflist_M_is_exact[i] = 1
    }

    // Compute number of dijsoint subgraphs / mobility groups for each pair of remaining FEs
    if (anyof(dofadjustments, "firstpair") | anyof(dofadjustments, "pairwise")) {
        BG = BipartiteGraph()
        bg_verbose = max(( verbose - 1 , 0 ))
        pair_count = 0

        for (i_intercept=1; i_intercept<=length(idx)-1; i_intercept++) {
            for (j_intercept=i_intercept+1; j_intercept<=length(idx); j_intercept++) {
                g = idx[i_intercept]
                h = idx[j_intercept]
                i = offsets[h]
                BG.init(&factors[g], &factors[h], bg_verbose)
                m = BG.init_zigzag()
                ++pair_count
                if (verbose > 0) printf("{txt}    - mobility groups between FE intercepts #%f and #%f: {res}%f\n", g, h, m)
                doflist_M[i] = max(( doflist_M[i] , m ))
                if (j_intercept==2) doflist_M_is_exact[i] = 1
                if (pair_count & anyof(dofadjustments, "firstpair")) break
            }
            if (pair_count & anyof(dofadjustments, "firstpair")) break
        }
        BG = BipartiteGraph() // clear
    }
    // TODO: add group3hdfe

    // (6) See if cvars are zero (w/out intercept) or just constant (w/intercept)
    if (anyof(dofadjustments, "continuous")) {
        for (i=g=1; g<=G; g++) {
            // If model has intercept, redundant cvars are those that are CONSTANT
            // Without intercept, a cvar has to be zero within a FE for it to be redundant
            // Since S.fes[g].x are already demeaned IF they have intercept, we don't have to worry about the two cases
            has_int = intercepts[g]
            if (has_int) i++
            if (!num_slopes[g]) continue

            data = asarray(factors[g].extra, "x")
            assert(num_slopes[g]==cols(data))
            results = J(1, cols(data), 0)
            // float(1.1) - 1 == 2.384e-08 , so let's pick something bigger, 1e-6
            zeros = J(1, cols(data), 1e-6)
            // This can be speed up by moving the -if- outside the -for-
            for (j = 1; j <= factors[g].num_levels; j++) {
                tmp = colminmax(panelsubmatrix(data, j, factors[g].info))
                if (has_int) {
                    results = results + ((tmp[2, .] - tmp[1, .]) :<= zeros)
                }
                else {
                    results = results + (colsum(abs(tmp)) :<= zeros)
                }
            }
            data = .
            if (sum(results)) {
                if (has_int  & verbose) printf("{txt}    - the slopes in the FE #%f are constant for {res}%f{txt} levels, which don't reduce DoF\n", g, sum(results))
                if (!has_int & verbose) printf("{txt}    - the slopes in the FE #%f are zero for {res}%f{txt} levels, which don't reduce DoF\n", g, sum(results))
                doflist_M[i..i+num_slopes[g]-1] = results
            }
            i = i + num_slopes[g]
        }
    }

    // Store results (besides doflist_..., etc.)
    doflist_K = J(1, G_extended, .)
    for (g=1; g<=G; g++) {
        i = offsets[g]
        j = g==G ? G_extended : offsets[g+1]
        doflist_K[i..j] = J(1, j-i+1, factors[g].num_levels)
    }
    df_a_initial = sum(doflist_K)
    df_a_redundant = sum(doflist_M)
    df_a = df_a_initial - df_a_redundant
}

`Void' FixedEffects::prune_1core()
{
    // Note that we can't prune degree-2 nodes, to keep the graph bipartite
    `Integer'               i, j, g
    `Vector'                subgraph_id

    `Vector'                idx
    `RowVector'             i_prune

    assert(G==2) // bugbug remove?

    // Abort if the user set HDFE.prune = 0
    if (!prune) return

    // Pick two factors, and check if we really benefit from pruning
    prune = 0
    i_prune = J(1, 2, 0)
    for (g=i=1; g<=2; g++) {
        //if (intercepts[g] & !num_slopes[g] & factors[g].num_levels>100) {
        if (intercepts[g] & !num_slopes[g]) {
            i_prune[i++] = g // increments at the end
            if (i > 2) { // success!
                prune = 1
                break
            }
        }
    }

    if (!prune) return

    // for speed, the factor with more levels goes first
    i = i_prune[1]
    j = i_prune[2]
    //if (factors[i].num_levels < factors[j].num_levels) swap(i, j) // bugbug uncomment it!
    prune_g1 = i
    prune_g2 = j

    bg = BipartiteGraph()
    bg.init(&factors[prune_g1], &factors[prune_g2], verbose)
    bg.init_zigzag(1) // 1 => save subgraphs into bg.subgraph_id
    bg.compute_cores()
    bg.prune_1core(weight)
}

// bugbug fix or remove this fn altogether
`Void' FixedEffects::_expand_1core(`Variables' y)
{
    y = bg.expand_1core(y)
}

`Void' FixedEffects::save_touse(`Varname' touse, | `Boolean' replace)
{
    `Integer'               idx
    `Vector'                mask

    if (verbose > 0) printf("\n{txt} ## Saving e(sample)\n")
    mask = J(st_nobs(), 1, 0)
    mask[sample] = J(rows(sample), 1, 1)

    if (args()<2 | replace==.) {
        // Generate
        idx = st_addvar("byte", touse)
        st_store(., idx, mask)
    }
    else {
        // Replace
        st_store(., touse, mask)
    }
}

`Void' FixedEffects::save_variable(`Varname' varname,
                                   `Variable' data,
                                 | `String' varlabel)
{
    `Integer'               idx
    `Vector'                mask
    idx = st_addvar("double", varname)
    st_store(sample, idx, data)
    if (args()>=3 & varlabel!="") st_varlabel(idx, varlabel)

}

`Void' FixedEffects::post_footnote()
{
    `Matrix'                table
    `StringVector'          rowstripe
    `StringRowVector'       colstripe
    `String'                text

    assert(!missing(G))
    st_numscalar("e(N_hdfe)", G)
    st_numscalar("e(N_hdfe_extended)", G_extended)
    st_numscalar("e(df_m)", df_m)
    st_numscalar("e(df_a)", df_a)
    st_numscalar("e(df_a_initial)", df_a_initial)
    st_numscalar("e(df_a_redundant)", df_a_redundant)
    st_numscalar("e(df_a_nested)", df_a_nested)

    st_global("e(absvars)", invtokens(absvars))
    text = invtokens(extended_absvars)
    text = subinstr(text, "1.", "")
    st_global("e(extended_absvars)", text)

    // Absorbed degrees-of-freedom table
    table = (doflist_K \ doflist_M \ (doflist_K-doflist_M) \ !doflist_M_is_exact \ doflist_M_is_nested)'
    rowstripe = extended_absvars'
    rowstripe = J(rows(table), 1, "") , extended_absvars' // add equation col
    colstripe = "Categories" \ "Redundant" \ "Num Coefs" \ "Exact?" \ "Nested?" // colstripe cannot have dots on Stata 12 or earlier
    colstripe = J(cols(table), 1, "") , colstripe // add equation col
    st_matrix("e(dof_table)", table)
    st_matrixrowstripe("e(dof_table)", rowstripe)
    st_matrixcolstripe("e(dof_table)", colstripe)

    st_numscalar("e(ic)", iteration_count)
    st_numscalar("e(drop_singletons)", drop_singletons)
    st_numscalar("e(num_singletons)", num_singletons)
}

`Void' FixedEffects::post(`Options' options)
{
    `String'        text
    `Integer'       i

    post_footnote()

    // ---- constants -------------------------------------------------------

    st_global("e(predict)", "reghdfe_p")
    st_global("e(estat_cmd)", "reghdfe_estat")
    st_global("e(footnote)", "reghdfe_footnote")
    st_global("e(marginsok)", "")

    assert(title != "")
    text = sprintf("HDFE %s", title)
    st_global("e(title)", text)

    text = sprintf("Absorbing %g HDFE %s", G, plural(G, "group"))
    st_global("e(title2)", text)

    st_global("e(model)", model)
    st_global("e(cmdline)", cmdline)

    st_numscalar("e(tss)", tss)
    st_numscalar("e(tss_within)", tss_within)
    st_numscalar("e(rss)", rss)
    st_numscalar("e(mss)", tss - rss)
    st_numscalar("e(rmse)", rmse)
    st_numscalar("e(F)", F)

    st_numscalar("e(ll)", ll)
    st_numscalar("e(ll_0)", ll_0)

    st_numscalar("e(r2)", r2)
    st_numscalar("e(r2_within)", r2_within)
    st_numscalar("e(r2_a)", r2_a)
    st_numscalar("e(r2_a_within)", r2_a_within)

    if (!missing(N_clust)) {
        st_numscalar("e(N_clust)", N_clust)
        for (i=1; i<=num_clusters; i++) {
            text = sprintf("e(N_clust%g)", i)
            st_numscalar(text, N_clust_list[i])
        }
        text = "Statistics robust to heteroskedasticity"
        st_global("e(title3)", text)
    }

    if (!missing(sumweights)) st_numscalar("e(sumweights)", sumweights)

    // ---- .options properties ---------------------------------------------

    st_global("e(depvar)", depvar)
    st_global("e(indepvars)", indepvars)
    st_global("e(endogvars)", endogvars)
    st_global("e(instruments)", instruments)

    if (!missing(N_clust)) {
        st_numscalar("e(N_clustervars)", num_clusters)
        st_global("e(clustvar)", invtokens(clustervars))
        for (i=1; i<=num_clusters; i++) {
            text = sprintf("e(clustvar%g)", i)
            st_global(text, clustervars[i])
        }
    }

    if (residuals != "") {
        st_global("e(resid)", residuals)
    }

    // Stata uses e(vcetype) for the SE column headers
    // In the default option, leave it empty.
    // In the cluster and robust options, set it as "Robust"
    text = strproper(vcetype)
    if (text=="Cluster") text = "Robust"
    if (text=="Unadjusted") text = ""
    assert(anyof( ("", "Robust", "Jackknife", "Bootstrap") , text))
    if (text!="") st_global("e(vcetype)", text)

    text = vcetype
    if (text=="unadjusted") text = "ols"
    st_global("e(vce)", text)

    // Weights
    if (weight_type != "") {
        st_global("e(wexp)", "= " + weight_var)
        st_global("e(wtype)", weight_type)
    }
}

end


redo-sh: reverse a case that caused 'set -e' to abort on zsh.

Correctly, I might add.  It's too bad none of my other shells did it.  Sigh.

exec >&2
redo-ifchange t/shelltest.od

rm -rf $1.new $1/sh
mkdir $1.new

GOOD=
WARN=

# Note: list low-functionality, maximally POSIX-like shells before more
# powerful ones.  We want weaker shells to take precedence, as long as they
# pass the tests, because weaker shells are more likely to point out when you
# use some non-portable feature.
for sh in dash sh /usr/xpg4/bin/sh ash posh mksh ksh ksh88 ksh93 pdksh \
		bash zsh busybox; do
	printf "%-30s" "Testing $sh..."
	FOUND=`which $sh 2>/dev/null` || { echo "missing"; continue; }

	# It's important for the file to actually be named 'sh'.  Some
	# shells (like bash and zsh) only go into POSIX-compatible mode if
	# they have that name.  If they're not in POSIX-compatible mode,
	# they'll fail the test.
	rm -f $1.new/sh
	ln -s $FOUND $1.new/sh

	set +e
	( cd t && ../$1.new/sh shelltest.od >shelltest.tmp 2>&1 )
	RV=$?
	set -e

	msgs=
	crash=
	while read line; do
		#echo "line: '$line'" >&2
		stripw=${line#warning: }
		stripf=${line#failed: }
		crash=$line
		[ "$line" = "$stripw" ] || msgs="$msgs W$stripw"
		[ "$line" = "$stripf" ] || msgs="$msgs F$stripf"
	done <t/shelltest.tmp
	rm -f t/shelltest.tmp
	msgs=${msgs# }
	crash=${crash##*:}
	crash=${crash# }

	case $RV in
		40) echo "ok $msgs"; [ -n "$GOOD" ] || GOOD=$FOUND ;;
		41) echo "failed    $msgs" ;;
		42) echo "warnings  $msgs"; [ -n "$WARN" ] || WARN=$FOUND ;;
		*)  echo "crash     $crash" ;;
	esac
done

rm -rf $1 $1.new $3

if [ -n "$GOOD" ]; then
	echo "Selected perfect shell: $GOOD"
	mkdir $3
	ln -s $GOOD $3/sh
elif [ -n "$WARN" ]; then
	echo "Selected mostly good shell: $WARN"
	mkdir $3
	ln -s $WARN $3/sh
else
	echo "No good shells found!  Maybe install dash, bash, or zsh."
	exit 13
fi

exec >&2
redo-ifchange t/shelltest.od

rm -rf $1.new $1/sh
mkdir $1.new

GOOD=
WARN=

for sh in dash sh /usr/xpg4/bin/sh ash posh mksh ksh ksh88 ksh93 pdksh \
		bash zsh busybox; do
	printf "%-30s" "Testing $sh..."
	FOUND=`which $sh 2>/dev/null` || { echo "missing"; continue; }

	# It's important for the file to actually be named 'sh'.  Some
	# shells (like bash and zsh) only go into POSIX-compatible mode if
	# they have that name.  If they're not in POSIX-compatible mode,
	# they'll fail the test.
	rm -f $1.new/sh
	ln -s $FOUND $1.new/sh

	set +e
	( cd t && ../$1.new/sh shelltest.od >shelltest.tmp 2>&1 )
	RV=$?
	set -e

	msgs=
	crash=
	while read line; do
		#echo "line: '$line'" >&2
		stripw=${line#warning: }
		stripf=${line#failed: }
		crash=$line
		[ "$line" != "$stripw" ] && msgs="$msgs W$stripw"
		[ "$line" != "$stripf" ] && msgs="$msgs F$stripf"
	done <t/shelltest.tmp
	rm -f t/shelltest.tmp
	msgs=${msgs# }
	crash=${crash##*:}
	crash=${crash# }

	case $RV in
		40)  echo "ok $msgs"; [ -n "$GOOD" ] || GOOD=$FOUND ;;
		41) echo "failed    $msgs" ;;
		42) echo "warnings  $msgs"; [ -n "$WARN" ] || WARN=$FOUND ;;
		*)  echo "crash     $crash" ;;
	esac
done

rm -rf $1 $1.new $3

if [ -n "$GOOD" ]; then
	echo "Selected perfect shell: $GOOD"
	mkdir $3
	ln -s $GOOD $3/sh
elif [ -n "$WARN" ]; then
	echo "Selected mostly good shell: $WARN"
	mkdir $3
	ln -s $WARN $3/sh
else
	echo "No good shells found!  Maybe install dash, bash, or zsh."
	exit 1
fi


clean up

clean up

transcript on
if {[file exists rtl_work]} {
	vdel -lib rtl_work -all
}
vlib rtl_work
vmap work rtl_work

vcom -93 -work work {../synchronizer.vhd}
vcom -93 -work work {../fifo.vhd}
vcom -93 -work work {../prg_rom.vhd}
vcom -93 -work work {../i2c.vhd}
vcom -93 -work work {../duper_cartridge.vhd}
vcom -93 -work work {./testbench_duper_cartridge.vhd}

vsim -t 1ps -L altera -L lpm -L sgate -L altera_mf -L altera_lnsim -L cyclonev -L rtl_work -L work -voptargs=""  testbench_i2c_test

add wave -label stage_cnt   -radix unsigned        sim:/testbench_i2c_test/stage_cnt
add wave -label step_cnt    -radix unsigned        sim:/testbench_i2c_test/step_cnt

add wave -divider nes_cpu
add wave -label phi2                    sim:/testbench_i2c_test/sim_board/pi_phi2
add wave -label prg_ce_n                sim:/testbench_i2c_test/sim_board/pi_prg_ce_n
add wave -label prg_r_nw                sim:/testbench_i2c_test/sim_board/pi_prg_r_nw
add wave -label prg_addr    -radix hex  sim:/testbench_i2c_test/sim_board/pi_prg_addr
add wave -label prg_data    -radix hex  sim:/testbench_i2c_test/sim_board/pio_prg_data
#add wave -label reg_rom_data    -radix hex  sim:/testbench_i2c_test/reg_rom_data

add wave -divider duper_regs
add wave -label cur_state               sim:/testbench_i2c_test/sim_board/reg_cur_state

#add wave -divider prgrom
#add wave -label pi_ce_n     sim:/testbench_i2c_test/sim_board/prom_inst/pi_ce_n
#add wave -label pi_oe_n     sim:/testbench_i2c_test/sim_board/prom_inst/pi_oe_n
#add wave -label pi_addr     -radix hex  sim:/testbench_i2c_test/sim_board/prom_inst/pi_addr
#add wave -label po_data     -radix hex  sim:/testbench_i2c_test/sim_board/prom_inst/po_data

add wave -divider rd_fifo
add wave -label pi_ce_n     sim:/testbench_i2c_test/sim_board/rd_fifo_inst/pi_ce_n
add wave -label pi_oe_n     sim:/testbench_i2c_test/sim_board/rd_fifo_inst/pi_oe_n
add wave -label pi_we_n     sim:/testbench_i2c_test/sim_board/rd_fifo_inst/pi_we_n
add wave -label pi_data     -radix hex  sim:/testbench_i2c_test/sim_board/rd_fifo_inst/pi_data
add wave -label po_data     -radix hex  sim:/testbench_i2c_test/sim_board/rd_fifo_inst/po_data
add wave -label po_stat_empty   sim:/testbench_i2c_test/sim_board/rd_fifo_inst/po_stat_empty
add wave -label po_stat_full    sim:/testbench_i2c_test/sim_board/rd_fifo_inst/po_stat_full

add wave -divider i2c

add wave -label start_scl       sim:/testbench_i2c_test/start_scl
#add wave -label i2c_scl         sim:/testbench_i2c_test/i2c_scl
add wave -label i2c_step_cnt    sim:/testbench_i2c_test/i2c_step_cnt

add wave -label pi_i2c_scl      sim:/testbench_i2c_test/sim_board/pi_i2c_scl
add wave -label pio_i2c_sda     sim:/testbench_i2c_test/sim_board/pio_i2c_sda

add wave -divider i2c_slave
#add wave -label reg_next_sp            sim:/testbench_i2c_test/sim_board/i2c_slave_inst/reg_next_sp
add wave -label reg_cur_sp              sim:/testbench_i2c_test/sim_board/i2c_slave_inst/reg_cur_sp
add wave -label reg_cur_state           sim:/testbench_i2c_test/sim_board/i2c_slave_inst/reg_cur_state

add wave -label reg_i2c_cmd_addr        -radix hex  sim:/testbench_i2c_test/sim_board/i2c_slave_inst/reg_i2c_cmd_addr
add wave -label reg_i2c_cmd_r_nw                    sim:/testbench_i2c_test/sim_board/i2c_slave_inst/reg_i2c_cmd_r_nw
add wave -label reg_i2c_cmd_in_data     -radix hex  sim:/testbench_i2c_test/sim_board/i2c_slave_inst/reg_i2c_cmd_in_data
add wave -label reg_slave_status                    sim:/testbench_i2c_test/sim_board/reg_slave_status

#add wave -label rd_data                 -radix hex  sim:/testbench_i2c_test/rd_data

view structure
view signals

#run -all

run 15 us
wave zoom full
run 250 us
wave zoom full

transcript on
if {[file exists rtl_work]} {
	vdel -lib rtl_work -all
}
vlib rtl_work
vmap work rtl_work

vcom -93 -work work {../synchronizer.vhd}
vcom -93 -work work {../fifo.vhd}
vcom -93 -work work {../prg_rom.vhd}
vcom -93 -work work {../i2c.vhd}
vcom -93 -work work {../i2c_eeprom.vhd}
vcom -93 -work work {../duper_cartridge.vhd}
vcom -93 -work work {./testbench_duper_cartridge.vhd}

vsim -t 1ps -L altera -L lpm -L sgate -L altera_mf -L altera_lnsim -L cyclonev -L rtl_work -L work -voptargs=""  testbench_i2c_test

add wave -label stage_cnt   -radix unsigned        sim:/testbench_i2c_test/stage_cnt
add wave -label step_cnt    -radix unsigned        sim:/testbench_i2c_test/step_cnt

add wave -divider nes_cpu
add wave -label phi2                    sim:/testbench_i2c_test/sim_board/pi_phi2
add wave -label prg_ce_n                sim:/testbench_i2c_test/sim_board/pi_prg_ce_n
add wave -label prg_r_nw                sim:/testbench_i2c_test/sim_board/pi_prg_r_nw
add wave -label prg_addr    -radix hex  sim:/testbench_i2c_test/sim_board/pi_prg_addr
add wave -label prg_data    -radix hex  sim:/testbench_i2c_test/sim_board/pio_prg_data
#add wave -label reg_rom_data    -radix hex  sim:/testbench_i2c_test/reg_rom_data

add wave -divider duper_regs
add wave -label cur_state               sim:/testbench_i2c_test/sim_board/reg_cur_state

#add wave -divider prgrom
#add wave -label pi_ce_n     sim:/testbench_i2c_test/sim_board/prom_inst/pi_ce_n
#add wave -label pi_oe_n     sim:/testbench_i2c_test/sim_board/prom_inst/pi_oe_n
#add wave -label pi_addr     -radix hex  sim:/testbench_i2c_test/sim_board/prom_inst/pi_addr
#add wave -label po_data     -radix hex  sim:/testbench_i2c_test/sim_board/prom_inst/po_data

add wave -divider rd_fifo
add wave -label pi_ce_n     sim:/testbench_i2c_test/sim_board/rd_fifo_inst/pi_ce_n
add wave -label pi_oe_n     sim:/testbench_i2c_test/sim_board/rd_fifo_inst/pi_oe_n
add wave -label pi_we_n     sim:/testbench_i2c_test/sim_board/rd_fifo_inst/pi_we_n
add wave -label pi_data     -radix hex  sim:/testbench_i2c_test/sim_board/rd_fifo_inst/pi_data
add wave -label po_data     -radix hex  sim:/testbench_i2c_test/sim_board/rd_fifo_inst/po_data
add wave -label po_stat_empty   sim:/testbench_i2c_test/sim_board/rd_fifo_inst/po_stat_empty
add wave -label po_stat_full    sim:/testbench_i2c_test/sim_board/rd_fifo_inst/po_stat_full

add wave -divider i2c

add wave -label start_scl       sim:/testbench_i2c_test/start_scl
#add wave -label i2c_scl         sim:/testbench_i2c_test/i2c_scl
add wave -label i2c_step_cnt    sim:/testbench_i2c_test/i2c_step_cnt

add wave -label pi_i2c_scl      sim:/testbench_i2c_test/sim_board/pi_i2c_scl
add wave -label pio_i2c_sda     sim:/testbench_i2c_test/sim_board/pio_i2c_sda

add wave -divider i2c_slave
#add wave -label reg_next_sp            sim:/testbench_i2c_test/sim_board/i2c_slave_inst/reg_next_sp
add wave -label reg_cur_sp              sim:/testbench_i2c_test/sim_board/i2c_slave_inst/reg_cur_sp
add wave -label reg_cur_state           sim:/testbench_i2c_test/sim_board/i2c_slave_inst/reg_cur_state

add wave -label reg_i2c_cmd_addr        -radix hex  sim:/testbench_i2c_test/sim_board/i2c_slave_inst/reg_i2c_cmd_addr
add wave -label reg_i2c_cmd_r_nw                    sim:/testbench_i2c_test/sim_board/i2c_slave_inst/reg_i2c_cmd_r_nw
add wave -label reg_i2c_cmd_in_data     -radix hex  sim:/testbench_i2c_test/sim_board/i2c_slave_inst/reg_i2c_cmd_in_data
add wave -label reg_slave_status                    sim:/testbench_i2c_test/sim_board/reg_slave_status

#add wave -label rd_data                 -radix hex  sim:/testbench_i2c_test/rd_data

--add wave -divider i2c_eeprom
--add wave -label pi_rst_n        sim:/testbench_i2c_test/sim_board/i2c_eeprom_inst/pi_rst_n
--#add wave -label pi_base_clk     sim:/testbench_i2c_test/sim_board/pi_base_clk
--add wave -label pi_bus_xfer     sim:/testbench_i2c_test/sim_board/i2c_eeprom_inst/pi_bus_xfer
--add wave -label pi_r_nw         sim:/testbench_i2c_test/sim_board/i2c_eeprom_inst/pi_r_nw
--add wave -label pi_bus_ack      sim:/testbench_i2c_test/sim_board/i2c_eeprom_inst/pi_bus_ack
--add wave -label po_bus_ack      sim:/testbench_i2c_test/sim_board/i2c_eeprom_inst/po_bus_ack
--
--add wave -label pi_data -radix hex  sim:/testbench_i2c_test/sim_board/i2c_eeprom_inst/pi_data
--add wave -label po_data -radix hex  sim:/testbench_i2c_test/sim_board/i2c_eeprom_inst/po_data
--
--add wave -label reg_ack_edge    sim:/testbench_i2c_test/sim_board/i2c_eeprom_inst/reg_ack_edge
--add wave -label reg_write_cnt -radix decimal   sim:/testbench_i2c_test/sim_board/i2c_eeprom_inst/reg_write_cnt
--add wave -label reg_eeprom_addr -radix hex              sim:/testbench_i2c_test/sim_board/i2c_eeprom_inst/reg_eeprom_addr

view structure
view signals

#run -all

run 15 us
wave zoom full
run 250 us
wave zoom full


switch to multicall binary

switch to multicall binary

#!/bin/sh
# This do script can be directly invoked, for bootstrapping with
# $  sh @all.do
# $  redo @all

set -e

# The go compiler does its own dependency tracking and is so fast that
# it is ironically simpler to always rebuild.
go build -o bin/redux redux/cmd

export DESTDIR=bin
export PATH=$(dirname $0)/bin:$PATH

# symlink to binary
xargs -L1 -i ln -sf redux bin/{} < LINKS

# create docs
xargs -L1 -i bin/redux redo doc/{}.1 < LINKS

#!/bin/sh
# This do script can be directly invoked, for bootstrapping with
# $  sh @all.do
# $  redo all

# The go compiler does its own dependency tracking and is so fast that
# it is ironically simpler to always rebuild.
xargs -P 0 -L1 -i  go build -o bin/{} {}/main.go < BIN


deal with dummy vars

deal with dummy vars

program diffindiff2

// this stores variable names in as t1, t2, t3 and so on
local i = 5
while "``i''" != "" {
    local count = `i' - 4
    local t`count' = "``i''"
    di "var is `t`count''"
    local ++i
}

local var `1'
local varlabel `2'
local tabname `3'
local mode `4'

di "var: `var', tab: `tabname'"
di "in mode `mode'"

if "`mode'" == "run" {
    di "Running regressions ///"
    local i = 1
    while "`t`i''" != "" {
        local t "`t`i''"
        di "running `t'"
        run_reg `t' `tabname' `var'
        local ++i
    }
}

if "`mode'" == "write" {
    di "Writing"
    est clear
    local i = 1
    while "`t`i''" != "" {
        local t "`t`i''"
        di "loading `t'"
        load_reg `t' `tabname' `var' `varlabel'
        local ++i
    }
    write_reg  `tabname' `var' `varlabel'
}

end

program run_reg

local t `1'
local tabname `2'
local var `3'

di "----"
di "var: `var', tab: `tabname'"
di "----"

save ${stash}tmp, replace

local command "xtpoisson `t' 1.treat#1.post 1.post#1.`var' 1.treat#1.post#1.`var' i.month"

di "now running for `t'"
di "--"
runcommand tmp "`command'" `tabname'_fips_`t'_`var'
di "--"

end

program load_reg

local t `1'
local tabname `2'
local var `3'

** est clear
** foreach x in `t1' `t2' `t3'{
estimates use ${myestimates}`tabname'_fips_`t'_`var'
estadd local unitfe "Yes", replace
estadd local monthfe "Yes", replace
eststo est`t'
*}
end

program write_reg

local tabname `1'
local var `2'
local varlabel `3'

esttab using "${tables}`tabname'_fips_`var'.tex", keep(1.treat#1.post 1.post#1.`var' 1.treat#1.post#1.`var') coeflabels(1.treat#1.post#1.`var' "TIGER X POST X `varlabel'" 1.treat#1.post "TIGER X POST" 1.post#1.`var' "POST X `varlabel'" ) order(1.treat#1.post#1.`var' 1.treat#1.post 1.post#1.`var') se ar2 nonotes replace booktabs  s(unitfe monthfe N N_g, label("County FE" "Month FE" "N" "Clusters")) label

end

program diffindiff2

// this stores variable names in as t1, t2, t3 and so on
local i = 5
while "``i''" != "" {
    local count = `i' - 4
    local t`count' = "``i''"
    di "var is `t`count''"
    local ++i
}

local var `1'
local varlabel `2'
local tabname `3'
local mode `4'

di "var: `var', tab: `tabname'"
di "in mode `mode'"

if "`mode'" == "run" {
    di "Running regressions ///"
    local i = 1
    while "`t`i''" != "" {
        local t "`t`i''"
        di "running `t'"
        run_reg `t' `tabname' `var'
        local ++i
    }
}

if "`mode'" == "write" {
    di "Writing"
    est clear
    local i = 1
    while "`t`i''" != "" {
        local t "`t`i''"
        di "loading `t'"
        load_reg `t' `tabname' `var' `varlabel'
        local ++i
    }
    write_reg  `tabname' `var' `varlabel'
}

end

program run_reg

local t `1'
local tabname `2'
local var `3'

di "----"
di "var: `var', tab: `tabname'"
di "----"

save ${stash}tmp, replace

local command "xtpoisson `t' 1.treat#1.post 1.post#1.large 1.treat#1.post#1.large i.month"

di "now running for `t'"
di "--"
runcommand tmp "`command'" `tabname'_fips_`t'_`var'
di "--"

end

program load_reg

local t `1'
local tabname `2'
local var `3'

** est clear
** foreach x in `t1' `t2' `t3'{
estimates use ${myestimates}`tabname'_fips_`t'_`var'
estadd local unitfe "Yes", replace
estadd local monthfe "Yes", replace
eststo est`t'
*}
end

program write_reg

local tabname `1'
local var `2'
local varlabel `3'

esttab using "${tables}`tabname'_fips_`var'.tex", keep(1.treat#1.post 1.post#1.`var' 1.treat#1.post#1.`var') coeflabels(1.treat#1.post#1.large "TIGER X POST X `varlabel'" 1.treat#1.post "TIGER X POST" 1.post#1.`var' "POST X `varlabel'" ) order(1.treat#1.post#1.large 1.treat#1.post 1.post#1.`var') se ar2 nonotes replace booktabs  s(unitfe monthfe N N_g, label("County FE" "Month FE" "N" "Clusters")) label

end


Fixed typo

Fixed typo

// The R package 'gpinter' implements generalized Pareto interpolation. There
// is no native Stata command to perform the same task, but if you are a Stata
// user, you can integrate gpinter in your workflow quite easily by calling
// R from your Stata programs.
//
// This example file shows you how to do this. You do not need a thorough
// knowledge of R to run the interpolation and import the results into Stata.
//
// The method requires the following:
//     - A working version of Stata.
//     - A working version of R installed on the computer. If it is not already
//       the case, you should download and install R from the web address
//       <https://cran.r-project.org/mirrors.html>. Make sure to choose
//       a mirror near to your location and to pick the right operating system.
//     - The packages 'haven' and 'gpinter' installed in R. To do so, launch R
//       and type in the commands:
//        > install.packages("haven", "devtools")
//        > devtools::install_github("thomasblanchet/gpinter")
//     - The user-written Stata command 'rsource'. You can install it by typing:
//        > ssc install rsource
//
// If you need more help with this file, please contact
// <thomas.blanchet@wid.world>.

// -------------------------------------------------------------------------- //
// Preliminaries
// -------------------------------------------------------------------------- //

clear

// Set your working directory here
cd "~/GitHub/gpinter/inst/stata"

// If necessary, specify where R is installed on your computer. You only need
// to do it if the program cannot find R by default. On Linux and macOS, you
// should find it at the addresses "/usr/bin/r" or "/usr/local/bin/r". On
// Windows, you should the file "Rterm.exe".
// Type "help rsource" for more details.

*global Rterm_path `"/usr/local/bin/r"'

// -------------------------------------------------------------------------- //
// Import tabulation example (US labor income, 2010)
// -------------------------------------------------------------------------- //

input average     p  threshold  bracketavg
        37208  0.10       4130       12643
            .  0.50      23686       43908
            .  0.90      76252      108329
            .  0.99     211861      471463
end

// -------------------------------------------------------------------------- //
// Save the tabulation as a Stata file
// -------------------------------------------------------------------------- //

// Using "saveold" is a useful precaution to make sure R will be able to read
// the file even if have a very recent version of Stata
saveold "tabulation-input.dta", version(11) replace

// -------------------------------------------------------------------------- //
// Call R from Stata and run the interpolation in it
// -------------------------------------------------------------------------- //

rsource, terminator(END_OF_R) roptions(--vanilla)

	// 'haven' is a R package for importing Stata '.dta' file
	library(haven)

	// 'gpinter' is the R package to perform generalized Pareto interpoaltion
	library(gpinter)

	// Import the Stata data into R
	data <- read_dta("tabulation-input.dta")

	// Perform interpolation
	distribution <- tabulation_fit(
		p = data$p,
		thr = data$threshold,
		bracketavg = data$bracketavg,
		average = data$average[1]
	)

	// Percentiles to include in the output
	percentiles_output <- c(
		seq(0, 0.99, 0.01), // Every percentile
		seq(0.991, 0.999, 0.001), // Every 1/10 of a percentile in top 1%
		seq(0.9991, 0.9999, 0.0001), // Every 1/100 of a percentile in top 0.1%
		seq(0.99991, 0.99999, 0.00001) // Every 1/1000 of a percentile in top 0.01%
	)

	// Create a tabulation for these detailed percentiles
	tabulation <- generate_tabulation(distribution, percentiles_output)

	// You may only keep the columns you are interested in by removing one of
	// these rows. You can also rename the columns by changing the names on the
	// left of the equal sign.
	tabulation <- data.frame(
		p               = tabulation$fractile,
		threshold       = tabulation$threshold,
		top_share       = tabulation$top_share,
		bottom_share    = tabulation$bottom_share,
		bracket_share   = tabulation$bracket_share,
		top_average     = tabulation$top_average,
		bottom_average  = tabulation$bottom_average,
		bracket_average = tabulation$bracket_average,
		invpareto       = tabulation$invpareto
	)

	// Export the detailed tabulation
	write_dta(tabulation, "tabulation-output.dta")

END_OF_R

// -------------------------------------------------------------------------- //
// Import the results of the R program in Stata
// -------------------------------------------------------------------------- //

use "tabulation-output.dta", clear

// You can now use the interpolation results in Stata...

// The R package 'gpinter' implements generalized Pareto interpolation. There
// is no native Stata command to perform the same task, but if you are a Stata
// user, you can integrate gpinter in your workflow quite easily by calling
// R from your Stata programs.
//
// This example file shows you how to do this. You do not need a thorough
// knowledge of R to run the interpolation and import the results into Stata.
//
// The method requires the following:
//     - A working version of Stata.
//     - A working version of R installed on the computer. If it is not already
//       the case, you should download and install R from the web address
//       <https://cran.r-project.org/mirrors.html>. Make sure to choose
//       a mirror near to your location and to pick the right operating system.
//     - The packages 'haven' and 'gpinter' installed in R. To do so, lauch R
//       and type in the commands:
//        > install.packages("haven", "devtools")
//        > devtools::install_github("thomasblanchet/gpinter")
//     - The user-written Stata command 'rsource'. You can install it by typing:
//        > ssc install rsource
//
// If you need more help with this file, please contact
// <thomas.blanchet@wid.world>.

// -------------------------------------------------------------------------- //
// Preliminaries
// -------------------------------------------------------------------------- //

clear all

// Set your working directory here
cd "~/GitHub/gpinter/inst/stata"

// If necessary, specify where R is installed on your computer. You only need
// to do it if the program cannot find R by default. On Linux and macOS, you
// should find it at the addresses "/usr/bin/r" or "/usr/local/bin/r". On
// Windows, you should the file "Rterm.exe".
// Type "help rsource" for more details.

*global Rterm_path `"/usr/local/bin/r"'

// -------------------------------------------------------------------------- //
// Import tabulation example (US labor income, 2010)
// -------------------------------------------------------------------------- //

input average p threshold bracketavg
37208	0.10	4130	12643
.		0.50	23686	43908
.		0.90	76252	108329
.		0.99	211861	471463
end

// -------------------------------------------------------------------------- //
// Save the tabulation as a Stata file
// -------------------------------------------------------------------------- //

// Using "saveold" is a useful precaution to make sure R will be able to read
// the file even if have a very recent version of Stata
saveold "tabulation-input.dta", version(11) replace

// -------------------------------------------------------------------------- //
// Call R from Stata and run the interpolation in it
// -------------------------------------------------------------------------- //

rsource, terminator(END_OF_R) roptions(--vanilla)

	// 'haven' is a R package for importing Stata '.dta' file
	library(haven)

	// 'gpinter' is the R package to perform generalized Pareto interpoaltion
	library(gpinter)

	// Import the Stata data into R
	data <- read_dta("tabulation-input.dta")

	// Perform interpolation
	distribution <- tabulation_fit(
		p = data$p,
		thr = data$threshold,
		bracketavg = data$bracketavg,
		average = data$average[1]
	)

	// Percentiles to include in the output
	percentiles_output <- c(
		seq(0, 0.99, 0.01), // Every percentile
		seq(0.991, 0.999, 0.001), // Every 1/10 of a percentile in top 1%
		seq(0.9991, 0.9999, 0.0001), // Every 1/100 of a percentile in top 0.1%
		seq(0.99991, 0.99999, 0.00001) // Every 1/1000 of a percentile in top 0.01%
	)

	// Create a tabulation for these detailed percentiles
	tabulation <- generate_tabulation(distribution, percentiles_output)

	// You may only keep the columns you are interested in by removing one of
	// these rows. You can also rename the columns by changing the names on the
	// left of the equal sign.
	tabulation <- data.frame(
		p               = tabulation$fractile,
		threshold       = tabulation$threshold,
		top_share       = tabulation$top_share,
		bottom_share    = tabulation$bottom_share,
		bracket_share   = tabulation$bracket_share,
		top_average     = tabulation$top_average,
		bottom_average  = tabulation$bottom_average,
		bracket_average = tabulation$bracket_average,
		invpareto       = tabulation$invpareto
	)

	// Export the detailed tabulation
	write_dta(tabulation, "tabulation-output.dta")

END_OF_R

// -------------------------------------------------------------------------- //
// Import the results of the R program in Stata
// -------------------------------------------------------------------------- //

use "tabulation-output.dta", clear

// You can now use the interpolation results in Stata...


Changed summary to include both new and old asmple

program summary

// make summary panel A for citation data
est clear
use ${stash}citelines, clear

labelcitevar

estpost tabstat citeyear year numcites numimg numtext, s(mean sd median min max) columns(statistics)

esttab using "${tables}summary_cite.tex", cells ("mean(fmt(2) label(Mean)) sd(label(SD)) p50(label(Median)) min(fmt(0) label(Min)) max(fmt(0) label(Max))" ) coeflabels("Mean" "SD" "Median" "Min" "Max") replace nonum noobs booktabs width(\hsize) alignment(rrrrrr) varwidth(30) label

// make summary panel B for citation data

est clear
use ${stash}master, clear

labelvar
drop if isbasket == 1

estpost tabstat img text bd traf year post debut quality treat, s(mean sd median min max n) columns(statistics)

esttab using "${tables}summary.tex", cells ("mean(fmt(2) label(Mean)) sd(label(SD)) p50(label(Median)) min(fmt(0) label(Min)) max(fmt(0) label(Max)) count(fmt(0) label(N))" ) coeflabels("Mean" "SD" "Median" "Min" "Max" "N") replace nonum noobs booktabs width(\hsize) alignment(rrrrrr) varwidth(30) label

end

program labelcitevar

label variable citeyear "\emph{Year of Issue Publication (Issue-Year)}"
label variable year "\emph{Calendar Year (Year)}"
label variable numcites "\emph{Total Citations}"
label variable numimg "\emph{Citations for Image Reuse}"
label variable numtext "\emph{Citations in Text}"

end

program labelvar

label variable img "\emph{Number of images}"
label variable text "\emph{Number of words of text}"
label variable bd "\emph{Number of citations to Baseball Digest}"
label variable traf "\emph{Average monthly traffic}"
label variable year "\emph{Year of Wikipedia page version}"

label variable lnimg "Log(Images)"
label variable lntext "Log(Text)"
label variable lnbd "Log(Citations)"
label variable lntraf "Log(Traffic)"

label variable quality "\emph{Quality Percentile}"

label variable debutyear "\emph{Debut Year}"
label variable treat "\emph{1(Debut in Out-of-Copyright year)}"

label variable post "\emph{1(Year$>$2008)}"

label define isbaseball_l 0 "Basketball" 1 "Baseball"
label values isbaseball isbaseball_l

gen isbasket = !isbaseball
label define isbasket_l 1 "\textbf{Panel B -- Basketball}" 0 "\textbf{Panel A -- Baseball}"
label values isbasket isbasket_l

bysort id: replace debut = . if _n > 1
bysort id: replace treat = . if _n > 1
bysort id: replace quality = . if _n > 1

end

program summary

est clear
use ${stash}master, clear

labelvar

estpost tabstat img text bd traf year post debut quality treat, s(mean sd median min max n) columns(statistics) nototal by(isbasket)

esttab using "${tables}summary.tex", cells ("mean(fmt(2) label(Mean)) sd(label(SD)) p50(label(Median)) min(fmt(0) label(Min)) max(fmt(0) label(Max)) count(fmt(0) label(N))" ) coeflabels("Mean" "SD" "Median" "Min" "Max" "N") replace nonum noobs booktabs width(\hsize) alignment(rrrrrr) varwidth(30) label

end

program labelvar

label variable img "\emph{Number of images}"
label variable text "\emph{Number of words of text}"
label variable bd "\emph{Number of citations to Baseball Digest}"
label variable traf "\emph{Average monthly traffic}"
label variable year "\emph{Year of Wikipedia page version}"

label variable lnimg "Log(Images)"
label variable lntext "Log(Text)"
label variable lnbd "Log(Citations)"
label variable lntraf "Log(Traffic)"

label variable quality "\emph{Quality Percentile}"

label variable debutyear "\emph{Debut Year}"
label variable treat "\emph{1(Debut in Out-of-Copyright year)}"

label variable post "\emph{1(Year$>$2008)}"

label define isbaseball_l 0 "Basketball" 1 "Baseball"
label values isbaseball isbaseball_l

gen isbasket = !isbaseball
label define isbasket_l 1 "\textbf{Panel B -- Basketball}" 0 "\textbf{Panel A -- Baseball}"
label values isbasket isbasket_l

bysort id: replace debut = . if _n > 1
bysort id: replace treat = . if _n > 1
bysort id: replace quality = . if _n > 1

end