(minor)

(minor)

Replace syntax workaround with new -ms_add_comma- from ftools.

*! version 5.3.5 26jan2018

program reghdfe, eclass
	* Intercept old+version
	cap syntax, version old
	if !c(rc) {
		reghdfe_old, version
		exit
	}

	* Intercept old
	cap syntax anything(everything) [fw aw pw/], [*] old
	if !c(rc) {
		di as error "(running historical version of reghdfe)"
		if ("`weight'"!="") local weightexp [`weight'=`exp']
		reghdfe_old `anything' `weightexp', `options'
		exit
	}

	* Aux. subcommands
	cap syntax, [*]
	if inlist("`options'", "check", "compile", "reload", "update", "version", "requirements") {
		if ("`options'"=="compile") loc args force
		if ("`options'"=="check") loc options compile
		if ("`options'"=="update") {
			loc args 1
			loc options reload
		}
		loc subcmd = proper("`options'")
		`subcmd' `args'
	}
	else if replay() {
		Replay `0'
	}
	else {
		Cleanup 0
		Compile // takes 0.01s to run this useful check (ensures .mlib exists)
		cap noi Estimate `0'
		Cleanup `c(rc)'
	}
end

program Compile
	args force

	* Check dependencies
	ftools, check // in case lftools.mlib does not exist or is outdated
	ms_get_version ftools, min_version("2.32.0")
	ms_get_version reghdfe // save local package_version
	loc list_objects "FixedEffects() fixed_effects() BipartiteGraph()"
	loc list_functions "reghdfe_*() transform_*() accelerate_*() panelmean() panelsolve_*() lsmr()"
	loc list_misc "weighted_quadcolsum() safe_divide() check_convergence() precompute_inv_xx() _st_data_wrapper()"
	// TODO: prefix everything with reghdfe_*

	ms_compile_mata, ///
		package(reghdfe) ///
		version(`package_version') ///
		fun("`list_objects' `list_functions' `list_misc'") ///
		verbose ///
		`force'
end

program Reload
	* Internal debugging tool.
	* Updates dependencies and reghdfe from local path or from github
	* Usage:
	* 	reghdfe, update // from c:\git\..
	* 	reghdfe, reload // from github

	args online
	if ("`online'" == "") loc online 0

	di as text _n "{bf:reghdfe: updating required packages}"
	di as text "{hline 64}"

	* -ftools- https://github.com/sergiocorreia/ftools/
	cap ado uninstall ftools
	if (`online') net install ftools, from("https://github.com/sergiocorreia/ftools/raw/master/src/")
	if (!`online') net install ftools, from("c:\git\ftools\src")
	di as text "{hline 64}"
	ftools, compile
	di as text "{hline 64}"

	* Update -reghdfe-
	di as text _n  _n "{bf:reghdfe: updating self}"
	di as text "{hline 64}"
	qui ado uninstall reghdfe
	if (`online') net install reghdfe, from("https://github.com/sergiocorreia/reghdfe/raw/master/src/")
	if (!`online') net install reghdfe, from("c:\git\reghdfe\src")
	qui which reghdfe
	di as text "{hline 64}"
	reghdfe, compile
	di as text "{hline 64}"

	* Cleaning up
	di as text _n "{bf:Note:} You need to run {stata program drop _all} now."
end

program Version
	which reghdfe
	Requirements
end

program Requirements
	di as text _n "Required packages installed?"
	loc reqs ftools
	// ivreg2 avar tuples group3hdfe
	if (c(stata_version)<13) loc reqs `reqs' boottest

	loc ftools_github "https://github.com/sergiocorreia/ftools/raw/master/src/"

	loc error 0

	foreach req of local reqs {
		loc fn `req'.ado
		cap findfile `fn'
		if (_rc) {
			loc error 1
			di as text "{lalign 20:- `req'}" as error "not" _c
			di as text "    {stata ssc install `req':install from SSC}" _c
			if inlist("`req'", "ftools") {
				loc github ``req'_github'
				di as text `"    {stata `"net install `req', from(`"`github'"')"':install from github}"'
			}
			else {
				di as text // newline
			}
		}
		else {
			di as text "{lalign 20:- `req'}" as text "yes"
		}
	}

	if (`error') exit 601
end

program Cleanup
	args rc
	cap mata: mata drop HDFE
	cap mata: mata drop hdfe_*
	cap drop __temp_reghdfe_resid__
	cap matrix drop reghdfe_statsmatrix
	if (`rc' == 132) {
		di as text "- If you got the {it:parentheses unbalanced} error, note that IV/2SLS was moved to {help ivreghdfe}"
		di as smcl `"- Latest version: {browse "https://github.com/sergiocorreia/ivreghdfe":https://github.com/sergiocorreia/ivreghdfe}"'
		di as smcl `"- SSC version: {stata "net describe ivreghdfe, from(http://fmwww.bc.edu/RePEc/bocode/i)"}"'
		di as smcl `"- Note: the older functionality can still be accessed through the {it:old} option"'
	}
	if (`rc') exit `rc'
end

program Parse
	* Trim whitespace (caused by "///" line continuations; aesthetic only)
	mata: st_local("0", stritrim(st_local("0")))

	* Main syntax
	#d;
	syntax varlist(fv ts numeric) [if] [in] [aw pw fw/] , [

		/* Model */
		Absorb(string) NOAbsorb
		SUmmarize SUmmarize2(string asis) /* simulate implicit options */

		/* Standard Errors */
		VCE(string) CLuster(string)

		/* Diagnostic */
		Verbose(numlist min=1 max=1 >=-1 <=5 integer)
		TIMEit

		/* Speedup and memory Tricks */
		NOSAMPle /* do not save e(sample) */
		COMPACT /* use as little memory as possible but is slower */

		/* Undocumented */
		KEEPSINgletons
		OLD /* use latest v3 */
		NOTES(string) /* NOTES(key=value ...), will be stored on e() */

		] [*] /* capture optimization options, display options, etc. */
		;
	#d cr

	* Unused
	* SAVEcache
	* USEcache
	* CLEARcache

	* Convert options to boolean
	if ("`verbose'" == "") loc verbose 0
	loc timeit = ("`timeit'"!="")
	loc drop_singletons = ("`keepsingletons'" == "")
	loc compact = ("`compact'" != "")

	if (`timeit') timer on 29

	* Sanity checks
	if (`verbose'>-1 & "`keepsingletons'"!="") {
		loc url "http://scorreia.com/reghdfe/nested_within_cluster.pdf"
		loc msg "WARNING: Singleton observations not dropped; statistical significance is biased"
		di as error `"`msg' {browse "`url'":(link)}"'
	}
	if ("`cluster'"!="") {
		_assert ("`vce'"==""), msg("cannot specify both cluster() and vce()")
		loc vce cluster `cluster'
		loc cluster // clear it to avoid bugs in subsequent lines
	}

	* Split varlist into <depvar> and <indepvars>
	ms_parse_varlist `varlist'
	if (`verbose' > 0) {
		di as text _n "## Parsing varlist: {res}`varlist'"
		return list
	}
	loc depvar `r(depvar)'
	loc indepvars `r(indepvars)'
	loc fe_format "`r(fe_format)'"
	loc basevars `r(basevars)'

	* Parse Weights
	if ("`weight'"!="") {
		unab exp : `exp', min(1) max(1) // simple weights only
	}

	* Parse VCE
	ms_parse_vce, vce(`vce') weighttype(`weight')
	if (`verbose' > 0) {
		di as text _n "## Parsing vce({res}`vce'{txt})"
		sreturn list
	}
	loc vcetype = "`s(vcetype)'"
	loc num_clusters = `s(num_clusters)'
	loc clustervars = "`s(clustervars)'"
	loc base_clustervars = "`s(base_clustervars)'"
	loc vceextra = "`s(vceextra)'"

	* Select sample (except for absvars)
	loc varlist `depvar' `indepvars' `base_clustervars'
	tempvar touse
	marksample touse, strok // based on varlist + cluster + if + in + weight

	* Parse noabsorb
	_assert  ("`absorb'`noabsorb'" != ""), msg("option {bf:absorb()} or {bf:noabsorb} required")
	if ("`noabsorb'" != "") {
		_assert ("`absorb'" == ""), msg("{bf:absorb()} and {bf:noabsorb} are mutually exclusive")
		tempvar c
		gen byte `c' = 1
		loc absorb `c'
	}

	if (`timeit') timer off 29

	* Construct HDFE object
	// SYNTAX: fixed_effects(absvars | , touse, wtype, wtvar, dropsing, verbose)
	ms_add_comma, loc(absorb) cmd(`"`absorb'"') opt(`"`options'"')
	if (`timeit') timer on 20
	mata: HDFE = fixed_effects(`"`absorb'"', "`touse'", "`weight'", "`exp'", `drop_singletons', `verbose')
	if (`timeit') timer off 20
	mata: HDFE.cmdline = "reghdfe " + st_local("0")
	loc options `s(options)'

	mata: st_local("N", strofreal(HDFE.N))
	if (`N' == 0) error 2000

	* Fill out HDFE object
	* mata: HDFE.varlist = "`base_varlist'"
	mata: HDFE.depvar = "`depvar'"
	mata: HDFE.indepvars = "`indepvars'"
	mata: HDFE.vcetype = "`vcetype'"
	mata: HDFE.num_clusters = `num_clusters'
	mata: HDFE.clustervars = tokens("`clustervars'")
	mata: HDFE.base_clustervars = tokens("`base_clustervars'")
	mata: HDFE.vceextra = "`vceextra'"

	* Preserve memory
	mata: HDFE.compact = `compact'
	if (`compact') {
		loc panelvar "`_dta[_TSpanel]'"
		loc timevar "`_dta[_TStvar]'"
		mata: HDFE.panelvar = "`panelvar'"
		mata: HDFE.timevar = "`timevar'"
		c_local keepvars `basevars' `base_clustervars' `panelvar' `timevar' // `exp'
	}

	* Parse summarize
	if ("`summarize'" != "") {
		_assert ("`summarize2'" == ""), msg("summarize() syntax error")
		loc summarize2 mean min max  // default values
	}
	ParseSummarize `summarize2'
	mata: HDFE.summarize_stats = "`s(stats)'"
	mata: HDFE.summarize_quietly = `s(quietly)'

	* Parse misc options
	mata: HDFE.notes = `"`notes'"'
	mata: HDFE.store_sample = ("`nosample'"=="")
	mata: HDFE.timeit = `timeit'

	* Parse Coef Table Options (do this last!)
	_get_diopts diopts options, `options' // store in `diopts', and the rest back to `options'
	_assert (`"`options'"'==""), msg(`"invalid options: `options'"')
	if ("`hascons'"!="") di in ye "(option ignored: `hascons')"
	if ("`tsscons'"!="") di in ye "(option ignored: `tsscons')"
	mata: HDFE.diopts = `"`diopts'"'
end

program ParseSummarize, sclass
	sreturn clear
	syntax [namelist(name=stats)] , [QUIetly]
	local quietly = ("`quietly'"!="")
	sreturn loc stats "`stats'"
	sreturn loc quietly = `quietly'
end

// --------------------------------------------------------------------------

program Estimate, eclass
	ereturn clear

	* Parse and fill out HDFE object
	Parse `0'
	mata: st_local("timeit", strofreal(HDFE.timeit))
	mata: st_local("compact", strofreal(HDFE.compact))
	mata: st_local("verbose", strofreal(HDFE.verbose))

	* Compute degrees-of-freedom
	if (`timeit') timer on 21
	mata: HDFE.estimate_dof()
	if (`timeit') timer off 21

	* Save updated e(sample) (singletons reduce sample);
	* required to parse factor variables to partial out
	if (`timeit') timer on 29
	tempvar touse
	mata: HDFE.save_touse("`touse'")
	if (`timeit') timer off 29

	* Expand varlists
	if (`timeit') timer on 22
	mata: st_local("depvar", HDFE.depvar)
	mata: st_local("indepvars", HDFE.indepvars)
	if (`verbose' > 0) di as text _n "## Parsing and expanding indepvars: {res}`indepvars'"
	ms_expand_varlist `indepvars' if `touse'
	if (`verbose' > 0) return list
	mata: HDFE.fullindepvars = "`r(fullvarlist)'"
	mata: HDFE.indepvars = "`r(varlist)'"
	mata: HDFE.not_basevar = strtoreal(tokens("`r(not_omitted)'"))
	mata: HDFE.varlist = "`depvar' `r(varlist)'"
	if (`timeit') timer off 22

	* Stats
	mata: st_local("stats", HDFE.summarize_stats)
	if ("`stats'" != "") Stats `touse'

	* Condition number
	mata: HDFE.estimate_cond()

	* Preserve
	if (`compact') {
		if (`verbose' > 0) di as text "## Preserving dataset"
		preserve
		keep `keepvars'
	}

	* Partial out; save TSS of depvar
	if (`timeit') timer on 23
	// partial_out(Varlist/Matrix | , Save TSS if HDFE.tss is missing? [0], Standardize data? [1], First col is depvar? [1])
	mata: hdfe_variables = HDFE.partial_out(HDFE.varlist, 1, ., .)
	if (`timeit') timer off 23

	* Regress
	if (`timeit') timer on 24
	tempname b V N rank df_r
	mata: reghdfe_post_ols(HDFE, hdfe_variables, "`b'", "`V'", "`N'", "`rank'", "`df_r'")
	mata: hdfe_variables = .
	* Restore
	if (`compact') {
		if (`verbose' > 0) di as text "## Restoring dataset"
		restore
		mata: st_local("residuals", HDFE.residuals)
		if ("`residuals'" != "") mata: HDFE.save_variable(HDFE.residuals, HDFE.residuals_vector, "Residuals")
	}
	RegressOLS `touse' `b' `V' `N' `rank' `df_r'
	if (`timeit') timer off 24

	* (optional) Store FEs
	if (`timeit') timer on 29
	reghdfe_store_alphas
	if (`timeit') timer off 29

	* View estimation tables
	mata: st_local("diopts", HDFE.diopts)
	Replay, `diopts'

	if (`timeit') {
		di as text _n "{bf: Timer results:}"
		timer list
		di as text "Legend: 20: Create HDFE object; 21: Estimate DoF; 22: expand varlists; 23: partial out; 24: regress; 29: rest"
		di
	}
end

program RegressOLS, eclass
	args touse b V N rank df_r

	mata: st_local("store_sample", strofreal(HDFE.store_sample))
	if (`store_sample') loc esample "esample(`touse')"

	mata: st_local("indepvars", HDFE.fullindepvars)
	if ("`indepvars'" != "") {
		matrix colnames `b' = `indepvars'
		matrix colnames `V' = `indepvars'
		matrix rownames `V' = `indepvars'
		_ms_findomitted `b' `V'
		ereturn post `b' `V', `esample' buildfvinfo depname(`depvar')
	}
	else {
		ereturn post, `esample' buildfvinfo depname(`depvar')
	}

	ereturn scalar N       = `N'
	ereturn scalar rank    = `rank'
	ereturn scalar df_r    = `df_r'
	ereturn local  cmd     "reghdfe"
	mata: HDFE.post()

	* Post stats
	cap conf matrix reghdfe_statsmatrix
	if (!c(rc)) {
		ereturn matrix summarize = reghdfe_statsmatrix
		mata: st_local("summarize_quietly", strofreal(HDFE.summarize_quietly))
		ereturn scalar summarize_quietly = `summarize_quietly'
	}
end

program Replay, rclass
	syntax [, *]

	if `"`e(cmd)'"' != "reghdfe"  {
	        error 301
	}

	_get_diopts options, `options'
	reghdfe_header // _coef_table_header
	di ""
	_coef_table, `options' // ereturn display, `options'
	return add // adds r(level), r(table), etc. to ereturn (before the footnote deletes them)
	reghdfe_footnote

	* Replay stats
	if (e(summarize_quietly)==0) {
		di as text _n "{sf:Regression Summary Statistics:}" _c
		matlist e(summarize)', border(top bottom) rowtitle(Variable) // twidth(18)
	}
end

program Stats
	args touse
	* Optional weights
	mata: st_local("weight", sprintf("[%s=%s]", HDFE.weight_type, HDFE.weight_var))
	assert "`weight'" != ""
	if ("`weight'" == "[=]") loc weight
	loc weight : subinstr local weight "[pweight" "[aweight"

	mata: st_local("stats", HDFE.summarize_stats)
	mata: st_local("varlist", HDFE.varlist)
	mata: st_local("cvars", invtokens(HDFE.cvars))
	loc full_varlist `varlist' `cvars'

	* quick workaround b/c -tabstat- does not support factor variables
	fvrevar `full_varlist', list
	loc full_varlist `r(varlist)'

	qui tabstat `full_varlist' if `touse' `weight' , stat(`stats') col(stat) save
	matrix reghdfe_statsmatrix = r(StatTotal)
end

*! version 5.3.4 13jan2018

program reghdfe, eclass
	* Intercept old+version
	cap syntax, version old
	if !c(rc) {
		reghdfe_old, version
		exit
	}

	* Intercept old
	cap syntax anything(everything) [fw aw pw/], [*] old
	if !c(rc) {
		di as error "(running historical version of reghdfe)"
		if ("`weight'"!="") local weightexp [`weight'=`exp']
		reghdfe_old `anything' `weightexp', `options'
		exit
	}

	* Aux. subcommands
	cap syntax, [*]
	if inlist("`options'", "check", "compile", "reload", "update", "version", "requirements") {
		if ("`options'"=="compile") loc args force
		if ("`options'"=="check") loc options compile
		if ("`options'"=="update") {
			loc args 1
			loc options reload
		}
		loc subcmd = proper("`options'")
		`subcmd' `args'
	}
	else if replay() {
		Replay `0'
	}
	else {
		Cleanup 0
		Compile // takes 0.01s to run this useful check (ensures .mlib exists)
		cap noi Estimate `0'
		Cleanup `c(rc)'
	}
end

program Compile
	args force

	* Check dependencies
	ftools, check // in case lftools.mlib does not exist or is outdated
	ms_get_version ftools, min_version("2.30.1")
	ms_get_version reghdfe // save local package_version
	loc list_objects "FixedEffects() fixed_effects() BipartiteGraph()"
	loc list_functions "reghdfe_*() transform_*() accelerate_*() panelmean() panelsolve_*() lsmr()"
	loc list_misc "weighted_quadcolsum() safe_divide() check_convergence() precompute_inv_xx() _st_data_wrapper()"
	// TODO: prefix everything with reghdfe_*

	ms_compile_mata, ///
		package(reghdfe) ///
		version(`package_version') ///
		fun("`list_objects' `list_functions' `list_misc'") ///
		verbose ///
		`force'
end

program Reload
	* Internal debugging tool.
	* Updates dependencies and reghdfe from local path or from github
	* Usage:
	* 	reghdfe, update // from c:\git\..
	* 	reghdfe, reload // from github

	args online
	if ("`online'" == "") loc online 0

	di as text _n "{bf:reghdfe: updating required packages}"
	di as text "{hline 64}"

	* -ftools- https://github.com/sergiocorreia/ftools/
	cap ado uninstall ftools
	if (`online') net install ftools, from("https://github.com/sergiocorreia/ftools/raw/master/src/")
	if (!`online') net install ftools, from("c:\git\ftools\src")
	di as text "{hline 64}"
	ftools, compile
	di as text "{hline 64}"

	* Update -reghdfe-
	di as text _n  _n "{bf:reghdfe: updating self}"
	di as text "{hline 64}"
	qui ado uninstall reghdfe
	if (`online') net install reghdfe, from("https://github.com/sergiocorreia/reghdfe/raw/master/src/")
	if (!`online') net install reghdfe, from("c:\git\reghdfe\src")
	qui which reghdfe
	di as text "{hline 64}"
	reghdfe, compile
	di as text "{hline 64}"

	* Cleaning up
	di as text _n "{bf:Note:} You need to run {stata program drop _all} now."
end

program Version
	which reghdfe
	Requirements
end

program Requirements
	di as text _n "Required packages installed?"
	loc reqs ftools
	// ivreg2 avar tuples group3hdfe
	if (c(stata_version)<13) loc reqs `reqs' boottest

	loc ftools_github "https://github.com/sergiocorreia/ftools/raw/master/src/"

	loc error 0

	foreach req of local reqs {
		loc fn `req'.ado
		cap findfile `fn'
		if (_rc) {
			loc error 1
			di as text "{lalign 20:- `req'}" as error "not" _c
			di as text "    {stata ssc install `req':install from SSC}" _c
			if inlist("`req'", "ftools") {
				loc github ``req'_github'
				di as text `"    {stata `"net install `req', from(`"`github'"')"':install from github}"'
			}
			else {
				di as text // newline
			}
		}
		else {
			di as text "{lalign 20:- `req'}" as text "yes"
		}
	}

	if (`error') exit 601
end

program Cleanup
	args rc
	cap mata: mata drop HDFE
	cap mata: mata drop hdfe_*
	cap drop __temp_reghdfe_resid__
	cap matrix drop reghdfe_statsmatrix
	if (`rc' == 132) {
		di as text "- If you got the {it:parentheses unbalanced} error, note that IV/2SLS was moved to {help ivreghdfe}"
		di as smcl `"- Latest version: {browse "https://github.com/sergiocorreia/ivreghdfe":https://github.com/sergiocorreia/ivreghdfe}"'
		di as smcl `"- SSC version: {stata "net describe ivreghdfe, from(http://fmwww.bc.edu/RePEc/bocode/i)"}"'
		di as smcl `"- Note: the older functionality can still be accessed through the {it:old} option"'
	}
	if (`rc') exit `rc'
end

program Parse
	* Trim whitespace (caused by "///" line continuations; aesthetic only)
	mata: st_local("0", stritrim(st_local("0")))

	* Main syntax
	#d;
	syntax varlist(fv ts numeric) [if] [in] [aw pw fw/] , [

		/* Model */
		Absorb(string) NOAbsorb
		SUmmarize SUmmarize2(string asis) /* simulate implicit options */

		/* Standard Errors */
		VCE(string) CLuster(string)

		/* Diagnostic */
		Verbose(numlist min=1 max=1 >=-1 <=5 integer)
		TIMEit

		/* Speedup and memory Tricks */
		NOSAMPle /* do not save e(sample) */
		COMPACT /* use as little memory as possible but is slower */

		/* Undocumented */
		KEEPSINgletons
		OLD /* use latest v3 */
		NOTES(string) /* NOTES(key=value ...), will be stored on e() */

		] [*] /* capture optimization options, display options, etc. */
		;
	#d cr

	* Unused
	* SAVEcache
	* USEcache
	* CLEARcache

	* Convert options to boolean
	if ("`verbose'" == "") loc verbose 0
	loc timeit = ("`timeit'"!="")
	loc drop_singletons = ("`keepsingletons'" == "")
	loc compact = ("`compact'" != "")

	if (`timeit') timer on 29

	* Sanity checks
	if (`verbose'>-1 & "`keepsingletons'"!="") {
		loc url "http://scorreia.com/reghdfe/nested_within_cluster.pdf"
		loc msg "WARNING: Singleton observations not dropped; statistical significance is biased"
		di as error `"`msg' {browse "`url'":(link)}"'
	}
	if ("`cluster'"!="") {
		_assert ("`vce'"==""), msg("cannot specify both cluster() and vce()")
		loc vce cluster `cluster'
		loc cluster // clear it to avoid bugs in subsequent lines
	}

	* Split varlist into <depvar> and <indepvars>
	ms_parse_varlist `varlist'
	if (`verbose' > 0) {
		di as text _n "## Parsing varlist: {res}`varlist'"
		return list
	}
	loc depvar `r(depvar)'
	loc indepvars `r(indepvars)'
	loc fe_format "`r(fe_format)'"
	loc basevars `r(basevars)'

	* Parse Weights
	if ("`weight'"!="") {
		unab exp : `exp', min(1) max(1) // simple weights only
	}

	* Parse VCE
	ms_parse_vce, vce(`vce') weighttype(`weight')
	if (`verbose' > 0) {
		di as text _n "## Parsing vce({res}`vce'{txt})"
		sreturn list
	}
	loc vcetype = "`s(vcetype)'"
	loc num_clusters = `s(num_clusters)'
	loc clustervars = "`s(clustervars)'"
	loc base_clustervars = "`s(base_clustervars)'"
	loc vceextra = "`s(vceextra)'"

	* Select sample (except for absvars)
	loc varlist `depvar' `indepvars' `base_clustervars'
	tempvar touse
	marksample touse, strok // based on varlist + cluster + if + in + weight

	* Parse noabsorb
	_assert  ("`absorb'`noabsorb'" != ""), msg("option {bf:absorb()} or {bf:noabsorb} required")
	if ("`noabsorb'" != "") {
		_assert ("`absorb'" == ""), msg("{bf:absorb()} and {bf:noabsorb} are mutually exclusive")
		tempvar c
		gen byte `c' = 1
		loc absorb `c'
	}

	if (`timeit') timer off 29

	* Construct HDFE object
	// SYNTAX: fixed_effects(absvars | , touse, wtype, wtvar, dropsing, verbose)
	mata: st_local("comma", strpos(`"`absorb'"', ",") ? "" : ",")
	if (`timeit') timer on 20
	mata: HDFE = fixed_effects(`"`absorb' `comma' `options'"', "`touse'", "`weight'", "`exp'", `drop_singletons', `verbose')
	if (`timeit') timer off 20
	mata: HDFE.cmdline = "reghdfe " + st_local("0")
	loc options `s(options)'

	mata: st_local("N", strofreal(HDFE.N))
	if (`N' == 0) error 2000

	* Fill out HDFE object
	* mata: HDFE.varlist = "`base_varlist'"
	mata: HDFE.depvar = "`depvar'"
	mata: HDFE.indepvars = "`indepvars'"
	mata: HDFE.vcetype = "`vcetype'"
	mata: HDFE.num_clusters = `num_clusters'
	mata: HDFE.clustervars = tokens("`clustervars'")
	mata: HDFE.base_clustervars = tokens("`base_clustervars'")
	mata: HDFE.vceextra = "`vceextra'"

	* Preserve memory
	mata: HDFE.compact = `compact'
	if (`compact') {
		loc panelvar "`_dta[_TSpanel]'"
		loc timevar "`_dta[_TStvar]'"
		mata: HDFE.panelvar = "`panelvar'"
		mata: HDFE.timevar = "`timevar'"
		c_local keepvars `basevars' `base_clustervars' `panelvar' `timevar' // `exp'
	}

	* Parse summarize
	if ("`summarize'" != "") {
		_assert ("`summarize2'" == ""), msg("summarize() syntax error")
		loc summarize2 mean min max  // default values
	}
	ParseSummarize `summarize2'
	mata: HDFE.summarize_stats = "`s(stats)'"
	mata: HDFE.summarize_quietly = `s(quietly)'

	* Parse misc options
	mata: HDFE.notes = `"`notes'"'
	mata: HDFE.store_sample = ("`nosample'"=="")
	mata: HDFE.timeit = `timeit'

	* Parse Coef Table Options (do this last!)
	_get_diopts diopts options, `options' // store in `diopts', and the rest back to `options'
	_assert (`"`options'"'==""), msg(`"invalid options: `options'"')
	if ("`hascons'"!="") di in ye "(option ignored: `hascons')"
	if ("`tsscons'"!="") di in ye "(option ignored: `tsscons')"
	mata: HDFE.diopts = `"`diopts'"'
end

program ParseSummarize, sclass
	sreturn clear
	syntax [namelist(name=stats)] , [QUIetly]
	local quietly = ("`quietly'"!="")
	sreturn loc stats "`stats'"
	sreturn loc quietly = `quietly'
end

// --------------------------------------------------------------------------

program Estimate, eclass
	ereturn clear

	* Parse and fill out HDFE object
	Parse `0'
	mata: st_local("timeit", strofreal(HDFE.timeit))
	mata: st_local("compact", strofreal(HDFE.compact))
	mata: st_local("verbose", strofreal(HDFE.verbose))

	* Compute degrees-of-freedom
	if (`timeit') timer on 21
	mata: HDFE.estimate_dof()
	if (`timeit') timer off 21

	* Save updated e(sample) (singletons reduce sample);
	* required to parse factor variables to partial out
	if (`timeit') timer on 29
	tempvar touse
	mata: HDFE.save_touse("`touse'")
	if (`timeit') timer off 29

	* Expand varlists
	if (`timeit') timer on 22
	mata: st_local("depvar", HDFE.depvar)
	mata: st_local("indepvars", HDFE.indepvars)
	if (`verbose' > 0) di as text _n "## Parsing and expanding indepvars: {res}`indepvars'"
	ms_expand_varlist `indepvars' if `touse'
	if (`verbose' > 0) return list
	mata: HDFE.fullindepvars = "`r(fullvarlist)'"
	mata: HDFE.indepvars = "`r(varlist)'"
	mata: HDFE.not_basevar = strtoreal(tokens("`r(not_omitted)'"))
	mata: HDFE.varlist = "`depvar' `r(varlist)'"
	if (`timeit') timer off 22

	* Stats
	mata: st_local("stats", HDFE.summarize_stats)
	if ("`stats'" != "") Stats `touse'

	* Condition number
	mata: HDFE.estimate_cond()

	* Preserve
	if (`compact') {
		if (`verbose' > 0) di as text "## Preserving dataset"
		preserve
		keep `keepvars'
	}

	* Partial out; save TSS of depvar
	if (`timeit') timer on 23
	// partial_out(Varlist/Matrix | , Save TSS if HDFE.tss is missing? [0], Standardize data? [1], First col is depvar? [1])
	mata: hdfe_variables = HDFE.partial_out(HDFE.varlist, 1, ., .)
	if (`timeit') timer off 23

	* Regress
	if (`timeit') timer on 24
	tempname b V N rank df_r
	mata: reghdfe_post_ols(HDFE, hdfe_variables, "`b'", "`V'", "`N'", "`rank'", "`df_r'")
	mata: hdfe_variables = .
	* Restore
	if (`compact') {
		if (`verbose' > 0) di as text "## Restoring dataset"
		restore
		mata: st_local("residuals", HDFE.residuals)
		if ("`residuals'" != "") mata: HDFE.save_variable(HDFE.residuals, HDFE.residuals_vector, "Residuals")
	}
	RegressOLS `touse' `b' `V' `N' `rank' `df_r'
	if (`timeit') timer off 24

	* (optional) Store FEs
	if (`timeit') timer on 29
	reghdfe_store_alphas
	if (`timeit') timer off 29

	* View estimation tables
	mata: st_local("diopts", HDFE.diopts)
	Replay, `diopts'

	if (`timeit') {
		di as text _n "{bf: Timer results:}"
		timer list
		di as text "Legend: 20: Create HDFE object; 21: Estimate DoF; 22: expand varlists; 23: partial out; 24: regress; 29: rest"
		di
	}
end

program RegressOLS, eclass
	args touse b V N rank df_r

	mata: st_local("store_sample", strofreal(HDFE.store_sample))
	if (`store_sample') loc esample "esample(`touse')"

	mata: st_local("indepvars", HDFE.fullindepvars)
	if ("`indepvars'" != "") {
		matrix colnames `b' = `indepvars'
		matrix colnames `V' = `indepvars'
		matrix rownames `V' = `indepvars'
		_ms_findomitted `b' `V'
		ereturn post `b' `V', `esample' buildfvinfo depname(`depvar')
	}
	else {
		ereturn post, `esample' buildfvinfo depname(`depvar')
	}

	ereturn scalar N       = `N'
	ereturn scalar rank    = `rank'
	ereturn scalar df_r    = `df_r'
	ereturn local  cmd     "reghdfe"
	mata: HDFE.post()

	* Post stats
	cap conf matrix reghdfe_statsmatrix
	if (!c(rc)) {
		ereturn matrix summarize = reghdfe_statsmatrix
		mata: st_local("summarize_quietly", strofreal(HDFE.summarize_quietly))
		ereturn scalar summarize_quietly = `summarize_quietly'
	}
end

program Replay, rclass
	syntax [, *]

	if `"`e(cmd)'"' != "reghdfe"  {
	        error 301
	}

	_get_diopts options, `options'
	reghdfe_header // _coef_table_header
	di ""
	_coef_table, `options' // ereturn display, `options'
	return add // adds r(level), r(table), etc. to ereturn (before the footnote deletes them)
	reghdfe_footnote

	* Replay stats
	if (e(summarize_quietly)==0) {
		di as text _n "{sf:Regression Summary Statistics:}" _c
		matlist e(summarize)', border(top bottom) rowtitle(Variable) // twidth(18)
	}
end

program Stats
	args touse
	* Optional weights
	mata: st_local("weight", sprintf("[%s=%s]", HDFE.weight_type, HDFE.weight_var))
	assert "`weight'" != ""
	if ("`weight'" == "[=]") loc weight
	loc weight : subinstr local weight "[pweight" "[aweight"

	mata: st_local("stats", HDFE.summarize_stats)
	mata: st_local("varlist", HDFE.varlist)
	mata: st_local("cvars", invtokens(HDFE.cvars))
	loc full_varlist `varlist' `cvars'

	* quick workaround b/c -tabstat- does not support factor variables
	fvrevar `full_varlist', list
	loc full_varlist `r(varlist)'

	qui tabstat `full_varlist' if `touse' `weight' , stat(`stats') col(stat) save
	matrix reghdfe_statsmatrix = r(StatTotal)
end


Update Country_SpecificQs.do

Update Country_SpecificQs.do

** this program reads in the grand merge file and prints each country specific question code to the console
** read the grand_merge file
use "C:\Users\zoorobmj\Desktop\grand_merge.dta"

* this loop will get every country specific question
** to instead find any particular country, delete outer loop and replace pais with number equal to pais of interest (line 13)
*** plausibly one could toggle with r(c) == [num] if you wanted to, say, find questions asked in <5 countries, etc
**** note that if you do not neeed to know which qs go with which country, delete outer loop
***** and 2nd if statement for drastically improved performance.
foreach i of num 1/17 21/29 40 41 {
	display "These questions are for Country ", "`i'"
	foreach var of varlist q1-q11nr {
		capture quietly tab `var' pais
		* I know it's country specific if r(c) == 1
		if r(c) == 1 {
		quietly sum `var' if pais == `i'
		* if it's ctry specific AND has results for Nicaragua, it is a Nicaragua-only question
			if r(N) > 0 {
				display "`var'"
				}
		}
	}
}

** this program reads in the grand merge file and prints each country specific question code to the console
** read the grand_merge file
use "C:\Users\zoorobmj\Desktop\grand_merge.dta"

* this loop will get every country specific question
** to instead find any particular country, delete outer loop and replace pais with number equal to pais of interest (line 13)
*** plausibly one could toggle with r(c) == [num] if you wanted to, say, find questions asked in <5 countries, etc
foreach i of num 1/17 21/29 40 41 {
	foreach var of varlist q1-q11nr {
		capture quietly tab `var' pais
		* I know it's country specific if r(c) == 1
		if r(c) == 1 {
		quietly sum `var' if pais == `i'
		* if it's ctry specific AND has results for Nicaragua, it is a Nicaragua-only question
			if r(N) > 0 {
				display "`var'"
				}
		}
	}
}


a couple of to.do items

a couple of to.do items

= Honu: to.do: =

  o bloggish directories should have a "make wc" target that subtracts the word count of
    the template file.  Or, better, use sed to filter out tags.

  o bootstrap should check ~/.ssh to see whether to use savitzky.net or github.

  o Much of the configuration should be refactored to put more of the customization into
    Myrtle/Mathilda; Honu should become more of a framework.  A similar structure might
    work for MakeStuff.

  o Some of the configuration is obsolete, and is unlikely ever to be used again.

========================================================================================>
Work Log:
========

2015:
=====

1111We
  @ OpenSSH/Cookbook/Multiplexing - Wikibooks
    Makes ssh much faster, and circumvents dreamhost's connection limits.
    > mkdir ~/.ssh/controlmasters; chmod 700 ~/.ssh/controlmasters
  * Add the following to .ssh/config, in the global section at the top:
      ControlPath ~/.ssh/controlmasters/%r@%h:%p
      ControlPersist 10m
      ControlMaster auto
  @ https://github.com/blipvert/ljupdate/commits/master marginally more up to date
    * update emacs/depends.make

2016:
=====

0214Su
  * much hacking in gnome-setup and font setup over the last week or so.

0215Mo
  * added adobe-source-code-pro font, tell emacs to use system font.

... Really ought to copy the config-related stuff out of starport/Admin/admin.to.do

0626
  ~ alltray looked useful - lets you put an icon for any program into the tray.
    It starts the program if it's not already running, but fails in xmonad because it's
    waiting for the program to be iconified.
  * gsimplecal, fbpanel, and probably others have subdirectories in .config
    probably best is to make another directory called dotconfig, parallel to dotfiles

0627Mo
  * made the Makefiles independendent of Tools/Makefile.

0704Mo
  * git subtree add --prefix bin ../bin master
  * The idea is that if there's a pre-existing ~/bin, we can symlink into it.

0705Tu
  * regular dotfiles should start with "_" rather than ".", to make them visible.
    -> move dotfiles/.* -> dotfiles/_*; update dotfiles/Makefile

0716Sa
  . Some dotfiles may need to be modified by a script rather than simply symlinked.
    That's because things like username and email address change from one system to
    another.  In most cases, though, one can use include files generated by make install.

0719Tu
  * When site-dependent parameters can be in separate files (include files/modules).
    the various install targets can initialize them from templates.  A good place to start
    would be .xmonad/Local.hs.  Could also do this for .bashrc; a conditional include
    would make it unnecessary to generate the file if it's not needed. (0716)
    : for gitconfig - note that the file doesn't need to exist
      [include]
	  path = /path/to/file

0725Mo
  * color-coded workspace labels.  ws N makes an xclock with WS=N and the appropriate
    resistor color code.
  * the ability to make ad-hoc workspace labels might help.  Put the list in a file that
    can be read in either xmonad or the bottom-bars script.  Possibly a top dbar with
    vertical menu. (0719) -> add a second argument to ws.
  * Re-running ws N should grep ps for the appropriate color code and kill it.
    An alternative to xclock would be dzen -p.

0726Tu
  x use dzen for workspace labels?  Note that dzen is smaller, but only slightly.
    Problem is that it doesn't seem possible to have it shrinkwrap text.
  * separate widths for first and other top bars; make default 950; adjust ws

0827Sa
  * Allow MakeStuff as well as Tools.  Eventually it will be possible to drop Tools as
    an alternative.

0828Su
  ^ Config -> Honu.  The only real question is whether I want to make Honu a separate
    version, cleaned up and generalized for distribution. -> I don't think so.  It would
    be hard to keep them in sync.  The main me-specific things are the link pages in web,
    and some of the stuff in Archive, but there are also some of the dotfiles (e.g.
    .signature and .Xdefaults-*), stuff in the emacs config, and...  On second thought: I
    think I'd be a very uncomfortable about putting .mailrc, .gnus, and the bookmarks out
    there.  So there's going to have to be a rewrite anyway.  Those can go in -Personal-
  ^ Problematic stuff -- bookmarks, .mailrc, .gnus, etc. belongs in a separate git repo
    that's a subdirectory of local!  Can of course have dotfiles and dotconfig subdirs.
    Can have its own bootstrap, which would get the user-specific out of there, too.  In
    fact, call it local/local-Config or local/My-Honu or some such and derive it from the
    original, with all the old history.  Make a relatively clean start for Honu.

0829Mo
  o bootstrap and the first run of make should log messages, and print out the log on
    exit.

0904Su
  * clone from Config; clean up with bfg:
    78  alias bfg='java -jar ~/Downloads/bfg-1.12.13.jar'
    85  bfg --help
    87  git commit -m "deletions before bfg"
    88  bfg --delete-folders Archive
    91  bfg --delete-folders Archive
    92  bfg --delete-folders web
    94  bfg -D [._]mailrc
    95  bfg -D [._]gnus
    96  bfg -D [._]sig*
    97  bfg -D [hw].home.html
   107  git filter-branch --prune-empty
   110  git reflog expire --expire=now --all && git gc --prune=now --aggressive

0905Mo
  * Push to github.
  o .emacs:  generate, and load Honu/emacs/emacs.el and Honu/local/emacs.el
  o testing is needed:
    Set up a testing environment by setting USER and HOME, and putting a dummy sudo
    that simply records what it's supposed to do on $PATH.  Ought to have a dummy groups
    command as well. -> see MakeStuff for test framework.
  o Originals:  Should go into, e.g., local/orig  (ignore subdirectories of local)
  * Myrtle/Mathilda - install into the same directory as Honu and MakeStuff.
    symlink files and subdirs into Honu/local, or directly into $HOME if they don't
    overlap.
  o add scsitools to packages

0911Su
  * no links to bottom-bars, bottom-mobars
  o length on bottom-bars -- see ws for how to get this

0912Mo
  ^ At some point, we may want to move some of the settings most likely to be customized
    out into Myrtle/Mathilda.  That includes whether to pull down MakeStuff -- Honu should
    be completely stand-alone and not need much customization.

1226Mo
  ~ use roxterm as the preferred terminal rather than gnome-terminal.
    Can use roxterm --separate --name=wsN with wssetup

2017:
=====

0128Sa
  : roxterm doesn't work as well as I'd like; the profile is split up and some parts can't
    have their defaults configured (in particular whether to show the tab bar when there's
    only one tab present, which is the usual case the way I do it.

========================================================================================>|
Local Variables:
    fill-column:90
End:

= Honu: to.do: =

  o bloggish directories should have a "make wc" target that subtracts the word count of
    the template file.  Or, better, use sed to filter out tags.

  o bootstrap should check ~/.ssh to see whether to use savitzky.net or github.

========================================================================================>
Work Log:
========

2015:
=====

1111We
  @ OpenSSH/Cookbook/Multiplexing - Wikibooks
    Makes ssh much faster, and circumvents dreamhost's connection limits.
    > mkdir ~/.ssh/controlmasters; chmod 700 ~/.ssh/controlmasters
  * Add the following to .ssh/config, in the global section at the top:
      ControlPath ~/.ssh/controlmasters/%r@%h:%p
      ControlPersist 10m
      ControlMaster auto
  @ https://github.com/blipvert/ljupdate/commits/master marginally more up to date
    * update emacs/depends.make

2016:
=====

0214Su
  * much hacking in gnome-setup and font setup over the last week or so.

0215Mo
  * added adobe-source-code-pro font, tell emacs to use system font.

... Really ought to copy the config-related stuff out of starport/Admin/admin.to.do

0626
  ~ alltray looked useful - lets you put an icon for any program into the tray.
    It starts the program if it's not already running, but fails in xmonad because it's
    waiting for the program to be iconified.
  * gsimplecal, fbpanel, and probably others have subdirectories in .config
    probably best is to make another directory called dotconfig, parallel to dotfiles

0627Mo
  * made the Makefiles independendent of Tools/Makefile.

0704Mo
  * git subtree add --prefix bin ../bin master
  * The idea is that if there's a pre-existing ~/bin, we can symlink into it.

0705Tu
  * regular dotfiles should start with "_" rather than ".", to make them visible.
    -> move dotfiles/.* -> dotfiles/_*; update dotfiles/Makefile

0716Sa
  . Some dotfiles may need to be modified by a script rather than simply symlinked.
    That's because things like username and email address change from one system to
    another.  In most cases, though, one can use include files generated by make install.

0719Tu
  * When site-dependent parameters can be in separate files (include files/modules).
    the various install targets can initialize them from templates.  A good place to start
    would be .xmonad/Local.hs.  Could also do this for .bashrc; a conditional include
    would make it unnecessary to generate the file if it's not needed. (0716)
    : for gitconfig - note that the file doesn't need to exist
      [include]
	  path = /path/to/file

0725Mo
  * color-coded workspace labels.  ws N makes an xclock with WS=N and the appropriate
    resistor color code.
  * the ability to make ad-hoc workspace labels might help.  Put the list in a file that
    can be read in either xmonad or the bottom-bars script.  Possibly a top dbar with
    vertical menu. (0719) -> add a second argument to ws.
  * Re-running ws N should grep ps for the appropriate color code and kill it.
    An alternative to xclock would be dzen -p.

0726Tu
  x use dzen for workspace labels?  Note that dzen is smaller, but only slightly.
    Problem is that it doesn't seem possible to have it shrinkwrap text.
  * separate widths for first and other top bars; make default 950; adjust ws

0827Sa
  * Allow MakeStuff as well as Tools.  Eventually it will be possible to drop Tools as
    an alternative.

0828Su
  ^ Config -> Honu.  The only real question is whether I want to make Honu a separate
    version, cleaned up and generalized for distribution. -> I don't think so.  It would
    be hard to keep them in sync.  The main me-specific things are the link pages in web,
    and some of the stuff in Archive, but there are also some of the dotfiles (e.g.
    .signature and .Xdefaults-*), stuff in the emacs config, and...  On second thought: I
    think I'd be a very uncomfortable about putting .mailrc, .gnus, and the bookmarks out
    there.  So there's going to have to be a rewrite anyway.  Those can go in -Personal-
  ^ Problematic stuff -- bookmarks, .mailrc, .gnus, etc. belongs in a separate git repo
    that's a subdirectory of local!  Can of course have dotfiles and dotconfig subdirs.
    Can have its own bootstrap, which would get the user-specific out of there, too.  In
    fact, call it local/local-Config or local/My-Honu or some such and derive it from the
    original, with all the old history.  Make a relatively clean start for Honu.

0829Mo
  o bootstrap and the first run of make should log messages, and print out the log on
    exit.

0904Su
  * clone from Config; clean up with bfg:
    78  alias bfg='java -jar ~/Downloads/bfg-1.12.13.jar'
    85  bfg --help
    87  git commit -m "deletions before bfg"
    88  bfg --delete-folders Archive
    91  bfg --delete-folders Archive
    92  bfg --delete-folders web
    94  bfg -D [._]mailrc
    95  bfg -D [._]gnus
    96  bfg -D [._]sig*
    97  bfg -D [hw].home.html
   107  git filter-branch --prune-empty
   110  git reflog expire --expire=now --all && git gc --prune=now --aggressive

0905Mo
  * Push to github.
  o .emacs:  generate, and load Honu/emacs/emacs.el and Honu/local/emacs.el
  o testing is needed:
    Set up a testing environment by setting USER and HOME, and putting a dummy sudo
    that simply records what it's supposed to do on $PATH.  Ought to have a dummy groups
    command as well. -> see MakeStuff for test framework.
  o Originals:  Should go into, e.g., local/orig  (ignore subdirectories of local)
  * Myrtle/Mathilda - install into the same directory as Honu and MakeStuff.
    symlink files and subdirs into Honu/local, or directly into $HOME if they don't
    overlap.
  o add scsitools to packages

0911Su
  * no links to bottom-bars, bottom-mobars
  o length on bottom-bars -- see ws for how to get this

0912Mo
  ^ At some point, we may want to move some of the settings most likely to be customized
    out into Myrtle/Mathilda.  That includes whether to pull down MakeStuff -- Honu should
    be completely stand-alone and not need much customization.

1226Mo
  o use roxterm as the preferred terminal rather than gnome-terminal.
    Can use roxterm --separate --name=wsN with wssetup

========================================================================================>|
Local Variables:
    fill-column:90
End:


Even more normalization of output!

Even more normalization of output!

* settings.do
set linesize 119
* make all platforms consistent in their batch output;
* NOTE: this is autoincluded to all tests via makefile magic
* tip buried in http://web.stanford.edu/group/ssds/cgi-bin/drupal/files/Guides/Stata_Unix__2011.pdf
* ALSO these comments are *after* the set, because it affects how stata prints out comments.

* settings.do
* make all platforms consistent in their batch output;
* tip buried in http://web.standford.edu/group/ssds/cgi-bin/drupal/files/Guides/Stata_Unix__2011.pdf*/
* NOTE: this is autoincluded to all tests via makefile magic
set linesize 119