Fix #87

Fix #87

*! version 4.1.0 28feb2017

program reghdfe, eclass
	* Intercept old+version
	cap syntax, version old
	if !c(rc) {
		reghdfe_old, version
		exit
	}

	* Intercept old
	cap syntax anything(everything) [fw aw pw/], [*] old
	if !c(rc) {
		di as error "(running historical version of reghdfe)"
		if ("`weight'"!="") local weightexp [`weight'=`exp']
		reghdfe_old `anything' `weightexp', `options'
		exit
	}

	* Aux. subcommands
	cap syntax, [*]
	if inlist("`options'", "check", "compile", "reload", "update", "version", "requirements") {
		if ("`options'"=="compile") loc args force
		if ("`options'"=="check") loc options compile
		if ("`options'"=="update") {
			loc args 1
			loc options reload
		}
		loc subcmd = proper("`options'")
		`subcmd' `args'
	}
	else if replay() {
		Replay `0'
	}
	else {
		Cleanup 0
		Compile // takes 0.01s to run this useful check (ensures .mlib exists)
		cap noi Estimate `0'
		Cleanup `c(rc)'
	}
end

program Compile
	args force
	ftools, check // in case lftools.mlib does not exist or is outdated
	ms_get_version reghdfe // from moresyntax package; save local package_version

	loc list_objects "FixedEffects() fixed_effects() BipartiteGraph()"
	loc list_functions "reghdfe_*() transform_*() accelerate_*() panelmean() panelsolve_*() lsmr()"
	loc list_misc "weighted_quadcolsum() safe_divide() check_convergence()"
	// TODO: prefix everything with reghdfe_*

	ms_compile_mata, ///
		package(reghdfe) ///
		version(`package_version') ///
		fun("`list_objects' `list_functions' `list_misc'") ///
		verbose ///
		`force'
end

program Reload
	* Internal debugging tool.
	* Updates dependencies and reghdfe from local path or from github
	* Usage:
	* 	reghdfe, update // from c:\git\..
	* 	reghdfe, reload // from github

	args online
	if ("`online'" == "") loc online 0

	di as text _n "{bf:reghdfe: updating required packages}"
	di as text "{hline 64}"

	* -moresyntax- https://github.com/sergiocorreia/moresyntax/
	cap ado uninstall moresyntax
	if (`online') net install moresyntax, from("https://github.com/sergiocorreia/moresyntax/raw/master/src/")
	if (!`online') net install moresyntax, from("c:\git\moresyntax\src")
	di as text "{hline 64}"

	* -ftools- https://github.com/sergiocorreia/ftools/
	cap ado uninstall ftools
	if (`online') net install ftools, from("https://github.com/sergiocorreia/ftools/raw/master/src/")
	if (!`online') net install ftools, from("c:\git\ftools\src")
	di as text "{hline 64}"
	ftools, compile // requires moresyntax
	di as text "{hline 64}"

	* Update -reghdfe-
	di as text _n  _n "{bf:reghdfe: updating self}"
	di as text "{hline 64}"
	qui ado uninstall reghdfe
	if (`online') net install reghdfe, from("https://github.com/sergiocorreia/reghdfe/raw/version-4/src/")
	if (!`online') net install reghdfe, from("c:\git\reghdfe\src")
	qui which reghdfe
	di as text "{hline 64}"
	reghdfe, compile
	di as text "{hline 64}"

	* Cleaning up
	di as text _n "{bf:Note:} You need to run {stata program drop _all} now."
end

program Version
	which reghdfe
	Requirements
end

program Requirements
	di as text _n "Required packages installed?"
	loc reqs moresyntax ftools
	// ivreg2 avar tuples group3hdfe
	if (c(version)<13) loc reqs `reqs' boottest

	loc ftools_github "https://github.com/sergiocorreia/ftools/raw/master/src/"
	loc moresyntax_github "https://github.com/sergiocorreia/moresyntax/raw/master/src/"

	loc error 0

	foreach req of local reqs {
		loc fn `req'.ado
		if ("`req'"=="moresyntax") loc fn ms_get_version.ado
		cap findfile `fn'
		if (_rc) {
			loc error 1
			di as text "{lalign 20:- `req'}" as error "not" _c
			di as text "    {stata ssc install `req':install from SSC}" _c
			if inlist("`req'", "ftools", "moresyntax") {
				loc github ``req'_github'
				di as text `"    {stata `"net install `req', from(`"`github'"')"':install from github}"'
			}
			else {
				di as text // newline
			}
		}
		else {
			di as text "{lalign 20:- `req'}" as text "yes"
		}
	}

	if (`error') exit 601
end

program Cleanup
	args rc
	cap mata: mata drop HDFE
	cap mata: mata drop hdfe_*
	cap drop __temp_reghdfe_resid__
	cap matrix drop reghdfe_statsmatrix
	if (`rc') exit `rc'
end

program Parse
	* Trim whitespace (caused by "///" line continuations; aesthetic only)
	mata: st_local("0", stritrim(st_local("0")))

	* Main syntax
	#d;
	syntax anything(id=varlist equalok) [if] [in] [aw pw fw/] , [

		/* Model */
		Absorb(string) NOAbsorb
		RESiduals(name) RESiduals2 /* use _reghdfe_resid */
		SUmmarize SUmmarize2(string asis) /* simulate implicit options */

		/* Standard Errors */
		VCE(string) CLuster(string)

		/* Diagnostic */
		Verbose(numlist min=1 max=1 >=-1 <=5 integer)
		TIMEit

		/* Speedup and memory Tricks */
		NOSAMPle /* do not save e(sample) */

		/* Undocumented */
		KEEPSINgletons
		OLD /* use latest v3 */
		NOTES(string) /* NOTES(key=value ...), will be stored on e() */

		] [*] /* capture optimization options, display options, etc. */
		;
	#d cr

	* Unused
	* SAVEcache
	* USEcache
	* CLEARcache
	* COMPACT /* use as little memory as possible but is slower */

	* Convert options to boolean
	if ("`verbose'" == "") loc verbose 0
	loc timeit = ("`timeit'"!="")
	loc drop_singletons = ("`keepsingletons'"=="")

	if (`timeit') timer on 29

	* Sanity checks
	if (`verbose'>-1 & "`keepsingletons'"!="") {
		loc url "http://scorreia.com/reghdfe/nested_within_cluster.pdf"
		loc msg "WARNING: Singleton observations not dropped; statistical significance is biased"
		di as error `"`msg' {browse "`url'":(link)}"'
	}
	if ("`cluster'"!="") {
		_assert ("`vce'"==""), msg("cannot specify both cluster() and vce()")
		loc vce cluster `cluster'
		loc cluster // clear it to avoid bugs in subsequent lines
	}

	* Parse Varlist
	ms_fvunab `anything'
	ms_parse_varlist `s(varlist)'
	loc base_varlist "`s(basevars)'"
	foreach cat in depvar indepvars endogvars instruments {
		loc original_`cat' "`s(`cat')'"
	}
	loc model = cond("`s(instruments)'" == "", "ols", "iv")
	loc original_varlist = "`s(varlist)'" // no parens or equal

	* Parse Weights
	if ("`weight'"!="") {
		unab exp : `exp', min(1) max(1) // simple weights only
	}

	* Parse VCE
	ms_parse_vce, vce(`vce') weighttype(`weight')
	loc vcetype = "`s(vcetype)'"
	loc num_clusters = `s(num_clusters)'
	loc clustervars = "`s(clustervars)'"
	loc base_clustervars = "`s(base_clustervars)'"
	loc vceextra = "`s(vceextra)'"

	* Select sample (except for absvars)
	loc varlist `original_varlist' `base_clustervars'
	tempvar touse
	marksample touse, strok // based on varlist + cluster + if + in + weight

	* Parse noabsorb
	_assert  ("`absorb'`noabsorb'" != ""), msg("option {bf:absorb()} or {bf:noabsorb} required")
	if ("`noabsorb'" != "") {
		_assert ("`absorb'" == ""), msg("{bf:absorb()} and {bf:noabsorb} are mutually exclusive")
		tempvar c
		gen byte `c' = 1
		loc absorb `c'
	}

	if (`timeit') timer off 29

	* Construct HDFE object
	// SYNTAX: fixed_effects(absvars | , touse, wtype, wtvar, dropsing, verbose)
	mata: st_local("comma", strpos(`"`absorb'"', ",") ? "" : ",")
	if (`timeit') timer on 20
	mata: HDFE = fixed_effects(`"`absorb' `comma' `options'"', "`touse'", "`weight'", "`exp'", `drop_singletons', `verbose')
	if (`timeit') timer off 20
	mata: HDFE.cmdline = "reghdfe " + st_local("0")
	loc options `s(options)'

	mata: st_local("N", strofreal(HDFE.N))
	if (`N' == 0) error 2000

	* Fill out HDFE object
	mata: HDFE.varlist = "`base_varlist'"
	mata: HDFE.original_depvar = "`original_depvar'"
	mata: HDFE.original_indepvars = "`original_indepvars'"
	mata: HDFE.original_endogvars = "`original_endogvars'"
	mata: HDFE.original_instruments = "`original_instruments'"
	mata: HDFE.original_varlist = "`original_varlist'"
	mata: HDFE.model = "`model'"

	mata: HDFE.vcetype = "`vcetype'"
	mata: HDFE.num_clusters = `num_clusters'
	mata: HDFE.clustervars = tokens("`clustervars'")
	mata: HDFE.base_clustervars = tokens("`base_clustervars'")
	mata: HDFE.vceextra = "`vceextra'"

	* Parse summarize
	if ("`summarize'" != "") {
		_assert ("`summarize2'" == ""), msg("summarize() syntax error")
		loc summarize2 mean min max  // default values
	}
	ParseSummarize `summarize2'
	mata: HDFE.summarize_stats = "`s(stats)'"
	mata: HDFE.summarize_quietly = `s(quietly)'

	* Parse residuals
	if ("`residuals2'" != "") {
		_assert ("`residuals'" == ""), msg("residuals() syntax error")
		loc residuals _reghdfe_resid
		cap drop `residuals' // destructive!
	}
	else if ("`residuals'"!="") {
		conf new var `residuals'
	}
	mata: HDFE.residuals = "`residuals'"

	* Parse misc options
	mata: HDFE.notes = `"`notes'"'
	mata: HDFE.store_sample = ("`nosample'"=="")
	mata: HDFE.timeit = `timeit'

	* Parse Coef Table Options (do this last!)
	_get_diopts diopts options, `options' // store in `diopts', and the rest back to `options'
	_assert (`"`options'"'==""), msg(`"invalid options: `options'"')
	if ("`hascons'"!="") di in ye "(option ignored: `hascons')"
	if ("`tsscons'"!="") di in ye "(option ignored: `tsscons')"
	mata: HDFE.diopts = `"`diopts'"'
end

program ParseSummarize, sclass
	sreturn clear
	syntax [namelist(name=stats)] , [QUIetly]
	local quietly = ("`quietly'"!="")
	sreturn loc stats "`stats'"
	sreturn loc quietly = `quietly'
end

// --------------------------------------------------------------------------

program Estimate, eclass
	ereturn clear

	* Parse and fill out HDFE object
	Parse `0'
	mata: st_local("timeit", strofreal(HDFE.timeit))

	* Compute degrees-of-freedom
	if (`timeit') timer on 21
	mata: HDFE.estimate_dof()
	if (`timeit') timer off 21

	* Save updated e(sample) (singletons reduce sample);
	* required to parse factor variables to partial out
	if (`timeit') timer on 29
	tempvar touse
	mata: HDFE.save_touse("`touse'")
	if (`timeit') timer off 29

	* Expand varlists
	if (`timeit') timer on 22
	// BUGBUG: do we really need HDFE.varlist ? or can we create it by concatenating the others?
	loc varlist
	foreach cat in /*varlist*/ depvar indepvars endogvars instruments {
		mata: st_local("vars", HDFE.original_`cat')
		if ("`vars'" == "") continue
		// HACK: addbn replaces 0.foreign with 0bn.foreign , to prevent st_data() from loading a bunch of zeros
		ms_fvstrip `vars' if `touse', expand dropomit addbn onebyone
		// If we don't use onebyone, then 1.x 2.x ends up as 2.x
		loc vars "`r(varlist)'"
		loc varlist `varlist' `vars'
		mata: HDFE.`cat' = "`vars'"
	}
	mata: HDFE.varlist = "`varlist'"
	if (`timeit') timer off 22

	* Stats
	mata: st_local("stats", HDFE.summarize_stats)
	if ("`stats'" != "") Stats

	* Condition number
	mata: HDFE.estimate_cond()

	* Partial out; save TSS of depvar
	if (`timeit') timer on 23
	mata: hdfe_variables = HDFE.partial_out(HDFE.varlist, 1) // 1=Save TSS of first var if HDFE.tss is missing
	if (`timeit') timer off 23

	* Regress
	mata: assert(HDFE.model=="ols")
	if (`timeit') timer on 24
	RegressOLS `touse'
	if (`timeit') timer off 24

	* (optional) Store FEs
	if (`timeit') timer on 29
	mata: st_local("save_any_fe", strofreal(HDFE.save_any_fe))
	assert inlist(`save_any_fe', 0, 1)
	if (`save_any_fe') {
		_assert e(depvar) != "", msg("e(depvar) is empty")
		_assert e(resid) != "", msg("e(resid) is empty")
		confirm numeric var `e(depvar)', exact
		confirm numeric var `e(resid)', exact
		tempvar d
		if (e(rank)) {
			qui _predict double `d' if e(sample), xb
		}
		else {
			gen double `d' = 0
		}
		qui replace `d' = `e(depvar)' - `d' - `e(resid)' if e(sample)
		mata: HDFE.store_alphas("`d'")
		drop `d'

		// Drop resid if we don't want to save it; and update e(resid)
		cap drop __temp_reghdfe_resid__
		if (!c(rc)) ereturn local resid
	}
	if (`timeit') timer off 29

	* View estimation tables
	mata: st_local("diopts", HDFE.diopts)
	Replay, `diopts'

	// ~~ Preserve relevant dataset ~~-
	//
	// if (`compact') {
	// 	preserve
	// 	keep `varlist' `absorb'
	// 	loc N = c(N)
	// }
	// else {
	// 	qui cou if `touse'
	// 	loc N = r(N)
	// }

	if (`timeit') {
		di as text _n "{bf: Timer results:}"
		timer list
		di as text "Legend: 20: Create HDFE object; 21: Estimate DoF; 22: expand varlists; 23: partial out; 24: regress; 29: rest"
		di
	}
end

program RegressOLS, eclass
	args touse

	tempname b V N rank df_r
	mata: reghdfe_post_ols(HDFE, hdfe_variables, "`b'", "`V'", "`N'", "`rank'", "`df_r'")
	mata: st_local("indepvars", HDFE.indepvars)
	mata: hdfe_variables = .

	loc esample
	mata: st_local("store_sample", strofreal(HDFE.store_sample))
	if (`store_sample') loc esample "esample(`touse')"

	if ("`indepvars'" != "") {
		matrix colnames `b' = `indepvars'
		matrix colnames `V' = `indepvars'
		matrix rownames `V' = `indepvars'
		ereturn post `b' `V', `esample' buildfvinfo depname(`depvar')
	}
	else {
		ereturn post, `esample' buildfvinfo depname(`depvar')
	}

	ereturn scalar N       = `N'
	ereturn scalar rank    = `rank'
	ereturn scalar df_r    = `df_r'
	ereturn local  cmd     "reghdfe"
	mata: HDFE.post(HDFE)

	* Post stats
	cap conf matrix reghdfe_statsmatrix
	if (!c(rc)) {
		ereturn matrix summarize = reghdfe_statsmatrix
		mata: st_local("summarize_quietly", strofreal(HDFE.summarize_quietly))
		ereturn scalar summarize_quietly = `summarize_quietly'
	}
end

program Replay, rclass
	syntax [, *]
	_get_diopts options, `options'
	reghdfe_header // _coef_table_header
	di ""
	_coef_table, `options' // ereturn display, `options'
	return add // adds r(level), r(table), etc. to ereturn (before the footnote deletes them)
	reghdfe_footnote

	* Replay stats
	if (e(summarize_quietly)==0) {
		di as text _n "{sf:Regression Summary Statistics:}" _c
		matlist e(summarize)', border(top bottom) rowtitle(Variable) // twidth(18)
	}
end

program Stats
	* Optional weights
	mata: st_local("weight", sprintf("[%s=%s]", HDFE.weight_type, HDFE.weight_var))
	assert "`weight'" != ""
	if ("`weight'" == "[=]") loc weight
	loc weight : subinstr local weight "[pweight" "[aweight"

	mata: st_local("stats", HDFE.summarize_stats)
	mata: st_local("varlist", HDFE.varlist)
	mata: st_local("cvars", invtokens(HDFE.cvars))
	loc full_varlist `varlist' `cvars'

	qui tabstat `full_varlist' `weight' , stat(`stats') col(stat) save
	matrix reghdfe_statsmatrix = r(StatTotal)
end

*! version 4.1.0 28feb2017

program reghdfe, eclass
	* Intercept old+version
	cap syntax, version old
	if !c(rc) {
		reghdfe_old, version
		exit
	}

	* Intercept old
	cap syntax anything(everything) [fw aw pw/], [*] old
	if !c(rc) {
		di as error "(running historical version of reghdfe)"
		if ("`weight'"!="") local weightexp [`weight'=`exp']
		reghdfe_old `anything' `weightexp', `options'
		exit
	}

	* Aux. subcommands
	cap syntax, [*]
	if inlist("`options'", "check", "compile", "reload", "update", "version", "requirements") {
		if ("`options'"=="compile") loc args force
		if ("`options'"=="check") loc options compile
		if ("`options'"=="update") {
			loc args 1
			loc options reload
		}
		loc subcmd = proper("`options'")
		`subcmd' `args'
	}
	else if replay() {
		Replay `0'
	}
	else {
		Cleanup 0
		Compile // takes 0.01s to run this useful check (ensures .mlib exists)
		cap noi Estimate `0'
		Cleanup `c(rc)'
	}
end

program Compile
	args force
	ftools, check // in case lftools.mlib does not exist or is outdated
	ms_get_version reghdfe // from moresyntax package; save local package_version

	loc list_objects "FixedEffects() fixed_effects() BipartiteGraph()"
	loc list_functions "reghdfe_*() transform_*() accelerate_*() panelmean() panelsolve_*() lsmr()"
	loc list_misc "weighted_quadcolsum() safe_divide() check_convergence()"
	// TODO: prefix everything with reghdfe_*

	ms_compile_mata, ///
		package(reghdfe) ///
		version(`package_version') ///
		fun("`list_objects' `list_functions' `list_misc'") ///
		verbose ///
		`force'
end

program Reload
	* Internal debugging tool.
	* Updates dependencies and reghdfe from local path or from github
	* Usage:
	* 	reghdfe, update // from c:\git\..
	* 	reghdfe, reload // from github

	args online
	if ("`online'" == "") loc online 0

	di as text _n "{bf:reghdfe: updating required packages}"
	di as text "{hline 64}"

	* -moresyntax- https://github.com/sergiocorreia/moresyntax/
	cap ado uninstall moresyntax
	if (`online') net install moresyntax, from("https://github.com/sergiocorreia/moresyntax/raw/master/src/")
	if (!`online') net install moresyntax, from("c:\git\moresyntax\src")
	di as text "{hline 64}"

	* -ftools- https://github.com/sergiocorreia/ftools/
	cap ado uninstall ftools
	if (`online') net install ftools, from("https://github.com/sergiocorreia/ftools/raw/master/src/")
	if (!`online') net install ftools, from("c:\git\ftools\src")
	di as text "{hline 64}"
	ftools, compile // requires moresyntax
	di as text "{hline 64}"

	* Update -reghdfe-
	di as text _n  _n "{bf:reghdfe: updating self}"
	di as text "{hline 64}"
	qui ado uninstall reghdfe
	if (`online') net install reghdfe, from("https://github.com/sergiocorreia/reghdfe/raw/version-4/src/")
	if (!`online') net install reghdfe, from("c:\git\reghdfe\src")
	qui which reghdfe
	di as text "{hline 64}"
	reghdfe, compile
	di as text "{hline 64}"

	* Cleaning up
	di as text _n "{bf:Note:} You need to run {stata program drop _all} now."
end

program Version
	which reghdfe
	Requirements
end

program Requirements
	di as text _n "Required packages installed?"
	loc reqs moresyntax ftools
	// ivreg2 avar tuples group3hdfe
	if (c(version)<13) loc reqs `reqs' boottest

	loc ftools_github "https://github.com/sergiocorreia/ftools/raw/master/src/"
	loc moresyntax_github "https://github.com/sergiocorreia/moresyntax/raw/master/src/"

	loc error 0

	foreach req of local reqs {
		loc fn `req'.ado
		if ("`req'"=="moresyntax") loc fn ms_get_version.ado
		cap findfile `fn'
		if (_rc) {
			loc error 1
			di as text "{lalign 20:- `req'}" as error "not" _c
			di as text "    {stata ssc install `req':install from SSC}" _c
			if inlist("`req'", "ftools", "moresyntax") {
				loc github ``req'_github'
				di as text `"    {stata `"net install `req', from(`"`github'"')"':install from github}"'
			}
			else {
				di as text // newline
			}
		}
		else {
			di as text "{lalign 20:- `req'}" as text "yes"
		}
	}

	if (`error') exit 601
end

program Cleanup
	args rc
	cap mata: mata drop HDFE
	cap mata: mata drop hdfe_*
	cap drop __temp_reghdfe_resid__
	cap matrix drop reghdfe_statsmatrix
	if (`rc') exit `rc'
end

program Parse
	* Trim whitespace (caused by "///" line continuations; aesthetic only)
	mata: st_local("0", stritrim(st_local("0")))

	* Main syntax
	#d;
	syntax anything(id=varlist equalok) [if] [in] [aw pw fw/] , [

		/* Model */
		Absorb(string) NOAbsorb
		RESiduals(name) RESiduals2 /* use _reghdfe_resid */
		SUmmarize SUmmarize2(string asis) /* simulate implicit options */

		/* Standard Errors */
		VCE(string) CLuster(string)

		/* Diagnostic */
		Verbose(numlist min=1 max=1 >=-1 <=5 integer)
		TIMEit

		/* Speedup and memory Tricks */
		NOSAMPle /* do not save e(sample) */

		/* Undocumented */
		KEEPSINgletons
		OLD /* use latest v3 */
		NOTES(string) /* NOTES(key=value ...), will be stored on e() */

		] [*] /* capture optimization options, display options, etc. */
		;
	#d cr

	* Unused
	* SAVEcache
	* USEcache
	* CLEARcache
	* COMPACT /* use as little memory as possible but is slower */

	* Convert options to boolean
	if ("`verbose'" == "") loc verbose 0
	loc timeit = ("`timeit'"!="")
	loc drop_singletons = ("`keepsingletons'"=="")

	if (`timeit') timer on 29

	* Sanity checks
	if (`verbose'>-1 & "`keepsingletons'"!="") {
		loc url "http://scorreia.com/reghdfe/nested_within_cluster.pdf"
		loc msg "WARNING: Singleton observations not dropped; statistical significance is biased"
		di as error `"`msg' {browse "`url'":(link)}"'
	}
	if ("`cluster'"!="") {
		_assert ("`vce'"==""), msg("cannot specify both cluster() and vce()")
		loc vce cluster `cluster'
		loc cluster // clear it to avoid bugs in subsequent lines
	}

	* Parse Varlist
	ms_fvunab `anything'
	ms_parse_varlist `s(varlist)'
	loc base_varlist "`s(basevars)'"
	foreach cat in depvar indepvars endogvars instruments {
		loc original_`cat' "`s(`cat')'"
	}
	loc model = cond("`s(instruments)'" == "", "ols", "iv")
	loc original_varlist = "`s(varlist)'" // no parens or equal

	* Parse Weights
	if ("`weight'"!="") {
		unab exp : `exp', min(1) max(1) // simple weights only
	}

	* Parse VCE
	ms_parse_vce, vce(`vce') weighttype(`weight')
	loc vcetype = "`s(vcetype)'"
	loc num_clusters = `s(num_clusters)'
	loc clustervars = "`s(clustervars)'"
	loc base_clustervars = "`s(base_clustervars)'"
	loc vceextra = "`s(vceextra)'"

	* Select sample (except for absvars)
	loc varlist `original_varlist' `base_clustervars'
	tempvar touse
	marksample touse, strok // based on varlist + cluster + if + in + weight

	* Parse noabsorb
	_assert  ("`absorb'`noabsorb'" != ""), msg("option {bf:absorb()} or {bf:noabsorb} required")
	if ("`noabsorb'" != "") {
		_assert ("`absorb'" == ""), msg("{bf:absorb()} and {bf:noabsorb} are mutually exclusive")
		tempvar c
		gen byte `c' = 1
		loc absorb `c'
	}

	if (`timeit') timer off 29

	* Construct HDFE object
	// SYNTAX: fixed_effects(absvars | , touse, wtype, wtvar, dropsing, verbose)
	mata: st_local("comma", strpos(`"`absorb'"', ",") ? "" : ",")
	if (`timeit') timer on 20
	mata: HDFE = fixed_effects(`"`absorb' `comma' `options'"', "`touse'", "`weight'", "`exp'", `drop_singletons', `verbose')
	if (`timeit') timer off 20
	mata: HDFE.cmdline = "reghdfe " + st_local("0")
	loc options `s(options)'

	mata: st_local("N", strofreal(HDFE.N))
	if (`N' == 0) error 2000

	* Fill out HDFE object
	mata: HDFE.varlist = "`base_varlist'"
	mata: HDFE.original_depvar = "`original_depvar'"
	mata: HDFE.original_indepvars = "`original_indepvars'"
	mata: HDFE.original_endogvars = "`original_endogvars'"
	mata: HDFE.original_instruments = "`original_instruments'"
	mata: HDFE.original_varlist = "`original_varlist'"
	mata: HDFE.model = "`model'"

	mata: HDFE.vcetype = "`vcetype'"
	mata: HDFE.num_clusters = `num_clusters'
	mata: HDFE.clustervars = tokens("`clustervars'")
	mata: HDFE.base_clustervars = tokens("`base_clustervars'")
	mata: HDFE.vceextra = "`vceextra'"

	* Parse summarize
	if ("`summarize'" != "") {
		_assert ("`summarize2'" == ""), msg("summarize() syntax error")
		loc summarize2 mean min max  // default values
	}
	ParseSummarize `summarize2'
	mata: HDFE.summarize_stats = "`s(stats)'"
	mata: HDFE.summarize_quietly = `s(quietly)'

	* Parse residuals
	if ("`residuals2'" != "") {
		_assert ("`residuals'" == ""), msg("residuals() syntax error")
		loc residuals _reghdfe_resid
		cap drop `residuals' // destructive!
	}
	else if ("`residuals'"!="") {
		conf new var `residuals'
	}
	mata: HDFE.residuals = "`residuals'"

	* Parse misc options
	mata: HDFE.notes = `"`notes'"'
	mata: HDFE.store_sample = ("`nosample'"=="")
	mata: HDFE.timeit = `timeit'

	* Parse Coef Table Options (do this last!)
	_get_diopts diopts options, `options' // store in `diopts', and the rest back to `options'
	_assert (`"`options'"'==""), msg(`"invalid options: `options'"')
	if ("`hascons'"!="") di in ye "(option ignored: `hascons')"
	if ("`tsscons'"!="") di in ye "(option ignored: `tsscons')"
	mata: HDFE.diopts = `"`diopts'"'
end

program ParseSummarize, sclass
	sreturn clear
	syntax [namelist(name=stats)] , [QUIetly]
	local quietly = ("`quietly'"!="")
	sreturn loc stats "`stats'"
	sreturn loc quietly = `quietly'
end

// --------------------------------------------------------------------------

program Estimate, eclass
	ereturn clear

	* Parse and fill out HDFE object
	Parse `0'
	mata: st_local("timeit", strofreal(HDFE.timeit))

	* Compute degrees-of-freedom
	if (`timeit') timer on 21
	mata: HDFE.estimate_dof()
	if (`timeit') timer off 21

	* Save updated e(sample) (singletons reduce sample);
	* required to parse factor variables to partial out
	if (`timeit') timer on 29
	tempvar touse
	mata: HDFE.save_touse("`touse'")
	if (`timeit') timer off 29

	* Expand varlists
	if (`timeit') timer on 22
	// BUGBUG: do we really need HDFE.varlist ? or can we create it by concatenating the others?
	loc varlist
	foreach cat in /*varlist*/ depvar indepvars endogvars instruments {
		mata: st_local("vars", HDFE.original_`cat')
		if ("`vars'" == "") continue
		// HACK: addbn replaces 0.foreign with 0bn.foreign , to prevent st_data() from loading a bunch of zeros
		ms_fvstrip `vars' if `touse', expand dropomit addbn onebyone
		// If we don't use onebyone, then 1.x 2.x ends up as 2.x
		loc vars "`r(varlist)'"
		loc varlist `varlist' `vars'
		mata: HDFE.`cat' = "`vars'"
	}
	mata: HDFE.varlist = "`varlist'"
	if (`timeit') timer off 22

	* Stats
	mata: st_local("stats", HDFE.summarize_stats)
	if ("`stats'" != "") Stats

	* Condition number
	mata: HDFE.estimate_cond()

	* Partial out; save TSS of depvar
	if (`timeit') timer on 23
	mata: hdfe_variables = HDFE.partial_out(HDFE.varlist, 1) // 1=Save TSS of first var if HDFE.tss is missing
	if (`timeit') timer off 23

	* Regress
	mata: assert(HDFE.model=="ols")
	if (`timeit') timer on 24
	RegressOLS `touse'
	if (`timeit') timer off 24

	* (optional) Store FEs
	if (`timeit') timer on 29
	mata: st_local("save_any_fe", strofreal(HDFE.save_any_fe))
	assert inlist(`save_any_fe', 0, 1)
	if (`save_any_fe') {
		_assert e(depvar) != "", msg("e(depvar) is empty")
		_assert e(resid) != "", msg("e(resid) is empty")
		confirm numeric var `e(depvar)', exact
		confirm numeric var `e(resid)', exact
		tempvar d
		if (e(rank)) {
			qui _predict double `d' if e(sample), xb
		}
		else {
			gen double `d' = 0
		}
		qui replace `d' = `e(depvar)' - `d' - `e(resid)' if e(sample)
		mata: HDFE.store_alphas("`d'")
		drop `d'
		cap drop __temp_reghdfe_resid__ // in case we didn't intend to save resids
	}
	if (`timeit') timer off 29

	* View estimation tables
	mata: st_local("diopts", HDFE.diopts)
	Replay, `diopts'

	// ~~ Preserve relevant dataset ~~-
	//
	// if (`compact') {
	// 	preserve
	// 	keep `varlist' `absorb'
	// 	loc N = c(N)
	// }
	// else {
	// 	qui cou if `touse'
	// 	loc N = r(N)
	// }

	if (`timeit') {
		di as text _n "{bf: Timer results:}"
		timer list
		di as text "Legend: 20: Create HDFE object; 21: Estimate DoF; 22: expand varlists; 23: partial out; 24: regress; 29: rest"
		di
	}
end

program RegressOLS, eclass
	args touse

	tempname b V N rank df_r
	mata: reghdfe_post_ols(HDFE, hdfe_variables, "`b'", "`V'", "`N'", "`rank'", "`df_r'")
	mata: st_local("indepvars", HDFE.indepvars)
	mata: hdfe_variables = .

	loc esample
	mata: st_local("store_sample", strofreal(HDFE.store_sample))
	if (`store_sample') loc esample "esample(`touse')"

	if ("`indepvars'" != "") {
		matrix colnames `b' = `indepvars'
		matrix colnames `V' = `indepvars'
		matrix rownames `V' = `indepvars'
		ereturn post `b' `V', `esample' buildfvinfo depname(`depvar')
	}
	else {
		ereturn post, `esample' buildfvinfo depname(`depvar')
	}

	ereturn scalar N       = `N'
	ereturn scalar rank    = `rank'
	ereturn scalar df_r    = `df_r'
	ereturn local  cmd     "reghdfe"
	mata: HDFE.post(HDFE)

	* Post stats
	cap conf matrix reghdfe_statsmatrix
	if (!c(rc)) {
		ereturn matrix summarize = reghdfe_statsmatrix
		mata: st_local("summarize_quietly", strofreal(HDFE.summarize_quietly))
		ereturn scalar summarize_quietly = `summarize_quietly'
	}
end

program Replay, rclass
	syntax [, *]
	_get_diopts options, `options'
	reghdfe_header // _coef_table_header
	di ""
	_coef_table, `options' // ereturn display, `options'
	return add // adds r(level), r(table), etc. to ereturn (before the footnote deletes them)
	reghdfe_footnote

	* Replay stats
	if (e(summarize_quietly)==0) {
		di as text _n "{sf:Regression Summary Statistics:}" _c
		matlist e(summarize)', border(top bottom) rowtitle(Variable) // twidth(18)
	}
end

program Stats
	* Optional weights
	mata: st_local("weight", sprintf("[%s=%s]", HDFE.weight_type, HDFE.weight_var))
	assert "`weight'" != ""
	if ("`weight'" == "[=]") loc weight
	loc weight : subinstr local weight "[pweight" "[aweight"

	mata: st_local("stats", HDFE.summarize_stats)
	mata: st_local("varlist", HDFE.varlist)
	mata: st_local("cvars", invtokens(HDFE.cvars))
	loc full_varlist `varlist' `cvars'

	qui tabstat `full_varlist' `weight' , stat(`stats') col(stat) save
	matrix reghdfe_statsmatrix = r(StatTotal)
end


ieboilstart : Stata11/IC - first final draft

Earlier version of ieboilstart did not work properly in Stata 11 and in
Stata IC/Small. These changes have solved most of those issues and added
code that properly handle the cases where this is still not the case.

	*! version 2.1 1Nov2016  Kristoffer Bjarkefur kbjarkefur@worldbank.org

	capture program drop ieboilstart2
	program ieboilstart2 , rclass

	noi di "Syntax works!"

	qui {

		*This is used to indicate in the do file how the command was specified
		local full_specification `0'

		syntax using ,  Versionnumber(string) [noclear maxvar(numlist) matsize(numlist) QUIetdo CUSTom(string) setmem(string) ]

		version 11.0

		/*********************************

			Check user specifed version
			and apply it is valid

		*********************************/

		local stata_versions "11.0 11.1 11.2 12.0 12.1 13.0 13.1 14.0 14.1"

		if `:list versionnumber in stata_versions' == 0 {

			di as error "{phang}Only recent major releases are allowed. One decimal must always be included. The releases currently allowed are:{break}`stata_versions'{p_end}"
			di ""
			error 198
			exit
		}

		*Set the version specfied in the command
		version `versionnumber'

		/*********************************

			Check file name

		*********************************/

		*Parse out the file name
		tokenize `"`using'"'
		local 		textfile "`2'"

		*Find index for where the file type suffix start
		local dot_index 	= strpos("`textfile'",".")

		*Extract the file index
		local file_suffix 	= substr("`textfile'", `dot_index', .)

		*If no file format suffix is specified, use the default .xlsx
		if "`file_suffix'" == "" {

			local textfile `"`textfile'.do"'
		}

		*If a file format suffix is specified make sure that it is one of the two allowed.
		else if "`file_suffix'" != ".do" {

			noi display as error "{phang}The file format specified in using is other than .do. Only the .do format is allowed. If no format is specified .do is assumed.{p_end}"
			error 198
		}

		/*********************************

			Check settings related to older
			versions and Stata 11

		*********************************/

		*Test that maxvar is not set in combination when using Stata IC
		if 	!(c(MP) == 1 | c(SE) == 1) & "`maxvar'" != "" {

			di as error "{phang}In Stata IC the maximum number of variables allowed is fixed at 2,047 and maxvar() is therefore not allowed.{p_end}"
			error 198
		}

		if 	"`setmem'" != ""	{

			if `versionnumber' >= 12  {

				di as error "{phang}Option setmem() is only allowed when setting the version number to 11. Setmem() is only applicable in Stata 11 or eralier, but those versions wont be able to run this file as the version number is set to 12 or higher.{p_end}"
				error 198
			}

			*Split the memory setting string into value and byte unit
			local mem_strlen 	= strlen("`setmem'") -1
			local mem_number	= substr("`setmem'",  1 , `mem_strlen')
			local mem_bytetype	= substr("`setmem'", -1 , 1)

			*Test that the number part is a number and a integer
			cap confirm integer number `mem_number'
			if _rc {

				di as error "{phang}The value in setmem(`setmem') must be a number (followed by either b, k, m or g). See {help memory} for details.{p_end}"
				error _rc
			}

			*Test that the byte type is not anything else than b, k, m or g
			if !`=inlist(lower("`mem_bytetype'"), "b", "k", "m", "g")' {

				di as error "{phang}The last character in setmem(`setmem') must be either b, k, m or g. See {help memory} for details.{p_end}""
				error 7

			}
		}
		else {

			*Default value is 50M. This is probably too little, but
			*it will safely work. And users can manually increase this
			local setmem "50M"

		}

		/*********************************

			Check input for maxvar and matsize
			if specified, other wise set
			maximum value allowed.

		*********************************/

		*Setting maxvar requires a cleared memory. Therefore
		*maxvar() and noclear cannot be used at the same time.
		if "`maxvar'" != "" & "`clear'" != "" {

			di as error "{phang}It is not possible to set allowed maximum numbers of variables without clearing the data. noclear and maxvar() can therefore not be specified at the same time{p_end}"
			di ""
			error 198
			exit
		}

		foreach maxlocal in maxvar matsize {

			*Set locals with the max and min values fox maxvar and matsize
			if "`maxlocal'" == "maxvar" {
				local max 32767
				local min 2048
			}
			if "`maxlocal'" == "matsize" {
				local max 11000
				local min 10
			}

			*Test if user set a value for this value
			if "``maxlocal''" != "" {

				*If user specified a value, test that it is between the min and the max
				if !(``maxlocal'' >= `min' & ``maxlocal'' <= `max') {

					di as error "{phang}`maxlocal' must be between `min' and `max' (inclusive) if you are using Stata SE and Stata MP. You entered ``maxlocal''.{p_end}"
					di ""
					*Throw appropriate error if below or above error
					if ``maxlocal'' < `min' error 910
					if ``maxlocal'' > `max' error 912
					exit
				}
			}

			else {
				*User did not specify value, use ieboilstart's defaults:

				if "`maxlocal'" == "maxvar" {
					*Set maxvar to max value allowed as this is often an issue when working with large survey data
					local `maxlocal' `max'
				}
				if "`maxlocal'" == "matsize" {
					*Set to the default as the maximum is rarely requered.
					local `maxlocal' 400
				}
			}
		}

		/*********************************

			Prepare a the dofile strings with
			all code (settings and comments)
			that is going in to the dofile

		*********************************/

		*******************************
		*Messages in dofiles generated
		*in Stata 11 and in Stata IC

		*Stata IC
		local dofile_string1 ""
		if 	!(c(MP) == 1 | c(SE) == 1) {
			local dofile_string1 `dofile_string1' **This dofile does not inlcude the setting maxvar as it ;
			local dofile_string1 `dofile_string1' * was generated in Stata IC or Small. If the dofile would ;
			local dofile_string1 `dofile_string1' * be generated in Stata MP or SE, then then maxvar is ;
			local dofile_string1 `dofile_string1' * included but for IC and Small it is ignored. ;
			local dofile_string1 `dofile_string1' * ;
		}

		*Stata 11
		local dofile_string2 ""
		if 	c(stata_version) < 12 {
			local dofile_string2 `dofile_string2' **This dofile does not inlcude modern memory settings as it ;
			local dofile_string2 `dofile_string2' * was generated in Stata 11. It only includes set memory that ;
			local dofile_string2 `dofile_string2' * is ignored by Stata >=12. Dofiles generated in Stata >=12 ;
			local dofile_string2 `dofile_string2' * includes both. ;
			local dofile_string2 `dofile_string2' * ;
		}

		*******************
		*Maxvar and matsize

		local dofile_string3 ""

		*Set verison number
		local dofile_string3 `dofile_string3' version `versionnumber' ;

		*Set basic memory limits
		if "`clear'" == "" {
			local dofile_string3 `dofile_string3' clear all ;

			**Setting maxvar not allowed in Stata IC. If Stata IC creates the file these lines
			* are excluded. The "if (c(MP) == 1 | c(SE) == 1) {" is still included in the dofile
			* so that a dofile generated by MP or SE still works on IC. In such a case you are
			* likely to run in to the varaible count limit in Stata IC, but that error message
			* is more informative then when trying to set maxvar in Stata IC.
			if 	(c(MP) == 1 | c(SE) == 1) {
				local dofile_string3 `dofile_string3' if (c(MP) == 1 | c(SE) == 1) { ;
				local dofile_string3 `dofile_string3' set maxvar 	`maxvar' ;
				local dofile_string3 `dofile_string3' } ;
			}
		}

		local dofile_string3 `dofile_string3'  set matsize 	`matsize' ;

		****************
		*Memory	settings

		local dofile_string4 ""

		*New memory settings was introduced in Stata 12.
		if c(stata_version) >= 12 {

			*For compatibility with Stata 11 the do file includes and if/else
			*condition testing version number. The memoroy settings introduced
			*in Stata 12 will be applied to Stata version more recent than
			*Stata 11, and set memory will be applied to Stata 11.
			local dofile_string4 `dofile_string4'  if c(stata_version) >= 12 { ;

			*Set advanced memory limits
			local dofile_string4 `dofile_string4' set niceness	5 ;

			if "`clear'" == "" {
				*These settings cannot be modified with data in memory
				local dofile_string4 `dofile_string4' set min_memory	0 ;
				local dofile_string4 `dofile_string4' set max_memory	. ;

				*Set segment size to the largest value allowed by the operative system
				if c(bit) == 64 {
					local dofile_string4 `dofile_string4' set segmentsize	32m ;
				}
				else {
					local dofile_string4 `dofile_string4' set segmentsize	16m ;
				}
			}
			local dofile_string4 `dofile_string4'  } ;

			*Inlcude an else statement that allows compatibility with Stata 11. This
			*might cause errors when running the rest of the code for this project,
			*but by using setmem() it might be possible to solve those errors.
			local dofile_string4 `dofile_string4'  else { ;
			local dofile_string4 `dofile_string4'  set memory `setmem' ;
			local dofile_string4 `dofile_string4'  } ;

		}
		else {

			*If this dofile is generated in Stata 11 then only the old
			*way of setting memory is included. This will be ignored by
			*more recent versios of Stata
			local dofile_string4 `dofile_string4'  set memory `setmem' ;

		}

		*********************
		*Set default settings
		local dofile_string4 `dofile_string4' set more 			off ;
		local dofile_string4 `dofile_string4' pause 			on 	;
		local dofile_string4 `dofile_string4' set varabbrev 	off ;

		/*********************************

			Add custom lines of code

		*********************************/

		if "`custom'" != "" {

			local dofile_string5 ""

			*Add a comment that list where
			local dofile_string5 `dofile_string5' **The code below was added manually by the user to iboilstart. If ;
			local dofile_string5 `dofile_string5' * there are any issues please reconfigure ieboilstart. There is a ;
			local dofile_string5 `dofile_string5' * limit of less than 244 characters in the code in Stata 11, and ;
			local dofile_string5 `dofile_string5' * 512 in later versions.;

			*di "`dofile_string5'"
			*di `=strlen("`dofile_string5'")'

			*Create a local with the rowlabel input to be tokenized
			local custom_code_lines `custom'
			local dofile_string6 ""

			while "`custom_code_lines'" != "" {

				*Parsing name and label pair
				gettoken code_line custom_code_lines : custom_code_lines, parse("@")

				*Removing leadning or trailing spaces
				local code_line = trim("`code_line'")

				local dofile_string6 `dofile_string6' `code_line';

				*Parse char is not removed by gettoken
				local custom_code_lines = subinstr("`custom_code_lines'" ,"@","",1)
			}
		}

		/*********************************

			Create the dofile

		*********************************/

		*Create a local with the date
		local		date = subinstr(c(current_date)," ","",.)

		*Write the title rows defined above
		capture file close settingsfile

		*Set up the file to be written
		file open  settingsfile using "`textfile'", text write replace

		*Suppress output from dofile if option qietdo is applied
		if "`quietdo'" != "" file write settingsfile "quietly {" _n

		*Write the standard message in the opening file.
		file write settingsfile ///
																					_n ///
			`" 	/*************************************************"' 				_n ///
			`" 	This file is was prepared by command ieboilstart "' 				_n ///
			`" 	on `date' by username: `c(username)'. "'							_n ///
																					_n ///
			`" 	Run this file to set the recommended settings for "' 				_n ///
			`" 	this project to your instance of Stata"' 							_n ///
			`" 	*************************************************/"' 				_n ///
																					_n ///
			`"	*This dofile was generated by specifying ieboilstart like this:"' 	_n ///
			`"	*ieboilstart `full_specification' "'								_n ///
																					_n ///
			"	*See help file for ieboilstart for details on these settings"		_n ///
			_n

		** Repeat through the possible 6 strings from above. The reason multiple strings
		*  are created is that in older versions of Stata, the maximum string length
		*  used in a string operation, like gettoken or subinstr(), is limited to 214.
		forvalues strNum = 1/6 {

			** Write each setting prepared above to the dofile.
			while ("`dofile_string`strNum''" != "") {

				*noi di `=strlen("`dofile_string`strNum''")'
				*noi di "`dofile_string`strNum''"

				local tempstring `dofile_string`strNum''

				*Tokenize the first setting
				gettoken setting tempstring : tempstring, parse(";")

				*Write it to file
				file write settingsfile "	`setting'" _n

				*Remove the parse character
				local dofile_string`strNum' = subinstr("`tempstring'",";","",1)
			}
		}

		*Close quiet bracket if option quietdo is used.
		if "`quietdo'" != "" file write settingsfile "}" _n

		*Close file
		file close settingsfile

		/*********************************

			Create return value and output message

		*********************************/

		return local ieboil_dofile "`textfile'"

		noi di ""
		noi di "{phang}{err:DISCLAIMER:} Due to how settings works in Stata, it is impossible to create the exact same coding environment for all users regardless of what version of Stata each user is using. Hence, this command only attempts to harmonize the settings as much as possible across users collaborating on the same code, but no guarantee can be given that all commands will always behave identical unless the exact same version of Stata is used.{p_end}"
		noi di ""
		noi di "{phang}For a similar reason, ieboilstart only prepares a dofile with all recommended settings. This dofile {ul:MUST} be executed using the command {it:include} and not with {it:do}, otherwise not all settings will be set properly. See {help ieboilstart :help ieboilstart} for more details on how to correctly execute the dofile ieboilstart has prepared. If you do not run the dofile, ieboilstart does nothing to harmonize your settings.{p_end}"
		noi di ""
		noi di "{phang}dofile with all settings saved to: `textfile'{p_end}"
	}
	end

	*! version 2.1 1Nov2016  Kristoffer Bjarkefur kbjarkefur@worldbank.org

	capture program drop ieboilstart2
	program ieboilstart2 , rclass

	noi di "Syntax works!"

	qui {

		syntax using ,  Versionnumber(string) [noclear maxvar(numlist) matsize(numlist) QUIetdo CUSTom(string)]

		*Potentional updates
		*set trace on
		*set scrollbufsize #

		version 11.0
		version `versionnumber'

		/*********************************

			Check file name

		*********************************/

		*Parse out the file name
		tokenize `"`using'"'
		local 		textfile "`2'"

		*Find index for where the file type suffix start
		local dot_index 	= strpos("`textfile'",".")

		*Extract the file index
		local file_suffix 	= substr("`textfile'", `dot_index', .)

		*If no file format suffix is specified, use the default .xlsx
		if "`file_suffix'" == "" {

			local textfile `"`textfile'.do"'
		}

		*If a file format suffix is specified make sure that it is one of the two allowed.
		else if "`file_suffix'" != ".do" {

			noi display as error "{phang}The file format specified in using is other than .do. Only the .do format is allowed. If no format is specified .do is assumed.{p_end}"
			error 198
		}

		/*********************************

			Check version

		*********************************/

		if "`maxvar'" != "" & "`clear'" != "" {

			di as error "{phang}It is not possible to set allowed maximum numbers of variables without clearing the data. noclear and maxvar() can therefore not be specified at the same time{p_end}"
			di ""
			error 198
			exit
		}

		local stata_versions "11.0 11.1 11.2 12.0 12.1 13.0 13.1 14.0 14.1"

		if `:list versionnumber in stata_versions' == 0 {

			di as error "{phang}Only recent major releases are allowed. One decimal must always be included. The releases currently allowed are:{break}`stata_versions'{p_end}"
			di ""
			error 198
			exit
		}

		if 	!(c(MP) == 1 | c(SE) == 1)	{

			//If using IC test that maxvar and maxsize is not used

		}

		/*********************************

			Check input for maxvar and matsize
			if specified, other wise set
			maximum value allowed.

		*********************************/

		foreach maxlocal in maxvar matsize {

			if "`maxlocal'" == "maxvar" {

				local max 32767
				local min 2048
			}

			if "`maxlocal'" == "matsize" {

				local max 11000
				local min 10
			}

			*Test if user set maxvar
			if "``maxlocal''" != "" {

				if ``maxlocal'' >= `min' & ``maxlocal'' <= `max' {

					local `maxlocal' ``maxlocal''
				}
				else {

					di as error "{phang}`maxlocal' must be between `min' and `max' (inclusive) if you are using Stata SE and Stata MP. You entered ``maxlocal''.{p_end}"
					if ``maxlocal'' < `min' {
						di ""
						error 910
					}
					else {
						di ""
						error 912
					}
					exit
				}
			}
			else {

				*User did not specify value, use ieboilstart's defaults
				if "`maxlocal'" == "maxvar" {

					*Set maxvar to max value allowed as this is often an issue when working with large survey data
					local `maxlocal' 32767

				}

				if "`maxlocal'" == "matsize" {

					*Set to the default as the maximum is rarely requered.
					local `maxlocal' 400

				}
			}
		}

		/*********************************

			Prepare a local with all settings

		*********************************/

		*******************
		*Maxvar and matsize

		local return_string1 ""

		*Set verison number
		local return_string1 `return_string1' version `versionnumber' ;

		*Set basic memory limits
		if "`clear'" == "" {
			local return_string1 `return_string1' clear all ;
			local return_string1 `return_string1' if (c(MP) == 1 | c(SE) == 1) { ;
			local return_string1 `return_string1' set maxvar 	`maxvar' ;
			local return_string1 `return_string1' } ;
		}

		local return_string1 `return_string1'  set matsize 	`matsize' ;

		*******
		*Memory

		local return_string2 ""

		*Settings that were introduced in Stata 12
		local return_string2 `return_string2'  if c(stata_version) > 12 { ;

		*Set advanced memory limits
		local return_string2 `return_string2' set niceness	5 ;

		if "`clear'" == "" {
			*These settings cannot be modified with data in memory
			local return_string2 `return_string2' set min_memory	0 ;
			local return_string2 `return_string2' set max_memory	. ;

			if c(stata_version) >= 12 {
				if c(bit) == 64 {
					local return_string2 `return_string2' set segmentsize	32m ;
				}
				else {
					local return_string2 `return_string2' set segmentsize	16m ;
				}
			}
			else {
				local return_string2 `return_string2' set segmentsize	16m ;
			}
		}

		local return_string2 `return_string2'  } ;
		local return_string2 `return_string2'  else { ;
		local return_string2 `return_string2'  set memory 200m ;
		local return_string2 `return_string2'  } ;

		*********************
		*Set default settings
		local return_string2 `return_string2' set more 			off ;
		local return_string2 `return_string2' pause 			on 	;
		local return_string2 `return_string2' set varabbrev 	off ;

		/*********************************

			Add custom lines of code

		*********************************/

		if "`custom'" != "" {

			local return_string3 ""

			*Add a comment that list where
			local return_string3 `return_string3' **The code below was added manually by the user to iboilstart. If ;
			local return_string3 `return_string3' * there are any issues please reconfigure ieboilstart. There is a ;
			local return_string3 `return_string3' * limit of less than 244 characters in the code in version 11, and ;
			local return_string3 `return_string3' * 512 in later versions.;

			di "`return_string3'"
			di `=strlen("`return_string3'")'

			*Create a local with the rowlabel input to be tokenized
			local custom_code_lines `custom'
			local return_string4 ""

			while "`custom_code_lines'" != "" {

				*Parsing name and label pair
				gettoken code_line custom_code_lines : custom_code_lines, parse("@")

				*Removing leadning or trailing spaces
				local code_line = trim("`code_line'")

				local return_string4 `return_string4' `code_line';

				*Parse char is not removed by gettoken
				local custom_code_lines = subinstr("`custom_code_lines'" ,"@","",1)
			}
		}

		/*********************************

			Create the do-file

		*********************************/

		*Create a local with the date
		local		date = subinstr(c(current_date)," ","",.)

		*Write the title rows defined above
		capture file close settingsfile

		*Set up the file to be written
		file open  settingsfile using "`textfile'", text write replace

		if "`quietdo'" != "" file write settingsfile "quietly {" _n

		*Write the standard message in the opening file.
		file write settingsfile ///
																				_n ///
			`" 	/*************************************************"' 			_n ///
			`" 	This file is was prepared by command ieboilstart "' 			_n ///
			`" 	on `date'."'													_n ///
																				_n ///
			`" 	Run this file to set the recommended settings for "' 			_n ///
			`" 	this project to your instance of Stata"' 						_n ///
			`" 	*************************************************/"' 			_n ///
																				_n ///
																				_n ///
			"	*See help file for ieboilstart for details on these settings"	_n ///
			_n

		*Repeat through the possible 4 strings above
		forvalues strNum = 1/4 {

			** Write each setting prepared above to the do-file.
			while ("`return_string`strNum''" != "") {

				*noi di `=strlen("`return_string`strNum''")'
				*noi di "`return_string`strNum''"

				local tempstring `return_string`strNum''

				di "`return_string'

				*Tokenize the first setting
				gettoken setting tempstring : tempstring, parse(";")

				*Write it to file
				file write settingsfile "	`setting'" _n

				*Remove the parse character
				local return_string`strNum' = subinstr("`tempstring'",";","",1)
			}
		}

		if "`quietdo'" != "" file write settingsfile "}" _n

		*Close file
		file close settingsfile

		/*********************************

			Create return value and output message

		*********************************/

		return local ieboil_dofile "`textfile'"

		noi di ""
		noi di "{phang}{err:DISCLAIMER:} Due to how settings works in Stata, it is impossible to create the exact same coding environment for all users regardless of what version of Stata each user is using. Hence, this command only attempts to harmonize the settings as much as possible across users collaborating on the same code, but no guarantee can be given that all commands will always behave identical unless the exact same version of Stata is used.{p_end}"
		noi di ""
		noi di "{phang}For a similar reason, ieboilstart only prepares a do-file with all recommended settings. This do-file {ul:MUST} be executed using the command {it:include} and not with {it:do}, otherwise not all settings will be set properly. See {help ieboilstart :help ieboilstart} for more details on how to correctly execute the do-file ieboilstart has prepared. If you do not run the do-file, ieboilstart does nothing to harmonize your settings.{p_end}"
		noi di ""
		noi di "{phang}Do-file with all settings saved to: `textfile'{p_end}"
	}
	end


Removed pause breaks

Removed pause breaks

	*Set standardized settings
	ieboilstart, v(11.1)
	`r(version)'

	*Create filepaths
	global ietoolkit "C:\Users\wb503680\Box Sync\DIME WOrk\GitFolder\ietoolkit"
	*global ietoolkit "C:\Users\wb462869\Box Sync\Stata\Stata work\Commands\ietoolkit"
	global testOutput "$ietoolkit/test/ieimpgraphOutput"

	*Set cd folder to catch and cd leakage
	cd "$ietoolkit\test\cdjunk"

	*Load the command
	do "$ietoolkit\iegraph.do"

	pause on

	**********************
	*Graph 1 - single treatment

	*open an example data set and create a mock dummy
	sysuse auto, clear
	gen Treatment = (weight >= 3019.5)

	*Run the impact regression
	reg price  Treatment
	*Create the graph
	iegraph Treatment, noconfbars title("Treatment effect on price")  save("$testOutput/Graph1.gph") yzero
	iegraph Treatment, title("Treatment effect on price") save("$testOutput/Graph1.gph") yzero
	**********************
	*Graph 2 - multiple treatment

	*open an example data set and create mutually exclusive mock dummies
	sysuse auto, clear
	tab rep78, gen(d_)

	*Run the impact regression
	reg price	d_2 d_3 weight length gear_ratio
	*Create the graph

	iegraph 	d_2 d_3,  title("Treatment effect on price") save("$testOutput/Graph2.gph") yzero

	reg price	d_2 d_3 d_4  weight length gear_ratio
	iegraph 	d_2 d_3 d_4, title("Treatment effect") save("$testOutput/Graph4.gph") yzero note("Stata iegraph command")

	reg price	d_2 d_3 d_4 d_5  weight length gear_ratio
	iegraph 	d_2 d_3 d_4 d_5, noconfbars title("Treatment effect on price") save("$testOutput/Graph2.gph") yzero

	iegraph d_2 d_3,  title("Treatment effect on price") save("$testOutput/Graph2.gph") yzero
	iegraph d_2 d_3 d_4,  title("Treatment effect") save("$testOutput/Graph4.gph") yzero note(Stata iegraph command)
	iegraph d_2 d_3 d_4 d_5,  noconfbars title("Treatment effect on price") save("$testOutput/Graph2.gph") yzero

	iegraph d_2 d_3,  title("Treatment effect on price") save("$testOutput/Graph2.gph") yzero
	iegraph d_2 d_3 d_4,  title("Treatment effect") save("$testOutput/Graph4.gph") yzero note(Stata iegraph command)
	iegraph d_2 d_3 d_4 d_5,  noconfbars title("Treatment effect on price") save("$testOutput/Graph2.gph") yzero

	*Set standardized settings
	ieboilstart, v(11.1)
	`r(version)'

	*Create filepaths
	global ietoolkit "C:\Users\wb503680\Box Sync\DIME WOrk\GitFolder\ietoolkit"
	*global ietoolkit "C:\Users\wb462869\Box Sync\Stata\Stata work\Commands\ietoolkit"
	global testOutput "$ietoolkit/test/ieimpgraphOutput"

	*Set cd folder to catch and cd leakage
	cd "$ietoolkit\test\cdjunk"

	*Load the command
	do "$ietoolkit\iegraph.do"

	pause on

	**********************
	*Graph 1 - single treatment

	*open an example data set and create a mock dummy
	sysuse auto, clear
	gen Treatment = (weight >= 3019.5)

	*Run the impact regression
	reg price  Treatment
	*Create the graph
	iegraph Treatment, noconfbars title("Treatment effect on price")  save("$testOutput/Graph1.gph") yzero grey
	pause
	iegraph Treatment, title("Treatment effect on price") save("$testOutput/Graph1.gph") yzero grey
	pause

	**********************
	*Graph 2 - multiple treatment

	*open an example data set and create mutually exclusive mock dummies
	sysuse auto, clear
	tab rep78, gen(d_)

	*Run the impact regression
	reg price	d_2 d_3 weight length gear_ratio
	*Create the graph

	iegraph 	d_2 d_3,  title("Treatment effect on price") save("$testOutput/Graph2.gph") yzero

	reg price	d_2 d_3 d_4  weight length gear_ratio
	iegraph 	d_2 d_3 d_4, title("Treatment effect") save("$testOutput/Graph4.gph") yzero note("Stata iegraph command")

	reg price	d_2 d_3 d_4 d_5  weight length gear_ratio
	iegraph 	d_2 d_3 d_4 d_5, noconfbars title("Treatment effect on price") save("$testOutput/Graph2.gph") yzero

	iegraph d_2 d_3,  title("Treatment effect on price") save("$testOutput/Graph2.gph") yzero
	iegraph d_2 d_3 d_4,  title("Treatment effect") save("$testOutput/Graph4.gph") yzero note(Stata iegraph command)
	iegraph d_2 d_3 d_4 d_5,  noconfbars title("Treatment effect on price") save("$testOutput/Graph2.gph") yzero


t/110-compile: remove hard dependency on /usr/include/stdio.h.

This was testing source files outside the current build tree, but if
you don't have the file at all, that's not a good reason to fail the
test.

# This test is meant to confirm some basic redo functionality
# related to static files not in the build tree. But if your
# system doesn't happen to have stdio.h in the usual location,
# let's not explode just for that.
stdio=/usr/include/stdio.h
[ -e "$stdio" ] || stdio=

redo-ifchange CC hello.c $stdio
redo-ifcreate stdio.h
../sleep 3
./CC hello.c

redo-ifchange CC hello.c /usr/include/stdio.h
redo-ifcreate stdio.h
../sleep 3
./CC hello.c


get words instead of characters

get words instead of characters

program make_merge

insheet using ${stash}tmp_word.csv, clear

renamevar

merge m:1 wikihandle using ${stash}bbk_master, nogen

merge 1:1 wikihandle year using ${stash}traf, nogen

merge 1:1 wikihandle year using ${stash}rev, keep(master match) nogen

egen id = group(wikihandle)
xtset id year

gen post = (year>2008)

gen treat = (debutyear < 1964)

local outcomes "img text bd traf numrev numuser numregusers numnew_wiki numnewreg_wiki numnewreg_page numserious1 numserious3 numserious21 numserious278 numserious1851 avgsize numcountry numregion numtimezone numnewregion numnewregion_wiki"

foreach x in `outcomes'{
    gen ln`x' = ln(`x'+1)
}

drop if debut < 1944 | debut > 1984

genqual

gen decade = round(debut, 10)
egen dy = group(decade year)
egen qy = group(quality year)

save ${stash}master, replace

end

program renamevar

rename v1 wikihandle
rename v2 year
rename v3 user
rename v4 revid
rename v5 size
rename v6 text
rename v7 img
rename v8 bd

end

program genqual

gen quality = .
_pctile numallstar, p(33.333, 66.667 90)

replace qual = 1 if numallstar <= `r(r1)' & isb == 1
replace qual = 2 if numallstar > `r(r1)'  & numall <=`r(r2)' & isb == 1
replace qual = 3 if numallstar > `r(r2)'  & numall <=`r(r3)' & isb == 1
replace qual = 4 if numallstar > `r(r3)' & isb == 1

_pctile minutesrank, p(33.333, 66.667 90)

replace qual = 1 if minutesrank <= `r(r1)' & isb == 0
replace qual = 2 if minutesrank > `r(r1)' & minutesrank <=`r(r2)' & isb == 0
replace qual = 3 if minutesrank > `r(r2)' & minutesrank <=`r(r3)' & isb == 0
replace qual = 4 if minutesrank > `r(r3)' & isb == 0

end

program make_merge

insheet using ${stash}tmp.csv, clear

renamevar

merge m:1 wikihandle using ${stash}bbk_master, nogen

merge 1:1 wikihandle year using ${stash}traf, nogen

merge 1:1 wikihandle year using ${stash}rev, keep(master match) nogen

egen id = group(wikihandle)
xtset id year

gen post = (year>2008)

gen treat = (debutyear < 1964)

local outcomes "img text bd traf numrev numuser numregusers numnew_wiki numnewreg_wiki numnewreg_page numserious1 numserious3 numserious21 numserious278 numserious1851 avgsize numcountry numregion numtimezone numnewregion numnewregion_wiki"

foreach x in `outcomes'{
    gen ln`x' = ln(`x'+1)
}

drop if debut < 1944 | debut > 1984

genqual

gen decade = round(debut, 10)
egen dy = group(decade year)
egen qy = group(quality year)

save ${stash}master, replace

end

program renamevar

rename v1 wikihandle
rename v2 year
rename v3 user
rename v4 revid
rename v5 size
rename v6 text
rename v7 img
rename v8 bd

end

program genqual

gen quality = .
_pctile numallstar, p(33.333, 66.667 90)

replace qual = 1 if numallstar <= `r(r1)' & isb == 1
replace qual = 2 if numallstar > `r(r1)'  & numall <=`r(r2)' & isb == 1
replace qual = 3 if numallstar > `r(r2)'  & numall <=`r(r3)' & isb == 1
replace qual = 4 if numallstar > `r(r3)' & isb == 1

_pctile minutesrank, p(33.333, 66.667 90)

replace qual = 1 if minutesrank <= `r(r1)' & isb == 0
replace qual = 2 if minutesrank > `r(r1)' & minutesrank <=`r(r2)' & isb == 0
replace qual = 3 if minutesrank > `r(r2)' & minutesrank <=`r(r3)' & isb == 0
replace qual = 4 if minutesrank > `r(r3)' & isb == 0

end


redo: add -DDEBUG_FATAL

redo: add -DDEBUG_FATAL

# jgfs2
# (c) 2013 Justin Gottula
# The source code of this project is distributed under the terms of the
# simplified BSD license. See the LICENSE file for details.

exec >&2

TARGET=$1
TARGET_BASE=$2
OUTPUT=$3

export CC="ccache gcc"
export CXX="ccache g++"
export AR=ar

export CFLAGS="-std=gnu11 -O0 -ggdb -Wall -Wextra -Wno-unused-parameter \
-Wno-unused-function -include stddef.h -include stdbool.h -include stdint.h \
-Isrc"
export CXXFLAGS="-std=c++11 -O0 -ggdb -Wall -Wextra -Wno-unused-parameter \
-Wno-unused-function -include cstddef -include cstdint -Isrc"

DEFINES="-D_GNU_SOURCE -D_FILE_OFFSET_BITS=64 -DFUSE_USE_VERSION=26 \
-DDEBUG_FATAL"

LIB_OUT="bin/libjgfs2.a"
LIB_SRC=(lib/*.c lib/*/*.c)
LIB_OBJS=${LIB_SRC[@]//.c/.o}
LIB_LIBS=(-lm -lbsd -luuid)

FUSE_OUT="bin/fuse.jgfs2"
FUSE_SRC=(src/fuse/*.c)
FUSE_OBJS=${FUSE_SRC[@]//.c/.o}
FUSE_LIBS=(-lfuse)

TEST_OUT="bin/test.jgfs2"
TEST_SRC=(src/test/*.c)
TEST_OBJS=${TEST_SRC[@]//.c/.o}
TEST_LIBS=()

VIEW_OUT="bin/view.jgfs2"
VIEW_SRC=(src/view/*.c)
VIEW_OBJS=${VIEW_SRC[@]//.c/.o}
VIEW_LIBS=()

MKFS_OUT="bin/mkfs.jgfs2"
MKFS_SRC=(src/mkfs/*.c)
MKFS_OBJS=${MKFS_SRC[@]//.c/.o}
MKFS_LIBS=()

FSCK_OUT="bin/fsck.jgfs2"
FSCK_SRC=(src/fsck/*.c)
FSCK_OBJS=${FSCK_SRC[@]//.c/.o}
FSCK_LIBS=()

DEFRAG_OUT="bin/defrag.jgfs2"
DEFRAG_SRC=(src/defrag/*.c)
DEFRAG_OBJS=${DEFRAG_SRC[@]//.c/.o}
DEFRAG_LIBS=()

FSCTL_OUT="bin/fsctl.jgfs2"
FSCTL_SRC=(src/fsctl/*.c)
FSCTL_OBJS=${FSCTL_SRC[@]//.c/.o}
FSCTL_LIBS=()

ATTR_OUT="bin/attr.jgfs2"
ATTR_SRC=(src/attr/*.c)
ATTR_OBJS=${ATTR_SRC[@]//.c/.o}
ATTR_LIBS=()

function target_cxx_dep {
	$CXX $CXXFLAGS $DEFINES -o${TARGET//.o/.dep} -MM -MG ${TARGET//.o/.cxx}
}

function target_cxx {
	target_cxx_dep
	read DEPS <${TARGET//.o/.dep}
	redo-ifchange ${DEPS#*:}

	$CXX $CXXFLAGS $DEFINES -o$OUTPUT -c ${TARGET//.o/.cxx}
}

function target_cxx_link {
	redo-ifchange $OBJS

	$CXX $CFXXLAGS $LIBS -o$OUTPUT $OBJS
}

function target_c_dep {
	$CC $CFLAGS $DEFINES -o${TARGET//.o/.dep} -MM -MG ${TARGET//.o/.c}
}

function target_c {
	target_c_dep
	read DEPS <${TARGET//.o/.dep}
	redo-ifchange ${DEPS#*:}

	$CC $CFLAGS $DEFINES -o$OUTPUT -c ${TARGET//.o/.c}
}

function target_c_link {
	redo-ifchange $OBJS

	$CC $CFLAGS $LIBS -o$OUTPUT $OBJS
}

function target_lib {
	redo-ifchange $OBJS

	$AR rcs $OUTPUT $OBJS
}

case "$TARGET" in
all)
	redo lib fuse test view mkfs fsck defrag fsctl attr ;;
lib)
	redo-ifchange $LIB_OUT ;;
fuse)
	redo-ifchange $FUSE_OUT ;;
test)
	redo-ifchange $TEST_OUT ;;
view)
	redo-ifchange $VIEW_OUT ;;
mkfs)
	redo-ifchange $MKFS_OUT ;;
fsck)
	redo-ifchange $FSCK_OUT ;;
defrag)
	redo-ifchange $DEFRAG_OUT ;;
fsctl)
	redo-ifchange $FSCTL_OUT ;;
attr)
	redo-ifchange $ATTR_OUT ;;
$LIB_OUT)
	LIBS=""
	OBJS="${LIB_OBJS[@]}"
	target_lib
	;;
$FUSE_OUT)
	LIBS="${FUSE_LIBS[@]} ${LIB_LIBS[@]}"
	OBJS="${FUSE_OBJS[@]} $LIB_OUT"
	target_c_link
	;;
$TEST_OUT)
	LIBS="${TEST_LIBS[@]} ${LIB_LIBS[@]}"
	OBJS="${TEST_OBJS[@]} $LIB_OUT"
	target_c_link
	;;
$VIEW_OUT)
	LIBS="${VIEW_LIBS[@]} ${LIB_LIBS[@]}"
	OBJS="${VIEW_OBJS[@]} $LIB_OUT"
	target_c_link
	;;
$MKFS_OUT)
	LIBS="${MKFS_LIBS[@]} ${LIB_LIBS[@]}"
	OBJS="${MKFS_OBJS[@]} $LIB_OUT"
	target_c_link
	;;
$FSCK_OUT)
	LIBS="${FSCK_LIBS[@]} ${LIB_LIBS[@]}"
	OBJS="${FSCK_OBJS[@]} $LIB_OUT"
	target_c_link
	;;
$DEFRAG_OUT)
	LIBS="${DEFRAG_LIBS[@]} ${LIB_LIBS[@]}"
	OBJS="${DEFRAG_OBJS[@]} $LIB_OUT"
	target_c_link
	;;
$FSCTL_OUT)
	LIBS="${FSCTL_LIBS[@]} ${LIB_LIBS[@]}"
	OBJS="${FSCTL_OBJS[@]} $LIB_OUT"
	target_c_link
	;;
$ATTR_OUT)
	LIBS="${ATTR_LIBS[@]} ${LIB_LIBS[@]}"
	OBJS="${ATTR_OBJS[@]} $LIB_OUT"
	target_c_link
	;;
*.o)
	if [[ -e "${TARGET//.o/.cxx}" ]]; then
		target_cxx
	elif [[ -e "${TARGET//.o/.c}" ]]; then
		target_c
	else
		echo "cannot find source file for '$TARGET'"
		exit 1
	fi
	;;
clean)
	rm -rf $(find bin/ -type f)
	rm -rf $(find src/ lib/ -type f -iname *.o)
	rm -rf $(find src/ lib/ -type f -iname *.dep)
	;;
backup)
	cd .. && tar -acvf backup/jgfs2-$(date +'%Y%m%d-%H%M').tar.xz jgfs2/
	;;
*)
	echo "unknown target '$TARGET'"
	exit 1
	;;
esac

# jgfs2
# (c) 2013 Justin Gottula
# The source code of this project is distributed under the terms of the
# simplified BSD license. See the LICENSE file for details.

exec >&2

TARGET=$1
TARGET_BASE=$2
OUTPUT=$3

export CC="ccache gcc"
export CXX="ccache g++"
export AR=ar

export CFLAGS="-std=gnu11 -O0 -ggdb -Wall -Wextra -Wno-unused-parameter \
-Wno-unused-function -include stddef.h -include stdbool.h -include stdint.h \
-Isrc"
export CXXFLAGS="-std=c++11 -O0 -ggdb -Wall -Wextra -Wno-unused-parameter \
-Wno-unused-function -include cstddef -include cstdint -Isrc"

DEFINES="-D_GNU_SOURCE -D_FILE_OFFSET_BITS=64 -DFUSE_USE_VERSION=26"

LIB_OUT="bin/libjgfs2.a"
LIB_SRC=(lib/*.c lib/*/*.c)
LIB_OBJS=${LIB_SRC[@]//.c/.o}
LIB_LIBS=(-lm -lbsd -luuid)

FUSE_OUT="bin/fuse.jgfs2"
FUSE_SRC=(src/fuse/*.c)
FUSE_OBJS=${FUSE_SRC[@]//.c/.o}
FUSE_LIBS=(-lfuse)

TEST_OUT="bin/test.jgfs2"
TEST_SRC=(src/test/*.c)
TEST_OBJS=${TEST_SRC[@]//.c/.o}
TEST_LIBS=()

VIEW_OUT="bin/view.jgfs2"
VIEW_SRC=(src/view/*.c)
VIEW_OBJS=${VIEW_SRC[@]//.c/.o}
VIEW_LIBS=()

MKFS_OUT="bin/mkfs.jgfs2"
MKFS_SRC=(src/mkfs/*.c)
MKFS_OBJS=${MKFS_SRC[@]//.c/.o}
MKFS_LIBS=()

FSCK_OUT="bin/fsck.jgfs2"
FSCK_SRC=(src/fsck/*.c)
FSCK_OBJS=${FSCK_SRC[@]//.c/.o}
FSCK_LIBS=()

DEFRAG_OUT="bin/defrag.jgfs2"
DEFRAG_SRC=(src/defrag/*.c)
DEFRAG_OBJS=${DEFRAG_SRC[@]//.c/.o}
DEFRAG_LIBS=()

FSCTL_OUT="bin/fsctl.jgfs2"
FSCTL_SRC=(src/fsctl/*.c)
FSCTL_OBJS=${FSCTL_SRC[@]//.c/.o}
FSCTL_LIBS=()

ATTR_OUT="bin/attr.jgfs2"
ATTR_SRC=(src/attr/*.c)
ATTR_OBJS=${ATTR_SRC[@]//.c/.o}
ATTR_LIBS=()

function target_cxx_dep {
	$CXX $CXXFLAGS $DEFINES -o${TARGET//.o/.dep} -MM -MG ${TARGET//.o/.cxx}
}

function target_cxx {
	target_cxx_dep
	read DEPS <${TARGET//.o/.dep}
	redo-ifchange ${DEPS#*:}

	$CXX $CXXFLAGS $DEFINES -o$OUTPUT -c ${TARGET//.o/.cxx}
}

function target_cxx_link {
	redo-ifchange $OBJS

	$CXX $CFXXLAGS $LIBS -o$OUTPUT $OBJS
}

function target_c_dep {
	$CC $CFLAGS $DEFINES -o${TARGET//.o/.dep} -MM -MG ${TARGET//.o/.c}
}

function target_c {
	target_c_dep
	read DEPS <${TARGET//.o/.dep}
	redo-ifchange ${DEPS#*:}

	$CC $CFLAGS $DEFINES -o$OUTPUT -c ${TARGET//.o/.c}
}

function target_c_link {
	redo-ifchange $OBJS

	$CC $CFLAGS $LIBS -o$OUTPUT $OBJS
}

function target_lib {
	redo-ifchange $OBJS

	$AR rcs $OUTPUT $OBJS
}

case "$TARGET" in
all)
	redo lib fuse test view mkfs fsck defrag fsctl attr ;;
lib)
	redo-ifchange $LIB_OUT ;;
fuse)
	redo-ifchange $FUSE_OUT ;;
test)
	redo-ifchange $TEST_OUT ;;
view)
	redo-ifchange $VIEW_OUT ;;
mkfs)
	redo-ifchange $MKFS_OUT ;;
fsck)
	redo-ifchange $FSCK_OUT ;;
defrag)
	redo-ifchange $DEFRAG_OUT ;;
fsctl)
	redo-ifchange $FSCTL_OUT ;;
attr)
	redo-ifchange $ATTR_OUT ;;
$LIB_OUT)
	LIBS=""
	OBJS="${LIB_OBJS[@]}"
	target_lib
	;;
$FUSE_OUT)
	LIBS="${FUSE_LIBS[@]} ${LIB_LIBS[@]}"
	OBJS="${FUSE_OBJS[@]} $LIB_OUT"
	target_c_link
	;;
$TEST_OUT)
	LIBS="${TEST_LIBS[@]} ${LIB_LIBS[@]}"
	OBJS="${TEST_OBJS[@]} $LIB_OUT"
	target_c_link
	;;
$VIEW_OUT)
	LIBS="${VIEW_LIBS[@]} ${LIB_LIBS[@]}"
	OBJS="${VIEW_OBJS[@]} $LIB_OUT"
	target_c_link
	;;
$MKFS_OUT)
	LIBS="${MKFS_LIBS[@]} ${LIB_LIBS[@]}"
	OBJS="${MKFS_OBJS[@]} $LIB_OUT"
	target_c_link
	;;
$FSCK_OUT)
	LIBS="${FSCK_LIBS[@]} ${LIB_LIBS[@]}"
	OBJS="${FSCK_OBJS[@]} $LIB_OUT"
	target_c_link
	;;
$DEFRAG_OUT)
	LIBS="${DEFRAG_LIBS[@]} ${LIB_LIBS[@]}"
	OBJS="${DEFRAG_OBJS[@]} $LIB_OUT"
	target_c_link
	;;
$FSCTL_OUT)
	LIBS="${FSCTL_LIBS[@]} ${LIB_LIBS[@]}"
	OBJS="${FSCTL_OBJS[@]} $LIB_OUT"
	target_c_link
	;;
$ATTR_OUT)
	LIBS="${ATTR_LIBS[@]} ${LIB_LIBS[@]}"
	OBJS="${ATTR_OBJS[@]} $LIB_OUT"
	target_c_link
	;;
*.o)
	if [[ -e "${TARGET//.o/.cxx}" ]]; then
		target_cxx
	elif [[ -e "${TARGET//.o/.c}" ]]; then
		target_c
	else
		echo "cannot find source file for '$TARGET'"
		exit 1
	fi
	;;
clean)
	rm -rf $(find bin/ -type f)
	rm -rf $(find src/ lib/ -type f -iname *.o)
	rm -rf $(find src/ lib/ -type f -iname *.dep)
	;;
backup)
	cd .. && tar -acvf backup/jgfs2-$(date +'%Y%m%d-%H%M').tar.xz jgfs2/
	;;
*)
	echo "unknown target '$TARGET'"
	exit 1
	;;
esac


[iegitaddmd ?] quotations marks in skipfolder() values

{smcl}
{* 5 Nov 2019}{...}
{hline}
help for {hi:iegitaddmd}
{hline}

{title:Title}

{cmd:iegitaddmd} {hline 2} Creates a placeholder file in subfolders of a GitHub repository folder, which allows committing folder structures with empty folders.

{phang2}For a more descriptive discussion on the intended usage and work flow of this
command please see the {browse "https://dimewiki.worldbank.org/wiki/Iegitaddmd":DIME Wiki}.

{title:Syntax}

{phang} {cmdab:iegitaddmd} , {opt folder(full_file_path)} [{opt comparefolder(full_file_path)} {cmd:customfile(}{it:{help filename}}{cmd:)} {opt all} {opt skip} {opt replace} {opt skipfolders(folder_name)}  {opt auto:matic} {opt dry:run}]

{marker opts}{...}
{synoptset 28}{...}
{synopthdr:options}
{synoptline}
{synopt :{opt folder(full_file_path)}}Specifies the folder path to the project folder clone where placeholder files will be created{p_end}
{synopt :{opt comparefolder(full_file_path)}}Specifies the top folder of a folder tree to be compared with {opt folder()}{p_end}
{synopt :{cmd:customfile(}{it:{help filename}}{cmd:)}}Specifies a file saved on disk that is used instead of the default file as placeholder{p_end}
{synopt :{cmd:all}}Creates the placeholder file in every subfolder of {opt folder()}, whether empty or not{p_end}
{synopt :{opt skip}}If option {opt all} is used and a folder has a file with same name as placeholder file, then nothing is done{p_end}
{synopt :{opt replace}}If option {opt all} is used and a folder has a file with same name as placeholder file, then the file is overwritten{p_end}
{synopt :{opt skipfolders(folder_name)}}List of folders to be skipped. The folder {it:.git} is always added to this list, even when option is not used{p_end}
{synopt :{opt auto:matic}}Makes the command create placeholder files without prompting the user for each file{p_end}
{synopt :{opt dry:run}}Makes the command list all the files that would have been created without this option.{p_end}
{synoptline}

{marker desc}
{title:Description}

{pstd}GitHub does not sync empty folders, or folder that only contain ignored files. However, it is common in research projects that these types of folders needs to be shared anyways. {cmd:iegitaddmd} provides a solution to this by creating placeholder files in in these folders. This placeholder file can then be shared to GitHub and that way the folder is also shared. {cmd:iegitaddmd} have two ways to identify which those folders are that are developed with the two use cases described below in mind.{p_end}

{pstd}{bf:Use case 1.} It is common in the beginning of a research project that a standardized folder structure is set up (for example with {help iefolder}) where some folders are created that are not yet needed. Examples of folders like that could be the output folders. If these folders are not shared at the time they are created, there is a risk that different team members name them differently which will be confusing and could lead to errors. {cmd:iegitaddmd} can solve this by scanning a folder and all its sub-folders for completely empty folders, and create a placeholder file in that folder.{p_end}

{pstd}The solution to use case 1 in {cmd:iegitaddmd} is a Stata adaptation of {it:Solution B} in {browse "http://bytefreaks.net/gnulinux/bash/how-to-add-automatically-all-empty-folders-in-git-repository" :this post}.{p_end}

{pstd}{bf:Use case 2.} A common method when transferring an already ongoing research project to GitHub is to copy all of the data work folder from wherever the files and folders where shared before (for example DropBox) to a clone of the repository. Then only the files that are meant to be shared on GitHub are uploaded to the cloud using a {it:.gitignore file}. Empty folders or folders where all files are ignored is then not shared through GitHub. {cmd:iegitaddmd} can solve this by scanning the DropBox folder (or wherever the project folder was shared before) for sub-folders that exist in that folder but not in the clone and crate that folder in the clone and put a placeholder in it.{p_end}

{pstd}The way you differentiate between these two use cases when using {cmd:iegitaddmd} is whether the option {opt comparefolder()} is used or not. If that option is used, then {cmd:iegitaddmd} will identify all folders that are in {opt comparefolder()} that is not in {opt folder()}, and create a placeholder file there. If {opt comparefolder()} is not used, then {cmd:iegitaddmd} will identify all completely empty folders (no files or subfolders) and create a placeholder file there.{p_end}

{pstd}The default placeholder file used if no file is specified using option {opt customfile()} is called README.md, which is a name and format recognized by GitHub.com so that the content of the file is displayed on GitHub.com when navigating to that folder using a browser. The placeholder file may be removed as	soon as files that are not ignored have been added to the folder, or even better is to keep the file but edit the content of it so it documents the content of that folder for other and future team members.{p_end}

{marker optslong}
{title:Options}

{phang}{opt folder(file_path)} is the folder path to the project folder clone where the placeholder files will be created. Unless option {opt comparefolder()} or {opt all} is used, then placeholder files will be created in all empty folders. See option {opt manual} for a way to manually confirm each folder before a placeholder file is created in it.{p_end}

{phang}{opt comparefolder(file_path)} is used to compare a file tree in a different location and create the folders with placeholder files in all folders that exist in {opt comparefolder()} but not in {opt folder()}. If a folder that does no exist in {opt folder()} has a subfolder, then both the folder and the subfolder will be created and the placeholder file is only crated in the sub-folder. This is enough to share both folder and the subfolder in GitHub. If the {opt comparefolder()} and {opt folder()} ends with folders with different names, for example {opt folder(${project}/datawork/baseline)} and {opt comparefolder(${project}/datawork)}, then the command will throw an error as it is likely that the paths do not point to corresponding folders in the two folder trees. See option {opt manual} for a way to manually confirm each folder before a placeholder file is created in it.{p_end}

{phang}{cmd:file(}{it:{help filename}}{cmd:)} allows the user to specify a file saved on
	the computer to be used as the placeholder file	instead of the default README.md file. This
	allows people and organizations to write their own template placeholder files according to their own
	preferences. We recommend that a file of type .md (markdown) and name README.md is used as
	GitHub.com recognizes this name and format and will display the content of the file when someone
	browse to that folder on GitHub.com. But this is not a technical requirement, any file type
	and name can be used as placeholder file.{p_end}

{phang}{opt all} creates the placeholder file in {opt folder()} and in every subfolder,
	regardless if they are empty or not. This allows the user to create a placeholder file in every
	folder that can be edited with documentation and instructions to the purpose and usage of
	each subfolder. This option can also be important when .{it:gitignore} is used, as {cmd:iegitaddmd} will not
	create files to subfolders that only has ignored file -- in which case the folder will not
	be synced by GitHub.{p_end}

{phang}{opt skip} and {opt replace} tells {cmd:iegitaddmd} what to do if the option {opt all} is used
	and if any of the folders contain a file with the same name as the placeholder file the command
	is trying to create. If a file with the same name as the placeholder file exist and neither of these options
	are used, then the command will throw an error. If {opt skip} is used, then nothing is done in the
	folder where the file with the same name exists and the command proceeds to the next folder.
	If {opt replace} is used then the file with the same name is overwritten with the new placeholder
	file before the command proceeds to the next folder.{p_end}

{phang}{opt skipfolders(folder_name)} can be used to tell {cmd:iegitaddmd} which folders
in which a placeholder file should never be created. The best example of this is the {it:.git}
folder in which placeholders never should be created. That name is always skipped regardless if
this option is used or not. In this option additional folders to be skipped can be listed. You
should not list the full folder path in this option, just the folder name. All folders with
that name will be skipped regardless of location in the project folder. The folder names may
be listed with or without quotation marks as long as there are no spaces in the names. If any of
the folder names has spaces, then quotation marks must be used for all folder names.

{phang}{opt auto:matic} can be used to speed up the creation of placeholders by telling the
command to not prompt the users for confirmation for each file before it is created. The
default is that the command is asking the user before creating each place holder file.
This option should only be used when you are confident you have specified the correct folder
paths. We recommend that you use the {opt dryrun} with this option to make sure that the
folder paths are correct.{p_end}

{phang}{opt dry:run} can be used to safely test that the folder paths are specified correctly
before any placeholder files are created. When this option it used the command simply lists
the file that would have been created if this option were not used. Once you are confident
that list is correct, you can remove this option and re-run the command and the files will
be created.{p_end}

{title:Example}

{pstd}{inp:global github_folder "C:\Users\JohnSmith\Documents\GitHub\ProjectA"}{break}{inp:iegitaddmd , folder({it:"$github_folder"})}{p_end}

{pstd}In the example above, there is a GitHub repository in the folder ProjectA. This
	repository has a folder structure where some folders are still empty but will later
	be populated with files. In order to have all folders, even the empty ones, synced on all
	collaborators' cloned local copies of the repository, the folders need to contain at least
	one file, which is being created by the command.{p_end}

{title:Acknowledgements}

{pstd}I would like to acknowledge the help in testing and proofreading I received in relation to this command and help file from (in alphabetic order):{p_end}
{pmore}Guadalupe Bedoya{break}Luiza Cardoso de Andrade{break}Mrijan Rimal{break}Benjamin Daniels{break}

{title:Author}

{phang}All commands in ietoolkit is developed by DIME Analytics at DECIE, The World Bank's unit for Development Impact Evaluations.

{phang}Main author: Kristoffer Bjarkefur, DIME Analytics, The World Bank Group

{phang}Please send bug-reports, suggestions and requests for clarifications
		 writing "ietoolkit iegitaddmd" in the subject line to:{break}
		 dimeanalytics@worldbank.org

{phang}You can also see the code, make comments to the code, see the version
		 history of the code, and submit additions or edits to the code through {browse "https://github.com/worldbank/ietoolkit":the GitHub repository of ietoolkit}.{p_end}

{smcl}
{* 5 Nov 2019}{...}
{hline}
help for {hi:iegitaddmd}
{hline}

{title:Title}

{cmd:iegitaddmd} {hline 2} Creates a placeholder file in subfolders of a GitHub repository folder, which allows committing folder structures with empty folders.

{phang2}For a more descriptive discussion on the intended usage and work flow of this
command please see the {browse "https://dimewiki.worldbank.org/wiki/Iegitaddmd":DIME Wiki}.

{title:Syntax}

{phang} {cmdab:iegitaddmd} , {opt folder(full_file_path)} [{opt comparefolder(full_file_path)} {cmd:customfile(}{it:{help filename}}{cmd:)} {opt all} {opt skip} {opt replace} {opt skipfolders(folder_name)}  {opt auto:matic} {opt dry:run}]

{marker opts}{...}
{synoptset 28}{...}
{synopthdr:options}
{synoptline}
{synopt :{opt folder(full_file_path)}}Specifies the folder path to the project folder clone where placeholder files will be created{p_end}
{synopt :{opt comparefolder(full_file_path)}}Specifies the top folder of a folder tree to be compared with {opt folder()}{p_end}
{synopt :{cmd:customfile(}{it:{help filename}}{cmd:)}}Specifies a file saved on disk that is used instead of the default file as placeholder{p_end}
{synopt :{cmd:all}}Creates the placeholder file in every subfolder of {opt folder()}, whether empty or not{p_end}
{synopt :{opt skip}}If option {opt all} is used and a folder has a file with same name as placeholder file, then nothing is done{p_end}
{synopt :{opt replace}}If option {opt all} is used and a folder has a file with same name as placeholder file, then the file is overwritten{p_end}
{synopt :{opt skipfolders(folder_name)}}List of folders to be skipped. The folder {it:.git} is always added to this list, even when option is not used{p_end}
{synopt :{opt auto:matic}}Makes the command create placeholder files without prompting the user for each file{p_end}
{synopt :{opt dry:run}}Makes the command list all the files that would have been created without this option.{p_end}
{synoptline}

{marker desc}
{title:Description}

{pstd}GitHub does not sync empty folders, or folder that only contain ignored files. However, it is common in research projects that these types of folders needs to be shared anyways. {cmd:iegitaddmd} provides a solution to this by creating placeholder files in in these folders. This placeholder file can then be shared to GitHub and that way the folder is also shared. {cmd:iegitaddmd} have two ways to identify which those folders are that are developed with the two use cases described below in mind.{p_end}

{pstd}{bf:Use case 1.} It is common in the beginning of a research project that a standardized folder structure is set up (for example with {help iefolder}) where some folders are created that are not yet needed. Examples of folders like that could be the output folders. If these folders are not shared at the time they are created, there is a risk that different team members name them differently which will be confusing and could lead to errors. {cmd:iegitaddmd} can solve this by scanning a folder and all its sub-folders for completely empty folders, and create a placeholder file in that folder.{p_end}

{pstd}The solution to use case 1 in {cmd:iegitaddmd} is a Stata adaptation of {it:Solution B} in {browse "http://bytefreaks.net/gnulinux/bash/how-to-add-automatically-all-empty-folders-in-git-repository" :this post}.{p_end}

{pstd}{bf:Use case 2.} A common method when transferring an already ongoing research project to GitHub is to copy all of the data work folder from wherever the files and folders where shared before (for example DropBox) to a clone of the repository. Then only the files that are meant to be shared on GitHub are uploaded to the cloud using a {it:.gitignore file}. Empty folders or folders where all files are ignored is then not shared through GitHub. {cmd:iegitaddmd} can solve this by scanning the DropBox folder (or wherever the project folder was shared before) for sub-folders that exist in that folder but not in the clone and crate that folder in the clone and put a placeholder in it.{p_end}

{pstd}The way you differentiate between these two use cases when using {cmd:iegitaddmd} is whether the option {opt comparefolder()} is used or not. If that option is used, then {cmd:iegitaddmd} will identify all folders that are in {opt comparefolder()} that is not in {opt folder()}, and create a placeholder file there. If {opt comparefolder()} is not used, then {cmd:iegitaddmd} will identify all completely empty folders (no files or subfolders) and create a placeholder file there.{p_end}

{pstd}The default placeholder file used if no file is specified using option {opt customfile()} is called README.md, which is a name and format recognized by GitHub.com so that the content of the file is displayed on GitHub.com when navigating to that folder using a browser. The placeholder file may be removed as	soon as files that are not ignored have been added to the folder, or even better is to keep the file but edit the content of it so it documents the content of that folder for other and future team members.{p_end}

{marker optslong}
{title:Options}

{phang}{opt folder(file_path)} is the folder path to the project folder clone where the placeholder files will be created. Unless option {opt comparefolder()} or {opt all} is used, then placeholder files will be created in all empty folders. See option {opt manual} for a way to manually confirm each folder before a placeholder file is created in it.{p_end}

{phang}{opt comparefolder(file_path)} is used to compare a file tree in a different location and create the folders with placeholder files in all folders that exist in {opt comparefolder()} but not in {opt folder()}. If a folder that does no exist in {opt folder()} has a subfolder, then both the folder and the subfolder will be created and the placeholder file is only crated in the sub-folder. This is enough to share both folder and the subfolder in GitHub. If the {opt comparefolder()} and {opt folder()} ends with folders with different names, for example {opt folder(${project}/datawork/baseline)} and {opt comparefolder(${project}/datawork)}, then the command will throw an error as it is likely that the paths do not point to corresponding folders in the two folder trees. See option {opt manual} for a way to manually confirm each folder before a placeholder file is created in it.{p_end}

{phang}{cmd:file(}{it:{help filename}}{cmd:)} allows the user to specify a file saved on
	the computer to be used as the placeholder file	instead of the default README.md file. This
	allows people and organizations to write their own template placeholder files according to their own
	preferences. We recommend that a file of type .md (markdown) and name README.md is used as
	GitHub.com recognizes this name and format and will display the content of the file when someone
	browse to that folder on GitHub.com. But this is not a technical requirement, any file type
	and name can be used as placeholder file.{p_end}

{phang}{opt all} creates the placeholder file in {opt folder()} and in every subfolder,
	regardless if they are empty or not. This allows the user to create a placeholder file in every
	folder that can be edited with documentation and instructions to the purpose and usage of
	each subfolder. This option can also be important when .{it:gitignore} is used, as {cmd:iegitaddmd} will not
	create files to subfolders that only has ignored file -- in which case the folder will not
	be synced by GitHub.{p_end}

{phang}{opt skip} and {opt replace} tells {cmd:iegitaddmd} what to do if the option {opt all} is used
	and if any of the folders contain a file with the same name as the placeholder file the command
	is trying to create. If a file with the same name as the placeholder file exist and neither of these options
	are used, then the command will throw an error. If {opt skip} is used, then nothing is done in the
	folder where the file with the same name exists and the command proceeds to the next folder.
	If {opt replace} is used then the file with the same name is overwritten with the new placeholder
	file before the command proceeds to the next folder.{p_end}

{phang}{opt skipfolders(folder_name)} can be used to tell {cmd:iegitaddmd} which folders
in which a placeholder file should never be created. The best example of this is the {it:.git}
folder in which placeholders never should be created. That name is always skipped regardless if
this option is used or not. In this option additional folders to be skipped can be listed. You
should not list the full folder path in this option, just the folder name. All folders with
that name will be skipped regardless of location in the project folder.

{phang}{opt auto:matic} can be used to speed up the creation of placeholders by telling the
command to not prompt the users for confirmation for each file before it is created. The
default is that the command is asking the user before creating each place holder file.
This option should only be used when you are confident you have specified the correct folder
paths. We recommend that you use the {opt dryrun} with this option to make sure that the
folder paths are correct.{p_end}

{phang}{opt dry:run} can be used to safely test that the folder paths are specified correctly
before any placeholder files are created. When this option it used the command simply lists
the file that would have been created if this option were not used. Once you are confident
that list is correct, you can remove this option and re-run the command and the files will
be created.{p_end}

{title:Example}

{pstd}{inp:global github_folder "C:\Users\JohnSmith\Documents\GitHub\ProjectA"}{break}{inp:iegitaddmd , folder({it:"$github_folder"})}{p_end}

{pstd}In the example above, there is a GitHub repository in the folder ProjectA. This
	repository has a folder structure where some folders are still empty but will later
	be populated with files. In order to have all folders, even the empty ones, synced on all
	collaborators' cloned local copies of the repository, the folders need to contain at least
	one file, which is being created by the command.{p_end}

{title:Acknowledgements}

{pstd}I would like to acknowledge the help in testing and proofreading I received in relation to this command and help file from (in alphabetic order):{p_end}
{pmore}Guadalupe Bedoya{break}Luiza Cardoso de Andrade{break}Mrijan Rimal{break}Benjamin Daniels{break}

{title:Author}

{phang}All commands in ietoolkit is developed by DIME Analytics at DECIE, The World Bank's unit for Development Impact Evaluations.

{phang}Main author: Kristoffer Bjarkefur, DIME Analytics, The World Bank Group

{phang}Please send bug-reports, suggestions and requests for clarifications
		 writing "ietoolkit iegitaddmd" in the subject line to:{break}
		 dimeanalytics@worldbank.org

{phang}You can also see the code, make comments to the code, see the version
		 history of the code, and submit additions or edits to the code through {browse "https://github.com/worldbank/ietoolkit":the GitHub repository of ietoolkit}.{p_end}


update 02_bench to do all tests

update 02_bench to do all tests

/*
Benchmark program

This dofile describes takes the output from 01_parellel_benchmark.do and
creates a plot and a table.

This is work-in-progress.
*/

clear all

set more off
set trace off

// Function to print data
mata:
void output_the_table() {
	psize    = st_data(.,"psize")
	avgtimes = st_data(.,2..st_nvar())
	testname = strlower(st_global("TEST"))
	fn       = "tables_and_figures/parallel_benchmarks_test="+testname+".tex"
	if (fileexists(fn)) unlink(fn)
	fh       = fopen(fn, "w")

	// fput(fh, "\begin{tabular}{*{2}{m{.2\textwidth}}*{2}{m{.25\textwidth}}}")
	fput(fh, "\centering\begin{tabular}{l*{3}{c}}")
	fput(fh, "\toprule")
	fput(fh, "Problem size & Serial & 2 Clusters & 4 Clusters\\\midrule")
	for(i=1;i<=length(psize);i++) {

		// Times
		fwrite(fh, sprintf("%3.0f", psize[i]))
		for(j=1;j<=cols(avgtimes);j++) {
			// ans = ans + " &  "  + sprintf("%5.2f", avgtimes[i,j])
			fwrite(fh, " &  "  + sprintf("%5.2fs", avgtimes[i,j]))
		}

		fput(fh, " \\    ")

		// Relative times
		tmp = 1:/(J(1,cols(avgtimes),min(avgtimes[i,.])) :/avgtimes[i,.])
		for(j=1;j<=length(tmp);j++) {
			fwrite(fh, " &  x"  + sprintf("%4.2f", tmp[j]))
		}

		if (i != length(psize)) fput(fh, " \\ \\")
		else fput(fh, " \\")
	}

	fput(fh, "\bottomrule")
	fput(fh, "\end{tabular}")
	fclose(fh)
}
end

// Parameters
global DATE         20161102
global nreps        1000
global TESTS          BOOTTEST SIMTEST
global PROCESSOR    "Intel Core i7-4790 CPU @ 3.60GHz  8"

m: st_global("filename", sprintf("%f_parallel-bechmark_nreps=%04.0f.dta",$DATE, $nreps))
cap mkdir "tables_and_figures", public

foreach testname of global TESTS {
global TEST `testname'
use "$filename", clear

// Getting the parameters
local statav : display %3.1f stata_version[1]
global whichstata = "Stata `=flavor[1]' `statav' on a `=os[1]' machine with an $PROCESSOR processor"

// Filtering data and stacking times and nclusters
keep if test == "$TEST"
mata: TIMES = st_data(.,"tot_pll")\st_data(.,"tot_serial")
mata: PSIZE = st_data(.,"problem_size")\st_data(.,"problem_size")
mata: NCLUS = st_data(.,"nclusters")\J(st_nobs(),1,1)
mata: st_local("n", strofreal(st_nobs()*2))

// Storing the data back into stata
drop _all
set obs `n'
gen double time    = .
gen double psize   = .
gen double nclusts = .

mata: st_store(.,.,(TIMES, PSIZE, NCLUS))

// Reshaping to be used as time series
collapse (mean) avg_time=time, by(nclusts psize)

reshape wide avg_time, i(psize) j(nclusts)
// Adding labels
lab var avg_time1 "Serial"
lab var avg_time2 "2 Clusters"
lab var avg_time4 "4 Clusters"
lab var psize "Problem size"

graph twoway (connected avg* psize), ///
	ytitle(Total Time (in secs.)) scheme(sj) ///
	legend(rows(1) lstyle(transparend) subtitle(Method)) ///
	note( ///
	"Each point represents `=$nreps' runs of the problem." ///
	"$whichstata" ///
	)

graph export "tables_and_figures/parallel_benchmarks_test=`=lower("$TEST")'.eps", replace

mata: output_the_table()

}

/*
Benchmark program

This dofile describes takes the output from 01_parellel_benchmark.do and
creates a plot and a table.

This is work-in-progress.
*/

clear all

set more off
set trace off

// Parameters
global DATE         20161102
global nreps        1000
global TEST         BOOTTEST
global PROCESSOR    "Intel Core i7-4790 CPU @ 3.60GHz  8"

m: st_global("filename", sprintf("%f_parallel-bechmark_nreps=%04.0f.dta",$DATE, $nreps))

use "$filename", clear

// Getting the parameters
local statav : display %3.1f stata_version[1]
global whichstata = "Stata `=flavor[1]' `statav' on a `=os[1]' machine with an $PROCESSOR processor"

// Filtering data and stacking times and nclusters
keep if test == "$TEST"
mata:
TIMES = st_data(.,"tot_pll")\st_data(.,"tot_serial")
PSIZE = st_data(.,"problem_size")\st_data(.,"problem_size")
NCLUS = st_data(.,"nclusters")\J(st_nobs(),1,1)
st_local("n", strofreal(st_nobs()*2))
end

// Storing the data back into stata
drop _all
set obs `n'
gen double time    = .
gen double psize   = .
gen double nclusts = .

mata: st_store(.,.,(TIMES, PSIZE, NCLUS))

// Reshaping to be used as time series
collapse (mean) avg_time=time, by(nclusts psize)

reshape wide avg_time, i(psize) j(nclusts)
// Adding labels
lab var avg_time1 "Serial"
lab var avg_time2 "2 Clusters"
lab var avg_time4 "4 Clusters"
lab var psize "Problem size"

graph twoway (connected avg* psize), ///
	ytitle(Total Time (in secs.)) scheme(sj) ///
	legend(rows(1) lstyle(transparend) subtitle(Method)) ///
	note( ///
	"Each point represents `=$nreps' runs of the problem." ///
	"$whichstata" ///
	)

graph export "tables_and_figures/parallel_benchmarks_test=`=lower("$TEST")'.eps", replace

// Function to print data
mata:
psize    = st_data(.,"psize")
avgtimes = st_data(.,2..st_nvar())
fn       = "tables_and_figures/parallel_benchmarks_test=`=lower("$TEST")'.tex"
if (fileexists(fn)) unlink(fn)
fh       = fopen(fn, "w")

// fput(fh, "\begin{tabular}{*{2}{m{.2\textwidth}}*{2}{m{.25\textwidth}}}")
fput(fh, "\centering\begin{tabular}{l*{3}{c}}")
fput(fh, "\toprule")
fput(fh, "Problem size & Serial & 2 Clusters & 4 Clusters\\\midrule")
for(i=1;i<=length(psize);i++) {

	// Times
	fwrite(fh, sprintf("%3.0f", psize[i]))
	for(j=1;j<=cols(avgtimes);j++) {
		// ans = ans + " &  "  + sprintf("%5.2f", avgtimes[i,j])
		fwrite(fh, " &  "  + sprintf("%5.2fs", avgtimes[i,j]))
	}

	fput(fh, " \\    ")

	// Relative times
	tmp = 1:/(J(1,cols(avgtimes),min(avgtimes[i,.])) :/avgtimes[i,.])
	for(j=1;j<=length(tmp);j++) {
		fwrite(fh, " &  x"  + sprintf("%4.2f", tmp[j]))
	}

	if (i != length(psize)) fput(fh, " \\ \\")
	else fput(fh, " \\")
}

fput(fh, "\bottomrule")
fput(fh, "\end{tabular}")
fclose(fh)
end