ieboilstart: clening up do-file

ieboilstart: clening up do-file

some simple cleaning up of do-file

	*! version 2.0 24Oct2016  Kristoffer Bjarkefur kbjarkefur@worldbank.org

	capture program drop ieboilstart2
	program ieboilstart2 , rclass

	qui {

		syntax using ,  Versionnumber(string) [noclear maxvar(numlist) matsize(numlist) QUIetdo CUSTom(string) ]

		*Potentional updates
		*set trace on
		*set scrollbufsize #

		version 10.0
		version `versionnumber'

		/*********************************

			Check file name

		*********************************/

		*Parse out the file name
		tokenize `"`using'"'
		local 		textfile "`2'"

		*Find index for where the file type suffix start
		local dot_index 	= strpos("`textfile'",".")

		*Extract the file index
		local file_suffix 	= substr("`textfile'", `dot_index', .)

		*If no file format suffix is specified, use the default .xlsx
		if "`file_suffix'" == "" {

			local textfile `"`textfile'.do"'
		}

		*If a file format suffix is specified make sure that it is one of the two allowed.
		else if "`file_suffix'" != ".do" {

			noi display as error "{phang}The file format specified in using is other than .do. Only the .do format is allowed. If no format is specified .do is assumed.{p_end}"
			error 198
		}

		/*********************************

			Check version

		*********************************/

		if "`maxvar'" != "" & "`clear'" != "" {

			di as error "{phang}It is not possible to set allowed maximum numbers of variables without clearing the data. noclear and maxvar() can therefore not be specified at the same time{p_end}"
			di ""
			error 198
			exit
		}

		local stata_versions "10.0 10.1 11.0 11.1 11.2 12.0 12.1 13.0 13.1 14.0 14.1"

		if `:list versionnumber in stata_versions' == 0 {

			di as error "{phang}Only recent major releases are allowed. One decimal must always be included. The releases currently allowed are:{break}`stata_versions'{p_end}"
			di ""
			error 198
			exit
		}

		/*********************************

			Check input for maxvar and matsize
			if specified, other wise set
			maximum value allowed.

		*********************************/

		local stata_types ic se mp
		foreach maxlocal in maxvar matsize {

			if "`maxlocal'" == "maxvar" {
				if c(MP) == 1 | c(SE) == 1 {
					local max 32767
					local min 2048
				}
				else {
					local max 2047
					local min 2047
				}
			}

			if "`maxlocal'" == "matsize" {
				if c(MP) == 1 | c(SE) == 1 {
					local max 11000
					local min 10
				}
				else {
					local max 800
					local min 10
				}
			}

			if c(MP) == 1 | c(SE) == 1 {
				local vusing "Stata SE and Stata MP"
			}
			else {
				local vusing "Stata IC"
			}

			*Test if user set maxvar
			if "``maxlocal''" != "" {

				if ``maxlocal'' >= `min' & ``maxlocal'' <= `max' {

					local `maxlocal' ``maxlocal''
				}
				else {

					di as error "{phang}`maxlocal' must be between `min' and `max' (inclusive) if you are using `vusing'. You entered ``maxlocal''.{p_end}"
					if ``maxlocal'' < `min' {
						di ""
						error 910
					}
					else {
						di ""
						error 912
					}
					exit
				}
			}
			else {

				*User did not specify value, use max value allowed
				local `maxlocal' `max'
			}
		}

		/*********************************

			Prepare a local with all settings

		*********************************/

		local return_string ""

		*Set verison number
		local return_string `return_string' version `versionnumber' ;

		*Set basic memory limits
		if "`clear'" == "" {
			local return_string `return_string' clear all ;
			local return_string `return_string' set maxvar 	`maxvar' ;
		}

		local return_string `return_string'  set matsize 	`matsize' ;

		*Set advanced memory limits
		local return_string `return_string' set niceness	5 ;

		if "`clear'" == "" {
			*These settings cannot be modified with data in memory
			local return_string `return_string' set min_memory	0 ;
			local return_string `return_string' set max_memory	. ;

			if c(bit) == 64 {
				local return_string `return_string' set segmentsize	32m ;
			}
			else {
				local return_string `return_string' set segmentsize	16m ;
			}
		}

		*Set default options
		local return_string `return_string' set more 		off ;
		local return_string `return_string' pause 			on 	;
		local return_string `return_string' set varabbrev 	off ;

		/*********************************

			Add custom lines of code

		*********************************/

		if "`custom'" != "" {

			*Add a comment that list where
			local return_string `return_string' ** The lines of code below was added manually when iboilstart ;
			local return_string `return_string' *  was used to create  this do-file. If there are any issues ;
			local return_string `return_string' *  with the code below, please reconfigure ieboilstart and ;
			local return_string `return_string' *  generate a new do-file. ;

			*Create a local with the rowlabel input to be tokenized
			local custom_code_lines `custom'

			while "`custom_code_lines'" != "" {

				*Parsing name and label pair
				gettoken code_line custom_code_lines : custom_code_lines, parse("@")

				*Removing leadning or trailing spaces
				local code_line = trim("`code_line'")

				local return_string `return_string' `code_line';

				*Parse char is not removed by gettoken
				local custom_code_lines = subinstr("`custom_code_lines'" ,"@","",1)
			}
		}

		/*********************************

			Create the do-file

		*********************************/

		*Create a local with the date
		local		date = subinstr(c(current_date)," ","",.)

		*Write the title rows defined above
		capture file close settingsfile

		*Set up the file to be written
		file open  settingsfile using "`textfile'", text write replace

		if "`quietdo'" != "" file write settingsfile "quietly {" _n

		*Write the standard message in the opening file.
		file write settingsfile ///
																		_n ///
			`" 	/*************************************************"' 	_n ///
			`" 	This file is was prepared by command ieboilstart "' 	_n ///
			`" 	on `date'."'											_n ///
																		_n ///
			`" 	Run this file to set the recommended settings for "' 	_n ///
			`" 	this project to your instance of Stata"' 				_n ///
			`" 	*************************************************/"' 	_n ///
																		_n ///
																				_n ///
			"	*See help file for ieboilstart for details on these settings"	_n ///
			_n

		** Write each setting prepared above to the do-file.
		while ("`return_string'" != "") {

			*Tokenize the first setting
			gettoken setting return_string : return_string, parse(";")

			*Write it to file
			file write settingsfile "	`setting'" _n

			*Remove the parse character
			local return_string = subinstr("`return_string'",";","",1)
		}

		if "`quietdo'" != "" file write settingsfile "}" _n

		*Close file
		file close settingsfile

		/*********************************

			Create return value and output message

		*********************************/

		return local ieboil_dofile "`textfile'"

		noi di "{phang}Due to how settings in Stata work, a command can not change any settings itself. ieboilstart therefore only prepares a do-file with all recommended settings. See {help ieboilstart :help ieboilstart} for how to apply the settings ieboilstart has prepared. If you do not run the do-file, ieboilstart does not affect any of your settings.{p_end}"
		noi di ""
		noi di "{phang}Do-file with all settings saved to: `textfile'{p_end}"
	}
	end

	*! version 2.0 24Oct2016  Kristoffer Bjarkefur kbjarkefur@worldbank.org

	capture program drop ieboilstart2
	program ieboilstart2 , rclass

	qui {

		syntax using ,  Versionnumber(string) [noclear maxvar(numlist) matsize(numlist) QUIetdo CUSTom(string) ]

		*set trace on
		*set scrollbufsize #

		version 10.0
		version `versionnumber'

		/*********************************

			Check file name

		*********************************/

		*Parse out the file name
		tokenize `"`using'"'
		local 		textfile "`2'"

		*Find index for where the file type suffix start
		local dot_index 	= strpos("`textfile'",".")

		*Extract the file index
		local file_suffix 	= substr("`textfile'", `dot_index', .)

		*If no file format suffix is specified, use the default .xlsx
		if "`file_suffix'" == "" {

			local textfile `"`textfile'.do"'
		}

		*If a file format suffix is specified make sure that it is one of the two allowed.
		else if "`file_suffix'" != ".do" {

			noi display as error "{phang}The file format specified in using is other than .do. Only the .do format is allowed. If no format is specified .do is assumed.{p_end}"
			error 198
		}

		/*********************************

			Check version

		*********************************/

		if "`maxvar'" != "" & "`clear'" != "" {

			di as error "{phang}It is not possible to set allowed maximum numbers of variables without clearing the data. noclear and maxvar() can therefore not be specified at the same time{p_end}"
			di ""
			error 198
			exit
		}

		local stata_versions "10.0 10.1 11.0 11.1 11.2 12.0 12.1 13.0 13.1 14.0 14.1"

		if `:list versionnumber in stata_versions' == 0 {

			di as error "{phang}Only recent major releases are allowed. One decimal must always be included. The releases currently allowed are:{break}`stata_versions'{p_end}"
			di ""
			error 198
			exit
		}

		/*********************************

			Check input for maxvar and matsize
			if specified, other wise set
			maximum value allowed.

		*********************************/

		local stata_types ic se mp
		foreach maxlocal in maxvar matsize {

			if "`maxlocal'" == "maxvar" {
				if c(MP) == 1 | c(SE) == 1 {
					local max 32767
					local min 2048
				}
				else {
					local max 2047
					local min 2047
				}
			}

			if "`maxlocal'" == "matsize" {
				if c(MP) == 1 | c(SE) == 1 {
					local max 11000
					local min 10
				}
				else {
					local max 800
					local min 10
				}
			}

			if c(MP) == 1 | c(SE) == 1 {
				local vusing "Stata SE and Stata MP"
			}
			else {
				local vusing "Stata IC"
			}

			*Test if user set maxvar
			if "``maxlocal''" != "" {

				if ``maxlocal'' >= `min' & ``maxlocal'' <= `max' {

					local `maxlocal' ``maxlocal''
				}
				else {

					di as error "{phang}`maxlocal' must be between `min' and `max' (inclusive) if you are using `vusing'. You entered ``maxlocal''.{p_end}"
					if ``maxlocal'' < `min' {
						di ""
						error 910
					}
					else {
						di ""
						error 912
					}
					exit
				}
			}
			else {

				*User did not specify value, use max value allowed
				local `maxlocal' `max'
			}
		}

		/*********************************

			Prepare a local with all settings

		*********************************/

		local return_string ""

		*Set verison number
		local return_string `return_string' version `versionnumber' ;

		*Set basic memory limits
		if "`clear'" == "" {
			local return_string `return_string' clear all ;
			local return_string `return_string' set maxvar 	`maxvar' ;
		}

		local return_string `return_string'  set matsize 	`matsize' ;

		*Set advanced memory limits
		local return_string `return_string' set niceness	5 ;

		if "`clear'" == "" {
			*These settings cannot be modified with data in memory
			local return_string `return_string' set min_memory	0 ;
			local return_string `return_string' set max_memory	. ;

			if c(bit) == 64 {
				local return_string `return_string' set segmentsize	32m ;
			}
			else {
				local return_string `return_string' set segmentsize	16m ;
			}
		}

		*Set default options
		local return_string `return_string' set more 		off ;
		local return_string `return_string' pause 			on 	;
		local return_string `return_string' set varabbrev 	off ;

		/*********************************

			Add custom lines of code

		*********************************/

		if "`custom'" != "" {

			*Add a comment that list where
			local return_string `return_string' ** The lines of code below was added manually when iboilstart ;
			local return_string `return_string' *  was used to create  this do-file. If there are any issues ;
			local return_string `return_string' *  with the code below, please reconfigure ieboilstart and ;
			local return_string `return_string' *  generate a new do-file. ;

			*Create a local with the rowlabel input to be tokenized
			local custom_code_lines `custom'

			while "`custom_code_lines'" != "" {

				*Parsing name and label pair
				gettoken code_line custom_code_lines : custom_code_lines, parse("@")

				*Removing leadning or trailing spaces
				local code_line = trim("`code_line'")

				local return_string `return_string' `code_line';

				*Parse char is not removed by gettoken
				local custom_code_lines = subinstr("`custom_code_lines'" ,"@","",1)
			}
		}

		/*********************************

			Create the do-file

		*********************************/

		*Create a local with the date
		local		date = subinstr(c(current_date)," ","",.)

		*Write the title rows defined above
		capture file close settingsfile

		*Set up the file to be written
		file open  settingsfile using "`textfile'", text write replace

		if "`quietdo'" != "" file write settingsfile "quietly {" _n

		*Write the standard message in the opening file.
		file write settingsfile ///
																		_n ///
			`" 	/*************************************************"' 	_n ///
			`" 	This file is was prepared by command ieboilstart "' 	_n ///
			`" 	on `date'."'											_n ///
																		_n ///
			`" 	Run this file to set the recommended settings for "' 	_n ///
			`" 	this project to your instance of Stata"' 				_n ///
			`" 	*************************************************/"' 	_n ///
																		_n ///
																				_n ///
			"	*See help file for ieboilstart for details on these settings"	_n ///
			_n

		** Write each setting prepared above to the do-file.
		while ("`return_string'" != "") {

			*Tokenize the first setting
			gettoken setting return_string : return_string, parse(";")

			*Write it to file
			file write settingsfile "	`setting'" _n

			*Remove the parse character
			local return_string = subinstr("`return_string'",";","",1)
		}

		if "`quietdo'" != "" file write settingsfile "}" _n

		*Close file
		file close settingsfile

		/*********************************

			Create return value and output message

		*********************************/

		return local ieboil_dofile "`textfile'"

		noi di "{phang}Due to how settings in Stata work, a command can not change any settings itself. ieboilstart therefore only prepares a do-file with all recommended settings. See {help ieboilstart :help ieboilstart} for how to apply the settings ieboilstart has prepared. If you do not run the do-file, ieboilstart does not affect any of your settings.{p_end}"
		noi di ""
		noi di "{phang}Do-file with all settings saved to: `textfile'{p_end}"
	}
	end


killer pic

killer pic

program cite_killer_pic

use ${stash}citelines, clear

keep citeyear numcites numimg numtext year

keep if year == 2008 | year == 2012
//keep if year == 2013

sort citeyear year
foreach x in cites img text{
    di "`x'"
    bysort citeyear: gen diff`x' = num`x'[2]-num`x'[1]
}

keep diff* citeyear

foreach x in cite img text{
    if "`x'" == "img"{
        local y "Image"
    }
    else if "`x'" == "text"{
        local y "Text"
    }
    else {
        local y "Cites"
    }

    tw (bar diff`x' citeyear if citeyear < 1964, barwidth(0.75) color(gs2)) (bar diff`x' citeyear if citeyear >= 1964, barwidth(0.75) color(gs8)),  title("") xtitle("Year of Baseball Digest Publication") ytitle("`y' Citations Added After Digitization") xsca(titlegap(3) range(1944 1984) noextend) ysca(titlegap(2) range(-0.2 30)) xline(1963.5, noextend lcolor(gs4)) xlabel(1944(4)1984) ylabel(0(5)30,nogrid) legend(label(1 "`nocopyright' (Before 1964)") label(2 "`incopyright' (After 1964)")) legend(order(1 2))

    graph export "${tables}cite_killer_`x'.eps", replace
    shell epstopdf "${tables}cite_killer_`x'.eps"
}

end

program cite_killer_pic

use ${stash}citelines, clear

keep citeyear numcites numimg numtext year

keep if year == 2008 | year == 2013
sort citeyear year
foreach x in cites img text{
    di "`x'"
    bysort citeyear: gen diff`x' = num`x'[2]-num`x'[1]
}

keep diff* citeyear

foreach x in img text{
    if "`x'" == "img"{
        local y "Image"
        local z "A"
    }
    else{
        local y "Text"
        local z "B"
    }
    tw (bar diff`x' citeyear if citeyear < 1964, barwidth(0.75) color(gs2)) (bar diff`x' citeyear if citeyear >= 1964, barwidth(0.75) color(gs8)),  title("Panel `z' : Increase in `y' Citations Post-Digitization") xtitle("Year of Baseball Digest Publication") ytitle("`y' Citations Added After Digitization") xsca(titlegap(3) range(1944 1984) noextend) ysca(titlegap(2) range(-0.2 30)) xline(1963.5, noextend lcolor(gs4)) xlabel(1944(4)1984) ylabel(0(5)30,nogrid) legend(label(1 "`nocopyright' (Before 1964)") label(2 "`incopyright' (After 1964)")) legend(order(1 2))

    graph export "${tables}cite_killer_`x'.eps", replace
    shell epstopdf "${tables}cite_killer_`x'.eps"
}

end


Bump version

Bump version

*! version 4.4.7 25sep2017

program reghdfe, eclass
	* Intercept old+version
	cap syntax, version old
	if !c(rc) {
		reghdfe_old, version
		exit
	}

	* Intercept old
	cap syntax anything(everything) [fw aw pw/], [*] old
	if !c(rc) {
		di as error "(running historical version of reghdfe)"
		if ("`weight'"!="") local weightexp [`weight'=`exp']
		reghdfe_old `anything' `weightexp', `options'
		exit
	}

	* Aux. subcommands
	cap syntax, [*]
	if inlist("`options'", "check", "compile", "reload", "update", "version", "requirements") {
		if ("`options'"=="compile") loc args force
		if ("`options'"=="check") loc options compile
		if ("`options'"=="update") {
			loc args 1
			loc options reload
		}
		loc subcmd = proper("`options'")
		`subcmd' `args'
	}
	else if replay() {
		Replay `0'
	}
	else {
		Cleanup 0
		Compile // takes 0.01s to run this useful check (ensures .mlib exists)
		cap noi Estimate `0'
		Cleanup `c(rc)'
	}
end

program Compile
	args force
	ftools, check // in case lftools.mlib does not exist or is outdated
	ms_get_version reghdfe // save local package_version

	loc list_objects "FixedEffects() fixed_effects() BipartiteGraph()"
	loc list_functions "reghdfe_*() transform_*() accelerate_*() panelmean() panelsolve_*() lsmr()"
	loc list_misc "weighted_quadcolsum() safe_divide() check_convergence() precompute_inv_xx()"
	// TODO: prefix everything with reghdfe_*

	ms_compile_mata, ///
		package(reghdfe) ///
		version(`package_version') ///
		fun("`list_objects' `list_functions' `list_misc'") ///
		verbose ///
		`force'
end

program Reload
	* Internal debugging tool.
	* Updates dependencies and reghdfe from local path or from github
	* Usage:
	* 	reghdfe, update // from c:\git\..
	* 	reghdfe, reload // from github

	args online
	if ("`online'" == "") loc online 0

	di as text _n "{bf:reghdfe: updating required packages}"
	di as text "{hline 64}"

	* -ftools- https://github.com/sergiocorreia/ftools/
	cap ado uninstall ftools
	if (`online') net install ftools, from("https://github.com/sergiocorreia/ftools/raw/master/src/")
	if (!`online') net install ftools, from("c:\git\ftools\src")
	di as text "{hline 64}"
	ftools, compile
	di as text "{hline 64}"

	* Update -reghdfe-
	di as text _n  _n "{bf:reghdfe: updating self}"
	di as text "{hline 64}"
	qui ado uninstall reghdfe
	if (`online') net install reghdfe, from("https://github.com/sergiocorreia/reghdfe/raw/master/src/")
	if (!`online') net install reghdfe, from("c:\git\reghdfe\src")
	qui which reghdfe
	di as text "{hline 64}"
	reghdfe, compile
	di as text "{hline 64}"

	* Cleaning up
	di as text _n "{bf:Note:} You need to run {stata program drop _all} now."
end

program Version
	which reghdfe
	Requirements
end

program Requirements
	di as text _n "Required packages installed?"
	loc reqs ftools
	// ivreg2 avar tuples group3hdfe
	if (c(stata_version)<13) loc reqs `reqs' boottest

	loc ftools_github "https://github.com/sergiocorreia/ftools/raw/master/src/"

	loc error 0

	foreach req of local reqs {
		loc fn `req'.ado
		cap findfile `fn'
		if (_rc) {
			loc error 1
			di as text "{lalign 20:- `req'}" as error "not" _c
			di as text "    {stata ssc install `req':install from SSC}" _c
			if inlist("`req'", "ftools") {
				loc github ``req'_github'
				di as text `"    {stata `"net install `req', from(`"`github'"')"':install from github}"'
			}
			else {
				di as text // newline
			}
		}
		else {
			di as text "{lalign 20:- `req'}" as text "yes"
		}
	}

	if (`error') exit 601
end

program Cleanup
	args rc
	cap mata: mata drop HDFE
	cap mata: mata drop hdfe_*
	cap drop __temp_reghdfe_resid__
	cap matrix drop reghdfe_statsmatrix
	if (`rc') exit `rc'
end

program Parse
	* Trim whitespace (caused by "///" line continuations; aesthetic only)
	mata: st_local("0", stritrim(st_local("0")))

	* Main syntax
	#d;
	syntax anything(id=varlist equalok) [if] [in] [aw pw fw/] , [

		/* Model */
		Absorb(string) NOAbsorb
		SUmmarize SUmmarize2(string asis) /* simulate implicit options */

		/* Standard Errors */
		VCE(string) CLuster(string)

		/* Diagnostic */
		Verbose(numlist min=1 max=1 >=-1 <=5 integer)
		TIMEit

		/* Speedup and memory Tricks */
		NOSAMPle /* do not save e(sample) */

		/* Undocumented */
		KEEPSINgletons
		OLD /* use latest v3 */
		NOTES(string) /* NOTES(key=value ...), will be stored on e() */

		] [*] /* capture optimization options, display options, etc. */
		;
	#d cr

	* Unused
	* SAVEcache
	* USEcache
	* CLEARcache
	* COMPACT /* use as little memory as possible but is slower */

	* Convert options to boolean
	if ("`verbose'" == "") loc verbose 0
	loc timeit = ("`timeit'"!="")
	loc drop_singletons = ("`keepsingletons'"=="")

	if (`timeit') timer on 29

	* Sanity checks
	if (`verbose'>-1 & "`keepsingletons'"!="") {
		loc url "http://scorreia.com/reghdfe/nested_within_cluster.pdf"
		loc msg "WARNING: Singleton observations not dropped; statistical significance is biased"
		di as error `"`msg' {browse "`url'":(link)}"'
	}
	if ("`cluster'"!="") {
		_assert ("`vce'"==""), msg("cannot specify both cluster() and vce()")
		loc vce cluster `cluster'
		loc cluster // clear it to avoid bugs in subsequent lines
	}

	* Parse Varlist
	ms_parse_varlist `anything'
	loc base_varlist "`r(basevars)'"
	foreach cat in depvar indepvars endogvars instruments {
		loc original_`cat' "`r(`cat')'"
	}
	loc model = cond("`r(instruments)'" == "", "ols", "iv")
	loc original_varlist = "`r(varlist)'" // no parens or equal sign

	* Parse Weights
	if ("`weight'"!="") {
		unab exp : `exp', min(1) max(1) // simple weights only
	}

	* Parse VCE
	ms_parse_vce, vce(`vce') weighttype(`weight')
	loc vcetype = "`s(vcetype)'"
	loc num_clusters = `s(num_clusters)'
	loc clustervars = "`s(clustervars)'"
	loc base_clustervars = "`s(base_clustervars)'"
	loc vceextra = "`s(vceextra)'"

	* Select sample (except for absvars)
	loc varlist `original_varlist' `base_clustervars'
	tempvar touse
	marksample touse, strok // based on varlist + cluster + if + in + weight

	* Parse noabsorb
	_assert  ("`absorb'`noabsorb'" != ""), msg("option {bf:absorb()} or {bf:noabsorb} required")
	if ("`noabsorb'" != "") {
		_assert ("`absorb'" == ""), msg("{bf:absorb()} and {bf:noabsorb} are mutually exclusive")
		tempvar c
		gen byte `c' = 1
		loc absorb `c'
	}

	if (`timeit') timer off 29

	* Construct HDFE object
	// SYNTAX: fixed_effects(absvars | , touse, wtype, wtvar, dropsing, verbose)
	mata: st_local("comma", strpos(`"`absorb'"', ",") ? "" : ",")
	if (`timeit') timer on 20
	mata: HDFE = fixed_effects(`"`absorb' `comma' `options'"', "`touse'", "`weight'", "`exp'", `drop_singletons', `verbose')
	if (`timeit') timer off 20
	mata: HDFE.cmdline = "reghdfe " + st_local("0")
	loc options `s(options)'

	mata: st_local("N", strofreal(HDFE.N))
	if (`N' == 0) error 2000

	* Fill out HDFE object
	mata: HDFE.varlist = "`base_varlist'"
	mata: HDFE.original_depvar = "`original_depvar'"
	mata: HDFE.original_indepvars = "`original_indepvars'"
	mata: HDFE.original_endogvars = "`original_endogvars'"
	mata: HDFE.original_instruments = "`original_instruments'"
	mata: HDFE.original_varlist = "`original_varlist'"
	mata: HDFE.model = "`model'"

	mata: HDFE.vcetype = "`vcetype'"
	mata: HDFE.num_clusters = `num_clusters'
	mata: HDFE.clustervars = tokens("`clustervars'")
	mata: HDFE.base_clustervars = tokens("`base_clustervars'")
	mata: HDFE.vceextra = "`vceextra'"

	* Parse summarize
	if ("`summarize'" != "") {
		_assert ("`summarize2'" == ""), msg("summarize() syntax error")
		loc summarize2 mean min max  // default values
	}
	ParseSummarize `summarize2'
	mata: HDFE.summarize_stats = "`s(stats)'"
	mata: HDFE.summarize_quietly = `s(quietly)'

	* Parse misc options
	mata: HDFE.notes = `"`notes'"'
	mata: HDFE.store_sample = ("`nosample'"=="")
	mata: HDFE.timeit = `timeit'

	* Parse Coef Table Options (do this last!)
	_get_diopts diopts options, `options' // store in `diopts', and the rest back to `options'
	_assert (`"`options'"'==""), msg(`"invalid options: `options'"')
	if ("`hascons'"!="") di in ye "(option ignored: `hascons')"
	if ("`tsscons'"!="") di in ye "(option ignored: `tsscons')"
	mata: HDFE.diopts = `"`diopts'"'
end

program ParseSummarize, sclass
	sreturn clear
	syntax [namelist(name=stats)] , [QUIetly]
	local quietly = ("`quietly'"!="")
	sreturn loc stats "`stats'"
	sreturn loc quietly = `quietly'
end

// --------------------------------------------------------------------------

program Estimate, eclass
	ereturn clear

	* Parse and fill out HDFE object
	Parse `0'
	mata: st_local("timeit", strofreal(HDFE.timeit))

	* Compute degrees-of-freedom
	if (`timeit') timer on 21
	mata: HDFE.estimate_dof()
	if (`timeit') timer off 21

	* Save updated e(sample) (singletons reduce sample);
	* required to parse factor variables to partial out
	if (`timeit') timer on 29
	tempvar touse
	mata: HDFE.save_touse("`touse'")
	if (`timeit') timer off 29

	* Expand varlists
	if (`timeit') timer on 22
	foreach cat in /*varlist*/ depvar indepvars endogvars instruments {
		mata: st_local("vars", HDFE.original_`cat')
		if ("`vars'" == "") continue

		loc varlist
		// Expand (need to do it after the dataset has been reduced in size)
		loc 0 `vars' if `touse'
		syntax varlist(ts fv numeric) if
		//di as error "VAR1=<`vars'>"
		//di as error "VAR2=<`varlist'>"

		// HACK: addbn replaces 0.foreign with 0bn.foreign , to prevent st_data() from loading a bunch of zeros
		ms_fvstrip `varlist' if `touse', expand dropomit addbn  // onebyone
		// If we don't use onebyone, then 1.x 2.x ends up as 2.x
		loc varlist "`r(varlist)'"
		//di as error "VAR3=<`varlist'>"
		loc allvars `allvars' `varlist'
		mata: HDFE.`cat' = "`varlist'"

		loc varlist // (optional) clear it up to prevent bugs
	}
	mata: HDFE.varlist = "`allvars'"
	if (`timeit') timer off 22

	* Stats
	mata: st_local("stats", HDFE.summarize_stats)
	if ("`stats'" != "") Stats

	* Condition number
	mata: HDFE.estimate_cond()

	* Partial out; save TSS of depvar
	if (`timeit') timer on 23
	mata: hdfe_variables = HDFE.partial_out(HDFE.varlist, 1) // 1=Save TSS of first var if HDFE.tss is missing
	if (`timeit') timer off 23

	* Regress
	mata: assert_msg(HDFE.model=="ols", "You can only run OLS with reghdfe; for IV see https://github.com/sergiocorreia/ivreg2_demo")
	if (`timeit') timer on 24
	RegressOLS `touse'
	if (`timeit') timer off 24

	* (optional) Store FEs
	if (`timeit') timer on 29
	reghdfe_store_alphas
	if (`timeit') timer off 29

	* View estimation tables
	mata: st_local("diopts", HDFE.diopts)
	Replay, `diopts'

	// ~~ Preserve relevant dataset ~~-
	//
	// if (`compact') {
	// 	preserve
	// 	keep `varlist' `absorb'
	// 	loc N = c(N)
	// }
	// else {
	// 	qui cou if `touse'
	// 	loc N = r(N)
	// }

	if (`timeit') {
		di as text _n "{bf: Timer results:}"
		timer list
		di as text "Legend: 20: Create HDFE object; 21: Estimate DoF; 22: expand varlists; 23: partial out; 24: regress; 29: rest"
		di
	}
end

program RegressOLS, eclass
	args touse

	tempname b V N rank df_r
	mata: reghdfe_post_ols(HDFE, hdfe_variables, "`b'", "`V'", "`N'", "`rank'", "`df_r'")
	mata: st_local("indepvars", HDFE.indepvars)
	mata: hdfe_variables = .

	loc esample
	mata: st_local("store_sample", strofreal(HDFE.store_sample))
	if (`store_sample') loc esample "esample(`touse')"

	if ("`indepvars'" != "") {
		matrix colnames `b' = `indepvars'
		matrix colnames `V' = `indepvars'
		matrix rownames `V' = `indepvars'
		_ms_findomitted `b' `V'
		ereturn post `b' `V', `esample' buildfvinfo depname(`depvar')
	}
	else {
		ereturn post, `esample' buildfvinfo depname(`depvar')
	}

	ereturn scalar N       = `N'
	ereturn scalar rank    = `rank'
	ereturn scalar df_r    = `df_r'
	ereturn local  cmd     "reghdfe"
	mata: HDFE.post()

	* Post stats
	cap conf matrix reghdfe_statsmatrix
	if (!c(rc)) {
		ereturn matrix summarize = reghdfe_statsmatrix
		mata: st_local("summarize_quietly", strofreal(HDFE.summarize_quietly))
		ereturn scalar summarize_quietly = `summarize_quietly'
	}
end

program Replay, rclass
	syntax [, *]

	if `"`e(cmd)'"' != "reghdfe"  {
	        error 301
	}

	_get_diopts options, `options'
	reghdfe_header // _coef_table_header
	di ""
	_coef_table, `options' // ereturn display, `options'
	return add // adds r(level), r(table), etc. to ereturn (before the footnote deletes them)
	reghdfe_footnote

	* Replay stats
	if (e(summarize_quietly)==0) {
		di as text _n "{sf:Regression Summary Statistics:}" _c
		matlist e(summarize)', border(top bottom) rowtitle(Variable) // twidth(18)
	}
end

program Stats
	* Optional weights
	mata: st_local("weight", sprintf("[%s=%s]", HDFE.weight_type, HDFE.weight_var))
	assert "`weight'" != ""
	if ("`weight'" == "[=]") loc weight
	loc weight : subinstr local weight "[pweight" "[aweight"

	mata: st_local("stats", HDFE.summarize_stats)
	mata: st_local("varlist", HDFE.varlist)
	mata: st_local("cvars", invtokens(HDFE.cvars))
	loc full_varlist `varlist' `cvars'

	qui tabstat `full_varlist' `weight' , stat(`stats') col(stat) save
	matrix reghdfe_statsmatrix = r(StatTotal)
end

*! version 4.4.6 18sep2017

program reghdfe, eclass
	* Intercept old+version
	cap syntax, version old
	if !c(rc) {
		reghdfe_old, version
		exit
	}

	* Intercept old
	cap syntax anything(everything) [fw aw pw/], [*] old
	if !c(rc) {
		di as error "(running historical version of reghdfe)"
		if ("`weight'"!="") local weightexp [`weight'=`exp']
		reghdfe_old `anything' `weightexp', `options'
		exit
	}

	* Aux. subcommands
	cap syntax, [*]
	if inlist("`options'", "check", "compile", "reload", "update", "version", "requirements") {
		if ("`options'"=="compile") loc args force
		if ("`options'"=="check") loc options compile
		if ("`options'"=="update") {
			loc args 1
			loc options reload
		}
		loc subcmd = proper("`options'")
		`subcmd' `args'
	}
	else if replay() {
		Replay `0'
	}
	else {
		Cleanup 0
		Compile // takes 0.01s to run this useful check (ensures .mlib exists)
		cap noi Estimate `0'
		Cleanup `c(rc)'
	}
end

program Compile
	args force
	ftools, check // in case lftools.mlib does not exist or is outdated
	ms_get_version reghdfe // save local package_version

	loc list_objects "FixedEffects() fixed_effects() BipartiteGraph()"
	loc list_functions "reghdfe_*() transform_*() accelerate_*() panelmean() panelsolve_*() lsmr()"
	loc list_misc "weighted_quadcolsum() safe_divide() check_convergence() precompute_inv_xx()"
	// TODO: prefix everything with reghdfe_*

	ms_compile_mata, ///
		package(reghdfe) ///
		version(`package_version') ///
		fun("`list_objects' `list_functions' `list_misc'") ///
		verbose ///
		`force'
end

program Reload
	* Internal debugging tool.
	* Updates dependencies and reghdfe from local path or from github
	* Usage:
	* 	reghdfe, update // from c:\git\..
	* 	reghdfe, reload // from github

	args online
	if ("`online'" == "") loc online 0

	di as text _n "{bf:reghdfe: updating required packages}"
	di as text "{hline 64}"

	* -ftools- https://github.com/sergiocorreia/ftools/
	cap ado uninstall ftools
	if (`online') net install ftools, from("https://github.com/sergiocorreia/ftools/raw/master/src/")
	if (!`online') net install ftools, from("c:\git\ftools\src")
	di as text "{hline 64}"
	ftools, compile
	di as text "{hline 64}"

	* Update -reghdfe-
	di as text _n  _n "{bf:reghdfe: updating self}"
	di as text "{hline 64}"
	qui ado uninstall reghdfe
	if (`online') net install reghdfe, from("https://github.com/sergiocorreia/reghdfe/raw/master/src/")
	if (!`online') net install reghdfe, from("c:\git\reghdfe\src")
	qui which reghdfe
	di as text "{hline 64}"
	reghdfe, compile
	di as text "{hline 64}"

	* Cleaning up
	di as text _n "{bf:Note:} You need to run {stata program drop _all} now."
end

program Version
	which reghdfe
	Requirements
end

program Requirements
	di as text _n "Required packages installed?"
	loc reqs ftools
	// ivreg2 avar tuples group3hdfe
	if (c(stata_version)<13) loc reqs `reqs' boottest

	loc ftools_github "https://github.com/sergiocorreia/ftools/raw/master/src/"

	loc error 0

	foreach req of local reqs {
		loc fn `req'.ado
		cap findfile `fn'
		if (_rc) {
			loc error 1
			di as text "{lalign 20:- `req'}" as error "not" _c
			di as text "    {stata ssc install `req':install from SSC}" _c
			if inlist("`req'", "ftools") {
				loc github ``req'_github'
				di as text `"    {stata `"net install `req', from(`"`github'"')"':install from github}"'
			}
			else {
				di as text // newline
			}
		}
		else {
			di as text "{lalign 20:- `req'}" as text "yes"
		}
	}

	if (`error') exit 601
end

program Cleanup
	args rc
	cap mata: mata drop HDFE
	cap mata: mata drop hdfe_*
	cap drop __temp_reghdfe_resid__
	cap matrix drop reghdfe_statsmatrix
	if (`rc') exit `rc'
end

program Parse
	* Trim whitespace (caused by "///" line continuations; aesthetic only)
	mata: st_local("0", stritrim(st_local("0")))

	* Main syntax
	#d;
	syntax anything(id=varlist equalok) [if] [in] [aw pw fw/] , [

		/* Model */
		Absorb(string) NOAbsorb
		SUmmarize SUmmarize2(string asis) /* simulate implicit options */

		/* Standard Errors */
		VCE(string) CLuster(string)

		/* Diagnostic */
		Verbose(numlist min=1 max=1 >=-1 <=5 integer)
		TIMEit

		/* Speedup and memory Tricks */
		NOSAMPle /* do not save e(sample) */

		/* Undocumented */
		KEEPSINgletons
		OLD /* use latest v3 */
		NOTES(string) /* NOTES(key=value ...), will be stored on e() */

		] [*] /* capture optimization options, display options, etc. */
		;
	#d cr

	* Unused
	* SAVEcache
	* USEcache
	* CLEARcache
	* COMPACT /* use as little memory as possible but is slower */

	* Convert options to boolean
	if ("`verbose'" == "") loc verbose 0
	loc timeit = ("`timeit'"!="")
	loc drop_singletons = ("`keepsingletons'"=="")

	if (`timeit') timer on 29

	* Sanity checks
	if (`verbose'>-1 & "`keepsingletons'"!="") {
		loc url "http://scorreia.com/reghdfe/nested_within_cluster.pdf"
		loc msg "WARNING: Singleton observations not dropped; statistical significance is biased"
		di as error `"`msg' {browse "`url'":(link)}"'
	}
	if ("`cluster'"!="") {
		_assert ("`vce'"==""), msg("cannot specify both cluster() and vce()")
		loc vce cluster `cluster'
		loc cluster // clear it to avoid bugs in subsequent lines
	}

	* Parse Varlist
	ms_parse_varlist `anything'
	loc base_varlist "`r(basevars)'"
	foreach cat in depvar indepvars endogvars instruments {
		loc original_`cat' "`r(`cat')'"
	}
	loc model = cond("`r(instruments)'" == "", "ols", "iv")
	loc original_varlist = "`r(varlist)'" // no parens or equal sign

	* Parse Weights
	if ("`weight'"!="") {
		unab exp : `exp', min(1) max(1) // simple weights only
	}

	* Parse VCE
	ms_parse_vce, vce(`vce') weighttype(`weight')
	loc vcetype = "`s(vcetype)'"
	loc num_clusters = `s(num_clusters)'
	loc clustervars = "`s(clustervars)'"
	loc base_clustervars = "`s(base_clustervars)'"
	loc vceextra = "`s(vceextra)'"

	* Select sample (except for absvars)
	loc varlist `original_varlist' `base_clustervars'
	tempvar touse
	marksample touse, strok // based on varlist + cluster + if + in + weight

	* Parse noabsorb
	_assert  ("`absorb'`noabsorb'" != ""), msg("option {bf:absorb()} or {bf:noabsorb} required")
	if ("`noabsorb'" != "") {
		_assert ("`absorb'" == ""), msg("{bf:absorb()} and {bf:noabsorb} are mutually exclusive")
		tempvar c
		gen byte `c' = 1
		loc absorb `c'
	}

	if (`timeit') timer off 29

	* Construct HDFE object
	// SYNTAX: fixed_effects(absvars | , touse, wtype, wtvar, dropsing, verbose)
	mata: st_local("comma", strpos(`"`absorb'"', ",") ? "" : ",")
	if (`timeit') timer on 20
	mata: HDFE = fixed_effects(`"`absorb' `comma' `options'"', "`touse'", "`weight'", "`exp'", `drop_singletons', `verbose')
	if (`timeit') timer off 20
	mata: HDFE.cmdline = "reghdfe " + st_local("0")
	loc options `s(options)'

	mata: st_local("N", strofreal(HDFE.N))
	if (`N' == 0) error 2000

	* Fill out HDFE object
	mata: HDFE.varlist = "`base_varlist'"
	mata: HDFE.original_depvar = "`original_depvar'"
	mata: HDFE.original_indepvars = "`original_indepvars'"
	mata: HDFE.original_endogvars = "`original_endogvars'"
	mata: HDFE.original_instruments = "`original_instruments'"
	mata: HDFE.original_varlist = "`original_varlist'"
	mata: HDFE.model = "`model'"

	mata: HDFE.vcetype = "`vcetype'"
	mata: HDFE.num_clusters = `num_clusters'
	mata: HDFE.clustervars = tokens("`clustervars'")
	mata: HDFE.base_clustervars = tokens("`base_clustervars'")
	mata: HDFE.vceextra = "`vceextra'"

	* Parse summarize
	if ("`summarize'" != "") {
		_assert ("`summarize2'" == ""), msg("summarize() syntax error")
		loc summarize2 mean min max  // default values
	}
	ParseSummarize `summarize2'
	mata: HDFE.summarize_stats = "`s(stats)'"
	mata: HDFE.summarize_quietly = `s(quietly)'

	* Parse misc options
	mata: HDFE.notes = `"`notes'"'
	mata: HDFE.store_sample = ("`nosample'"=="")
	mata: HDFE.timeit = `timeit'

	* Parse Coef Table Options (do this last!)
	_get_diopts diopts options, `options' // store in `diopts', and the rest back to `options'
	_assert (`"`options'"'==""), msg(`"invalid options: `options'"')
	if ("`hascons'"!="") di in ye "(option ignored: `hascons')"
	if ("`tsscons'"!="") di in ye "(option ignored: `tsscons')"
	mata: HDFE.diopts = `"`diopts'"'
end

program ParseSummarize, sclass
	sreturn clear
	syntax [namelist(name=stats)] , [QUIetly]
	local quietly = ("`quietly'"!="")
	sreturn loc stats "`stats'"
	sreturn loc quietly = `quietly'
end

// --------------------------------------------------------------------------

program Estimate, eclass
	ereturn clear

	* Parse and fill out HDFE object
	Parse `0'
	mata: st_local("timeit", strofreal(HDFE.timeit))

	* Compute degrees-of-freedom
	if (`timeit') timer on 21
	mata: HDFE.estimate_dof()
	if (`timeit') timer off 21

	* Save updated e(sample) (singletons reduce sample);
	* required to parse factor variables to partial out
	if (`timeit') timer on 29
	tempvar touse
	mata: HDFE.save_touse("`touse'")
	if (`timeit') timer off 29

	* Expand varlists
	if (`timeit') timer on 22
	foreach cat in /*varlist*/ depvar indepvars endogvars instruments {
		mata: st_local("vars", HDFE.original_`cat')
		if ("`vars'" == "") continue

		loc varlist
		// Expand (need to do it after the dataset has been reduced in size)
		loc 0 `vars' if `touse'
		syntax varlist(ts fv numeric) if
		//di as error "VAR1=<`vars'>"
		//di as error "VAR2=<`varlist'>"

		// HACK: addbn replaces 0.foreign with 0bn.foreign , to prevent st_data() from loading a bunch of zeros
		ms_fvstrip `varlist' if `touse', expand dropomit addbn  // onebyone
		// If we don't use onebyone, then 1.x 2.x ends up as 2.x
		loc varlist "`r(varlist)'"
		//di as error "VAR3=<`varlist'>"
		loc allvars `allvars' `varlist'
		mata: HDFE.`cat' = "`varlist'"

		loc varlist // (optional) clear it up to prevent bugs
	}
	mata: HDFE.varlist = "`allvars'"
	if (`timeit') timer off 22

	* Stats
	mata: st_local("stats", HDFE.summarize_stats)
	if ("`stats'" != "") Stats

	* Condition number
	mata: HDFE.estimate_cond()

	* Partial out; save TSS of depvar
	if (`timeit') timer on 23
	mata: hdfe_variables = HDFE.partial_out(HDFE.varlist, 1) // 1=Save TSS of first var if HDFE.tss is missing
	if (`timeit') timer off 23

	* Regress
	mata: assert_msg(HDFE.model=="ols", "You can only run OLS with reghdfe; for IV see https://github.com/sergiocorreia/ivreg2_demo")
	if (`timeit') timer on 24
	RegressOLS `touse'
	if (`timeit') timer off 24

	* (optional) Store FEs
	if (`timeit') timer on 29
	reghdfe_store_alphas
	if (`timeit') timer off 29

	* View estimation tables
	mata: st_local("diopts", HDFE.diopts)
	Replay, `diopts'

	// ~~ Preserve relevant dataset ~~-
	//
	// if (`compact') {
	// 	preserve
	// 	keep `varlist' `absorb'
	// 	loc N = c(N)
	// }
	// else {
	// 	qui cou if `touse'
	// 	loc N = r(N)
	// }

	if (`timeit') {
		di as text _n "{bf: Timer results:}"
		timer list
		di as text "Legend: 20: Create HDFE object; 21: Estimate DoF; 22: expand varlists; 23: partial out; 24: regress; 29: rest"
		di
	}
end

program RegressOLS, eclass
	args touse

	tempname b V N rank df_r
	mata: reghdfe_post_ols(HDFE, hdfe_variables, "`b'", "`V'", "`N'", "`rank'", "`df_r'")
	mata: st_local("indepvars", HDFE.indepvars)
	mata: hdfe_variables = .

	loc esample
	mata: st_local("store_sample", strofreal(HDFE.store_sample))
	if (`store_sample') loc esample "esample(`touse')"

	if ("`indepvars'" != "") {
		matrix colnames `b' = `indepvars'
		matrix colnames `V' = `indepvars'
		matrix rownames `V' = `indepvars'
		_ms_findomitted `b' `V'
		ereturn post `b' `V', `esample' buildfvinfo depname(`depvar')
	}
	else {
		ereturn post, `esample' buildfvinfo depname(`depvar')
	}

	ereturn scalar N       = `N'
	ereturn scalar rank    = `rank'
	ereturn scalar df_r    = `df_r'
	ereturn local  cmd     "reghdfe"
	mata: HDFE.post()

	* Post stats
	cap conf matrix reghdfe_statsmatrix
	if (!c(rc)) {
		ereturn matrix summarize = reghdfe_statsmatrix
		mata: st_local("summarize_quietly", strofreal(HDFE.summarize_quietly))
		ereturn scalar summarize_quietly = `summarize_quietly'
	}
end

program Replay, rclass
	syntax [, *]

	if `"`e(cmd)'"' != "reghdfe"  {
	        error 301
	}

	_get_diopts options, `options'
	reghdfe_header // _coef_table_header
	di ""
	_coef_table, `options' // ereturn display, `options'
	return add // adds r(level), r(table), etc. to ereturn (before the footnote deletes them)
	reghdfe_footnote

	* Replay stats
	if (e(summarize_quietly)==0) {
		di as text _n "{sf:Regression Summary Statistics:}" _c
		matlist e(summarize)', border(top bottom) rowtitle(Variable) // twidth(18)
	}
end

program Stats
	* Optional weights
	mata: st_local("weight", sprintf("[%s=%s]", HDFE.weight_type, HDFE.weight_var))
	assert "`weight'" != ""
	if ("`weight'" == "[=]") loc weight
	loc weight : subinstr local weight "[pweight" "[aweight"

	mata: st_local("stats", HDFE.summarize_stats)
	mata: st_local("varlist", HDFE.varlist)
	mata: st_local("cvars", invtokens(HDFE.cvars))
	loc full_varlist `varlist' `cvars'

	qui tabstat `full_varlist' `weight' , stat(`stats') col(stat) save
	matrix reghdfe_statsmatrix = r(StatTotal)
end