iegraph : add option varlabels

iegraph : add option varlabels

Add option to use the variable labels instead of the variable names in
the legend. + some small fomrat edits

cap	program drop	iegraph
	program define 	iegraph
	preserve
	syntax varlist, [noconfbars TItle(string) save(string) confbarsnone(varlist) varlables GREYscale yzero *]
	mat beta_ = e(b)

	local counter = 0
	//local textfile = "Mrijan"
	qui{
		if "`confbars'" 			!= "" {
		local CONFINT_BAR 	= 0
		}
		else if "`confbars'" 			== "" {
		local CONFINT_BAR 	= 1
		}

		local varTest : list confbarsnone in varlist

		if `varTest' == 0 {
					noi display as error "{phang} Variables defined in confbarsnone cannot be found in the graph variable list. {p_end}"
					noi display ""
					error 111
					 }

		foreach var of local varlist{
			cap assert inlist(`var',0,1) | missing(`var')
				if _rc {
					noi display as error "{phang} The variable `var' is not a dummy. Treatment variable needs to be a dummy(0 or 1) variable. {p_end}"
					noi display ""
					error 149
					}
				else {
					}

		local counter = `counter' + 1
		di `counter'

		scalar coeff_`var' = _b[`var']
		//noi display coeff_`var'

		scalar coeff_se_`var' = _se[`var']
		//noi display coeff_se_`var'

		count if e(sample) == 1 & `var' == 1 //Count one tmt group at the time
		scalar obs_`var' = r(N)
		//noi display obs_`var'

			}
	if "`save'" != "" {

        *Find index for where the file type suffix start
        local dot_index     = strpos("`save'",".") + 1

        *Extract the file index
        local file_suffix   = substr("`save'", `dot_index', .)
		//noi di "`file_suffix'"

		*
        local nonGPH_formats png tiff gph ps eps pdf wmf emf

        *If no file format suffix is specified, use the default .xlsx

        if `dot_index' == 1 | "`file_suffix'" == "gph" {
			local save_export = 0
		}

        *If a file format suffix is specified make sure that it is one of the two allowed.

        else if `:list file_suffix in nonGPH_formats' != 0  {
            local save_export = 1

			if ("`file_suffix'" == "wmf" | "`file_suffix'" == "emf") & "`c(os)'" != "Windows" {
				di as error "The file formats .wmf and .emf are only allowed when using Stata on a Windows computer."
                error 198
			}
		}
        else {

            di as error "You are not using a allowed file format in save(`save'). Only the following formats are allowed: gph `nonGPH_formats'"
            error 198
        }
    }

	//noi di `"iegraph `varlist', `noconfbars' title(`title') save(`save') `confbarsnone' `yzero' `anything'"'

	local count: word count `varlist'
	local graphCount = `count' + 1
	//tokenize "`varlist'"
	//noi di `count'

	//Make all vars tempvars (maybe do later)
	//Make sure that missing is properly handled
	tempvar anyTMT
	egen `anyTMT' = rowmax(`varlist') if e(sample) == 1
	tempvar control
	gen `control' = (`anyTMT' == 0) if !missing(`anyTMT')

	sum `e(depvar)' if e(sample) == 1 & `control' == 1
	scalar ctl_N		= r(N)
	scalar ctl_mean	  	= r(mean)
	scalar ctl_mean_sd 	= r(sd)

		foreach var of local varlist {
			scalar  conf_int_min_`var'   =	coeff_`var'-(1.96*coeff_se_`var') + ctl_mean
			scalar  conf_int_max_`var'   =	coeff_`var'+(1.96*coeff_se_`var') + ctl_mean

			test `var'
			local star_`var' " "

			scalar  pvalue =r(p)
			if pvalue < 0.10 {
				local star_`var' "*"
			}
			if pvalue < 0.05 {
				local star_`var' "**"
			}
			if pvalue < 0.01 {
				local star_`var' "***"
			}
			*noi display "`star_`var''"
			scalar tmt_mean_`var' = ctl_mean + coeff_`var'
		}

	tempname 	newHandle
	tempfile	newTextFile
	cap file close 	`newHandle'
	file open  	`newHandle' using "`newTextFile'", text write append

	*Write headers and control value
	file write `newHandle' ///
		"order" 	_tab "xLabeled"	_tab "mean" _tab "coeff" _tab "conf_int_min" _tab "conf_int_max" _tab "obs" _tab "star" _n 	///
		%9.3f (1) _tab "Control"  _tab %9.3f (ctl_mean)      _tab  		 			_tab 		 			  	  _tab 			 				_tab %9.3f (ctl_N) 	   _tab			  	  _n

	tempvar newCounter
	gen `newCounter' = 2  //First tmt group starts at 2 (1 is control)

	foreach var in `varlist' {

		if "`varlabels'" == "" {

			*Default is to use the varname in legend
			local var_legend "`var'"

		else {

			*Option to use the variable label in the legen instead
			local var_legend : variable label `var'

		}

	    if `: list var in confbarsnone' {
			file write `newHandle' 	%9.3f (`newCounter') _tab "`var_legend'"  ///
			_tab %9.3f (tmt_mean_`var') _tab  %9.3f (coeff_`var')  _tab _tab ///
			_tab %9.3f (obs_`var') _tab "`star_`var''"  _n

			replace `newCounter' = `newCounter' + 1
		}
		else {
			file write `newHandle' %9.3f (`newCounter') _tab "`var_legend'" ///
			_tab %9.3f (tmt_mean_`var') _tab  %9.3f (coeff_`var')  			///
			_tab %9.3f (conf_int_min_`var') _tab %9.3f (conf_int_max_`var')	///
			_tab %9.3f (obs_`var')   _tab "`star_`var''"  _n

			replace `newCounter' = `newCounter' + 1
		}
	}

	file close `newHandle'

	copy "`newTextFile'"  "mainMasterDofile.txt" , replace

	/*************************************

		Create the graph

	*************************************/

	insheet using mainMasterDofile.txt, clear

	local tmtGroupBars 	""
	local xAxisLabels 	`"xlabel( "'
	local legendLabels	`"lab(1 "`obsLabelControl'")"'
	local legendNumbers	""

	forval tmtGroupCount = 1/`graphCount' {

		************
		*Create the bar for this group

		if "`greyscale'" == "" {
			colorPicker `tmtGroupCount' `graphCount'
		}
		else {
			greyPicker `tmtGroupCount' `graphCount'
		}

		local tmtGroupBars `"`tmtGroupBars' (bar mean order if order == `tmtGroupCount', color("`r(color)'") lcolor(black) ) "'

		************
		*Create labels etc. for this group

		local obs 			= obs[`tmtGroupCount']
		local stars 		= star[`tmtGroupCount']
		local legendLabel 	= xlabeled[`tmtGroupCount']

		local xAxisLabels 	`"`xAxisLabels' `tmtGroupCount' "(N = `obs') `stars'" "'
		local legendLabels 	`"`legendLabels' lab(`tmtGroupCount' "`legendLabel'") "'
		local legendNumbers	`"`legendNumbers' `tmtGroupCount'"'
	}

	*Close or comple some strings
	local xAxisLabels `"`xAxisLabels' ,noticks labsize(medsmall)) "'
	local legendOption `"legend(order(`legendNumbers') `legendLabels')"'

	*Create the confidence interval bars
	if `CONFINT_BAR' == 0 {
		//local confIntGraph == `"(scatter mean order,  msym(none)  mlabs(medium) mlabpos(10) mlabcolor(black)), xtitle("") ytitle("`e(depvar)'") "'
		local confIntGraph = ""
	}
		else if `CONFINT_BAR' == 1 {
		local confIntGraph = `"(rcap conf_int_max conf_int_min order, lc(gs)) (scatter mean order,  msym(none)  mlabs(medium) mlabpos(10) mlabcolor(black))"'
	}

	local titleOption `" , xtitle("") ytitle("`e(depvar)'") "'

	if "`save'" != "" {
		local saveOption saving("`save'", replace)
	}

	if "`yzero'" != "" {

		gen maxvalue = max(mean , conf_int_max)

		sum maxvalue
		local max = `r(max)'

		local logmax = log10(`max')

		local logmax = round(`logmax') - 1

		local tenpower = 10 ^ (`logmax')

		local up = `tenpower' * ceil(`max' / `tenpower')

		local quarter = (`up') / 4

		noi di "up: `up'"
		noi di "quarter: `quarter'"

		local yzero_option ylabel(0(`quarter')`up')
	}

		//noi di `" graph twoway `tmtGroupBars' `confIntGraph' `titleOption' `options' `legendOption' `xAxisLabels' `saveOption' title("`title'") `yzero_option'  "'
	if `save_export' == 0 {
		graph twoway `tmtGroupBars' `confIntGraph' `titleOption'  `legendOption' `xAxisLabels' `saveOption' title("`title'") `yzero_option' `options'
		noi di `" graph twoway `tmtGroupBars' `confIntGraph' `titleOption' `options' `legendOption' `xAxisLabels' `saveOption' title("`title'") `yzero_option'  "'

		}
	else if `save_export' == 1 {
		graph twoway `tmtGroupBars' `confIntGraph' `titleOption'  `legendOption' `xAxisLabels' title("`title'") `yzero_option' `options'
		graph export "`save'", replace
		}

	restore
}

end

cap	program drop	colorPicker
	program define 	colorPicker , rclass

	args groupCount totalNumGroups

	if `totalNumGroups' == 2 {

		if `groupCount' == 1 return local color "215 25 28"
		if `groupCount' == 2 return local color "43 123 182"
	}
	else if `totalNumGroups' == 3 {

		if `groupCount' == 1 return local color "215 25 28"
		if `groupCount' == 2 return local color "255 255 191"
		if `groupCount' == 3 return local color "43 123 182"
	}
	else if `totalNumGroups' == 4 {

		if `groupCount' == 1 return local color "215 25 28"
		if `groupCount' == 2 return local color "255 255 191"
		if `groupCount' == 3 return local color "171 217 233"
		if `groupCount' == 4 return local color "43 123 182"
	}
	else {

		*For five or more colors we repeat the same pattern

		local colourNum = mod(`groupCount', 5)
		if `colourNum' == 1 return local color "215 25 28"
		if `colourNum' == 2 return local color "253 174 93"
		if `colourNum' == 3 return local color "255 255 191"
		if `colourNum' == 4 return local color "171 217 233"
		if `colourNum' == 0 return local color "43 123 182"

	}

end

cap	program drop	greyPicker
	program define 	greyPicker , rclass

	args groupCount totalNumGroups

	if `groupCount' == 1 {

		return local color "black"
	}
	else if `groupCount' == 2 & `totalNumGroups' <= 3 {

		return local color "gs14"
	}
	else {

		local grayscale =  round( (`groupCount'-1) * (100 / (`totalNumGroups'-1) ))

		return local color "`grayscale' `grayscale' `grayscale' `grayscale'"
	}

end

cap	program drop	iegraph
	program define 	iegraph
	preserve
	syntax varlist, [noconfbars TItle(string) save(string) confbarsnone(varlist) GREYscale yzero *]
	mat beta_ = e(b)

	local counter = 0
	//local textfile = "Mrijan"
	qui{
		if "`confbars'" 			!= "" {
		local CONFINT_BAR 	= 0
		}
		else if "`confbars'" 			== "" {
		local CONFINT_BAR 	= 1
		}

		local varTest : list confbarsnone in varlist

		if `varTest' == 0 {
					noi display as error "{phang} Variables defined in confbarsnone cannot be found in the graph variable list. {p_end}"
					noi display ""
					error 111
					 }

		foreach var of local varlist{
			cap assert inlist(`var',0,1) | missing(`var')
				if _rc {
					noi display as error "{phang} The variable `var' is not a dummy. Treatment variable needs to be a dummy(0 or 1) variable. {p_end}"
					noi display ""
					error 149
					}
				else {
					}

		local counter = `counter' + 1
		di `counter'

		scalar coeff_`var' = _b[`var']
		//noi display coeff_`var'

		scalar coeff_se_`var' = _se[`var']
		//noi display coeff_se_`var'

		count if e(sample) == 1 & `var' == 1 //Count one tmt group at the time
		scalar obs_`var' = r(N)
		//noi display obs_`var'

			}
	if "`save'" != "" {

        *Find index for where the file type suffix start
        local dot_index     = strpos("`save'",".") + 1

        *Extract the file index
        local file_suffix   = substr("`save'", `dot_index', .)
		//noi di "`file_suffix'"

		*
        local nonGPH_formats png tiff gph ps eps pdf wmf emf

        *If no file format suffix is specified, use the default .xlsx

        if `dot_index' == 1 | "`file_suffix'" == "gph" {
			local save_export = 0
		}

        *If a file format suffix is specified make sure that it is one of the two allowed.

        else if `:list file_suffix in nonGPH_formats' != 0  {
            local save_export = 1

			if ("`file_suffix'" == "wmf" | "`file_suffix'" == "emf") & "`c(os)'" != "Windows" {
				di as error "The file formats .wmf and .emf are only allowed when using Stata on a Windows computer."
                error 198
			}
		}
        else {

            di as error "You are not using a allowed file format in save(`save'). Only the following formats are allowed: gph `nonGPH_formats'"
            error 198
        }
    }

	//noi di `"iegraph `varlist', `noconfbars' title(`title') save(`save') `confbarsnone' `yzero' `anything'"'

	local count: word count `varlist'
	local graphCount = `count' + 1
	//tokenize "`varlist'"
	//noi di `count'

	//Make all vars tempvars (maybe do later)
	//Make sure that missing is properly handled
	tempvar anyTMT
	egen `anyTMT' = rowmax(`varlist') if e(sample) == 1
	tempvar control
	gen `control' = (`anyTMT' == 0) if !missing(`anyTMT')

	sum `e(depvar)' if e(sample) == 1 & `control' == 1
	scalar ctl_N		= r(N)
	scalar ctl_mean	  	= r(mean)
	scalar ctl_mean_sd 	= r(sd)

		foreach var of local varlist {
			scalar  conf_int_min_`var'   =	coeff_`var'-(1.96*coeff_se_`var') + ctl_mean
			scalar  conf_int_max_`var'   =	coeff_`var'+(1.96*coeff_se_`var') + ctl_mean

			test `var'
			local star_`var' " "

			scalar  pvalue =r(p)
			if pvalue < 0.10 {
				local star_`var' "*"
			}
			if pvalue < 0.05 {
				local star_`var' "**"
			}
			if pvalue < 0.01 {
				local star_`var' "***"
			}
			*noi display "`star_`var''"
			scalar tmt_mean_`var' = ctl_mean + coeff_`var'
		}

	tempname 	newHandle
	tempfile	newTextFile
	cap file close 	`newHandle'
	file open  	`newHandle' using "`newTextFile'", text write append

	*Write headers and control value
	file write `newHandle' ///
		"order" 	_tab "xLabeled"	_tab "mean" _tab "coeff" _tab "conf_int_min" _tab "conf_int_max" _tab "obs" _tab "star" _n 	///
		%9.3f (1) _tab "Control"  _tab %9.3f (ctl_mean)      _tab  		 			_tab 		 			  	  _tab 			 				_tab %9.3f (ctl_N) 	   _tab			  	  _n

	tempvar newCounter
	gen `newCounter' = 2  //First tmt group starts at 2 (1 is control)

	foreach var in `varlist' {
	     if `: list var in confbarsnone' {
		file write `newHandle' ///
			%9.3f (`newCounter') _tab "`var'" _tab %9.3f (tmt_mean_`var') _tab  %9.3f (coeff_`var')  _tab %9.3f  _tab _tab %9.3f (obs_`var')   _tab "`star_`var''"  _n

		replace `newCounter' = `newCounter' + 1
			}
		else {
		file write `newHandle' ///
			%9.3f (`newCounter') _tab "`var'" _tab %9.3f (tmt_mean_`var') _tab  %9.3f (coeff_`var')  _tab %9.3f (conf_int_min_`var') _tab %9.3f (conf_int_max_`var') _tab %9.3f (obs_`var')   _tab "`star_`var''"  _n

		replace `newCounter' = `newCounter' + 1
		}
	}

	file close `newHandle'

	copy "`newTextFile'"  "mainMasterDofile.txt" , replace

	/*************************************

		Create the graph

	*************************************/

	insheet using mainMasterDofile.txt, clear

	local tmtGroupBars 	""
	local xAxisLabels 	`"xlabel( "'
	local legendLabels	`"lab(1 "`obsLabelControl'")"'
	local legendNumbers	""

	forval tmtGroupCount = 1/`graphCount' {

		************
		*Create the bar for this group

		if "`greyscale'" == "" {
			colorPicker `tmtGroupCount' `graphCount'
		}
		else {
			greyPicker `tmtGroupCount' `graphCount'
		}

		local tmtGroupBars `"`tmtGroupBars' (bar mean order if order == `tmtGroupCount', color("`r(color)'") lcolor(black) ) "'

		************
		*Create labels etc. for this group

		local obs 			= obs[`tmtGroupCount']
		local stars 		= star[`tmtGroupCount']
		local legendLabel 	= xlabeled[`tmtGroupCount']

		local xAxisLabels 	`"`xAxisLabels' `tmtGroupCount' "(N = `obs') `stars'" "'
		local legendLabels 	`"`legendLabels' lab(`tmtGroupCount' "`legendLabel'") "'
		local legendNumbers	`"`legendNumbers' `tmtGroupCount'"'
	}

	*Close or comple some strings
	local xAxisLabels `"`xAxisLabels' ,noticks labsize(medsmall)) "'
	local legendOption `"legend(order(`legendNumbers') `legendLabels')"'

	*Create the confidence interval bars
	if `CONFINT_BAR' == 0 {
		//local confIntGraph == `"(scatter mean order,  msym(none)  mlabs(medium) mlabpos(10) mlabcolor(black)), xtitle("") ytitle("`e(depvar)'") "'
		local confIntGraph = ""
	}
		else if `CONFINT_BAR' == 1 {
		local confIntGraph = `"(rcap conf_int_max conf_int_min order, lc(gs)) (scatter mean order,  msym(none)  mlabs(medium) mlabpos(10) mlabcolor(black))"'
	}

	local titleOption `" , xtitle("") ytitle("`e(depvar)'") "'

	if "`save'" != "" {
		local saveOption saving("`save'", replace)
	}

	if "`yzero'" != "" {

		gen maxvalue = max(mean , conf_int_max)

		sum maxvalue
		local max = `r(max)'

		local logmax = log10(`max')

		local logmax = round(`logmax') - 1

		local tenpower = 10 ^ (`logmax')

		local up = `tenpower' * ceil(`max' / `tenpower')

		local quarter = (`up') / 4

		noi di "up: `up'"
		noi di "quarter: `quarter'"

		local yzero_option ylabel(0(`quarter')`up')
	}

		//noi di `" graph twoway `tmtGroupBars' `confIntGraph' `titleOption' `options' `legendOption' `xAxisLabels' `saveOption' title("`title'") `yzero_option'  "'
	if `save_export' == 0 {
		graph twoway `tmtGroupBars' `confIntGraph' `titleOption'  `legendOption' `xAxisLabels' `saveOption' title("`title'") `yzero_option' `options'
		noi di `" graph twoway `tmtGroupBars' `confIntGraph' `titleOption' `options' `legendOption' `xAxisLabels' `saveOption' title("`title'") `yzero_option'  "'

		}
	else if `save_export' == 1 {
		graph twoway `tmtGroupBars' `confIntGraph' `titleOption'  `legendOption' `xAxisLabels' title("`title'") `yzero_option' `options'
		graph export "`save'", replace
		}

	restore
}

end

cap	program drop	colorPicker
	program define 	colorPicker , rclass

	args groupCount totalNumGroups

	if `totalNumGroups' == 2 {

		if `groupCount' == 1 return local color "215 25 28"
		if `groupCount' == 2 return local color "43 123 182"
	}
	else if `totalNumGroups' == 3 {

		if `groupCount' == 1 return local color "215 25 28"
		if `groupCount' == 2 return local color "255 255 191"
		if `groupCount' == 3 return local color "43 123 182"
	}
	else if `totalNumGroups' == 4 {

		if `groupCount' == 1 return local color "215 25 28"
		if `groupCount' == 2 return local color "255 255 191"
		if `groupCount' == 3 return local color "171 217 233"
		if `groupCount' == 4 return local color "43 123 182"
	}
	else {

		*For five or more colors we repeat the same pattern

		local colourNum = mod(`groupCount', 5)
		if `colourNum' == 1 return local color "215 25 28"
		if `colourNum' == 2 return local color "253 174 93"
		if `colourNum' == 3 return local color "255 255 191"
		if `colourNum' == 4 return local color "171 217 233"
		if `colourNum' == 0 return local color "43 123 182"

	}

end

cap	program drop	greyPicker
	program define 	greyPicker , rclass

	args groupCount totalNumGroups

	if `groupCount' == 1 {

		return local color "black"
	}
	else if `groupCount' == 2 & `totalNumGroups' <= 3 {

		return local color "gs14"
	}
	else {

		local grayscale =  round( (`groupCount'-1) * (100 / (`totalNumGroups'-1) ))

		return local color "`grayscale' `grayscale' `grayscale' `grayscale'"
	}

end


fix bug

fix bug

*! version 1.19.0 26jul2017
*! PARALLEL: Stata module for parallel computing
*! by George G. Vega [cre,aut], Brian Quistorff [aut]
*!
*! Project website:
*!   https://github.com/gvegayon/parallel
*! Bug reports:
*!   https://github.com/gvegayon/parallel/issues
/*
Copyright (c) 2014  <George G. Vega> and <Brian Quistorff>

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/

// Syntax parser
program def parallel
    version 11.0

	// Checks wether if is parallel prefix or not
	if  (regexm(`"`0'"', "^(do|clean|break|version|append|printlog|viewlog|numprocessors)")) {
	/* If not prefix */
		parallel_`0'
	}
	else if (regexm(`"`0'"', "^(setclusters|initialize|init)")) {
		gettoken temp 0 : 0
		parallel_setclusters `0'
	}
	else if (regexm(`"`0'"', "^(bootstrap|bstrap|bs|simulate|sim)[,]?[\s ]?")) {
	/* Prefix bootstrap or simulate */
		local cmd = regexs(1)
    if inlist("`cmd'","bootstrap","bstrap") local cmd = "bs"
    if "`cmd'"=="simulate" local cmd = "sim"
		mata: st_local("0", regexr(st_local("0"), "^(bootstrap|bstrap|bs|simulate|sim)", ""))
		gettoken x 0 : 0, parse(":") bind
		local 0 = regexr(`"`0'"', "^[:]", "")
		gettoken x options : x, parse(",") bind

		gettoken 0 argopt : 0, parse(",") bind
		parallel_`cmd' `0', argopt(`argopt') `options'
	}
	else if (regexm(`"`0'"',"^([,]?.*[:])")) {
	/* if prefix */
		gettoken x 0 : 0, parse(":") bind
		mata: st_local("0", regexr(st_local("0"), "^[:]", ""))
		// Gets the options (if these exists) of parallel
		gettoken x options : x, parse(",") bind

		gettoken 0 argopt : 0, parse(",") bind
		parallel_do `0', `options' prefix argopt(`argopt')
	}
	else {
		local val = regexm(`"`0'"', "^([a-zA-Z0-9_]*).*")
		local subcommand = regexs(1)
		di as result `"-`subcommand'-"' as error " invalid parallel subcommand" as text " ({stata h parallel:{it:help parallel}})"
		exit 198
	}
end

/*
 For Windows we use an environment variable that could be redefined by a parent
   (both a plus and minus). Alternative is to sum output from command:
    WMIC CPU Get DeviceID,NumberOfCores,NumberOfLogicalProcessors

 For *nix see http://stackoverflow.com/questions/6481005/ for alternatives.
   Note: 'nproc --all' does not return the right result on some cloud providers where systems are shared.
*/
program parallel_numprocessors, rclass
	if "`c(os)'"=="Windows" {
		local nproc : env NUMBER_OF_PROCESSORS
	}
	else {
		tempfile nproc_out
		! getconf _NPROCESSORS_ONLN > "`nproc_out'"
		tempname nproc_out_fhandle
		file open `nproc_out_fhandle' using "`nproc_out'", read text
		file read `nproc_out_fhandle' nproc
		local r_eof = `r(eof)'
		file close `nproc_out_fhandle'
		_assert `r_eof'==0, msg("Wasn't able to read output from system")
	}

	local nproc = int(real("`nproc'"))
	_assert "`nproc'"!=".", msg("Wasn't able to interpret output from system")

	di "Number of logical processors: `nproc'"
	return scalar numprocessors = `nproc'
end

/* Returns the version of parallel */
program def parallel_version, rclass
	version 11.0
	di as result "parallel" as text " Stata module for parallel computing"
	di as result "vers" as text " 1.19.0 26jul2017"
	di as result "auth" as text " George G. Vega [cre,aut], Brian Quistorff [aut]"

	return local pll_vers = "1.19.0"
end

/* Take a look to logfiles */
program def parallel_printlog
	version 11.0
	syntax [anything(name=pll_instance)] , [Event(string)]
	parallel_checklog `pll_instance', e(`event') action(print)
end

program def parallel_viewlog
	version 11.0
	syntax [anything(name=pll_instance)] , [Event(string)]
	parallel_checklog `pll_instance', e(`event') action(view)
end

program def parallel_checklog
	version 11.0
	syntax [anything(name=pll_instance)] , [Event(string)] action(string)

	if ("`event'"=="") local event = "$LAST_PLL_ID"

	if ("`event'"!=".") {
		/* By default uses 1 */
		if ("`pll_instance'" == "") local pll_instance 1

		local pll_instance : di %04.0f `pll_instance'

		/* Does de file exists */
		local logname = "__pll`event'_do`pll_instance'.log"

		if (c(os) == "Windows") local logname = "`c(tmpdir)'`logname'"
		else local logname = "`c(tmpdir)'/`logname'"

		/* If the logfile does not exists, do nothing*/
		cap confirm file "`logname'"
		if (_rc) {
			di as result "No logfile for instance -`pll_instance'- of parallel process -`event'- found"
			exit
		}

		/* Showing the log in screen */
		if ("`action'"=="print") {
			di as result "{hline 80}"
			di as result %~80s "beginning of file -`logname'-"
			di as result "{hline 80}"
			type `"`logname'"'
			di as result "{hline 80}"
			di as result %~80s "end of file -`logname'-"
			di as result "{hline 80}"
		}
		else view `"`logname'"'
	}
	else {
		di as error "It seems that you haven't use -parallel- yet."
		exit 601
	}

end

////////////////////////////////////////////////////////////////////////////////
// Splits the dataset into clusters
program def parallel_spliter
	version 11.0
	syntax [namelist(name=xtstructure)] [,parallelid(string) sorting(integer 0) force(integer 0) keepusing(varlist) orig_cl_local(string)]
	//args xtstructure parallelid Sorting Force

	if length("$PLL_CLUSTERS") == 0 {
		di as err "Number of clusters not fixed." as text " Set the number of clusters with " _newline as err "-{cmd:parallel setclusters #}-"
		exit 198
	}

	//quietly {
		if (length("`xtstructure'")) {
			_assert `sorting' | `force', msg("Data not sorted") rc(5)

			/* Checking the data types */
			foreach var of varlist `xtstructure' {
				capture confirm string var `var'
				if (`=_rc') local numvars `numvars' `var'
				else local strvars `strvars' `var'
			}

			/* calculate max possible clusters */
			egen _`parallelid'grp = group(`xtstructure'), missing
			summ _`parallelid'grp, meanonly
			local max_n_cl = `r(max)'
		}
		else {
			local max_n_cl = _N
		}

		/* Do we have too many clusters? */
		if (`max_n_cl'<$PLL_CLUSTERS){
			c_local `orig_cl_local' ${PLL_CLUSTERS}
			global PLL_CLUSTERS = `max_n_cl'
			di "Small workload/num groups. Temporarily setting number of clusters to ${PLL_CLUSTERS}"
		}

		if (length("`xtstructure'")) {
			preserve
			keep _`parallelid'grp
			contract _`parallelid'grp
			sort _freq
			//Figuring out the mapping from groupID to cut that equalizes sizes is the "Partition Problem"
			//which is hard to solve exactly (it's NP-complete). A rough solution suffices here though.
			gen _`parallelid'cut = mod(_n, ${PLL_CLUSTERS}) + 1
			tempfile grp_to_cut_map
			qui save `"`grp_to_cut_map'"'
			restore

			qui merge m:1 _`parallelid'grp using `"`grp_to_cut_map'"', keepusing(_`parallelid'cut) nogenerate
			drop _`parallelid'grp
			sort `xtstructure'
		}
		else {
			gen _`parallelid'cut = min(${PLL_CLUSTERS}, ceil(_n*${PLL_CLUSTERS}/_N)) //numerical precision for big nubmers can make > PLL_CLUSTERS
		}

		if (length("`keepusing'")) {
			keep _`parallelid'cut `keepusing'
			gen __pllnobs`parallelid' = _n
		}

		qui save __pll`parallelid'_dataset, replace
		drop _all
	//}

end

//in a subprogram because some of the outputopts may have the same name as -parallel- options
program pll_remove_replace
	syntax, pll_argopt(string) pll_outputopts(string)

	//parse the original options
	foreach pll_outputopt of local pll_outputopts{
		local pll_outputopts_syn "`pll_outputopts_syn' `pll_outputopt'(string)"
	}
	local 0 : copy local pll_argopt
	syntax [anything(equalok everything)][, `pll_outputopts_syn' *]

	//remove the ones were are subbing out
	c_local argopt `"`anything', `options'"'

	//some may be null so only sub out real ones
	foreach pll_outputopt of local pll_outputopts{
		if "`pll_outputopt'"!="" local outputopts "`outputopts' `pll_outputopt'"
	}
	c_local outputopts `outputopts'
	/* old method (doesn't account for syntax eliding blank ones
	//make version of argopt that doesn't have the options that will be swapped out
	local argopt_orig : copy local argopt
	foreach outputopt of local outputopts{
		local argopt = regexr(`"`argopt'"',"`outputopt'\([^\)]*\)","")
	}*/
end

//allow some or all files to not have been created
//If at least one file was made, even if it was empty, make a file
// (same save dynamics as user program)
program pll_collect
	syntax, folder(string) parallelid(string) outputopts(string) argopt_orig(string)
	preserve

	foreach outputopt of local outputopts{
		mata: st_local("val", strofreal(regexm(st_local("argopt_orig"),st_local("outputopt")+"\(([^)]+)\)")))
		if `val'{
			mata: st_local("outfile", regexs(1))
			//get rid of quotes
			local outfile `outfile'
			local emptyok ""
			drop _all
			forval i=1/$PLL_CLUSTERS{
				cap append using `"`folder'__pll`parallelid'_out_`outputopt'`=strofreal(`i',"%04.0f")'"'
				if (_rc==0) local emptyok "emptyok"
			}

			qui save `"`outfile'"', replace `emptyok'
		}
	}
end

////////////////////////////////////////////////////////////////////////////////
// MAIN PROGRAM
// There are things we need to possibly restore on exit
// - Data/S_FN (this is taken care of by -preserve-)
// - PWD (non-indented capture block)
// - Temporaliy changed number of clusters (non-indented capture block)
// - Turn off the timer (do within catch for capture blocks)
program def parallel_do, rclass
	version 11.0

	#delimit ;
	syntax anything(name=dofile equalok everything)
		[, by(string)
		Keep
		KEEPLast
		prefix
		Force
		PROGrams(string)
		Mata
		NOGlobals
		KEEPTiming
		Seeds(string)
		NOData
		Randtype(string)
		Timeout(integer 60)
		PROCessors(integer 0)
		argopt(string)
		KEEPUsing(string)
		SETparallelid(string)
		OUTputopts(string)
		DETerministicoutput
		];
	#delimit cr

	if length("$PLL_CLUSTERS") == 0 {
		di as err "You haven't set the number of clusters" _n "Please set it with: {cmd:parallel setclusters} {it:#}"
		exit 198
	}

	if `c(noisily)'==0 & "`programs'"!="" {
		di as err "If you want to pass programs in memory (using the option programs)" _n "then the program must be run noisily."
		exit 198
	}

	// Initial checks
	foreach opt in macrolist keep keeplast prefix force mata noglobals keeptiming nodata deterministicoutput {
		local `opt' = length("``opt''") > 0
	}

	/* Randtype */
	if ("`randtype'" == "") local randtype = "datetime"

	/* If no data parsing has to be done (because of no data!) */
	if (!(c(N)*c(k))) local nodata 1

	if (!`keeptiming') {
		timer clear 97
		timer clear 98
		timer clear 99
	}

	timer on 98

	// Deletes last parallel instance ran
	local last_id = cond("`r(pll_id)'"!="","`r(pll_id)'", "$LAST_PLL_ID")
	if (`keeplast' & length("`last_id'")) cap parallel_clean, e(`last_id') nologs

	if length("`by'") != 0 {
		local sortlist: sortedby
		local sorting = regexm("`sortlist'","^`by'")
	}
	else local sorting = 0

	// Gets the directory where to work at
	if (!`prefix') {

		/* First checks if the file exists */
		mata: parallel_normalizepath(`"`dofile'"',1)
		local pll_dir = "`filedir'"
		local dofile = "`filename'"
	}
	else local pll_dir = c(pwd)+"/"

	local initialdir = c(pwd)
	qui cd "`pll_dir'"
	capture noisily { //Start capture block for PWD & $PLL_CLUSTERS

	/* Creates a unique ID for the process and secures it */
	if ("`setparallelid'"=="") mata: parallel_sandbox(5)
	else local parallelid = "`setparallelid'"

	/* Generates database clusters */
	if (!`nodata'){
		local sfn = "$S_FN" // be able to "fake" restore this
		preserve

		parallel_spliter `by' , parallelid(`parallelid') sorting(`sorting') force(`force') keepusing(`keepusing') orig_cl_local(orig_PLL_CLUSTERS)
	}

	global LAST_PLL_ID = "`parallelid'"
	global LAST_PLL_DIR = "`pll_dir'"
	global LAST_PLL_N = $PLL_CLUSTERS

	/* Starts building the files */
	local work_around_no_cwd = 0
	quietly {

		/* Saves mata objects */
		if (`mata') {
			mata: mata mlib create l__pll`parallelid'_mlib, replace
			cap mata: mata mlib add l__pll`parallelid'_mlib *()
			cap which l__pll`parallelid'_mlib.mlib
			if _rc!=0{
				local work_around_no_cwd = 1
				noi di "Note: In order to pass mata objects to the clusters, they are saved to a temporary mlib file in the current directly."
				noi di "      Your ado-path doesn't contain the current directory."
				noi di "      We have added the current directory to the end of the ado-path for the clusters.."
			}
			cap mata: mata matsave __pll`parallelid'_mata.mmat *, replace
			if (`=_rc') local matasave = 0
			else local matasave = 1
		}
		else local matasave = 0
	}

	//see if we handle extra output streams
	if "`outputopts'"!=""{
		local argopt_orig : copy local argopt
		pll_remove_replace, pll_argopt(`"`argopt'"') pll_outputopts(`outputopts')
	}

	/* Writing the dofile */
	mata: st_local("errornum", strofreal(parallel_write_do(strtrim(`"`dofile' `argopt'"'), "`parallelid'", $PLL_CLUSTERS, `prefix', `matasave', !`noglobals', "`seeds'", "`randtype'", `nodata', "`pll_dir'", "`programs'", `processors',`work_around_no_cwd',"`outputopts'")))
	if (`errornum') {
		/* Removes the sandbox file (unprotect the files) */
		mata: parallel_sandbox(2, "`parallelid'")

		error `errornum'
	}

	timer off 98
	cap timer list
	if (r(t98) == .) local pll_t_setu = 0
	else local pll_t_setu = r(t98)

	/* Running the dofiles */
	timer on 99
	mata: st_local("nerrors", strofreal(parallel_run("`parallelid'",$PLL_CLUSTERS,`"$PLL_STATA_PATH"',`=`timeout'*1000', `deterministicoutput', tokens("$PLL_HOSTNAMES"), "$PLL_SSH")))
	timer off 99

	cap timer list
	if (r(t99) == .) local pll_t_calc = 0
	else local pll_t_calc = r(t99)
	local pll_t_reps = r(nt99)

	timer on 97

	if ("`outputopts'"!=""  & !`nerrors'){
		pll_collect, folder("`pll_dir'") parallelid("`parallelid'") outputopts(`outputopts') argopt_orig(`"`argopt_orig'"')
	}

	// Paste the databases
	if (!`nodata' & !`nerrors') {
		parallel_fusion `parallelid', clusters($PLL_CLUSTERS) keepusing(`keepusing')
		cap drop _`parallelid'cut

		global S_FN = "`sfn'" // Restores original S_FN (file name) value
		restore, not
	}

	return scalar pll_n = $PLL_CLUSTERS

	/* Removes the sandbox file (unprotect the files) */
	if ("`setparallelid'" == "") {
		mata: parallel_sandbox(2, "`parallelid'")
		if (!`keep' & !`keeplast') parallel_clean, e("`parallelid'") nologs
	}

	} //End capture block
	if _rc {
		local orig_rc = _rc
		qui cd "`initialdir'"
		if "`orig_PLL_CLUSTERS'"!="" global PLL_CLUSTERS=`orig_PLL_CLUSTERS'
		cap timer off 97
		cap timer off 98
		cap timer off 99
		exit `orig_rc'
	}
	qui cd "`initialdir'"
	if "`orig_PLL_CLUSTERS'"!="" global PLL_CLUSTERS=`orig_PLL_CLUSTERS'

	timer off 97
	cap timer list
	if (r(t97) == .) local pll_t_fini = 0
	else local pll_t_fini = r(t97)

	qui timer list
	return local  pll_seeds="`pllseeds'"
	return scalar pll_errs = `nerrors'
	return local  pll_dir "`pll_dir'"
	return scalar pll_t_reps = `pll_t_reps'
	return scalar pll_t_setu = `pll_t_setu'
	return scalar pll_t_calc = `pll_t_calc'
	return scalar pll_t_fini = `pll_t_fini'
	return local pll_id = "`parallelid'"

	if `nerrors' {
		di as err "`nerrors' child processes encountered errors. Throwing last error."
		exit `pll_last_error'
	}
end

////////////////////////////////////////////////////////////////////////////////
// Cleans all files generated by parallel
// If you want to keep the logs, then specify -, nologs-
program def parallel_clean
	version 11.0
	syntax [, Event(string) All Force nologs]

	if (length("`event'") != 0 & length("`all'") != 0) {
		di as error `"invalid syntax: Using -pll_id- and -all- jointly is not allowed."'
		exit 198
	}
	local do_logs = ("`logs'"!="nologs")
	mata: parallel_clean("`event'", `=length("`all'") > 0', `=length("`force'") > 0', `do_logs')
end

////////////////////////////////////////////////////////////////////////////////
// Sets the number of clusters as a global macro
program parallel_setclusters
	version 11.0
	syntax [anything(name=nclusters)] [, Force Statapath(string asis) Gateway(string) Includefile(string) Hostnames(string) ssh(string) procexec(int 2)]

	_assert inlist(`procexec',0,1,2), msg("procexec() must be 0, 1, or 2") rc(198)
	cap parallel_numprocessors
	local nproc = int(real("`r(numprocessors)'"))
	if "`nclusters'"=="default" | "`nclusters'"==""{
		_assert `nproc'!=., msg("Couldn't determine number of available processors for default configuration.")
		local nclusters = max(floor(`nproc'*3/4),1)
	}
	else{
		local nclusters = int(real(`"`nclusters'"'))
		_assert (`nclusters'>0 & `nclusters'!=.),  msg(`"Not allowed: "#" Should be a positive number"') rc(109)
	}
	global USE_PROCEXEC = `procexec'
	global PLL_HOSTNAMES = "`hostnames'"
	global PLL_SSH = "`ssh'"
	if "$PLL_SSH"!="" global PLL_SSH = "$PLL_SSH "
  if "`hostnames'"!="" local nproc "." //don't use softmax when in cluster

	local force = (length("`force'")>0)
	mata: parallel_setclusters(`nclusters', `force', `nproc')
	mata: st_local("error", strofreal(parallel_setstatapath(`"`statapath'"', `force')))
	_assert (!`error'), msg("Can not set Stata directory, try using -statapath()- option") rc(`error')

	if "`c(mode)'"=="batch" & "`c(os)'"=="Windows" & "$USE_PROCEXEC"=="0" {
		if `"`gateway'"'=="" local gateway "pll_gateway.sh"

		cap confirm file `"`gateway'"'
		_assert `=!_rc', msg(`"On Windows batch-mode with command-line process execution, requires a gateway file is required but not found (path: `c(pwd)')"') rc(`=_rc')
		global PLL_GATEWAY_FNAME `"`gateway'"'
	}

	if `"`includefile'"'!=""{
		cap confirm file `"`includefile'"'
		_assert `=!_rc', msg("Can not find the include file (`includefile').") msg(`=_rc')
		global PLL_INCLUDE_FILE `"`includefile'"'
	}
end

////////////////////////////////////////////////////////////////////////////////
// Exports

// Exports a copy of programs
program def program_export
	version 11.0
	syntax using/ [,Programlist(string) Inname(string)]

	mata: program_export("`using'", "`programlist'", "`inname'")
end

////////////////////////////////////////////////////////////////////////////////
// Appends the clusterized dataset
program def parallel_fusion
	version 11.0
	syntax anything(name=parallelid) , clusters(integer) [keepusing(string)]

	cap use "__pll`parallelid'_dta0001.dta", clear
	if (_rc){
		di as err "No dataset for instance 0001."
		exit _rc
	}
	local sortlist: sortedby

	forval i = 2/`clusters' {
		cap append using `"__pll`parallelid'_dta`=string(`i',"%04.0f")'.dta"'
		if (_rc){
			di as err "No dataset for instance `=string(`i',"%04.0f")'."
			exit _rc
		}
	}

	/* If it just used a set of variables */
	if (length("`keepusing'")) {
		qui merge 1:1 __pllnobs`parallelid' using __pll`parallelid'_dataset, keep(3) nogen
		drop __pllnobs`parallelid'
	}

	//restore the sort
	if "`sortlist'"!="" sort `sortlist'

end

////////////////////////////////////////////////////////////////////////////////
// Checks whether the user pressed break inside a loop
program def parallel_break
	version 11.0
	mata: parallel_break()
end

*! version 1.19.0 26jul2017
*! PARALLEL: Stata module for parallel computing
*! by George G. Vega [cre,aut], Brian Quistorff [aut]
*!
*! Project website:
*!   https://github.com/gvegayon/parallel
*! Bug reports:
*!   https://github.com/gvegayon/parallel/issues
/*
Copyright (c) 2014  <George G. Vega> and <Brian Quistorff>

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/

// Syntax parser
program def parallel
    version 11.0

	// Checks wether if is parallel prefix or not
	if  (regexm(`"`0'"', "^(do|clean|break|version|append|printlog|viewlog|numprocessors)")) {
	/* If not prefix */
		parallel_`0'
	}
	else if (regexm(`"`0'"', "^(setclusters|initialize|init)")) {
		parallel_setclusters
	}
	else if (regexm(`"`0'"', "^(bootstrap|bstrap|bs|simulate|sim)[,]?[\s ]?")) {
	/* Prefix bootstrap or simulate */
		local cmd = regexs(1)
    if inlist("`cmd'","bootstrap","bstrap") local cmd = "bs"
    if "`cmd'"=="simulate" local cmd = "sim"
		mata: st_local("0", regexr(st_local("0"), "^(bootstrap|bstrap|bs|simulate|sim)", ""))
		gettoken x 0 : 0, parse(":") bind
		local 0 = regexr(`"`0'"', "^[:]", "")
		gettoken x options : x, parse(",") bind

		gettoken 0 argopt : 0, parse(",") bind
		parallel_`cmd' `0', argopt(`argopt') `options'
	}
	else if (regexm(`"`0'"',"^([,]?.*[:])")) {
	/* if prefix */
		gettoken x 0 : 0, parse(":") bind
		mata: st_local("0", regexr(st_local("0"), "^[:]", ""))
		// Gets the options (if these exists) of parallel
		gettoken x options : x, parse(",") bind

		gettoken 0 argopt : 0, parse(",") bind
		parallel_do `0', `options' prefix argopt(`argopt')
	}
	else {
		local val = regexm(`"`0'"', "^([a-zA-Z0-9_]*).*")
		local subcommand = regexs(1)
		di as result `"-`subcommand'-"' as error " invalid parallel subcommand" as text " ({stata h parallel:{it:help parallel}})"
		exit 198
	}
end

/*
 For Windows we use an environment variable that could be redefined by a parent
   (both a plus and minus). Alternative is to sum output from command:
    WMIC CPU Get DeviceID,NumberOfCores,NumberOfLogicalProcessors

 For *nix see http://stackoverflow.com/questions/6481005/ for alternatives.
   Note: 'nproc --all' does not return the right result on some cloud providers where systems are shared.
*/
program parallel_numprocessors, rclass
	if "`c(os)'"=="Windows" {
		local nproc : env NUMBER_OF_PROCESSORS
	}
	else {
		tempfile nproc_out
		! getconf _NPROCESSORS_ONLN > "`nproc_out'"
		tempname nproc_out_fhandle
		file open `nproc_out_fhandle' using "`nproc_out'", read text
		file read `nproc_out_fhandle' nproc
		local r_eof = `r(eof)'
		file close `nproc_out_fhandle'
		_assert `r_eof'==0, msg("Wasn't able to read output from system")
	}

	local nproc = int(real("`nproc'"))
	_assert "`nproc'"!=".", msg("Wasn't able to interpret output from system")

	di "Number of logical processors: `nproc'"
	return scalar numprocessors = `nproc'
end

/* Returns the version of parallel */
program def parallel_version, rclass
	version 11.0
	di as result "parallel" as text " Stata module for parallel computing"
	di as result "vers" as text " 1.19.0 26jul2017"
	di as result "auth" as text " George G. Vega [cre,aut], Brian Quistorff [aut]"

	return local pll_vers = "1.19.0"
end

/* Take a look to logfiles */
program def parallel_printlog
	version 11.0
	syntax [anything(name=pll_instance)] , [Event(string)]
	parallel_checklog `pll_instance', e(`event') action(print)
end

program def parallel_viewlog
	version 11.0
	syntax [anything(name=pll_instance)] , [Event(string)]
	parallel_checklog `pll_instance', e(`event') action(view)
end

program def parallel_checklog
	version 11.0
	syntax [anything(name=pll_instance)] , [Event(string)] action(string)

	if ("`event'"=="") local event = "$LAST_PLL_ID"

	if ("`event'"!=".") {
		/* By default uses 1 */
		if ("`pll_instance'" == "") local pll_instance 1

		local pll_instance : di %04.0f `pll_instance'

		/* Does de file exists */
		local logname = "__pll`event'_do`pll_instance'.log"

		if (c(os) == "Windows") local logname = "`c(tmpdir)'`logname'"
		else local logname = "`c(tmpdir)'/`logname'"

		/* If the logfile does not exists, do nothing*/
		cap confirm file "`logname'"
		if (_rc) {
			di as result "No logfile for instance -`pll_instance'- of parallel process -`event'- found"
			exit
		}

		/* Showing the log in screen */
		if ("`action'"=="print") {
			di as result "{hline 80}"
			di as result %~80s "beginning of file -`logname'-"
			di as result "{hline 80}"
			type `"`logname'"'
			di as result "{hline 80}"
			di as result %~80s "end of file -`logname'-"
			di as result "{hline 80}"
		}
		else view `"`logname'"'
	}
	else {
		di as error "It seems that you haven't use -parallel- yet."
		exit 601
	}

end

////////////////////////////////////////////////////////////////////////////////
// Splits the dataset into clusters
program def parallel_spliter
	version 11.0
	syntax [namelist(name=xtstructure)] [,parallelid(string) sorting(integer 0) force(integer 0) keepusing(varlist) orig_cl_local(string)]
	//args xtstructure parallelid Sorting Force

	if length("$PLL_CLUSTERS") == 0 {
		di as err "Number of clusters not fixed." as text " Set the number of clusters with " _newline as err "-{cmd:parallel setclusters #}-"
		exit 198
	}

	//quietly {
		if (length("`xtstructure'")) {
			_assert `sorting' | `force', msg("Data not sorted") rc(5)

			/* Checking the data types */
			foreach var of varlist `xtstructure' {
				capture confirm string var `var'
				if (`=_rc') local numvars `numvars' `var'
				else local strvars `strvars' `var'
			}

			/* calculate max possible clusters */
			egen _`parallelid'grp = group(`xtstructure'), missing
			summ _`parallelid'grp, meanonly
			local max_n_cl = `r(max)'
		}
		else {
			local max_n_cl = _N
		}

		/* Do we have too many clusters? */
		if (`max_n_cl'<$PLL_CLUSTERS){
			c_local `orig_cl_local' ${PLL_CLUSTERS}
			global PLL_CLUSTERS = `max_n_cl'
			di "Small workload/num groups. Temporarily setting number of clusters to ${PLL_CLUSTERS}"
		}

		if (length("`xtstructure'")) {
			preserve
			keep _`parallelid'grp
			contract _`parallelid'grp
			sort _freq
			//Figuring out the mapping from groupID to cut that equalizes sizes is the "Partition Problem"
			//which is hard to solve exactly (it's NP-complete). A rough solution suffices here though.
			gen _`parallelid'cut = mod(_n, ${PLL_CLUSTERS}) + 1
			tempfile grp_to_cut_map
			qui save `"`grp_to_cut_map'"'
			restore

			qui merge m:1 _`parallelid'grp using `"`grp_to_cut_map'"', keepusing(_`parallelid'cut) nogenerate
			drop _`parallelid'grp
			sort `xtstructure'
		}
		else {
			gen _`parallelid'cut = min(${PLL_CLUSTERS}, ceil(_n*${PLL_CLUSTERS}/_N)) //numerical precision for big nubmers can make > PLL_CLUSTERS
		}

		if (length("`keepusing'")) {
			keep _`parallelid'cut `keepusing'
			gen __pllnobs`parallelid' = _n
		}

		qui save __pll`parallelid'_dataset, replace
		drop _all
	//}

end

//in a subprogram because some of the outputopts may have the same name as -parallel- options
program pll_remove_replace
	syntax, pll_argopt(string) pll_outputopts(string)

	//parse the original options
	foreach pll_outputopt of local pll_outputopts{
		local pll_outputopts_syn "`pll_outputopts_syn' `pll_outputopt'(string)"
	}
	local 0 : copy local pll_argopt
	syntax [anything(equalok everything)][, `pll_outputopts_syn' *]

	//remove the ones were are subbing out
	c_local argopt `"`anything', `options'"'

	//some may be null so only sub out real ones
	foreach pll_outputopt of local pll_outputopts{
		if "`pll_outputopt'"!="" local outputopts "`outputopts' `pll_outputopt'"
	}
	c_local outputopts `outputopts'
	/* old method (doesn't account for syntax eliding blank ones
	//make version of argopt that doesn't have the options that will be swapped out
	local argopt_orig : copy local argopt
	foreach outputopt of local outputopts{
		local argopt = regexr(`"`argopt'"',"`outputopt'\([^\)]*\)","")
	}*/
end

//allow some or all files to not have been created
//If at least one file was made, even if it was empty, make a file
// (same save dynamics as user program)
program pll_collect
	syntax, folder(string) parallelid(string) outputopts(string) argopt_orig(string)
	preserve

	foreach outputopt of local outputopts{
		mata: st_local("val", strofreal(regexm(st_local("argopt_orig"),st_local("outputopt")+"\(([^)]+)\)")))
		if `val'{
			mata: st_local("outfile", regexs(1))
			//get rid of quotes
			local outfile `outfile'
			local emptyok ""
			drop _all
			forval i=1/$PLL_CLUSTERS{
				cap append using `"`folder'__pll`parallelid'_out_`outputopt'`=strofreal(`i',"%04.0f")'"'
				if (_rc==0) local emptyok "emptyok"
			}

			qui save `"`outfile'"', replace `emptyok'
		}
	}
end

////////////////////////////////////////////////////////////////////////////////
// MAIN PROGRAM
// There are things we need to possibly restore on exit
// - Data/S_FN (this is taken care of by -preserve-)
// - PWD (non-indented capture block)
// - Temporaliy changed number of clusters (non-indented capture block)
// - Turn off the timer (do within catch for capture blocks)
program def parallel_do, rclass
	version 11.0

	#delimit ;
	syntax anything(name=dofile equalok everything)
		[, by(string)
		Keep
		KEEPLast
		prefix
		Force
		PROGrams(string)
		Mata
		NOGlobals
		KEEPTiming
		Seeds(string)
		NOData
		Randtype(string)
		Timeout(integer 60)
		PROCessors(integer 0)
		argopt(string)
		KEEPUsing(string)
		SETparallelid(string)
		OUTputopts(string)
		DETerministicoutput
		];
	#delimit cr

	if length("$PLL_CLUSTERS") == 0 {
		di as err "You haven't set the number of clusters" _n "Please set it with: {cmd:parallel setclusters} {it:#}"
		exit 198
	}

	if `c(noisily)'==0 & "`programs'"!="" {
		di as err "If you want to pass programs in memory (using the option programs)" _n "then the program must be run noisily."
		exit 198
	}

	// Initial checks
	foreach opt in macrolist keep keeplast prefix force mata noglobals keeptiming nodata deterministicoutput {
		local `opt' = length("``opt''") > 0
	}

	/* Randtype */
	if ("`randtype'" == "") local randtype = "datetime"

	/* If no data parsing has to be done (because of no data!) */
	if (!(c(N)*c(k))) local nodata 1

	if (!`keeptiming') {
		timer clear 97
		timer clear 98
		timer clear 99
	}

	timer on 98

	// Deletes last parallel instance ran
	local last_id = cond("`r(pll_id)'"!="","`r(pll_id)'", "$LAST_PLL_ID")
	if (`keeplast' & length("`last_id'")) cap parallel_clean, e(`last_id') nologs

	if length("`by'") != 0 {
		local sortlist: sortedby
		local sorting = regexm("`sortlist'","^`by'")
	}
	else local sorting = 0

	// Gets the directory where to work at
	if (!`prefix') {

		/* First checks if the file exists */
		mata: parallel_normalizepath(`"`dofile'"',1)
		local pll_dir = "`filedir'"
		local dofile = "`filename'"
	}
	else local pll_dir = c(pwd)+"/"

	local initialdir = c(pwd)
	qui cd "`pll_dir'"
	capture noisily { //Start capture block for PWD & $PLL_CLUSTERS

	/* Creates a unique ID for the process and secures it */
	if ("`setparallelid'"=="") mata: parallel_sandbox(5)
	else local parallelid = "`setparallelid'"

	/* Generates database clusters */
	if (!`nodata'){
		local sfn = "$S_FN" // be able to "fake" restore this
		preserve

		parallel_spliter `by' , parallelid(`parallelid') sorting(`sorting') force(`force') keepusing(`keepusing') orig_cl_local(orig_PLL_CLUSTERS)
	}

	global LAST_PLL_ID = "`parallelid'"
	global LAST_PLL_DIR = "`pll_dir'"
	global LAST_PLL_N = $PLL_CLUSTERS

	/* Starts building the files */
	local work_around_no_cwd = 0
	quietly {

		/* Saves mata objects */
		if (`mata') {
			mata: mata mlib create l__pll`parallelid'_mlib, replace
			cap mata: mata mlib add l__pll`parallelid'_mlib *()
			cap which l__pll`parallelid'_mlib.mlib
			if _rc!=0{
				local work_around_no_cwd = 1
				noi di "Note: In order to pass mata objects to the clusters, they are saved to a temporary mlib file in the current directly."
				noi di "      Your ado-path doesn't contain the current directory."
				noi di "      We have added the current directory to the end of the ado-path for the clusters.."
			}
			cap mata: mata matsave __pll`parallelid'_mata.mmat *, replace
			if (`=_rc') local matasave = 0
			else local matasave = 1
		}
		else local matasave = 0
	}

	//see if we handle extra output streams
	if "`outputopts'"!=""{
		local argopt_orig : copy local argopt
		pll_remove_replace, pll_argopt(`"`argopt'"') pll_outputopts(`outputopts')
	}

	/* Writing the dofile */
	mata: st_local("errornum", strofreal(parallel_write_do(strtrim(`"`dofile' `argopt'"'), "`parallelid'", $PLL_CLUSTERS, `prefix', `matasave', !`noglobals', "`seeds'", "`randtype'", `nodata', "`pll_dir'", "`programs'", `processors',`work_around_no_cwd',"`outputopts'")))
	if (`errornum') {
		/* Removes the sandbox file (unprotect the files) */
		mata: parallel_sandbox(2, "`parallelid'")

		error `errornum'
	}

	timer off 98
	cap timer list
	if (r(t98) == .) local pll_t_setu = 0
	else local pll_t_setu = r(t98)

	/* Running the dofiles */
	timer on 99
	mata: st_local("nerrors", strofreal(parallel_run("`parallelid'",$PLL_CLUSTERS,`"$PLL_STATA_PATH"',`=`timeout'*1000', `deterministicoutput', tokens("$PLL_HOSTNAMES"), "$PLL_SSH")))
	timer off 99

	cap timer list
	if (r(t99) == .) local pll_t_calc = 0
	else local pll_t_calc = r(t99)
	local pll_t_reps = r(nt99)

	timer on 97

	if ("`outputopts'"!=""  & !`nerrors'){
		pll_collect, folder("`pll_dir'") parallelid("`parallelid'") outputopts(`outputopts') argopt_orig(`"`argopt_orig'"')
	}

	// Paste the databases
	if (!`nodata' & !`nerrors') {
		parallel_fusion `parallelid', clusters($PLL_CLUSTERS) keepusing(`keepusing')
		cap drop _`parallelid'cut

		global S_FN = "`sfn'" // Restores original S_FN (file name) value
		restore, not
	}

	return scalar pll_n = $PLL_CLUSTERS

	/* Removes the sandbox file (unprotect the files) */
	if ("`setparallelid'" == "") {
		mata: parallel_sandbox(2, "`parallelid'")
		if (!`keep' & !`keeplast') parallel_clean, e("`parallelid'") nologs
	}

	} //End capture block
	if _rc {
		local orig_rc = _rc
		qui cd "`initialdir'"
		if "`orig_PLL_CLUSTERS'"!="" global PLL_CLUSTERS=`orig_PLL_CLUSTERS'
		cap timer off 97
		cap timer off 98
		cap timer off 99
		exit `orig_rc'
	}
	qui cd "`initialdir'"
	if "`orig_PLL_CLUSTERS'"!="" global PLL_CLUSTERS=`orig_PLL_CLUSTERS'

	timer off 97
	cap timer list
	if (r(t97) == .) local pll_t_fini = 0
	else local pll_t_fini = r(t97)

	qui timer list
	return local  pll_seeds="`pllseeds'"
	return scalar pll_errs = `nerrors'
	return local  pll_dir "`pll_dir'"
	return scalar pll_t_reps = `pll_t_reps'
	return scalar pll_t_setu = `pll_t_setu'
	return scalar pll_t_calc = `pll_t_calc'
	return scalar pll_t_fini = `pll_t_fini'
	return local pll_id = "`parallelid'"

	if `nerrors' {
		di as err "`nerrors' child processes encountered errors. Throwing last error."
		exit `pll_last_error'
	}
end

////////////////////////////////////////////////////////////////////////////////
// Cleans all files generated by parallel
// If you want to keep the logs, then specify -, nologs-
program def parallel_clean
	version 11.0
	syntax [, Event(string) All Force nologs]

	if (length("`event'") != 0 & length("`all'") != 0) {
		di as error `"invalid syntax: Using -pll_id- and -all- jointly is not allowed."'
		exit 198
	}
	local do_logs = ("`logs'"!="nologs")
	mata: parallel_clean("`event'", `=length("`all'") > 0', `=length("`force'") > 0', `do_logs')
end

////////////////////////////////////////////////////////////////////////////////
// Sets the number of clusters as a global macro
program parallel_setclusters
	version 11.0
	syntax [anything(name=nclusters)] [, Force Statapath(string asis) Gateway(string) Includefile(string) Hostnames(string) ssh(string) procexec(int 2)]

	_assert inlist(`procexec',0,1,2), msg("procexec() must be 0, 1, or 2") rc(198)
	cap parallel_numprocessors
	local nproc = int(real("`r(numprocessors)'"))
	if "`nclusters'"=="default" | "`nclusters'"==""{
		_assert `nproc'!=., msg("Couldn't determine number of available processors for default configuration.")
		local nclusters = max(floor(`nproc'*3/4),1)
	}
	else{
		local nclusters = int(real(`"`nclusters'"'))
		_assert (`nclusters'>0 & `nclusters'!=.),  msg(`"Not allowed: "#" Should be a positive number"') rc(109)
	}
	global USE_PROCEXEC = `procexec'
	global PLL_HOSTNAMES = "`hostnames'"
	global PLL_SSH = "`ssh'"
	if "$PLL_SSH"!="" global PLL_SSH = "$PLL_SSH "
  if "`hostnames'"!="" local nproc "." //don't use softmax when in cluster

	local force = (length("`force'")>0)
	mata: parallel_setclusters(`nclusters', `force', `nproc')
	mata: st_local("error", strofreal(parallel_setstatapath(`"`statapath'"', `force')))
	_assert (!`error'), msg("Can not set Stata directory, try using -statapath()- option") rc(`error')

	if "`c(mode)'"=="batch" & "`c(os)'"=="Windows" & "$USE_PROCEXEC"=="0" {
		if `"`gateway'"'=="" local gateway "pll_gateway.sh"

		cap confirm file `"`gateway'"'
		_assert `=!_rc', msg(`"On Windows batch-mode with command-line process execution, requires a gateway file is required but not found (path: `c(pwd)')"') rc(`=_rc')
		global PLL_GATEWAY_FNAME `"`gateway'"'
	}

	if `"`includefile'"'!=""{
		cap confirm file `"`includefile'"'
		_assert `=!_rc', msg("Can not find the include file (`includefile').") msg(`=_rc')
		global PLL_INCLUDE_FILE `"`includefile'"'
	}
end

////////////////////////////////////////////////////////////////////////////////
// Exports

// Exports a copy of programs
program def program_export
	version 11.0
	syntax using/ [,Programlist(string) Inname(string)]

	mata: program_export("`using'", "`programlist'", "`inname'")
end

////////////////////////////////////////////////////////////////////////////////
// Appends the clusterized dataset
program def parallel_fusion
	version 11.0
	syntax anything(name=parallelid) , clusters(integer) [keepusing(string)]

	cap use "__pll`parallelid'_dta0001.dta", clear
	if (_rc){
		di as err "No dataset for instance 0001."
		exit _rc
	}
	local sortlist: sortedby

	forval i = 2/`clusters' {
		cap append using `"__pll`parallelid'_dta`=string(`i',"%04.0f")'.dta"'
		if (_rc){
			di as err "No dataset for instance `=string(`i',"%04.0f")'."
			exit _rc
		}
	}

	/* If it just used a set of variables */
	if (length("`keepusing'")) {
		qui merge 1:1 __pllnobs`parallelid' using __pll`parallelid'_dataset, keep(3) nogen
		drop __pllnobs`parallelid'
	}

	//restore the sort
	if "`sortlist'"!="" sort `sortlist'

end

////////////////////////////////////////////////////////////////////////////////
// Checks whether the user pressed break inside a loop
program def parallel_break
	version 11.0
	mata: parallel_break()
end


minor cleanup

minor cleanup

*! version 4.0.5 dev - 16feb2017

program reghdfe, eclass
	* Intercept old+version
	cap syntax, version old
	if !c(rc) {
		reghdfe_old, version
		exit
	}

	* Intercept old
	cap syntax anything(everything) [fw aw pw/], [*] old
	if !c(rc) {
		di as error "(running historical version of reghdfe)"
		if ("`weight'"!="") local weightexp [`weight'=`exp']
		reghdfe_old `anything' `weightexp', `options'
		exit
	}

	* Aux. subcommands
	cap syntax, [*]
	if inlist("`options'", "check", "compile", "reload", "update", "version") {
		if ("`options'"=="compile") loc args force
		if ("`options'"=="check") loc options compile
		if ("`options'"=="update") {
			loc args 1
			loc options reload
		}
		loc subcmd = proper("`options'")
		`subcmd' `args'
	}
	else if replay() {
		Replay `0'
	}
	else {
		Cleanup 0
		Compile // takes 0.01s to run this useful check (ensures .mlib exists)
		cap noi Estimate `0'
		Cleanup `c(rc)'
	}
end

// --------------------------------------------------------------------------

program Compile
	args force
	ftools, check // in case lftools.mlib does not exist or is outdated
	ms_get_version reghdfe // from moresyntax package; save local package_version

	loc list_objects "FixedEffects() fixed_effects() FE_Options() FE_Output() BipartiteGraph()"
	loc list_functions "reghdfe_*() transform_*() accelerate_*() panelmean() panelsolve_*()"
	loc list_misc "weighted_quadcolsum() safe_divide() check_convergence()"
	// TODO: prefix everything with reghdfe_*

	ms_compile_mata, ///
		package(reghdfe) ///
		version(`package_version') ///
		fun("`list_objects' `list_functions' `list_misc'") ///
		verbose ///
		`force'
end

// --------------------------------------------------------------------------

program Reload
	* Internal debugging tool.
	* Updates dependencies and reghdfe from local path or from github
	* Usage:
	* 	reghdfe, update // from c:\git\..
	* 	reghdfe, reload // from github

	args online
	if ("`online'" == "") loc online 0

	di as text _n "{bf:reghdfe: updating required packages}"
	di as text "{hline 64}"

	* -moresyntax- https://github.com/sergiocorreia/moresyntax/
	cap ado uninstall moresyntax
	if (`online') net install moresyntax, from("https://github.com/sergiocorreia/moresyntax/raw/master/src/")
	if (!`online') net install moresyntax, from("c:\git\moresyntax\src")
	di as text "{hline 64}"

	* -ftools- https://github.com/sergiocorreia/ftools/
	cap ado uninstall ftools
	if (`online') net install ftools, from("https://github.com/sergiocorreia/ftools/raw/master/src/")
	if (!`online') net install ftools, from("c:\git\ftools\src")
	di as text "{hline 64}"
	ftools, compile // requires moresyntax
	di as text "{hline 64}"

	* Update -reghdfe-
	di as text _n  _n "{bf:reghdfe: updating self}"
	di as text "{hline 64}"
	qui ado uninstall reghdfe
	if (`online') net install reghdfe, from("https://github.com/sergiocorreia/reghdfe/raw/version-4/src/")
	if (!`online') net install reghdfe, from("c:\git\reghdfe\src")
	qui which reghdfe
	di as text "{hline 64}"
	reghdfe, compile
	di as text "{hline 64}"

	* Cleaning up
	di as text _n "{bf:Note:} You need to run {stata program drop _all} now."
end

// --------------------------------------------------------------------------

program Version
	which reghdfe

	di as text _n "Dependencies installed?"
	local dependencies moresyntax ftools
	// ivreg2 avar tuples group3hdfe boottest
	foreach dependency of local dependencies {
		loc fn `dependency'.ado
		if ("`dependency'"=="moresyntax") loc fn ms_get_version.ado
		cap findfile `fn'
		if (_rc) {
			di as text "{lalign 20:- `dependency'}" as error "not"
		}
		else {
			di as text "{lalign 20:- `dependency'}" as result "yes"
		}
	}
end

// --------------------------------------------------------------------------

program Cleanup
	args rc
	cap mata: mata drop HDFE
	cap mata: mata drop hdfe_*
	cap matrix drop reghdfe_statsmatrix
	if (`rc') exit `rc'
end

program Estimate, eclass
	ereturn clear

* Parse and fill out HDFE object
	mata: HDFE = FixedEffects()
	reghdfe_parse `0' // store results on HDFE, HDFE.options, and HDFE.output
	loc OPT HDFE.options
	loc OUT HDFE.output

* Select sample
	marksample touse, strok // based on locals for varlist+cluster+absvars+if+in+weight (injected by -reghdfe_parse-)
	// -strok- because clustervars can be string

* Construct HDFE object
	// SYNTAX: fixed_effects(absvars | , touse, weighttype, weightvar, dropsing, verbose, object)
	mata: HDFE = fixed_effects(`OPT'.absorb, "`touse'", `OPT'.weight_type, `OPT'.weight_var, `OPT'.drop_singletons, HDFE.verbose, HDFE)
	mata: st_local("N", strofreal(HDFE.N))
	if (`N' == 0) error 2000

* Compute degrees-of-freedom
	mata: HDFE.estimate_dof()

* Save updated e(sample) (singletons reduce sample);
* required to parse factor variables to partial out
	mata: HDFE.save_touse("`touse'", 1)

* Expand varlists
	foreach cat in varlist depvar indepvars endogvars instruments {
		mata: st_local("vars", `OPT'.original_`cat')
		if ("`vars'" == "") continue
		// HACK: addbn replaces 0.foreign with 0bn.foreign , to prevent st_data() from loading a bunch of zeros
		ms_fvstrip `vars' if `touse', expand dropomit addbn onebyone
		// If we don't use onebyone, then 1.x 2.x ends up as 2.x
		loc vars "`r(varlist)'"
		mata: `OPT'.`cat' = "`vars'"
	}

* Stats
	mata: st_local("stats", `OPT'.summarize_stats)
	if ("`stats'" != "") Stats

* Partial out; and save TSS of depvar
	mata: hdfe_variables = HDFE.partial_out(`OPT'.varlist, 1) // 1=Save TSS of first var if HDFE.output.tss is missing

* Regress
	Regress `touse'
	mata: st_local("diopts", `OPT'.diopts)
	Replay, `diopts'

// ~~ Preserve relevant dataset ~~
	//
	// if (`compact') {
	// 	preserve
	// 	keep `varlist' `absorb'
	// 	loc N = c(N)
	// }
	// else {
	// 	qui cou if `touse'
	// 	loc N = r(N)
	// }
end

// -------------------------------------------------------------
// Run OLS regressions with partialled-out variables
// -------------------------------------------------------------
program Regress, eclass
	args touse

	tempname b V N rank df_r
	// Carefully split y from X, without duplicating the memory used
	mata: hdfe_y = hdfe_variables[., 1]
	mata: hdfe_variables = cols(hdfe_variables)==1 ? J(rows(hdfe_variables), 0, .) : hdfe_variables[., 2..cols(hdfe_variables)]
	mata: reghdfe_post_ols(HDFE, hdfe_y, hdfe_variables, "`b'", "`V'", "`N'", "`rank'", "`df_r'")
	mata: st_local("indepvars", HDFE.options.indepvars)
	mata: hdfe_y = hdfe_variables = .

	if ("`indepvars'" != "") {
		matrix colnames `b' = `indepvars'
		matrix colnames `V' = `indepvars'
		matrix rownames `V' = `indepvars'
		ereturn post `b' `V', esample(`touse') buildfvinfo depname(`depvar')
	}
	else {
		ereturn post, esample(`touse') buildfvinfo depname(`depvar')
	}

	ereturn scalar N       = `N'
	ereturn scalar rank    = `rank'
	ereturn scalar df_r    = `df_r'
	ereturn local  cmd     "reghdfe"
	mata: HDFE.output.post(HDFE.options)

	* Post stats
	cap conf matrix reghdfe_statsmatrix
	if (!c(rc)) {
		ereturn matrix summarize = reghdfe_statsmatrix
		mata: st_local("summarize_quietly", strofreal(HDFE.options.summarize_quietly))
		ereturn scalar summarize_quietly = `summarize_quietly'
	}
end

program Replay
	syntax [, *]
	_get_diopts options, `options'
	reghdfe_header // _coef_table_header
	di ""
	_coef_table, `options' // ereturn display, `options'
	reghdfe_footnote

	* Replay stats
	if (e(summarize_quietly)==0) {
		di as text _n "{sf:Regression Summary Statistics:}" _c
		matlist e(summarize)', border(top bottom) rowtitle(Variable) // twidth(18)
	}
end

// -----------------------------------------------------------------------------
// Matrix of summary statistics
// -----------------------------------------------------------------------------
program Stats
	* Optional weights
	mata: st_local("weight", sprintf("[%s=%s]", HDFE.weighttype, HDFE.weightvar))
	assert "`weight'" != ""
	if ("`weight'" == "[=]") loc weight
	loc weight : subinstr local weight "[pweight" "[aweight"

	mata: st_local("stats", HDFE.options.summarize_stats)
	mata: st_local("varlist", HDFE.options.varlist)
	mata: st_local("cvars", invtokens(HDFE.cvars))
	loc full_varlist `varlist' `cvars'

	qui tabstat `full_varlist' `weight' , stat(`stats') col(stat) save
	matrix reghdfe_statsmatrix = r(StatTotal)
end

*! version 4.0.5 dev - 16feb2017

program reghdfe, eclass
	* Intercept old+version
	cap syntax, version old
	if !c(rc) {
		reghdfe_old, version
		exit
	}

	* Intercept old
	cap syntax anything(everything) [fw aw pw/], [*] old
	if !c(rc) {
		di as error "(running historical version of reghdfe)"
		if ("`weight'"!="") local weightexp [`weight'=`exp']
		reghdfe_old `anything' `weightexp', `options'
		exit
	}

	* Aux. subcommands
	cap syntax, [*]
	if inlist("`options'", "check", "compile", "reload", "update", "version") {
		if ("`options'"=="compile") loc args force
		if ("`options'"=="check") loc options compile
		if ("`options'"=="update") {
			loc args 1
			loc options reload
		}
		loc subcmd = proper("`options'")
		`subcmd' `args'
	}
	else if replay() {
		Replay `0'
	}
	else {
		Cleanup 0
		Compile // takes 0.01s to run this useful check (ensures .mlib exists)
		cap noi Estimate `0'
		Cleanup `c(rc)'
	}
end

// --------------------------------------------------------------------------

program Compile
	args force
	ftools, check // in case lftools.mlib does not exist or is outdated
	ms_get_version reghdfe // from moresyntax package; save local package_version

	loc list_objects "FixedEffects() fixed_effects() FE_Options() FE_Output() BipartiteGraph()"
	loc list_functions "reghdfe_*() transform_*() accelerate_*() panelmean() panelsolve_*()"
	loc list_misc "weighted_quadcolsum() safe_divide() check_convergence()"
	// TODO: prefix everything with reghdfe_*

	ms_compile_mata, ///
		package(reghdfe) ///
		version(`package_version') ///
		fun("`list_objects' `list_functions' `list_misc'") ///
		verbose ///
		`force'
end

program Requirements

end

program Reload
	* Internal debugging tool.
	* Updates dependencies and reghdfe from local path or from github
	* Usage:
	* 	reghdfe, update // from c:\git\..
	* 	reghdfe, reload // from github

	args online
	if ("`online'" == "") loc online 0

	di as text _n "{bf:reghdfe: updating required packages}"
	di as text "{hline 64}"

	* -moresyntax- https://github.com/sergiocorreia/moresyntax/
	cap ado uninstall moresyntax
	if (`online') net install moresyntax, from("https://github.com/sergiocorreia/moresyntax/raw/master/src/")
	if (!`online') net install moresyntax, from("c:\git\moresyntax\src")
	di as text "{hline 64}"

	* -ftools- https://github.com/sergiocorreia/ftools/
	cap ado uninstall ftools
	if (`online') net install ftools, from("https://github.com/sergiocorreia/ftools/raw/master/src/")
	if (!`online') net install ftools, from("c:\git\ftools\src")
	di as text "{hline 64}"
	ftools, compile // requires moresyntax
	di as text "{hline 64}"

	* Update -reghdfe-
	di as text _n  _n "{bf:reghdfe: updating self}"
	di as text "{hline 64}"
	qui ado uninstall reghdfe
	if (`online') net install reghdfe, from("https://github.com/sergiocorreia/reghdfe/raw/version-4/src/")
	if (!`online') net install reghdfe, from("c:\git\reghdfe\src")
	qui which reghdfe
	di as text "{hline 64}"
	reghdfe, compile
	di as text "{hline 64}"

	* Cleaning up
	di as text _n "{bf:Note:} You need to run {stata program drop _all} now."
end

program Version
	which reghdfe

	di as text _n "Dependencies installed?"
	local dependencies moresyntax ftools ivreg2 avar tuples
	foreach dependency of local dependencies {
		cap findfile `dependency'.ado
		if (_rc) {
			di as text "{lalign 20:- `dependency'}" as error "not"
		}
		else {
			di as text "{lalign 20:- `dependency'}" as result "yes"
		}
	}

end

// --------------------------------------------------------------------------

program Cleanup
	args rc
	cap mata: mata drop HDFE
	cap mata: mata drop hdfe_*
	cap matrix drop reghdfe_statsmatrix
	if (`rc') exit `rc'
end

program Estimate, eclass
	ereturn clear

* Parse and fill out HDFE object
	mata: HDFE = FixedEffects()
	reghdfe_parse `0' // store results on HDFE, HDFE.options, and HDFE.output
	loc OPT HDFE.options
	loc OUT HDFE.output

* Select sample
	marksample touse, strok // based on locals for varlist+cluster+absvars+if+in+weight (injected by -reghdfe_parse-)
	// -strok- because clustervars can be string

* Construct HDFE object
	// SYNTAX: fixed_effects(absvars | , touse, weighttype, weightvar, dropsing, verbose, object)
	mata: HDFE = fixed_effects(`OPT'.absorb, "`touse'", `OPT'.weight_type, `OPT'.weight_var, `OPT'.drop_singletons, HDFE.verbose, HDFE)
	mata: st_local("N", strofreal(HDFE.N))
	if (`N' == 0) error 2000

* Compute degrees-of-freedom
	mata: HDFE.estimate_dof()

* Save updated e(sample) (singletons reduce sample);
* required to parse factor variables to partial out
	mata: HDFE.save_touse("`touse'", 1)

* Expand varlists
	foreach cat in varlist depvar indepvars endogvars instruments {
		mata: st_local("vars", `OPT'.original_`cat')
		if ("`vars'" == "") continue
		// HACK: addbn replaces 0.foreign with 0bn.foreign , to prevent st_data() from loading a bunch of zeros
		ms_fvstrip `vars' if `touse', expand dropomit addbn onebyone
		// If we don't use onebyone, then 1.x 2.x ends up as 2.x
		loc vars "`r(varlist)'"
		mata: `OPT'.`cat' = "`vars'"
	}

* Stats
	mata: st_local("stats", `OPT'.summarize_stats)
	if ("`stats'" != "") Stats

* Partial out; and save TSS of depvar
	mata: hdfe_variables = HDFE.partial_out(`OPT'.varlist, 1) // 1=Save TSS of first var if HDFE.output.tss is missing

* Regress
	Regress `touse'
	mata: st_local("diopts", `OPT'.diopts)
	Replay, `diopts'

// ~~ Preserve relevant dataset ~~
	//
	// if (`compact') {
	// 	preserve
	// 	keep `varlist' `absorb'
	// 	loc N = c(N)
	// }
	// else {
	// 	qui cou if `touse'
	// 	loc N = r(N)
	// }
end

// -------------------------------------------------------------
// Run OLS regressions with partialled-out variables
// -------------------------------------------------------------
program Regress, eclass
	args touse

	tempname b V N rank df_r
	// Carefully split y from X, without duplicating the memory used
	mata: hdfe_y = hdfe_variables[., 1]
	mata: hdfe_variables = cols(hdfe_variables)==1 ? J(rows(hdfe_variables), 0, .) : hdfe_variables[., 2..cols(hdfe_variables)]
	mata: reghdfe_post_ols(HDFE, hdfe_y, hdfe_variables, "`b'", "`V'", "`N'", "`rank'", "`df_r'")
	mata: st_local("indepvars", HDFE.options.indepvars)
	mata: hdfe_y = hdfe_variables = .

	if ("`indepvars'" != "") {
		matrix colnames `b' = `indepvars'
		matrix colnames `V' = `indepvars'
		matrix rownames `V' = `indepvars'
		ereturn post `b' `V', esample(`touse') buildfvinfo depname(`depvar')
	}
	else {
		ereturn post, esample(`touse') buildfvinfo depname(`depvar')
	}

	ereturn scalar N       = `N'
	ereturn scalar rank    = `rank'
	ereturn scalar df_r    = `df_r'
	ereturn local  cmd     "reghdfe"
	mata: HDFE.output.post(HDFE.options)

	* Post stats
	cap conf matrix reghdfe_statsmatrix
	if (!c(rc)) {
		ereturn matrix summarize = reghdfe_statsmatrix
		mata: st_local("summarize_quietly", strofreal(HDFE.options.summarize_quietly))
		ereturn scalar summarize_quietly = `summarize_quietly'
	}
end

program Replay
	syntax [, *]
	_get_diopts options, `options'
	reghdfe_header // _coef_table_header
	di ""
	_coef_table, `options' // ereturn display, `options'
	reghdfe_footnote

	* Replay stats
	if (e(summarize_quietly)==0) {
		di as text _n "{sf:Regression Summary Statistics:}" _c
		matlist e(summarize)', border(top bottom) rowtitle(Variable) // twidth(18)
	}
end

// -----------------------------------------------------------------------------
// Matrix of summary statistics
// -----------------------------------------------------------------------------
program Stats
	* Optional weights
	mata: st_local("weight", sprintf("[%s=%s]", HDFE.weighttype, HDFE.weightvar))
	assert "`weight'" != ""
	if ("`weight'" == "[=]") loc weight
	loc weight : subinstr local weight "[pweight" "[aweight"

	mata: st_local("stats", HDFE.options.summarize_stats)
	mata: st_local("varlist", HDFE.options.varlist)
	mata: st_local("cvars", invtokens(HDFE.cvars))
	loc full_varlist `varlist' `cvars'

	qui tabstat `full_varlist' `weight' , stat(`stats') col(stat) save
	matrix reghdfe_statsmatrix = r(StatTotal)
end


fix type mismatch on prep mergelist with blank MyMergeList

* Set FILENAME - static filenames are removed after each run.
local c_time_date = c(current_date) + c(current_time)
local c_time_date = subinstr("`c_time_date'", " ", "", .)
local c_time_date = subinstr("`c_time_date'", ":", "", .)
* Edit this filename to rename the resulting . file. Removing c_time_date will deleted previous versions when run.
global FILENAME "UCIMerge_`c_time_date'.dta"

* Rename Variables
capture program drop prefix
program define prefix
	display as text "`0'"
	foreach i of var * {
	rename `i' `0'_`i'
	}
end

* Convenience method to create value lables from string variables
capture program drop encode_value_labels
program define encode_value_labels
  rename `0' tmp_`0'
  encode tmp_`0', generate(`0')
  drop tmp_`0'
end

* Bring transformed source into the master dataset
* Create master if it doesn't exist - otherwise, merge
capture program drop merge_with_master
program define merge_with_master
	capture confirm file "$FILENAME"
	if _rc!=0 {
		save "$FILENAME"
	}
	else {
		merge 1:1 UCINumeric year using "$FILENAME", nogenerate
		save "$FILENAME", replace
	}
	check_merge
end

* If the file source file doesn't already exist then download and create it.
capture program drop use_or_import_source_excel
program define use_or_import_source_excel
	args PREFIX URL
	capture confirm file "source/`PREFIX'.dta"
	if _rc!=0 {
		noisily display "Importing `PREFIX' from web."
		import excel "`URL'", firstrow clear
		compress
		save "source/`PREFIX'.dta"
	}
	else {
		use "source/`PREFIX'.dta", clear
	}
end

capture program drop use_or_import_source
program define use_or_import_source
	args PREFIX URL
	capture confirm file "source/`PREFIX'.dta"
	if _rc!=0 {
		noisily display "Importing `PREFIX' from web."
		use "`URL'", clear
		compress
		save "source/`PREFIX'.dta"
	}
	else {
		use "source/`PREFIX'.dta", clear
	}
end

* Set up the mergefile for the current source
capture program drop prep_mergefile
program define prep_mergefile
	args MERGEWITH MERGEUSING merge

	capture confirm file "source/merge.dta"
	if _rc!=0 {
		cache_mergefile `merge'
	}
	else {
		use "source/merge.dta", clear
	}

	rename SourceLongName Name
	rename SourceNumericCode Numeric
	rename SourceCountryCode Alpha
	drop if Deprecated == 1
	drop if Source != "`MERGEWITH'"
	keep UCI* `MERGEUSING'
	save `merge', replace
end

* Merge IDs into current dataset
capture program drop merge_ids
program define merge_ids
	args PREFIX YEARVAR COUNTRYVAR MERGEUSING merge

	rename `PREFIX'_`YEARVAR' year
	rename `PREFIX'_`COUNTRYVAR' `MERGEUSING'

	merge m:1 `MERGEUSING' using `merge', keep(match master) nogenerate
	rename `MERGEUSING' `PREFIX'_`COUNTRYVAR'
end

capture program drop check_merge
program define check_merge
	isid UCIName year
	isid UCIAlpha year
	isid UCINumeric year
	isid UCIName UCIAlpha UCINumeric year
end

* Compiles mergefile from CSV sources
capture program drop cache_mergefile
program define cache_mergefile
	args merge

	* Combine merge lists
	import delimited "UCIMergeList.csv", clear case(preserve) numericcols(1 3 6) stringcols(2 4 5)
	save `merge', replace

	capture import delimited "private/MyMergeList.csv", clear case(preserve) numericcols(1 3 6) stringcols(2 4 5)
	if _rc ==0 {
		append using "`merge'"
		save `merge', replace
	}

	* Align merge lists with UCI Country Codes and Names
	import delimited "UCICountries.csv", clear case(preserve)
	capture labmask UCINumeric, values(UCIName)
	if _rc!=0 {
		ssc install labutil
		labmask UCINumeric, values(UCIName)
	}
	merge 1:m UCINumeric using `merge', keep(match ) nogenerate
	compress
	save "source/merge.dta", replace
end

* Removes Previous Compiled Dataset
capture rm "$FILENAME"

* Forces compile of merge list with every run
capture rm "source/merge.dta"

* Set FILENAME - static filenames are removed after each run.
local c_time_date = c(current_date) + c(current_time)
local c_time_date = subinstr("`c_time_date'", " ", "", .)
local c_time_date = subinstr("`c_time_date'", ":", "", .)
* Edit this filename to rename the resulting . file. Removing c_time_date will deleted previous versions when run.
global FILENAME "UCIMerge_`c_time_date'.dta"

* Rename Variables
capture program drop prefix
program define prefix
	display as text "`0'"
	foreach i of var * {
	rename `i' `0'_`i'
	}
end

* Convenience method to create value lables from string variables
capture program drop encode_value_labels
program define encode_value_labels
  rename `0' tmp_`0'
  encode tmp_`0', generate(`0')
  drop tmp_`0'
end

* Bring transformed source into the master dataset
* Create master if it doesn't exist - otherwise, merge
capture program drop merge_with_master
program define merge_with_master
	capture confirm file "$FILENAME"
	if _rc!=0 {
		save "$FILENAME"
	}
	else {
		merge 1:1 UCINumeric year using "$FILENAME", nogenerate
		save "$FILENAME", replace
	}
	check_merge
end

* If the file source file doesn't already exist then download and create it.
capture program drop use_or_import_source_excel
program define use_or_import_source_excel
	args PREFIX URL
	capture confirm file "source/`PREFIX'.dta"
	if _rc!=0 {
		noisily display "Importing `PREFIX' from web."
		import excel "`URL'", firstrow clear
		compress
		save "source/`PREFIX'.dta"
	}
	else {
		use "source/`PREFIX'.dta", clear
	}
end

capture program drop use_or_import_source
program define use_or_import_source
	args PREFIX URL
	capture confirm file "source/`PREFIX'.dta"
	if _rc!=0 {
		noisily display "Importing `PREFIX' from web."
		use "`URL'", clear
		compress
		save "source/`PREFIX'.dta"
	}
	else {
		use "source/`PREFIX'.dta", clear
	}
end

* Set up the mergefile for the current source
capture program drop prep_mergefile
program define prep_mergefile
	args MERGEWITH MERGEUSING merge

	capture confirm file "source/merge.dta"
	if _rc!=0 {
		cache_mergefile `merge'
	}
	else {
		use "source/merge.dta", clear
	}

	rename SourceLongName Name
	rename SourceNumericCode Numeric
	rename SourceCountryCode Alpha
	drop if Deprecated == 1
	drop if Source != "`MERGEWITH'"
	keep UCI* `MERGEUSING'
	save `merge', replace
end

* Merge IDs into current dataset
capture program drop merge_ids
program define merge_ids
	args PREFIX YEARVAR COUNTRYVAR MERGEUSING merge

	rename `PREFIX'_`YEARVAR' year
	rename `PREFIX'_`COUNTRYVAR' `MERGEUSING'

	merge m:1 `MERGEUSING' using `merge', keep(match master) nogenerate
	rename `MERGEUSING' `PREFIX'_`COUNTRYVAR'
end

capture program drop check_merge
program define check_merge
	isid UCIName year
	isid UCIAlpha year
	isid UCINumeric year
	isid UCIName UCIAlpha UCINumeric year
end

* Compiles mergefile from CSV sources
capture program drop cache_mergefile
program define cache_mergefile
	args merge

	* Combine merge lists
	import delimited "UCIMergeList.csv", clear case(preserve) numericcols(1 3) stringcols(2 4)
	save `merge', replace

	capture import delimited "private/MyMergeList.csv", clear case(preserve) numericcols(1 3) stringcols(2 4)
	if _rc ==0 {
		append using "`merge'"
		save `merge', replace
	}

	* Align merge lists with UCI Country Codes and Names
	import delimited "UCICountries.csv", clear case(preserve)
	capture labmask UCINumeric, values(UCIName)
	if _rc!=0 {
		ssc install labutil
		labmask UCINumeric, values(UCIName)
	}
	merge 1:m UCINumeric using `merge', keep(match ) nogenerate
	compress
	save "source/merge.dta", replace
end

* Removes Previous Compiled Dataset
capture rm "$FILENAME"

* Forces compile of merge list with every run
capture rm "source/merge.dta"


internal debugging tool

internal debugging tool

*! version 4.0.5 dev - 16feb2017

program reghdfe, eclass
	* Intercept old+version
	cap syntax, version old
	if !c(rc) {
		reghdfe_old, version
		exit
	}

	* Intercept old
	cap syntax anything(everything) [fw aw pw/], [*] old
	if !c(rc) {
		di as error "(running historical version of reghdfe)"
		if ("`weight'"!="") local weightexp [`weight'=`exp']
		reghdfe_old `anything' `weightexp', `options'
		exit
	}

	* Aux. subcommands
	cap syntax, [*]
	if inlist("`options'", "check", "compile", "reload", "update", "version") {
		if ("`options'"=="compile") loc args force
		if ("`options'"=="check") loc options compile
		if ("`options'"=="update") {
			loc args 1
			loc options reload
		}
		loc subcmd = proper("`options'")
		`subcmd' `args'
	}
	else if replay() {
		Replay `0'
	}
	else {
		Cleanup 0
		Compile // takes 0.01s to run this useful check (ensures .mlib exists)
		cap noi Estimate `0'
		Cleanup `c(rc)'
	}
end

// --------------------------------------------------------------------------

program Compile
	args force
	ftools, check // in case lftools.mlib does not exist or is outdated
	ms_get_version reghdfe // from moresyntax package; save local package_version

	loc list_objects "FixedEffects() fixed_effects() FE_Options() FE_Output() BipartiteGraph()"
	loc list_functions "reghdfe_*() transform_*() accelerate_*() panelmean() panelsolve_*()"
	loc list_misc "weighted_quadcolsum() safe_divide() check_convergence()"
	// TODO: prefix everything with reghdfe_*

	ms_compile_mata, ///
		package(reghdfe) ///
		version(`package_version') ///
		fun("`list_objects' `list_functions' `list_misc'") ///
		verbose ///
		`force'
end

program Requirements

end

program Reload
	* Internal debugging tool.
	* Updates dependencies and reghdfe from local path or from github
	* Usage:
	* 	reghdfe, update // from c:\git\..
	* 	reghdfe, reload // from github

	args online
	assert inlist("`online'", "", "1")

	di as text _n "{bf:reghdfe: updating required packages}"
	di as text "{hline 64}"

	* -moresyntax- https://github.com/sergiocorreia/moresyntax/
	cap ado uninstall moresyntax
	if (`online') net install moresyntax, from("https://github.com/sergiocorreia/moresyntax/raw/master/src/")
	if (!`online') net install moresyntax, from("c:\git\moresyntax\src")
	di as text "{hline 64}"

	* -ftools- https://github.com/sergiocorreia/ftools/
	cap ado uninstall ftools
	if (`online') net install ftools, from("https://github.com/sergiocorreia/ftools/raw/master/src/")
	if (!`online') net install ftools, from("c:\git\ftools\src")
	di as text "{hline 64}"
	ftools, compile // requires moresyntax
	di as text "{hline 64}"

	* Update -reghdfe-
	di as text _n  _n "{bf:reghdfe: updating self}"
	di as text "{hline 64}"
	qui ado uninstall reghdfe
	if (`online') net install reghdfe, from("https://github.com/sergiocorreia/reghdfe/raw/version-4/src/")
	if (!`online') net install reghdfe, from("c:\git\reghdfe\src")
	qui which reghdfe
	di as text "{hline 64}"
	reghdfe, compile
	di as text "{hline 64}"

	* Cleaning up
	di as text _n "{bf:Note:} You need to run {stata program drop _all} now."
end

program Version
	which reghdfe

	di as text _n "Dependencies installed?"
	local dependencies moresyntax ftools ivreg2 avar tuples
	foreach dependency of local dependencies {
		cap findfile `dependency'.ado
		if (_rc) {
			di as text "{lalign 20:- `dependency'}" as error "not"
		}
		else {
			di as text "{lalign 20:- `dependency'}" as result "yes"
		}
	}

end

// --------------------------------------------------------------------------

program Cleanup
	args rc
	cap mata: mata drop HDFE
	cap mata: mata drop hdfe_*
	cap matrix drop reghdfe_statsmatrix
	if (`rc') exit `rc'
end

program Estimate, eclass
	ereturn clear

* Parse and fill out HDFE object
	mata: HDFE = FixedEffects()
	reghdfe_parse `0' // store results on HDFE, HDFE.options, and HDFE.output
	loc OPT HDFE.options
	loc OUT HDFE.output

* Select sample
	marksample touse, strok // based on locals for varlist+cluster+absvars+if+in+weight (injected by -reghdfe_parse-)
	// -strok- because clustervars can be string

* Construct HDFE object
	// SYNTAX: fixed_effects(absvars | , touse, weighttype, weightvar, dropsing, verbose, object)
	mata: HDFE = fixed_effects(`OPT'.absorb, "`touse'", `OPT'.weight_type, `OPT'.weight_var, `OPT'.drop_singletons, HDFE.verbose, HDFE)
	mata: st_local("N", strofreal(HDFE.N))
	if (`N' == 0) error 2000

* Compute degrees-of-freedom
	mata: HDFE.estimate_dof()

* Save updated e(sample) (singletons reduce sample);
* required to parse factor variables to partial out
	mata: HDFE.save_touse("`touse'", 1)

* Expand varlists
	foreach cat in varlist depvar indepvars endogvars instruments {
		mata: st_local("vars", `OPT'.original_`cat')
		if ("`vars'" == "") continue
		// HACK: addbn replaces 0.foreign with 0bn.foreign , to prevent st_data() from loading a bunch of zeros
		ms_fvstrip `vars' if `touse', expand dropomit addbn onebyone
		// If we don't use onebyone, then 1.x 2.x ends up as 2.x
		loc vars "`r(varlist)'"
		mata: `OPT'.`cat' = "`vars'"
	}

* Stats
	mata: st_local("stats", `OPT'.summarize_stats)
	if ("`stats'" != "") Stats

* Partial out; and save TSS of depvar
	mata: hdfe_variables = HDFE.partial_out(`OPT'.varlist, 1) // 1=Save TSS of first var if HDFE.output.tss is missing

* Regress
	Regress `touse'
	mata: st_local("diopts", `OPT'.diopts)
	Replay, `diopts'

// ~~ Preserve relevant dataset ~~
	//
	// if (`compact') {
	// 	preserve
	// 	keep `varlist' `absorb'
	// 	loc N = c(N)
	// }
	// else {
	// 	qui cou if `touse'
	// 	loc N = r(N)
	// }
end

// -------------------------------------------------------------
// Run OLS regressions with partialled-out variables
// -------------------------------------------------------------
program Regress, eclass
	args touse

	tempname b V N rank df_r
	// Carefully split y from X, without duplicating the memory used
	mata: hdfe_y = hdfe_variables[., 1]
	mata: hdfe_variables = cols(hdfe_variables)==1 ? J(rows(hdfe_variables), 0, .) : hdfe_variables[., 2..cols(hdfe_variables)]
	mata: reghdfe_post_ols(HDFE, hdfe_y, hdfe_variables, "`b'", "`V'", "`N'", "`rank'", "`df_r'")
	mata: st_local("indepvars", HDFE.options.indepvars)
	mata: hdfe_y = hdfe_variables = .

	if ("`indepvars'" != "") {
		matrix colnames `b' = `indepvars'
		matrix colnames `V' = `indepvars'
		matrix rownames `V' = `indepvars'
		ereturn post `b' `V', esample(`touse') buildfvinfo depname(`depvar')
	}
	else {
		ereturn post, esample(`touse') buildfvinfo depname(`depvar')
	}

	ereturn scalar N       = `N'
	ereturn scalar rank    = `rank'
	ereturn scalar df_r    = `df_r'
	ereturn local  cmd     "reghdfe"
	mata: HDFE.output.post(HDFE.options)

	* Post stats
	cap conf matrix reghdfe_statsmatrix
	if (!c(rc)) {
		ereturn matrix summarize = reghdfe_statsmatrix
		mata: st_local("summarize_quietly", strofreal(HDFE.options.summarize_quietly))
		ereturn scalar summarize_quietly = `summarize_quietly'
	}
end

program Replay
	syntax [, *]
	_get_diopts options, `options'
	reghdfe_header // _coef_table_header
	di ""
	_coef_table, `options' // ereturn display, `options'
	reghdfe_footnote

	* Replay stats
	if (e(summarize_quietly)==0) {
		di as text _n "{sf:Regression Summary Statistics:}" _c
		matlist e(summarize)', border(top bottom) rowtitle(Variable) // twidth(18)
	}
end

// -----------------------------------------------------------------------------
// Matrix of summary statistics
// -----------------------------------------------------------------------------
program Stats
	* Optional weights
	mata: st_local("weight", sprintf("[%s=%s]", HDFE.weighttype, HDFE.weightvar))
	assert "`weight'" != ""
	if ("`weight'" == "[=]") loc weight
	loc weight : subinstr local weight "[pweight" "[aweight"

	mata: st_local("stats", HDFE.options.summarize_stats)
	mata: st_local("varlist", HDFE.options.varlist)
	mata: st_local("cvars", invtokens(HDFE.cvars))
	loc full_varlist `varlist' `cvars'

	qui tabstat `full_varlist' `weight' , stat(`stats') col(stat) save
	matrix reghdfe_statsmatrix = r(StatTotal)
end

*! version 4.0.4 dev - 15feb2017

program reghdfe, eclass
	* Intercept old+version
	cap syntax, version old
	if !c(rc) {
		reghdfe_old, version
		exit
	}

	* Intercept old
	cap syntax anything(everything) [fw aw pw/], [*] old
	if !c(rc) {
		di as error "(running historical version of reghdfe)"
		if ("`weight'"!="") local weightexp [`weight'=`exp']
		reghdfe_old `anything' `weightexp', `options'
		exit
	}

	* Aux. subcommands
	cap syntax, [*]
	if inlist("`options'", "check", "compile", "requirements", "setup", "update", "version") {
		if ("`options'"=="compile") loc args force
		if ("`options'"=="check") loc options compile
		loc subcmd = proper("`options'")
		`subcmd' `args'
	}
	else if replay() {
		Replay `0'
	}
	else {
		Cleanup 0
		Compile // takes 0.01s to run this useful check (ensures .mlib exists)
		cap noi Estimate `0'
		Cleanup `c(rc)'
	}
end

// --------------------------------------------------------------------------

program Compile
	args force
	ftools, check // in case lftools.mlib does not exist or is outdated
	ms_get_version reghdfe // from moresyntax package; save local package_version

	loc list_objects "FixedEffects() fixed_effects() FE_Options() FE_Output() BipartiteGraph()"
	loc list_functions "reghdfe_*() transform_*() accelerate_*() panelmean() panelsolve_*()"
	loc list_misc "weighted_quadcolsum() safe_divide() check_convergence()"
	// TODO: prefix everything with reghdfe_*

	ms_compile_mata, ///
		package(reghdfe) ///
		version(`package_version') ///
		fun("`list_objects' `list_functions' `list_misc'") ///
		verbose ///
		`force'
end

program Requirements
	di as text _n "{bf:[reghdfe]} - updating required packages"
	di as text "{dup 64:=}"
	* -moresyntax- https://github.com/sergiocorreia/moresyntax/
	cap ado uninstall moresyntax
	//net install moresyntax, from(https://github.com/sergiocorreia/moresyntax/raw/master/source/)
	net install moresyntax, from(c:\git\moresyntax\src)
	di as text "{dup 64:-}"

	* -ftools- https://github.com/sergiocorreia/ftools/
	cap ado uninstall ftools
	//net install ftools, from(https://github.com/sergiocorreia/ftools/raw/master/source/)
	net install ftools, from(c:\git\ftools\src)
	di as text "{dup 64:-}"
	ftools, compile // requires moresyntax
	di as text "{dup 64:-}"
end

program Setup
	* Update requirements/dependencies
	Requirements

	* Compile -reghdfe-
	loc source "c:\git\reghdfe\src"
	di as text _n  _n "{bf:[reghdfe]} - compiling Mata code for {res}reghdfe"
	di as text "{dup 64:=}"
	reghdfe, compile
	di as text "{dup 64:-}"
end

program Update
	* Update requirements/dependencies
	Requirements

	* Update -reghdfe-
	loc source "c:\git\reghdfe\src"
	di as text _n  _n "{bf:[reghdfe]} - updating self from {res}`source'"
	di as text "{dup 64:=}"
	qui ado uninstall reghdfe
	net install reghdfe, from("`source'") // TODO: Change this
	qui which reghdfe
	di as text "{dup 64:-}"
	reghdfe, compile
	di as text "{dup 64:-}"
	di as text "{bf:Note:} You need to run {stata program drop _all} now."
end

program Version
	which reghdfe

	di as text _n "Dependencies installed?"
	local dependencies moresyntax ftools ivreg2 avar tuples
	foreach dependency of local dependencies {
		cap findfile `dependency'.ado
		if (_rc) {
			di as text "{lalign 20:- `dependency'}" as error "not"
		}
		else {
			di as text "{lalign 20:- `dependency'}" as result "yes"
		}
	}

end

// --------------------------------------------------------------------------

program Cleanup
	args rc
	cap mata: mata drop HDFE
	cap mata: mata drop hdfe_*
	cap matrix drop reghdfe_statsmatrix
	if (`rc') exit `rc'
end

program Estimate, eclass
	ereturn clear

* Parse and fill out HDFE object
	mata: HDFE = FixedEffects()
	reghdfe_parse `0' // store results on HDFE, HDFE.options, and HDFE.output
	loc OPT HDFE.options
	loc OUT HDFE.output

* Select sample
	marksample touse, strok // based on locals for varlist+cluster+absvars+if+in+weight (injected by -reghdfe_parse-)
	// -strok- because clustervars can be string

* Construct HDFE object
	// SYNTAX: fixed_effects(absvars | , touse, weighttype, weightvar, dropsing, verbose, object)
	mata: HDFE = fixed_effects(`OPT'.absorb, "`touse'", `OPT'.weight_type, `OPT'.weight_var, `OPT'.drop_singletons, HDFE.verbose, HDFE)
	mata: st_local("N", strofreal(HDFE.N))
	if (`N' == 0) error 2000

* Compute degrees-of-freedom
	mata: HDFE.estimate_dof()

* Save updated e(sample) (singletons reduce sample);
* required to parse factor variables to partial out
	mata: HDFE.save_touse("`touse'", 1)

* Expand varlists
	foreach cat in varlist depvar indepvars endogvars instruments {
		mata: st_local("vars", `OPT'.original_`cat')
		if ("`vars'" == "") continue
		// HACK: addbn replaces 0.foreign with 0bn.foreign , to prevent st_data() from loading a bunch of zeros
		ms_fvstrip `vars' if `touse', expand dropomit addbn onebyone
		// If we don't use onebyone, then 1.x 2.x ends up as 2.x
		loc vars "`r(varlist)'"
		mata: `OPT'.`cat' = "`vars'"
	}

* Stats
	mata: st_local("stats", `OPT'.summarize_stats)
	if ("`stats'" != "") Stats

* Partial out; and save TSS of depvar
	mata: hdfe_variables = HDFE.partial_out(`OPT'.varlist, 1) // 1=Save TSS of first var if HDFE.output.tss is missing

* Regress
	Regress `touse'
	mata: st_local("diopts", `OPT'.diopts)
	Replay, `diopts'

// ~~ Preserve relevant dataset ~~
	//
	// if (`compact') {
	// 	preserve
	// 	keep `varlist' `absorb'
	// 	loc N = c(N)
	// }
	// else {
	// 	qui cou if `touse'
	// 	loc N = r(N)
	// }
end

// -------------------------------------------------------------
// Run OLS regressions with partialled-out variables
// -------------------------------------------------------------
program Regress, eclass
	args touse

	tempname b V N rank df_r
	// Carefully split y from X, without duplicating the memory used
	mata: hdfe_y = hdfe_variables[., 1]
	mata: hdfe_variables = cols(hdfe_variables)==1 ? J(rows(hdfe_variables), 0, .) : hdfe_variables[., 2..cols(hdfe_variables)]
	mata: reghdfe_post_ols(HDFE, hdfe_y, hdfe_variables, "`b'", "`V'", "`N'", "`rank'", "`df_r'")
	mata: st_local("indepvars", HDFE.options.indepvars)
	mata: hdfe_y = hdfe_variables = .

	if ("`indepvars'" != "") {
		matrix colnames `b' = `indepvars'
		matrix colnames `V' = `indepvars'
		matrix rownames `V' = `indepvars'
		ereturn post `b' `V', esample(`touse') buildfvinfo depname(`depvar')
	}
	else {
		ereturn post, esample(`touse') buildfvinfo depname(`depvar')
	}

	ereturn scalar N       = `N'
	ereturn scalar rank    = `rank'
	ereturn scalar df_r    = `df_r'
	ereturn local  cmd     "reghdfe"
	mata: HDFE.output.post(HDFE.options)

	* Post stats
	cap conf matrix reghdfe_statsmatrix
	if (!c(rc)) {
		ereturn matrix summarize = reghdfe_statsmatrix
		mata: st_local("summarize_quietly", strofreal(HDFE.options.summarize_quietly))
		ereturn scalar summarize_quietly = `summarize_quietly'
	}
end

program Replay
	syntax [, *]
	_get_diopts options, `options'
	reghdfe_header // _coef_table_header
	di ""
	_coef_table, `options' // ereturn display, `options'
	reghdfe_footnote

	* Replay stats
	if (e(summarize_quietly)==0) {
		di as text _n "{sf:Regression Summary Statistics:}" _c
		matlist e(summarize)', border(top bottom) rowtitle(Variable) // twidth(18)
	}
end

// -----------------------------------------------------------------------------
// Matrix of summary statistics
// -----------------------------------------------------------------------------
program Stats
	* Optional weights
	mata: st_local("weight", sprintf("[%s=%s]", HDFE.weighttype, HDFE.weightvar))
	assert "`weight'" != ""
	if ("`weight'" == "[=]") loc weight
	loc weight : subinstr local weight "[pweight" "[aweight"

	mata: st_local("stats", HDFE.options.summarize_stats)
	mata: st_local("varlist", HDFE.options.varlist)
	mata: st_local("cvars", invtokens(HDFE.cvars))
	loc full_varlist `varlist' `cvars'

	qui tabstat `full_varlist' `weight' , stat(`stats') col(stat) save
	matrix reghdfe_statsmatrix = r(StatTotal)
end