iematch : improve duplicate custom name error message

*! version 5.3 14NOV2017  Kristoffer Bjarkefur kbjarkefur@worldbank.org

	cap program drop 	iematch
	    program define	iematch

		syntax  [if] [in]  , 					///
			GRPdummy(varname) 					///
			MATCHvar(varname) 					///
			[									///
			IDvar(varname)						///
			m1									///
			maxdiff(numlist max = 1 min = 1) 	///
			seedok						 		///
			MATCHIDname(string)					///
			MATCHDIffname(string)				///
			MATCHREsultname(string)				///
			MATCHCOuntname(string)				///
			replace								///
			]

		*****

	qui {

		* Set version
		version 11.0

		********************************
		*
		*	Gen temp sort and merge var
		*
		********************************

		tempvar originalSort ifinvar

		**Generate a variable that is used
		* to restore original sort.
		gen `originalSort' = _n

		preserve

			* String used to store why and how
			*  many observations are excluded.
			local obsexcludstring ""

			*Dummy local that is 1 if replace is used
			if "`replace'" == "" local REPLACE_USED 0
			if "`replace'" != "" local REPLACE_USED 1

			*Deal with if and in and display info.
			if "`if'`in'" != "" {

				* Gen dummy that is 1 when if/in is true. Then
				* count when this dummy is not 1. Ther is no way
				* to negate `if'`in'
				gen 	 `ifinvar'  = 1 `if'`in'
				count if `ifinvar' != 1
				if `r(N)' > 0 {

					*This is not an error just outputting the number
					local obsexcludstring "`obsexcludstring'`r(N)' observation(s) were excluded in {inp:if}/{inp:in}: (`if'`in').{break}"

				}

				*Drop variables excluded by `if'`in'
				keep `if'`in'
			}

		********************************
		*
		*	Checking ID var (and create if needed)
		*
		********************************

			*Checking set-up for the ID var used for _matchID
			if "`idvar'" == "" {

				*Copy this name
				local idvar _ID

				*Make sure that the default name _ID is not already used
				cap confirm variable `idvar'
				if _rc == 0 {

					*Test if option replace is used
					if `REPLACE_USED' == 1 {
						*Replace option is used, drop the variable with the name specified
						drop `idvar'
					}
					else {
						*Replace is not used, throw an error.
						di as error "{pstd}A variable with name `idvar' is already defined. Either drop this variable manually, use the replace option or specify a variable using idvar() that fully and uniquely identifies the data set.{p_end}"
						error 110
					}
				}

				*Generate the ID var from row number
				gen `idvar' = _n
			}

			*Make sure that idvar is uniquelly and fully identifying
			cap isid `idvar'
			if _rc != 0 {

				di as error "{pstd}Variable `idvar' specified in idvar() does not fully and uniquely identify the observations in the data set. `idvar' either has duplicates or missing values which is not allowed. Make `idvar' uniquly and fully idnetifying or exclude the varaibleas using {inp:if} or {inp:in}.{p_end}"
				error 450
			}

			*local indicating assume ID is string
			local IDtypeNumeric 0

			*Change local to 1 if ID actally string
			cap confirm numeric variable `idvar'
			if _rc == 0 local IDtypeNumeric 1

		********************************
		*
		*	Checking duplicates in variable names in options
		*
		********************************

			*Create a local of all varnames used in any option
			local allnewvars `idvar' `matchidname' `matchdiffname' `matchresultname' `matchcountname'

			*Check if there are any duplicates in the local of all varnames in options
			local dupnewvars : list dups allnewvars
			local dupnewvars : list uniq dupnewvars

			*Throw error if there were any duplicates
			if "`dupnewvars'" != "" {

				noi di as error "{pstd}The variable name(s) [`dupnewvars'] was used twice or more in the options that manually name the outcome varaibles. Go back and make sure that no name is used more than once.{p_end}"
				error 198
			}

		********************************
		*
		*	Test the names used for output vars
		*
		********************************

			iematchMatchVarCheck 	_matchResult 	`REPLACE_USED' "`matchresultname'"
			local matchResultName 	"`r(validVarName)'"

			iematchMatchVarCheck 	_matchID 		`REPLACE_USED' "`matchidname'"
			local matchIDname 		"`r(validVarName)'"

			iematchMatchVarCheck 	_matchDiff 		`REPLACE_USED' "`matchdiffname'"
			local matchDiffName 	"`r(validVarName)'"

			if "`m1'" != "" {
			   iematchMatchVarCheck _matchCount 	`REPLACE_USED' "`matchcountname'"
			   local matchCountName "`r(validVarName)'"
			}

			*Option matchcountname() is not allowed if it is not a many-one match
			if "`matchcountname'" != "" & "`m1'" == "" {
				di as error "{pstd}Option {inp:matchcountname()} is only allowed in combination with option {inp:m1}.{p_end}"
				error 198
			}

		********************************
		*
		*	Create and label output vars
		*
		********************************

			* MATCH RESULT VAR
			label define 	 matchLabel 0 "Not Matched" 1 "Matched" .i "Obs excluded due to if/in" 			///
										.g "Missing value in `grpdummy'" .m "Missing value in `matchvar'" 	///
										.d "No match within maxdiff", replace

			gen 			`matchResultName' = .
			label variable	`matchResultName' "Matched obs = 1, not mathched = 0, all other different missing values"
			label value 	`matchResultName' matchLabel

			* MATCH ID VAR
			if `IDtypeNumeric' == 1 gen  `matchIDname' = .	//Main ID var is numeric
			if `IDtypeNumeric' == 0 gen  `matchIDname' = "" //Main ID var is string

			if "`m1'" != "" label variable	`matchIDname' "The ID of the target var in each matched group"	//If many to one
			if "`m1'" == "" label variable	`matchIDname' "The ID of the target var in each matched pair"	//If one to one

			* MATCH DIFF VAR
			gen 	`matchDiffName' = .
			if "`m1'" != "" label variable	`matchDiffName' "The difference in matchvar() between base obs and the target obs it matched with" 	//If many to one
			if "`m1'" == "" label variable	`matchDiffName' "The difference in matchvar() between base and target obs in each pair" 			//If one to one

			*Match count, only many-one
			if "`m1'" != "" {
				gen 			`matchCountName' = .
				label variable	`matchCountName' "The number of base obs this target obs matched with"
			}

		********************************
		*
		*	Keep only relevant vars
		*
		********************************

			*Keep only input vars and output vars
			keep  `grpdummy' `idvar' `matchvar'	`originalSort' `matchResultName' `matchIDname' `matchDiffName' `matchCountName'

		********************************
		*
		*	Checking group dummy
		*
		********************************

			*Test that the group dummy is 1, 0 or missing
			cap assert inlist(`grpdummy', 1, 0, .) | `grpdummy' > .
			if _rc != 0 {

				di as error "{pstd}The variable in grpdummy(`grpdummy') is not a dummy variable. The variable is only allowed to have the values 1, 0 or missing. Observations with missing varaibles in the grpdummy are ignored by this command.{p_end}"
				error _rc
			}

			**********
			**Exclude obs with missing value in groupdummy

			*Count number of obs to be dropped and and output that number if more than zero
			count if missing(`grpdummy')
			if `r(N)' > 0 {

				*Prepare the local to be outputted with info on observations excluded
				local obsexcludstring  "`obsexcludstring'`r(N)' observation(s) were excluded due to missing value in  grpdummy(`grpdummy').{break}"
			}

			*Drop obs with missing value in groupvar
			drop if missing(`grpdummy')

		********************************
		*
		*	Checking match continous var
		*
		********************************

			*Test that the continous is numeric
			cap confirm numeric variable `matchvar'
			if _rc != 0 {

				noi di as error "{pstd}The variable to match on specified in matchvar(`matchvar') is not a numeric variable.{p_end}"
				error 109
			}

			**All variables with value 1 or 0 in the group dummy and
			* not excluded by if/in must have a value in matchvar
			count if missing(`matchvar')
			if `r(N)' > 0 {

				*Prepare the local to be outputted with info on observations excluded
				local obsexcludstring  "`obsexcludstring'`r(N)' observation(s) were excluded due to missing value in matchvar(`matchvar').{break}"
			}

			*Drop obs with missing values in match var
			drop if missing(`matchvar')

		********************************
		*
		*	Checking match var and group dummy are unique
		*
		********************************

			if "`seedok'" == "" {

				**test that there are no duplicates in match
				* var within each type of observation
				cap isid `matchvar' `grpdummy'

				if _rc != 0 {

					*Output the error message
					noi di as error "{pstd}There are base observations or target observations {...}"
					noi di as error "with duplicate values in matchvar(`matchvar'). To guarantee {...}"
					noi di as error "a replicable match you must {help seed:set a seed}. To supress {...}"
					noi di as error "this error message after you have set a the seed, or if a {...}"
					noi di as error "replicable match is not important to you, use option {inp:seedok}{p_end}"
					error 198

				}
			}

		********************************
		*
		*	Checking that there are more target
		*	obs than base obs in a 1-to-1 match
		*
		********************************

			if "`m1'" == "" {

				*Count number of base vars
				count if `grpdummy' == 1
				local numBaseObs `r(N)'

				*Count number of target vars
				count if `grpdummy' == 0
				local numTrgtObs `r(N)'

				*Test that there are more target
				cap assert `numTrgtObs' >= `numBaseObs'

				if _rc != 0 {

					noi di as error "{pstd}There are more base observations than target observations. This is not allowed in a one-to-one match. See option {inp:m1} for an alternative matching where it is allowed to have more base observations than target ovbservations.{p_end}"
					error _rc
				}
			}

		********************************
		*
		*	Output exclude string
		*
		********************************

			*The exclude string is created above but only displayed
			*after all testing of input is done.

			if "`obsexcludstring'" != "" {

				noi di ""
				noi di "{hline}"
				noi di ""
				noi di "{pstd}{ul:Observations are excluded from the matching for the following reasons:}{break}"
				noi di "`obsexcludstring'{p_end}"

			}

	********************************
	*	Start matching
	********************************

		********************************
		*
		*	Creating tempvar used in matching
		*
		********************************

		*Initiate the temporary variables used by this command
		tempvar prefID prefDiff matched

		gen `prefDiff' 		= .
		gen byte `matched' 	= 0

		*Allow the ID var used to be string
		if `IDtypeNumeric' == 1 {
			gen  `prefID' = .
		}
		else {
			gen  `prefID' = ""
		}

		** Generate the inverse of the matchvar to sort descending (gsort is too slow),
		*  a random var to seperate two values with the same match var, and the inverse
		*  of the random var for when sorting descending.
		tempvar rand invsort invrand

		sort `matchvar'
		gen  `invsort' = -1 * `matchvar'

		gen    `rand' = uniform()
		gen `invrand' = -1 * `rand'

		***********
		*Tempvars for matching

		*Diffvars, they are always numeric
		tempvar					 diffup   diffdo   valUp_0   valDo_0   valUp_1   valDo_1
		local 	 updownTempVars `diffup' `diffdo' `valUp_0' `valDo_0' `valUp_1' `valDo_1'

		foreach tempVar of local updownTempVars {

			gen `tempVar' = .
		}

		*ID vars, allowed to be both numeric and string
		tempvar					  	IDup   IDdo   IDup_0   IDdo_0   IDup_1   IDdo_1
		local 	 updownIDTempVars  `IDup' `IDdo' `IDup_0' `IDdo_0' `IDup_1' `IDdo_1'

		foreach tempVar of local updownIDTempVars {

			*Allow the ID var used to be string
			if `IDtypeNumeric' == 1 {
				gen  `tempVar' = .
			}
			else {
				gen  `tempVar' = ""
			}
		}

		***************************
		*
		*	One to one match
		*
		***************************

		if "`m1'" == "" {

			*Start outputting the countdown
			noi di ""
			noi di "{hline}{break}"
			noi di "{pstd}{ul:Matching one-to-one. Base observations left to match:}{p_end}"
			count if `grpdummy' == 1 & `matched' == 0

			*Create local to display "obs left to match" and to use in while loop
			local left2Match = `r(N)'
			noi di "{pstd}`left2Match' " _c

			*Match until no more observations to match.
			while (`left2Match' > 0) {

				**For all observations still to be matched, assign the preferred
				* match among the other unmatched observations
				qui updatePrefDiffPreffID `prefID' `prefDiff' `matchvar' `invsort' `idvar' `grpdummy' `matched' `rand' `invrand' `updownTempVars' `updownIDTempVars'

				*Restrict matches to within maxdiff() if that option is used.
				if "`maxdiff'" != "" {

					*Omit base observation from matching if diff is to big
					replace `matched' 			= 1		if `prefDiff' > `maxdiff' & `grpdummy' == 1

					*Indicate in result var that this obs did not have valid match within maxdiff()
					replace `matchResultName' 	= .d 	if `prefDiff' > `maxdiff' & `grpdummy' == 1

					*Removed preferred match
					if `IDtypeNumeric' == 1 {
						*IDvar is numeric
						replace `prefID'		= .		if `prefDiff' > `maxdiff'
					}
					else {
						*IDvar is string
						replace `prefID'		= ""	if `prefDiff' > `maxdiff'
					}
				}

				*If two observations mutually prefer each other, then indicate both of them as matched.
				replace `matched' = 1 if `matched' == 0 & `prefID' == `idvar'[_n-1] & `prefID'[_n-1] == `idvar'
				replace `matched' = 1 if `matched' == 0 & `prefID' == `idvar'[_n+1] & `prefID'[_n+1] == `idvar'

				*Update local to display "obs left to match" and to use in while loop
				count if `grpdummy' == 1 & `matched' == 0
				local left2Match = `r(N)'
				noi di "`left2Match' " _c

			}

			*End formatting for the "base obs left to match"
			noi di "{p_end}" _c
		}

		***************************
		*
		*	Many to one match
		*
		***************************
		else {

			**For all observations to be matched, assign the preferred
			* match among the other unmatched observations
			updatePrefDiffPreffID `prefID' `prefDiff' `matchvar' `invsort' `idvar' `grpdummy' `matched' `rand' `invrand' `updownTempVars' `updownIDTempVars'

			*Restrict matches to within maxdiff() if that option is used.
			if "`maxdiff'" != "" {

				*Indicate in result var that this obs did not have valid match within maxdiff()
				replace `matchResultName' 	= .d 	if `prefDiff' > `maxdiff' & `grpdummy' == 1

				*Removed preferred match
				if `IDtypeNumeric' == 1 {
					*IDvar is numeric
					replace `prefID'		= .		if `prefDiff' > `maxdiff'
				}
				else {
					*IDvar is string
					replace `prefID'		= ""	if `prefDiff' > `maxdiff'
				}
			}

			*Assign it's own ID as pref ID for all target vars
			replace `prefID'			=  `idvar' if `grpdummy' == 0

			* Replace the _matchCount var with number of base observations in each
			* match group. Each group is all base observation plus the target
			* observation, therefore (_N - 1)
			bys 	`prefID' 			:   replace `matchCountName' = _N - 1 if !missing(`prefID')

			**Replace prefID to missing for target obs that had no base
			* obs matched to it. T
			if `IDtypeNumeric' == 1 {
				*IDvar is numeric
				replace `prefID'		= .		if `matchCountName' == 0
			}
			else {
				*IDvar is string
				replace `prefID'		= ""	if `matchCountName' == 0
			}

			*Only target obs with base obs prefering it are matched
			replace	 `matched' 			= 1		if `matchCountName' != 0

			*Remove values for target obs that were not matched
			replace  `matchCountName'	= . 	if `matchCountName' == 0
		}

		*Update all return vars
		replace `matchDiffName' 	= `prefDiff'	if `matched' == 1
		replace `matchIDname' 		= `idvar' 		if `matched' == 1 & `grpdummy' == 0
		replace `matchIDname' 		= `prefID' 		if `matched' == 1 & `grpdummy' == 1

		*Remove the best match value in obs that did not have a match within maxdiff()
		replace `matchDiffName'  = . 				if `matchResultName' == .d

		*Matched observations are give value 1 in result var
		replace `matchResultName' = 1 				if `matched' == 1 & `matchResultName' != .d

		*Target obs not used
		replace `matchResultName' = 0 if `matched' == 0 & `grpdummy' == 0

		*only keep output vars
		keep `matchIDname' `matchDiffName' `originalSort' `matchResultName' `matchCountName'

		*Merge the results back to the original data set
		tempfile mergefile
		save 	`mergefile'

	restore

	***************************
	*
	*	Merge match results to
	*	original data and assign
	*	remaining missing values
	*	to the result var
	*
	***************************

	*Merge the results with the original data
	tempvar mergevar
	merge 1:1  `originalSort' using `mergefile', gen(`mergevar')

	*remaining missing values are listed in ascending order of importance.
	*Meaning that if a variable is both .m and .i then it will be .i as it
	*is assigned afterwards below.

	*Missing matching var
	replace `matchResultName' = .m if missing(`matchvar')

	*Msising dummy var
	replace `matchResultName' = .g if missing(`grpdummy')

	*Excluded in if/in
	if "`if'`in'" != "" {

		tempvar  ifinvar
		gen 	`ifinvar' = 1 `if'`in'
		replace `matchResultName' = .i if `ifinvar' != 1
	}

	*comress the variables generated
	compress  `matchIDname' `matchDiffName' `matchResultName' `matchCountName'

	*Output result table
	noi outputTable `matchResultName' `grpdummy'

	*Restore the oridinal sort
	sort `originalSort'

}

end

	*Check manually entered names for the return vars.
	cap program drop 	iematchMatchVarCheck
		program define 	iematchMatchVarCheck  , rclass

		args defaultName replace_used userName

		if "`defaultName'" == "_matchID" 		local optionName matchidname
		if "`defaultName'" == "_matchDiff" 		local optionName matchdiffname
		if "`defaultName'" == "_matchResult" 	local optionName matchresult
		if "`defaultName'" == "_matchCount" 	local optionName matchcount

		*All the default names
		local dfltNms _noMatchReason _matchDiff _matchResult _matchCount

		*The other two default names
		local othDfltNms : list dfltNms - defaultName

		*Creating two locals with the deafult name in each local
		local Oth1 : word 1 of `othDfltNms'
		local Oth2 : word 2 of `othDfltNms'
		local Oth3 : word 3 of `othDfltNms'

		*Testing that the option name is not the same as the other variable's default name.
		if ("`userName'" == "`Oth1'" | "`userName'" == "`Oth2'" | "`userName'" == "`Oth3'" | "`userName'" == "_ID")  {

			noi di as error "{pstd}The new name specified in `optionName'(`userName') is not allowed to be _ID, `Oth1', `Oth2', or `Oth3'{p_end}"
			error 198
		}

		*Test if name is set manually
		if "`userName'" != "" {
			*Use the manually set name
			local validMatchVarname `userName'
		}
		else {
			*No manually set name, use deafult name
			local validMatchVarname `defaultName'
		}

		*Make sure that the manually entered name is not already used
		cap confirm variable `validMatchVarname'
		if _rc == 0 {

			if `replace_used' == 1 {
				//Replace is used, drop the old var
				drop `validMatchVarname'
			}
			else {

				*Replace is not used. Prepare an error message and throw error

				if "`userName'" != "" {
					*Error message for user specified name s
					local nameErrorString "The variable name specified in `optionName'(`userName')"
				}
				else {
					*Error message for user specified name s
					local nameErrorString "A variable with name `validMatchVarname'"
				}

				*Trow error
				noi di as error "{pstd}`nameErrorString' is already defined in the data set. Either drop this variable, use the replace option or specify a another variable name using `optionName'()."
				error 110

			}
		}

		*Testing that the new name is valid by creating a variable that is dropped immedeatly afterwards
		cap gen `validMatchVarname' = .

		if _rc != 0 {

			noi di as error "{pstd}The variable name specified in `optionName'(`userName') is not a valid variable name."
			error _rc
		}
		drop `validMatchVarname'

		return local validVarName 	"`validMatchVarname'"

	end

	** This function is called to assign the preferred match for all unmatched
	*  observations to any other unmatched observation. Returns a variable pref that is the
	cap program drop 	updatePrefDiffPreffID
		program define 	updatePrefDiffPreffID

		args prefID prefDiff matchvar invsort idvar grpvar matched rand invrand diffup diffdo valUp_0 valDo_0 valUp_1 valDo_1 IDup IDdo IDup_0 IDdo_0 IDup_1 IDdo_1

		*****************
		*Update possible matches with higher rank value
		sort `matched' `matchvar' `rand'

		updateBestValID 0 `matchvar' `idvar' `grpvar' `matched' `valUp_0' `IDup_0'
		updateBestValID 1 `matchvar' `idvar' `grpvar' `matched' `valUp_1' `IDup_1'

		*****************
		*Update possible matches with lower rank value
		sort `matched' `invsort' `invrand'

		updateBestValID 0 `matchvar' `idvar' `grpvar' `matched' `valDo_0' `IDdo_0'
		updateBestValID 1 `matchvar' `idvar' `grpvar' `matched' `valDo_1' `IDdo_1'

		*****************
		*Create one varaible common for both groups with diff upwards
		replace `diffup' 	= abs(`matchvar' - `valUp_1') if `grpvar' == 1
		replace `diffup' 	= abs(`matchvar' - `valUp_0') if `grpvar' == 0

		*Create one varaible common for both groups with diff downwards
		replace `diffdo' 	= abs(`matchvar' - `valDo_1') if `grpvar' == 1
		replace `diffdo' 	= abs(`matchvar' - `valDo_0') if `grpvar' == 0

		*Create one varaible common for both groups with ID upwards
		replace `IDup' 		= `IDup_1' 	if `grpvar' == 1
		replace `IDup' 		= `IDup_0' 	if `grpvar' == 0

		*Create one varaible common for both groups with ID downwards
		replace `IDdo' 		= `IDdo_1' 	if `grpvar' == 1
		replace `IDdo' 		= `IDdo_0' 	if `grpvar' == 0

		replace `prefID' 	= `IDdo' 	if `matched' == 0 & `diffdo' <= `diffup'
		replace `prefID' 	= `IDup' 	if `matched' == 0 & `diffdo' >  `diffup'

		replace `prefDiff' 	= `diffdo' 	if `matched' == 0 & `diffdo' <= `diffup'
		replace `prefDiff' 	= `diffup' 	if `matched' == 0 & `diffdo' >  `diffup'

	end

	** This function goes through the observatioins from top to down and copying
	*  the match value of the closest observation of the other group above to
	*  (bestVal) and the ccoprresponding ID to (bestID) for obe group at the time.
	*  To go from bottuom to top sort with invsort and apply this command.
	cap program drop 	updateBestValID
		program define 	updateBestValID

		args grpvl matchval idvar grpvar matched bestVal bestID

		*Test if ID var is string or numeric
		local IDNumeric 0
		cap confirm numeric variable `idvar'
		if _rc == 0 local IDNumeric 1

		*Reset all values
		replace `bestVal' 	= .
		if `IDNumeric' == 1 replace `bestID' 	= .
		if `IDNumeric' == 0 replace `bestID' 	= ""

		*Set the match value and ID of observations that are in the other group
		replace `bestVal'   = `matchval' 		if `grpvar' != `grpvl' & `matched' == 0
		replace `bestID'  	= `idvar' 			if `grpvar' != `grpvl' & `matched' == 0

		*Fill in that value from the observation in the other group until getting to another observation of the other group
							replace `bestVal'   = `bestVal'[_n-1]	if `bestVal' == .  & `matched' == 0
		if `IDNumeric' == 1 replace `bestID'  	= `bestID'[_n-1] 	if `bestID'	 == .  & `matched' == 0
		if `IDNumeric' == 0 replace `bestID'  	= `bestID'[_n-1] 	if `bestID'	 == "" & `matched' == 0

	end

	** This function creates the output table.
	cap program drop 	outputTable
		program define 	outputTable

		qui {
			args resultVar grpdum

			*List the texts used in the first column in the
			local text_1 " 1 Matched"
			local text_0 " 0 Not matched"
			local text_d ".d No match within maxdiff()"
			local text_i ".i Excluded using if/in"
			local text_g ".g Missing grpdummy()"
			local text_m ".m Missing matchvar()"

			*The minimum width of the
			local firstColWidth = max(strlen(" `resultVar'") , strlen(" value and result"))

			levelsof `resultVar' , missing local(resultsUsed)

			foreach result of local resultsUsed {

				*Remove the . from the local to match the string
				local result = subinstr("`result'",".","",1)

				* Test if the text in the first column is longer
				* for this row than previous row
				local firstColWidth = max(`firstColWidth' , strlen("`text_`result''"))
			}

			* Create a local that indicates if there is column
			* for obs with missing value in tmt dummy
			local missingGrpDum 0
			count if missing(`grpdum')
			if `r(N)' > 0 local missingGrpDum 1

			*Set all locals that determins column width

			local C1 "{col 4}{c |}"

			di "`firstColWidth'"

			local col1width = `firstColWidth'  + 1
			local hli1 "{hline `col1width'}"
			local cen1 "{center `col1width':"

			local col2border = `firstColWidth' + 6
			local C2  "{col `col2border'}{c |}"
			local C2a "{col `col2border'}"

			local col3border = `firstColWidth' + 17
			local C3 "{col `col3border'}{c |}"

			local col4border = `firstColWidth' + 30
			local C4 "{col `col4border'}{c |}"

			local grpdumCentre 23 //Overspills automatically

			local lastT  	"{c RT}"
			local lastTdown	"{c RT}"
			local lastC  	"{c BRC}"

			if `missingGrpDum' {

				local lastT 		"{c +}{hline 9}{c RT}"
				local lastTdown 	"{c BT}{hline 9}{c RT}"
				local lastC 		"{c BT}{hline 9}{c BRC}"
				local missTitle 	"  missing "
				local grpdumCentre 	33

			}

			noi di ""
			noi di ""
			noi di "{hline}"
			noi di ""
			noi di "{pstd}{ul:Output of Matching Result:}{p_end}"
			noi di ""
			noi di "{col 5}`cen1'`resultVar'}`C2a' {centre `grpdumCentre':`grpdum' }"
			noi di "{col 4}{c LT}`hli1'{c +}{hline `grpdumCentre'}{c RT}"
			noi di "`C1'`cen1'value and result}`C2' 1 (base)   0 (target) `missTitle'{c |}"
			noi di "{col 4}{c LT}`hli1'{c +}{hline 10}{c +}{hline 12}`lastT'"

			*This is the roworder for the output table
			local resultTableOrder 1 0 .d .i .g .m

			foreach result of local resultTableOrder {

				*Test if this result was used
				if `:list result in resultsUsed' {

					*Count base observations in this result type
					qui count if `resultVar' == `result' & `grpdum' == 1
					local numBase = trim("`: display %16.0gc  `r(N)''")

					*Count target observations in this result type
					qui count if `resultVar' == `result' & `grpdum' == 0
					local numTarg = trim("`: display %16.0gc  `r(N)''")

					*Count observations that are neither base or target in this result type
					if `missingGrpDum' {

						qui count if `resultVar' == `result' & missing(`grpdum')
						local numMiss = trim("`: display %16.0gc  `r(N)''")

						local missCol "{ralign 9 :`numMiss' }{c |}"

					}

					*Prepare the local that displays the columns with numbers
					local numCols "`C2'{ralign 10 :`numBase' }`C3'{ralign 12 :`numTarg' }`C4'`missCol'"

					*Remove the . from the local to match the string
					local result = subinstr("`result'",".","",1)

					*Output the table row
					noi di "`C1'`text_`result''`numCols'"

				}
			}

			*Output line before N per group row
			noi di "{col 4}{c LT}`hli1'{c +}{hline 10}{c +}{hline 12}`lastT'"

			*Caculate N per group row for base observations
			qui count if `grpdum' == 1
			local numBase = trim("`: display %16.0gc  `r(N)''")

			*Caculate N per group row for target observations
			qui count if `grpdum' == 0
			local numTarg = trim("`: display %16.0gc  `r(N)''")

			*Caculate N per group row for observations neither base or target
			if `missingGrpDum' {

				qui count if missing(`grpdum')
				local numMiss = trim("`: display %16.0gc  `r(N)''")

				local missCol "{ralign 9 :`numMiss' }{c |}"

			}

			*Output N per group row
			noi di "`C1'`cen1'N per group}`C2'{ralign 10 :`numBase' }`C3'{ralign 12 :`numTarg' }`C4'`missCol'"

			*Output line before total N row
			noi di "{col 4}{c LT}`hli1'{c +}{hline 10}{c BT}{hline 12}`lastTdown'"

			*Count total N
			qui count
			local numTot = trim("`: display %16.0gc  `r(N)''")

			*Output total N row
			noi di "`C1'`cen1'Total N}`C2'{centre `grpdumCentre':`numTot'}`C4'"

			*Output bottom line
			noi di "{col 4}{c BLC}`hli1'{c BT}{hline `grpdumCentre'}{c BRC}"
		}
	end

*! version 5.3 14NOV2017  Kristoffer Bjarkefur kbjarkefur@worldbank.org

	cap program drop 	iematch
	    program define	iematch

		syntax  [if] [in]  , 					///
			GRPdummy(varname) 					///
			MATCHvar(varname) 					///
			[									///
			IDvar(varname)						///
			m1									///
			maxdiff(numlist max = 1 min = 1) 	///
			seedok						 		///
			MATCHIDname(string)					///
			MATCHDIffname(string)				///
			MATCHREsultname(string)				///
			MATCHCOuntname(string)				///
			replace								///
			]

		*****

	qui {

		* Set version
		version 11.0

		********************************
		*
		*	Gen temp sort and merge var
		*
		********************************

		tempvar originalSort ifinvar

		**Generate a variable that is used
		* to restore original sort.
		gen `originalSort' = _n

		preserve

			* String used to store why and how
			*  many observations are excluded.
			local obsexcludstring ""

			*Dummy local that is 1 if replace is used
			if "`replace'" == "" local REPLACE_USED 0
			if "`replace'" != "" local REPLACE_USED 1

			*Deal with if and in and display info.
			if "`if'`in'" != "" {

				* Gen dummy that is 1 when if/in is true. Then
				* count when this dummy is not 1. Ther is no way
				* to negate `if'`in'
				gen 	 `ifinvar'  = 1 `if'`in'
				count if `ifinvar' != 1
				if `r(N)' > 0 {

					*This is not an error just outputting the number
					local obsexcludstring "`obsexcludstring'`r(N)' observation(s) were excluded in {inp:if}/{inp:in}: (`if'`in').{break}"

				}

				*Drop variables excluded by `if'`in'
				keep `if'`in'
			}

		********************************
		*
		*	Checking ID var (and create if needed)
		*
		********************************

			*Checking set-up for the ID var used for _matchID
			if "`idvar'" == "" {

				*Copy this name
				local idvar _ID

				*Make sure that the default name _ID is not already used
				cap confirm variable `idvar'
				if _rc == 0 {

					*Test if option replace is used
					if `REPLACE_USED' == 1 {
						*Replace option is used, drop the variable with the name specified
						drop `idvar'
					}
					else {
						*Replace is not used, throw an error.
						di as error "{pstd}A variable with name `idvar' is already defined. Either drop this variable manually, use the replace option or specify a variable using idvar() that fully and uniquely identifies the data set.{p_end}"
						error 110
					}
				}

				*Generate the ID var from row number
				gen `idvar' = _n
			}

			*Make sure that idvar is uniquelly and fully identifying
			cap isid `idvar'
			if _rc != 0 {

				di as error "{pstd}Variable `idvar' specified in idvar() does not fully and uniquely identify the observations in the data set. `idvar' either has duplicates or missing values which is not allowed. Make `idvar' uniquly and fully idnetifying or exclude the varaibleas using {inp:if} or {inp:in}.{p_end}"
				error 450
			}

			*local indicating assume ID is string
			local IDtypeNumeric 0

			*Change local to 1 if ID actally string
			cap confirm numeric variable `idvar'
			if _rc == 0 local IDtypeNumeric 1

		********************************
		*
		*	Checking duplicates in variable names in options
		*
		********************************

			*Create a local of all varnames used in any option
			local allnewvars `idvar' `matchidname' `matchdiffname' `matchresultname' `matchcountname'

			*Check if there are any duplicates in the local of all varnames in options
			local dupnewvars : list dups allnewvars

			*Throw error if there were any duplicates
			if "`dupnewvars'" != "" {

				noi di as error "{pstd}The same new variable name was used twice or more in the options generating a new variable. Go back and check syntax.{p_end}"
				error 198
			}

		********************************
		*
		*	Test the names used for output vars
		*
		********************************

			iematchMatchVarCheck 	_matchResult 	`REPLACE_USED' "`matchresultname'"
			local matchResultName 	"`r(validVarName)'"

			iematchMatchVarCheck 	_matchID 		`REPLACE_USED' "`matchidname'"
			local matchIDname 		"`r(validVarName)'"

			iematchMatchVarCheck 	_matchDiff 		`REPLACE_USED' "`matchdiffname'"
			local matchDiffName 	"`r(validVarName)'"

			if "`m1'" != "" {
			   iematchMatchVarCheck _matchCount 	`REPLACE_USED' "`matchcountname'"
			   local matchCountName "`r(validVarName)'"
			}

			*Option matchcountname() is not allowed if it is not a many-one match
			if "`matchcountname'" != "" & "`m1'" == "" {
				di as error "{pstd}Option {inp:matchcountname()} is only allowed in combination with option {inp:m1}.{p_end}"
				error 198
			}

		********************************
		*
		*	Create and label output vars
		*
		********************************

			* MATCH RESULT VAR
			label define 	 matchLabel 0 "Not Matched" 1 "Matched" .i "Obs excluded due to if/in" 			///
										.g "Missing value in `grpdummy'" .m "Missing value in `matchvar'" 	///
										.d "No match within maxdiff", replace

			gen 			`matchResultName' = .
			label variable	`matchResultName' "Matched obs = 1, not mathched = 0, all other different missing values"
			label value 	`matchResultName' matchLabel

			* MATCH ID VAR
			if `IDtypeNumeric' == 1 gen  `matchIDname' = .	//Main ID var is numeric
			if `IDtypeNumeric' == 0 gen  `matchIDname' = "" //Main ID var is string

			if "`m1'" != "" label variable	`matchIDname' "The ID of the target var in each matched group"	//If many to one
			if "`m1'" == "" label variable	`matchIDname' "The ID of the target var in each matched pair"	//If one to one

			* MATCH DIFF VAR
			gen 	`matchDiffName' = .
			if "`m1'" != "" label variable	`matchDiffName' "The difference in matchvar() between base obs and the target obs it matched with" 	//If many to one
			if "`m1'" == "" label variable	`matchDiffName' "The difference in matchvar() between base and target obs in each pair" 			//If one to one

			*Match count, only many-one
			if "`m1'" != "" {
				gen 			`matchCountName' = .
				label variable	`matchCountName' "The number of base obs this target obs matched with"
			}

		********************************
		*
		*	Keep only relevant vars
		*
		********************************

			*Keep only input vars and output vars
			keep  `grpdummy' `idvar' `matchvar'	`originalSort' `matchResultName' `matchIDname' `matchDiffName' `matchCountName'

		********************************
		*
		*	Checking group dummy
		*
		********************************

			*Test that the group dummy is 1, 0 or missing
			cap assert inlist(`grpdummy', 1, 0, .) | `grpdummy' > .
			if _rc != 0 {

				di as error "{pstd}The variable in grpdummy(`grpdummy') is not a dummy variable. The variable is only allowed to have the values 1, 0 or missing. Observations with missing varaibles in the grpdummy are ignored by this command.{p_end}"
				error _rc
			}

			**********
			**Exclude obs with missing value in groupdummy

			*Count number of obs to be dropped and and output that number if more than zero
			count if missing(`grpdummy')
			if `r(N)' > 0 {

				*Prepare the local to be outputted with info on observations excluded
				local obsexcludstring  "`obsexcludstring'`r(N)' observation(s) were excluded due to missing value in  grpdummy(`grpdummy').{break}"
			}

			*Drop obs with missing value in groupvar
			drop if missing(`grpdummy')

		********************************
		*
		*	Checking match continous var
		*
		********************************

			*Test that the continous is numeric
			cap confirm numeric variable `matchvar'
			if _rc != 0 {

				noi di as error "{pstd}The variable to match on specified in matchvar(`matchvar') is not a numeric variable.{p_end}"
				error 109
			}

			**All variables with value 1 or 0 in the group dummy and
			* not excluded by if/in must have a value in matchvar
			count if missing(`matchvar')
			if `r(N)' > 0 {

				*Prepare the local to be outputted with info on observations excluded
				local obsexcludstring  "`obsexcludstring'`r(N)' observation(s) were excluded due to missing value in matchvar(`matchvar').{break}"
			}

			*Drop obs with missing values in match var
			drop if missing(`matchvar')

		********************************
		*
		*	Checking match var and group dummy are unique
		*
		********************************

			if "`seedok'" == "" {

				**test that there are no duplicates in match
				* var within each type of observation
				cap isid `matchvar' `grpdummy'

				if _rc != 0 {

					*Output the error message
					noi di as error "{pstd}There are base observations or target observations {...}"
					noi di as error "with duplicate values in matchvar(`matchvar'). To guarantee {...}"
					noi di as error "a replicable match you must {help seed:set a seed}. To supress {...}"
					noi di as error "this error message after you have set a the seed, or if a {...}"
					noi di as error "replicable match is not important to you, use option {inp:seedok}{p_end}"
					error 198

				}
			}

		********************************
		*
		*	Checking that there are more target
		*	obs than base obs in a 1-to-1 match
		*
		********************************

			if "`m1'" == "" {

				*Count number of base vars
				count if `grpdummy' == 1
				local numBaseObs `r(N)'

				*Count number of target vars
				count if `grpdummy' == 0
				local numTrgtObs `r(N)'

				*Test that there are more target
				cap assert `numTrgtObs' >= `numBaseObs'

				if _rc != 0 {

					noi di as error "{pstd}There are more base observations than target observations. This is not allowed in a one-to-one match. See option {inp:m1} for an alternative matching where it is allowed to have more base observations than target ovbservations.{p_end}"
					error _rc
				}
			}

		********************************
		*
		*	Output exclude string
		*
		********************************

			*The exclude string is created above but only displayed
			*after all testing of input is done.

			if "`obsexcludstring'" != "" {

				noi di ""
				noi di "{hline}"
				noi di ""
				noi di "{pstd}{ul:Observations are excluded from the matching for the following reasons:}{break}"
				noi di "`obsexcludstring'{p_end}"

			}

	********************************
	*	Start matching
	********************************

		********************************
		*
		*	Creating tempvar used in matching
		*
		********************************

		*Initiate the temporary variables used by this command
		tempvar prefID prefDiff matched

		gen `prefDiff' 		= .
		gen byte `matched' 	= 0

		*Allow the ID var used to be string
		if `IDtypeNumeric' == 1 {
			gen  `prefID' = .
		}
		else {
			gen  `prefID' = ""
		}

		** Generate the inverse of the matchvar to sort descending (gsort is too slow),
		*  a random var to seperate two values with the same match var, and the inverse
		*  of the random var for when sorting descending.
		tempvar rand invsort invrand

		sort `matchvar'
		gen  `invsort' = -1 * `matchvar'

		gen    `rand' = uniform()
		gen `invrand' = -1 * `rand'

		***********
		*Tempvars for matching

		*Diffvars, they are always numeric
		tempvar					 diffup   diffdo   valUp_0   valDo_0   valUp_1   valDo_1
		local 	 updownTempVars `diffup' `diffdo' `valUp_0' `valDo_0' `valUp_1' `valDo_1'

		foreach tempVar of local updownTempVars {

			gen `tempVar' = .
		}

		*ID vars, allowed to be both numeric and string
		tempvar					  	IDup   IDdo   IDup_0   IDdo_0   IDup_1   IDdo_1
		local 	 updownIDTempVars  `IDup' `IDdo' `IDup_0' `IDdo_0' `IDup_1' `IDdo_1'

		foreach tempVar of local updownIDTempVars {

			*Allow the ID var used to be string
			if `IDtypeNumeric' == 1 {
				gen  `tempVar' = .
			}
			else {
				gen  `tempVar' = ""
			}
		}

		***************************
		*
		*	One to one match
		*
		***************************

		if "`m1'" == "" {

			*Start outputting the countdown
			noi di ""
			noi di "{hline}{break}"
			noi di "{pstd}{ul:Matching one-to-one. Base observations left to match:}{p_end}"
			count if `grpdummy' == 1 & `matched' == 0

			*Create local to display "obs left to match" and to use in while loop
			local left2Match = `r(N)'
			noi di "{pstd}`left2Match' " _c

			*Match until no more observations to match.
			while (`left2Match' > 0) {

				**For all observations still to be matched, assign the preferred
				* match among the other unmatched observations
				qui updatePrefDiffPreffID `prefID' `prefDiff' `matchvar' `invsort' `idvar' `grpdummy' `matched' `rand' `invrand' `updownTempVars' `updownIDTempVars'

				*Restrict matches to within maxdiff() if that option is used.
				if "`maxdiff'" != "" {

					*Omit base observation from matching if diff is to big
					replace `matched' 			= 1		if `prefDiff' > `maxdiff' & `grpdummy' == 1

					*Indicate in result var that this obs did not have valid match within maxdiff()
					replace `matchResultName' 	= .d 	if `prefDiff' > `maxdiff' & `grpdummy' == 1

					*Removed preferred match
					if `IDtypeNumeric' == 1 {
						*IDvar is numeric
						replace `prefID'		= .		if `prefDiff' > `maxdiff'
					}
					else {
						*IDvar is string
						replace `prefID'		= ""	if `prefDiff' > `maxdiff'
					}
				}

				*If two observations mutually prefer each other, then indicate both of them as matched.
				replace `matched' = 1 if `matched' == 0 & `prefID' == `idvar'[_n-1] & `prefID'[_n-1] == `idvar'
				replace `matched' = 1 if `matched' == 0 & `prefID' == `idvar'[_n+1] & `prefID'[_n+1] == `idvar'

				*Update local to display "obs left to match" and to use in while loop
				count if `grpdummy' == 1 & `matched' == 0
				local left2Match = `r(N)'
				noi di "`left2Match' " _c

			}

			*End formatting for the "base obs left to match"
			noi di "{p_end}" _c
		}

		***************************
		*
		*	Many to one match
		*
		***************************
		else {

			**For all observations to be matched, assign the preferred
			* match among the other unmatched observations
			updatePrefDiffPreffID `prefID' `prefDiff' `matchvar' `invsort' `idvar' `grpdummy' `matched' `rand' `invrand' `updownTempVars' `updownIDTempVars'

			*Restrict matches to within maxdiff() if that option is used.
			if "`maxdiff'" != "" {

				*Indicate in result var that this obs did not have valid match within maxdiff()
				replace `matchResultName' 	= .d 	if `prefDiff' > `maxdiff' & `grpdummy' == 1

				*Removed preferred match
				if `IDtypeNumeric' == 1 {
					*IDvar is numeric
					replace `prefID'		= .		if `prefDiff' > `maxdiff'
				}
				else {
					*IDvar is string
					replace `prefID'		= ""	if `prefDiff' > `maxdiff'
				}
			}

			*Assign it's own ID as pref ID for all target vars
			replace `prefID'			=  `idvar' if `grpdummy' == 0

			* Replace the _matchCount var with number of base observations in each
			* match group. Each group is all base observation plus the target
			* observation, therefore (_N - 1)
			bys 	`prefID' 			:   replace `matchCountName' = _N - 1 if !missing(`prefID')

			**Replace prefID to missing for target obs that had no base
			* obs matched to it. T
			if `IDtypeNumeric' == 1 {
				*IDvar is numeric
				replace `prefID'		= .		if `matchCountName' == 0
			}
			else {
				*IDvar is string
				replace `prefID'		= ""	if `matchCountName' == 0
			}

			*Only target obs with base obs prefering it are matched
			replace	 `matched' 			= 1		if `matchCountName' != 0

			*Remove values for target obs that were not matched
			replace  `matchCountName'	= . 	if `matchCountName' == 0
		}

		*Update all return vars
		replace `matchDiffName' 	= `prefDiff'	if `matched' == 1
		replace `matchIDname' 		= `idvar' 		if `matched' == 1 & `grpdummy' == 0
		replace `matchIDname' 		= `prefID' 		if `matched' == 1 & `grpdummy' == 1

		*Remove the best match value in obs that did not have a match within maxdiff()
		replace `matchDiffName'  = . 				if `matchResultName' == .d

		*Matched observations are give value 1 in result var
		replace `matchResultName' = 1 				if `matched' == 1 & `matchResultName' != .d

		*Target obs not used
		replace `matchResultName' = 0 if `matched' == 0 & `grpdummy' == 0

		*only keep output vars
		keep `matchIDname' `matchDiffName' `originalSort' `matchResultName' `matchCountName'

		*Merge the results back to the original data set
		tempfile mergefile
		save 	`mergefile'

	restore

	***************************
	*
	*	Merge match results to
	*	original data and assign
	*	remaining missing values
	*	to the result var
	*
	***************************

	*Merge the results with the original data
	tempvar mergevar
	merge 1:1  `originalSort' using `mergefile', gen(`mergevar')

	*remaining missing values are listed in ascending order of importance.
	*Meaning that if a variable is both .m and .i then it will be .i as it
	*is assigned afterwards below.

	*Missing matching var
	replace `matchResultName' = .m if missing(`matchvar')

	*Msising dummy var
	replace `matchResultName' = .g if missing(`grpdummy')

	*Excluded in if/in
	if "`if'`in'" != "" {

		tempvar  ifinvar
		gen 	`ifinvar' = 1 `if'`in'
		replace `matchResultName' = .i if `ifinvar' != 1
	}

	*comress the variables generated
	compress  `matchIDname' `matchDiffName' `matchResultName' `matchCountName'

	*Output result table
	noi outputTable `matchResultName' `grpdummy'

	*Restore the oridinal sort
	sort `originalSort'

}

end

	*Check manually entered names for the return vars.
	cap program drop 	iematchMatchVarCheck
		program define 	iematchMatchVarCheck  , rclass

		args defaultName replace_used userName

		if "`defaultName'" == "_matchID" 		local optionName matchidname
		if "`defaultName'" == "_matchDiff" 		local optionName matchdiffname
		if "`defaultName'" == "_matchResult" 	local optionName matchresult
		if "`defaultName'" == "_matchCount" 	local optionName matchcount

		*All the default names
		local dfltNms _noMatchReason _matchDiff _matchResult _matchCount

		*The other two default names
		local othDfltNms : list dfltNms - defaultName

		*Creating two locals with the deafult name in each local
		local Oth1 : word 1 of `othDfltNms'
		local Oth2 : word 2 of `othDfltNms'
		local Oth3 : word 3 of `othDfltNms'

		*Testing that the option name is not the same as the other variable's default name.
		if ("`userName'" == "`Oth1'" | "`userName'" == "`Oth2'" | "`userName'" == "`Oth3'" | "`userName'" == "_ID")  {

			noi di as error "{pstd}The new name specified in `optionName'(`userName') is not allowed to be _ID, `Oth1', `Oth2', or `Oth3'{p_end}"
			error 198
		}

		*Test if name is set manually
		if "`userName'" != "" {
			*Use the manually set name
			local validMatchVarname `userName'
		}
		else {
			*No manually set name, use deafult name
			local validMatchVarname `defaultName'
		}

		*Make sure that the manually entered name is not already used
		cap confirm variable `validMatchVarname'
		if _rc == 0 {

			if `replace_used' == 1 {
				//Replace is used, drop the old var
				drop `validMatchVarname'
			}
			else {

				*Replace is not used. Prepare an error message and throw error

				if "`userName'" != "" {
					*Error message for user specified name s
					local nameErrorString "The variable name specified in `optionName'(`userName')"
				}
				else {
					*Error message for user specified name s
					local nameErrorString "A variable with name `validMatchVarname'"
				}

				*Trow error
				noi di as error "{pstd}`nameErrorString' is already defined in the data set. Either drop this variable, use the replace option or specify a another variable name using `optionName'()."
				error 110

			}
		}

		*Testing that the new name is valid by creating a variable that is dropped immedeatly afterwards
		cap gen `validMatchVarname' = .

		if _rc != 0 {

			noi di as error "{pstd}The variable name specified in `optionName'(`userName') is not a valid variable name."
			error _rc
		}
		drop `validMatchVarname'

		return local validVarName 	"`validMatchVarname'"

	end

	** This function is called to assign the preferred match for all unmatched
	*  observations to any other unmatched observation. Returns a variable pref that is the
	cap program drop 	updatePrefDiffPreffID
		program define 	updatePrefDiffPreffID

		args prefID prefDiff matchvar invsort idvar grpvar matched rand invrand diffup diffdo valUp_0 valDo_0 valUp_1 valDo_1 IDup IDdo IDup_0 IDdo_0 IDup_1 IDdo_1

		*****************
		*Update possible matches with higher rank value
		sort `matched' `matchvar' `rand'

		updateBestValID 0 `matchvar' `idvar' `grpvar' `matched' `valUp_0' `IDup_0'
		updateBestValID 1 `matchvar' `idvar' `grpvar' `matched' `valUp_1' `IDup_1'

		*****************
		*Update possible matches with lower rank value
		sort `matched' `invsort' `invrand'

		updateBestValID 0 `matchvar' `idvar' `grpvar' `matched' `valDo_0' `IDdo_0'
		updateBestValID 1 `matchvar' `idvar' `grpvar' `matched' `valDo_1' `IDdo_1'

		*****************
		*Create one varaible common for both groups with diff upwards
		replace `diffup' 	= abs(`matchvar' - `valUp_1') if `grpvar' == 1
		replace `diffup' 	= abs(`matchvar' - `valUp_0') if `grpvar' == 0

		*Create one varaible common for both groups with diff downwards
		replace `diffdo' 	= abs(`matchvar' - `valDo_1') if `grpvar' == 1
		replace `diffdo' 	= abs(`matchvar' - `valDo_0') if `grpvar' == 0

		*Create one varaible common for both groups with ID upwards
		replace `IDup' 		= `IDup_1' 	if `grpvar' == 1
		replace `IDup' 		= `IDup_0' 	if `grpvar' == 0

		*Create one varaible common for both groups with ID downwards
		replace `IDdo' 		= `IDdo_1' 	if `grpvar' == 1
		replace `IDdo' 		= `IDdo_0' 	if `grpvar' == 0

		replace `prefID' 	= `IDdo' 	if `matched' == 0 & `diffdo' <= `diffup'
		replace `prefID' 	= `IDup' 	if `matched' == 0 & `diffdo' >  `diffup'

		replace `prefDiff' 	= `diffdo' 	if `matched' == 0 & `diffdo' <= `diffup'
		replace `prefDiff' 	= `diffup' 	if `matched' == 0 & `diffdo' >  `diffup'

	end

	** This function goes through the observatioins from top to down and copying
	*  the match value of the closest observation of the other group above to
	*  (bestVal) and the ccoprresponding ID to (bestID) for obe group at the time.
	*  To go from bottuom to top sort with invsort and apply this command.
	cap program drop 	updateBestValID
		program define 	updateBestValID

		args grpvl matchval idvar grpvar matched bestVal bestID

		*Test if ID var is string or numeric
		local IDNumeric 0
		cap confirm numeric variable `idvar'
		if _rc == 0 local IDNumeric 1

		*Reset all values
		replace `bestVal' 	= .
		if `IDNumeric' == 1 replace `bestID' 	= .
		if `IDNumeric' == 0 replace `bestID' 	= ""

		*Set the match value and ID of observations that are in the other group
		replace `bestVal'   = `matchval' 		if `grpvar' != `grpvl' & `matched' == 0
		replace `bestID'  	= `idvar' 			if `grpvar' != `grpvl' & `matched' == 0

		*Fill in that value from the observation in the other group until getting to another observation of the other group
							replace `bestVal'   = `bestVal'[_n-1]	if `bestVal' == .  & `matched' == 0
		if `IDNumeric' == 1 replace `bestID'  	= `bestID'[_n-1] 	if `bestID'	 == .  & `matched' == 0
		if `IDNumeric' == 0 replace `bestID'  	= `bestID'[_n-1] 	if `bestID'	 == "" & `matched' == 0

	end

	** This function creates the output table.
	cap program drop 	outputTable
		program define 	outputTable

		qui {
			args resultVar grpdum

			*List the texts used in the first column in the
			local text_1 " 1 Matched"
			local text_0 " 0 Not matched"
			local text_d ".d No match within maxdiff()"
			local text_i ".i Excluded using if/in"
			local text_g ".g Missing grpdummy()"
			local text_m ".m Missing matchvar()"

			*The minimum width of the
			local firstColWidth = max(strlen(" `resultVar'") , strlen(" value and result"))

			levelsof `resultVar' , missing local(resultsUsed)

			foreach result of local resultsUsed {

				*Remove the . from the local to match the string
				local result = subinstr("`result'",".","",1)

				* Test if the text in the first column is longer
				* for this row than previous row
				local firstColWidth = max(`firstColWidth' , strlen("`text_`result''"))
			}

			* Create a local that indicates if there is column
			* for obs with missing value in tmt dummy
			local missingGrpDum 0
			count if missing(`grpdum')
			if `r(N)' > 0 local missingGrpDum 1

			*Set all locals that determins column width

			local C1 "{col 4}{c |}"

			di "`firstColWidth'"

			local col1width = `firstColWidth'  + 1
			local hli1 "{hline `col1width'}"
			local cen1 "{center `col1width':"

			local col2border = `firstColWidth' + 6
			local C2  "{col `col2border'}{c |}"
			local C2a "{col `col2border'}"

			local col3border = `firstColWidth' + 17
			local C3 "{col `col3border'}{c |}"

			local col4border = `firstColWidth' + 30
			local C4 "{col `col4border'}{c |}"

			local grpdumCentre 23 //Overspills automatically

			local lastT  	"{c RT}"
			local lastTdown	"{c RT}"
			local lastC  	"{c BRC}"

			if `missingGrpDum' {

				local lastT 		"{c +}{hline 9}{c RT}"
				local lastTdown 	"{c BT}{hline 9}{c RT}"
				local lastC 		"{c BT}{hline 9}{c BRC}"
				local missTitle 	"  missing "
				local grpdumCentre 	33

			}

			noi di ""
			noi di ""
			noi di "{hline}"
			noi di ""
			noi di "{pstd}{ul:Output of Matching Result:}{p_end}"
			noi di ""
			noi di "{col 5}`cen1'`resultVar'}`C2a' {centre `grpdumCentre':`grpdum' }"
			noi di "{col 4}{c LT}`hli1'{c +}{hline `grpdumCentre'}{c RT}"
			noi di "`C1'`cen1'value and result}`C2' 1 (base)   0 (target) `missTitle'{c |}"
			noi di "{col 4}{c LT}`hli1'{c +}{hline 10}{c +}{hline 12}`lastT'"

			*This is the roworder for the output table
			local resultTableOrder 1 0 .d .i .g .m

			foreach result of local resultTableOrder {

				*Test if this result was used
				if `:list result in resultsUsed' {

					*Count base observations in this result type
					qui count if `resultVar' == `result' & `grpdum' == 1
					local numBase = trim("`: display %16.0gc  `r(N)''")

					*Count target observations in this result type
					qui count if `resultVar' == `result' & `grpdum' == 0
					local numTarg = trim("`: display %16.0gc  `r(N)''")

					*Count observations that are neither base or target in this result type
					if `missingGrpDum' {

						qui count if `resultVar' == `result' & missing(`grpdum')
						local numMiss = trim("`: display %16.0gc  `r(N)''")

						local missCol "{ralign 9 :`numMiss' }{c |}"

					}

					*Prepare the local that displays the columns with numbers
					local numCols "`C2'{ralign 10 :`numBase' }`C3'{ralign 12 :`numTarg' }`C4'`missCol'"

					*Remove the . from the local to match the string
					local result = subinstr("`result'",".","",1)

					*Output the table row
					noi di "`C1'`text_`result''`numCols'"

				}
			}

			*Output line before N per group row
			noi di "{col 4}{c LT}`hli1'{c +}{hline 10}{c +}{hline 12}`lastT'"

			*Caculate N per group row for base observations
			qui count if `grpdum' == 1
			local numBase = trim("`: display %16.0gc  `r(N)''")

			*Caculate N per group row for target observations
			qui count if `grpdum' == 0
			local numTarg = trim("`: display %16.0gc  `r(N)''")

			*Caculate N per group row for observations neither base or target
			if `missingGrpDum' {

				qui count if missing(`grpdum')
				local numMiss = trim("`: display %16.0gc  `r(N)''")

				local missCol "{ralign 9 :`numMiss' }{c |}"

			}

			*Output N per group row
			noi di "`C1'`cen1'N per group}`C2'{ralign 10 :`numBase' }`C3'{ralign 12 :`numTarg' }`C4'`missCol'"

			*Output line before total N row
			noi di "{col 4}{c LT}`hli1'{c +}{hline 10}{c BT}{hline 12}`lastTdown'"

			*Count total N
			qui count
			local numTot = trim("`: display %16.0gc  `r(N)''")

			*Output total N row
			noi di "`C1'`cen1'Total N}`C2'{centre `grpdumCentre':`numTot'}`C4'"

			*Output bottom line
			noi di "{col 4}{c BLC}`hli1'{c BT}{hline `grpdumCentre'}{c BRC}"
		}
	end


(minor) Tweak comment

(minor) Tweak comment

*! version 5.3.3 20dec2018

program reghdfe, eclass
	* Intercept old+version
	cap syntax, version old
	if !c(rc) {
		reghdfe_old, version
		exit
	}

	* Intercept old
	cap syntax anything(everything) [fw aw pw/], [*] old
	if !c(rc) {
		di as error "(running historical version of reghdfe)"
		if ("`weight'"!="") local weightexp [`weight'=`exp']
		reghdfe_old `anything' `weightexp', `options'
		exit
	}

	* Aux. subcommands
	cap syntax, [*]
	if inlist("`options'", "check", "compile", "reload", "update", "version", "requirements") {
		if ("`options'"=="compile") loc args force
		if ("`options'"=="check") loc options compile
		if ("`options'"=="update") {
			loc args 1
			loc options reload
		}
		loc subcmd = proper("`options'")
		`subcmd' `args'
	}
	else if replay() {
		Replay `0'
	}
	else {
		Cleanup 0
		Compile // takes 0.01s to run this useful check (ensures .mlib exists)
		cap noi Estimate `0'
		Cleanup `c(rc)'
	}
end

program Compile
	args force

	* Check dependencies
	ftools, check // in case lftools.mlib does not exist or is outdated
	ms_get_version ftools, min_version("2.30.1")
	ms_get_version reghdfe // save local package_version
	loc list_objects "FixedEffects() fixed_effects() BipartiteGraph()"
	loc list_functions "reghdfe_*() transform_*() accelerate_*() panelmean() panelsolve_*() lsmr()"
	loc list_misc "weighted_quadcolsum() safe_divide() check_convergence() precompute_inv_xx() _st_data_wrapper()"
	// TODO: prefix everything with reghdfe_*

	ms_compile_mata, ///
		package(reghdfe) ///
		version(`package_version') ///
		fun("`list_objects' `list_functions' `list_misc'") ///
		verbose ///
		`force'
end

program Reload
	* Internal debugging tool.
	* Updates dependencies and reghdfe from local path or from github
	* Usage:
	* 	reghdfe, update // from c:\git\..
	* 	reghdfe, reload // from github

	args online
	if ("`online'" == "") loc online 0

	di as text _n "{bf:reghdfe: updating required packages}"
	di as text "{hline 64}"

	* -ftools- https://github.com/sergiocorreia/ftools/
	cap ado uninstall ftools
	if (`online') net install ftools, from("https://github.com/sergiocorreia/ftools/raw/master/src/")
	if (!`online') net install ftools, from("c:\git\ftools\src")
	di as text "{hline 64}"
	ftools, compile
	di as text "{hline 64}"

	* Update -reghdfe-
	di as text _n  _n "{bf:reghdfe: updating self}"
	di as text "{hline 64}"
	qui ado uninstall reghdfe
	if (`online') net install reghdfe, from("https://github.com/sergiocorreia/reghdfe/raw/master/src/")
	if (!`online') net install reghdfe, from("c:\git\reghdfe\src")
	qui which reghdfe
	di as text "{hline 64}"
	reghdfe, compile
	di as text "{hline 64}"

	* Cleaning up
	di as text _n "{bf:Note:} You need to run {stata program drop _all} now."
end

program Version
	which reghdfe
	Requirements
end

program Requirements
	di as text _n "Required packages installed?"
	loc reqs ftools
	// ivreg2 avar tuples group3hdfe
	if (c(stata_version)<13) loc reqs `reqs' boottest

	loc ftools_github "https://github.com/sergiocorreia/ftools/raw/master/src/"

	loc error 0

	foreach req of local reqs {
		loc fn `req'.ado
		cap findfile `fn'
		if (_rc) {
			loc error 1
			di as text "{lalign 20:- `req'}" as error "not" _c
			di as text "    {stata ssc install `req':install from SSC}" _c
			if inlist("`req'", "ftools") {
				loc github ``req'_github'
				di as text `"    {stata `"net install `req', from(`"`github'"')"':install from github}"'
			}
			else {
				di as text // newline
			}
		}
		else {
			di as text "{lalign 20:- `req'}" as text "yes"
		}
	}

	if (`error') exit 601
end

program Cleanup
	args rc
	cap mata: mata drop HDFE
	cap mata: mata drop hdfe_*
	cap drop __temp_reghdfe_resid__
	cap matrix drop reghdfe_statsmatrix
	if (`rc' == 132) {
		di as text "- If you got the {it:parentheses unbalanced} error, note that IV/2SLS was moved to {help ivreghdfe}"
		di as smcl `"- Latest version: {browse "https://github.com/sergiocorreia/ivreghdfe":https://github.com/sergiocorreia/ivreghdfe}"'
		di as smcl `"- SSC version: {stata "net describe ivreghdfe, from(http://fmwww.bc.edu/RePEc/bocode/i)"}"'
		di as smcl `"- Note: the older functionality can still be accessed through the {it:old} option"'
	}
	if (`rc') exit `rc'
end

program Parse
	* Trim whitespace (caused by "///" line continuations; aesthetic only)
	mata: st_local("0", stritrim(st_local("0")))

	* Main syntax
	#d;
	syntax varlist(fv ts numeric) [if] [in] [aw pw fw/] , [

		/* Model */
		Absorb(string) NOAbsorb
		SUmmarize SUmmarize2(string asis) /* simulate implicit options */

		/* Standard Errors */
		VCE(string) CLuster(string)

		/* Diagnostic */
		Verbose(numlist min=1 max=1 >=-1 <=5 integer)
		TIMEit

		/* Speedup and memory Tricks */
		NOSAMPle /* do not save e(sample) */
		COMPACT /* use as little memory as possible but is slower */

		/* Undocumented */
		KEEPSINgletons
		OLD /* use latest v3 */
		NOTES(string) /* NOTES(key=value ...), will be stored on e() */

		] [*] /* capture optimization options, display options, etc. */
		;
	#d cr

	* Unused
	* SAVEcache
	* USEcache
	* CLEARcache

	* Convert options to boolean
	if ("`verbose'" == "") loc verbose 0
	loc timeit = ("`timeit'"!="")
	loc drop_singletons = ("`keepsingletons'" == "")
	loc compact = ("`compact'" != "")

	if (`timeit') timer on 29

	* Sanity checks
	if (`verbose'>-1 & "`keepsingletons'"!="") {
		loc url "http://scorreia.com/reghdfe/nested_within_cluster.pdf"
		loc msg "WARNING: Singleton observations not dropped; statistical significance is biased"
		di as error `"`msg' {browse "`url'":(link)}"'
	}
	if ("`cluster'"!="") {
		_assert ("`vce'"==""), msg("cannot specify both cluster() and vce()")
		loc vce cluster `cluster'
		loc cluster // clear it to avoid bugs in subsequent lines
	}

	* Split varlist into <depvar> and <indepvars>
	ms_parse_varlist `varlist'
	if (`verbose' > 0) {
		di as text _n "## Parsing varlist: {res}`varlist'"
		return list
	}
	loc depvar `r(depvar)'
	loc indepvars `r(indepvars)'
	loc fe_format "`r(fe_format)'"
	loc basevars `r(basevars)'

	* Parse Weights
	if ("`weight'"!="") {
		unab exp : `exp', min(1) max(1) // simple weights only
	}

	* Parse VCE
	ms_parse_vce, vce(`vce') weighttype(`weight')
	if (`verbose' > 0) {
		di as text _n "## Parsing vce({res}`vce'{txt})"
		sreturn list
	}
	loc vcetype = "`s(vcetype)'"
	loc num_clusters = `s(num_clusters)'
	loc clustervars = "`s(clustervars)'"
	loc base_clustervars = "`s(base_clustervars)'"
	loc vceextra = "`s(vceextra)'"

	* Select sample (except for absvars)
	loc varlist `depvar' `indepvars' `base_clustervars'
	tempvar touse
	marksample touse, strok // based on varlist + cluster + if + in + weight

	* Parse noabsorb
	_assert  ("`absorb'`noabsorb'" != ""), msg("option {bf:absorb()} or {bf:noabsorb} required")
	if ("`noabsorb'" != "") {
		_assert ("`absorb'" == ""), msg("{bf:absorb()} and {bf:noabsorb} are mutually exclusive")
		tempvar c
		gen byte `c' = 1
		loc absorb `c'
	}

	if (`timeit') timer off 29

	* Construct HDFE object
	// SYNTAX: fixed_effects(absvars | , touse, wtype, wtvar, dropsing, verbose)
	mata: st_local("comma", strpos(`"`absorb'"', ",") ? "" : ",")
	if (`timeit') timer on 20
	mata: HDFE = fixed_effects(`"`absorb' `comma' `options'"', "`touse'", "`weight'", "`exp'", `drop_singletons', `verbose')
	if (`timeit') timer off 20
	mata: HDFE.cmdline = "reghdfe " + st_local("0")
	loc options `s(options)'

	mata: st_local("N", strofreal(HDFE.N))
	if (`N' == 0) error 2000

	* Fill out HDFE object
	* mata: HDFE.varlist = "`base_varlist'"
	mata: HDFE.depvar = "`depvar'"
	mata: HDFE.indepvars = "`indepvars'"
	mata: HDFE.vcetype = "`vcetype'"
	mata: HDFE.num_clusters = `num_clusters'
	mata: HDFE.clustervars = tokens("`clustervars'")
	mata: HDFE.base_clustervars = tokens("`base_clustervars'")
	mata: HDFE.vceextra = "`vceextra'"

	* Preserve memory
	mata: HDFE.compact = `compact'
	if (`compact') {
		loc panelvar "`_dta[_TSpanel]'"
		loc timevar "`_dta[_TStvar]'"
		mata: HDFE.panelvar = "`panelvar'"
		mata: HDFE.timevar = "`timevar'"
		c_local keepvars `basevars' `base_clustervars' `panelvar' `timevar' // `exp'
	}

	* Parse summarize
	if ("`summarize'" != "") {
		_assert ("`summarize2'" == ""), msg("summarize() syntax error")
		loc summarize2 mean min max  // default values
	}
	ParseSummarize `summarize2'
	mata: HDFE.summarize_stats = "`s(stats)'"
	mata: HDFE.summarize_quietly = `s(quietly)'

	* Parse misc options
	mata: HDFE.notes = `"`notes'"'
	mata: HDFE.store_sample = ("`nosample'"=="")
	mata: HDFE.timeit = `timeit'

	* Parse Coef Table Options (do this last!)
	_get_diopts diopts options, `options' // store in `diopts', and the rest back to `options'
	_assert (`"`options'"'==""), msg(`"invalid options: `options'"')
	if ("`hascons'"!="") di in ye "(option ignored: `hascons')"
	if ("`tsscons'"!="") di in ye "(option ignored: `tsscons')"
	mata: HDFE.diopts = `"`diopts'"'
end

program ParseSummarize, sclass
	sreturn clear
	syntax [namelist(name=stats)] , [QUIetly]
	local quietly = ("`quietly'"!="")
	sreturn loc stats "`stats'"
	sreturn loc quietly = `quietly'
end

// --------------------------------------------------------------------------

program Estimate, eclass
	ereturn clear

	* Parse and fill out HDFE object
	Parse `0'
	mata: st_local("timeit", strofreal(HDFE.timeit))
	mata: st_local("compact", strofreal(HDFE.compact))
	mata: st_local("verbose", strofreal(HDFE.verbose))

	* Compute degrees-of-freedom
	if (`timeit') timer on 21
	mata: HDFE.estimate_dof()
	if (`timeit') timer off 21

	* Save updated e(sample) (singletons reduce sample);
	* required to parse factor variables to partial out
	if (`timeit') timer on 29
	tempvar touse
	mata: HDFE.save_touse("`touse'")
	if (`timeit') timer off 29

	* Expand varlists
	if (`timeit') timer on 22
	mata: st_local("depvar", HDFE.depvar)
	mata: st_local("indepvars", HDFE.indepvars)
	if (`verbose' > 0) di as text _n "## Parsing and expanding indepvars: {res}`indepvars'"
	ms_expand_varlist `indepvars' if `touse'
	if (`verbose' > 0) return list
	mata: HDFE.fullindepvars = "`r(fullvarlist)'"
	mata: HDFE.indepvars = "`r(varlist)'"
	mata: HDFE.not_basevar = strtoreal(tokens("`r(not_omitted)'"))
	mata: HDFE.varlist = "`depvar' `r(varlist)'"
	if (`timeit') timer off 22

	* Stats
	mata: st_local("stats", HDFE.summarize_stats)
	if ("`stats'" != "") Stats `touse'

	* Condition number
	mata: HDFE.estimate_cond()

	* Preserve
	if (`compact') {
		if (`verbose' > 0) di as text "## Preserving dataset"
		preserve
		keep `keepvars'
	}

	* Partial out; save TSS of depvar
	if (`timeit') timer on 23
	// partial_out(Varlist/Matrix | , Save TSS if HDFE.tss is missing? [0], Standardize data? [1], First col is depvar? [1])
	mata: hdfe_variables = HDFE.partial_out(HDFE.varlist, 1, ., .)
	if (`timeit') timer off 23

	* Regress
	if (`timeit') timer on 24
	tempname b V N rank df_r
	mata: reghdfe_post_ols(HDFE, hdfe_variables, "`b'", "`V'", "`N'", "`rank'", "`df_r'")
	mata: hdfe_variables = .
	* Restore
	if (`compact') {
		if (`verbose' > 0) di as text "## Restoring dataset"
		restore
		mata: st_local("residuals", HDFE.residuals)
		if ("`residuals'" != "") mata: HDFE.save_variable(HDFE.residuals, HDFE.residuals_vector, "Residuals")
	}
	RegressOLS `touse' `b' `V' `N' `rank' `df_r'
	if (`timeit') timer off 24

	* (optional) Store FEs
	if (`timeit') timer on 29
	reghdfe_store_alphas
	if (`timeit') timer off 29

	* View estimation tables
	mata: st_local("diopts", HDFE.diopts)
	Replay, `diopts'

	if (`timeit') {
		di as text _n "{bf: Timer results:}"
		timer list
		di as text "Legend: 20: Create HDFE object; 21: Estimate DoF; 22: expand varlists; 23: partial out; 24: regress; 29: rest"
		di
	}
end

program RegressOLS, eclass
	args touse b V N rank df_r

	mata: st_local("store_sample", strofreal(HDFE.store_sample))
	if (`store_sample') loc esample "esample(`touse')"

	mata: st_local("indepvars", HDFE.fullindepvars)
	if ("`indepvars'" != "") {
		matrix colnames `b' = `indepvars'
		matrix colnames `V' = `indepvars'
		matrix rownames `V' = `indepvars'
		_ms_findomitted `b' `V'
		ereturn post `b' `V', `esample' buildfvinfo depname(`depvar')
	}
	else {
		ereturn post, `esample' buildfvinfo depname(`depvar')
	}

	ereturn scalar N       = `N'
	ereturn scalar rank    = `rank'
	ereturn scalar df_r    = `df_r'
	ereturn local  cmd     "reghdfe"
	mata: HDFE.post()

	* Post stats
	cap conf matrix reghdfe_statsmatrix
	if (!c(rc)) {
		ereturn matrix summarize = reghdfe_statsmatrix
		mata: st_local("summarize_quietly", strofreal(HDFE.summarize_quietly))
		ereturn scalar summarize_quietly = `summarize_quietly'
	}
end

program Replay, rclass
	syntax [, *]

	if `"`e(cmd)'"' != "reghdfe"  {
	        error 301
	}

	_get_diopts options, `options'
	reghdfe_header // _coef_table_header
	di ""
	_coef_table, `options' // ereturn display, `options'
	return add // adds r(level), r(table), etc. to ereturn (before the footnote deletes them)
	reghdfe_footnote

	* Replay stats
	if (e(summarize_quietly)==0) {
		di as text _n "{sf:Regression Summary Statistics:}" _c
		matlist e(summarize)', border(top bottom) rowtitle(Variable) // twidth(18)
	}
end

program Stats
	args touse
	* Optional weights
	mata: st_local("weight", sprintf("[%s=%s]", HDFE.weight_type, HDFE.weight_var))
	assert "`weight'" != ""
	if ("`weight'" == "[=]") loc weight
	loc weight : subinstr local weight "[pweight" "[aweight"

	mata: st_local("stats", HDFE.summarize_stats)
	mata: st_local("varlist", HDFE.varlist)
	mata: st_local("cvars", invtokens(HDFE.cvars))
	loc full_varlist `varlist' `cvars'

	* quick workaround b/c -tabstat- does not support factor variables
	fvrevar `full_varlist', list
	loc full_varlist `r(varlist)'

	qui tabstat `full_varlist' if `touse' `weight' , stat(`stats') col(stat) save
	matrix reghdfe_statsmatrix = r(StatTotal)
end

*! version 5.3.3 20dec2018

program reghdfe, eclass
	* Intercept old+version
	cap syntax, version old
	if !c(rc) {
		reghdfe_old, version
		exit
	}

	* Intercept old
	cap syntax anything(everything) [fw aw pw/], [*] old
	if !c(rc) {
		di as error "(running historical version of reghdfe)"
		if ("`weight'"!="") local weightexp [`weight'=`exp']
		reghdfe_old `anything' `weightexp', `options'
		exit
	}

	* Aux. subcommands
	cap syntax, [*]
	if inlist("`options'", "check", "compile", "reload", "update", "version", "requirements") {
		if ("`options'"=="compile") loc args force
		if ("`options'"=="check") loc options compile
		if ("`options'"=="update") {
			loc args 1
			loc options reload
		}
		loc subcmd = proper("`options'")
		`subcmd' `args'
	}
	else if replay() {
		Replay `0'
	}
	else {
		Cleanup 0
		Compile // takes 0.01s to run this useful check (ensures .mlib exists)
		cap noi Estimate `0'
		Cleanup `c(rc)'
	}
end

program Compile
	args force

	* Check dependencies
	ftools, check // in case lftools.mlib does not exist or is outdated
	ms_get_version ftools, min_version("2.30.1")
	ms_get_version reghdfe // save local package_version
	loc list_objects "FixedEffects() fixed_effects() BipartiteGraph()"
	loc list_functions "reghdfe_*() transform_*() accelerate_*() panelmean() panelsolve_*() lsmr()"
	loc list_misc "weighted_quadcolsum() safe_divide() check_convergence() precompute_inv_xx() _st_data_wrapper()"
	// TODO: prefix everything with reghdfe_*

	ms_compile_mata, ///
		package(reghdfe) ///
		version(`package_version') ///
		fun("`list_objects' `list_functions' `list_misc'") ///
		verbose ///
		`force'
end

program Reload
	* Internal debugging tool.
	* Updates dependencies and reghdfe from local path or from github
	* Usage:
	* 	reghdfe, update // from c:\git\..
	* 	reghdfe, reload // from github

	args online
	if ("`online'" == "") loc online 0

	di as text _n "{bf:reghdfe: updating required packages}"
	di as text "{hline 64}"

	* -ftools- https://github.com/sergiocorreia/ftools/
	cap ado uninstall ftools
	if (`online') net install ftools, from("https://github.com/sergiocorreia/ftools/raw/master/src/")
	if (!`online') net install ftools, from("c:\git\ftools\src")
	di as text "{hline 64}"
	ftools, compile
	di as text "{hline 64}"

	* Update -reghdfe-
	di as text _n  _n "{bf:reghdfe: updating self}"
	di as text "{hline 64}"
	qui ado uninstall reghdfe
	if (`online') net install reghdfe, from("https://github.com/sergiocorreia/reghdfe/raw/master/src/")
	if (!`online') net install reghdfe, from("c:\git\reghdfe\src")
	qui which reghdfe
	di as text "{hline 64}"
	reghdfe, compile
	di as text "{hline 64}"

	* Cleaning up
	di as text _n "{bf:Note:} You need to run {stata program drop _all} now."
end

program Version
	which reghdfe
	Requirements
end

program Requirements
	di as text _n "Required packages installed?"
	loc reqs ftools
	// ivreg2 avar tuples group3hdfe
	if (c(stata_version)<13) loc reqs `reqs' boottest

	loc ftools_github "https://github.com/sergiocorreia/ftools/raw/master/src/"

	loc error 0

	foreach req of local reqs {
		loc fn `req'.ado
		cap findfile `fn'
		if (_rc) {
			loc error 1
			di as text "{lalign 20:- `req'}" as error "not" _c
			di as text "    {stata ssc install `req':install from SSC}" _c
			if inlist("`req'", "ftools") {
				loc github ``req'_github'
				di as text `"    {stata `"net install `req', from(`"`github'"')"':install from github}"'
			}
			else {
				di as text // newline
			}
		}
		else {
			di as text "{lalign 20:- `req'}" as text "yes"
		}
	}

	if (`error') exit 601
end

program Cleanup
	args rc
	cap mata: mata drop HDFE
	cap mata: mata drop hdfe_*
	cap drop __temp_reghdfe_resid__
	cap matrix drop reghdfe_statsmatrix
	if (`rc' == 132) {
		di as text "- If you got the {it:parentheses unbalanced} error, note that IV/2SLS was moved to {help ivreghdfe}"
		di as smcl `"- Latest version: {browse "https://github.com/sergiocorreia/ivreghdfe":https://github.com/sergiocorreia/ivreghdfe}"'
		di as smcl `"- SSC version: {stata "net describe ivreghdfe, from(http://fmwww.bc.edu/RePEc/bocode/i)"}"'
		di as smcl `"- Note: the older functionality can still be accessed through the {it:old} option"'
	}
	if (`rc') exit `rc'
end

program Parse
	* Trim whitespace (caused by "///" line continuations; aesthetic only)
	mata: st_local("0", stritrim(st_local("0")))

	* Main syntax
	#d;
	syntax varlist(fv ts numeric) [if] [in] [aw pw fw/] , [

		/* Model */
		Absorb(string) NOAbsorb
		SUmmarize SUmmarize2(string asis) /* simulate implicit options */

		/* Standard Errors */
		VCE(string) CLuster(string)

		/* Diagnostic */
		Verbose(numlist min=1 max=1 >=-1 <=5 integer)
		TIMEit

		/* Speedup and memory Tricks */
		NOSAMPle /* do not save e(sample) */
		COMPACT /* use as little memory as possible but is slower */

		/* Undocumented */
		KEEPSINgletons
		OLD /* use latest v3 */
		NOTES(string) /* NOTES(key=value ...), will be stored on e() */

		] [*] /* capture optimization options, display options, etc. */
		;
	#d cr

	* Unused
	* SAVEcache
	* USEcache
	* CLEARcache

	* Convert options to boolean
	if ("`verbose'" == "") loc verbose 0
	loc timeit = ("`timeit'"!="")
	loc drop_singletons = ("`keepsingletons'" == "")
	loc compact = ("`compact'" != "")

	if (`timeit') timer on 29

	* Sanity checks
	if (`verbose'>-1 & "`keepsingletons'"!="") {
		loc url "http://scorreia.com/reghdfe/nested_within_cluster.pdf"
		loc msg "WARNING: Singleton observations not dropped; statistical significance is biased"
		di as error `"`msg' {browse "`url'":(link)}"'
	}
	if ("`cluster'"!="") {
		_assert ("`vce'"==""), msg("cannot specify both cluster() and vce()")
		loc vce cluster `cluster'
		loc cluster // clear it to avoid bugs in subsequent lines
	}

	* Split varlist into <depvar> and <indepvars>
	ms_parse_varlist `varlist'
	if (`verbose' > 0) {
		di as text _n "## Parsing varlist: {res}`varlist'"
		return list
	}
	loc depvar `r(depvar)'
	loc indepvars `r(indepvars)'
	loc fe_format "`r(fe_format)'"
	loc basevars `r(basevars)'

	* Parse Weights
	if ("`weight'"!="") {
		unab exp : `exp', min(1) max(1) // simple weights only
	}

	* Parse VCE
	ms_parse_vce, vce(`vce') weighttype(`weight')
	if (`verbose' > 0) {
		di as text _n "## Parsing vce({res}`vce'{txt})"
		sreturn list
	}
	loc vcetype = "`s(vcetype)'"
	loc num_clusters = `s(num_clusters)'
	loc clustervars = "`s(clustervars)'"
	loc base_clustervars = "`s(base_clustervars)'"
	loc vceextra = "`s(vceextra)'"

	* Select sample (except for absvars)
	loc varlist `depvar' `indepvars' `base_clustervars'
	tempvar touse
	marksample touse, strok // based on varlist + cluster + if + in + weight

	* Parse noabsorb
	_assert  ("`absorb'`noabsorb'" != ""), msg("option {bf:absorb()} or {bf:noabsorb} required")
	if ("`noabsorb'" != "") {
		_assert ("`absorb'" == ""), msg("{bf:absorb()} and {bf:noabsorb} are mutually exclusive")
		tempvar c
		gen byte `c' = 1
		loc absorb `c'
	}

	if (`timeit') timer off 29

	* Construct HDFE object
	// SYNTAX: fixed_effects(absvars | , touse, wtype, wtvar, dropsing, verbose)
	mata: st_local("comma", strpos(`"`absorb'"', ",") ? "" : ",")
	if (`timeit') timer on 20
	mata: HDFE = fixed_effects(`"`absorb' `comma' `options'"', "`touse'", "`weight'", "`exp'", `drop_singletons', `verbose')
	if (`timeit') timer off 20
	mata: HDFE.cmdline = "reghdfe " + st_local("0")
	loc options `s(options)'

	mata: st_local("N", strofreal(HDFE.N))
	if (`N' == 0) error 2000

	* Fill out HDFE object
	* mata: HDFE.varlist = "`base_varlist'"
	mata: HDFE.depvar = "`depvar'"
	mata: HDFE.indepvars = "`indepvars'"
	mata: HDFE.vcetype = "`vcetype'"
	mata: HDFE.num_clusters = `num_clusters'
	mata: HDFE.clustervars = tokens("`clustervars'")
	mata: HDFE.base_clustervars = tokens("`base_clustervars'")
	mata: HDFE.vceextra = "`vceextra'"

	* Preserve memory
	mata: HDFE.compact = `compact'
	if (`compact') {
		loc panelvar "`_dta[_TSpanel]'"
		loc timevar "`_dta[_TStvar]'"
		mata: HDFE.panelvar = "`panelvar'"
		mata: HDFE.timevar = "`timevar'"
		c_local keepvars `basevars' `base_clustervars' `panelvar' `timevar' // `exp'
	}

	* Parse summarize
	if ("`summarize'" != "") {
		_assert ("`summarize2'" == ""), msg("summarize() syntax error")
		loc summarize2 mean min max  // default values
	}
	ParseSummarize `summarize2'
	mata: HDFE.summarize_stats = "`s(stats)'"
	mata: HDFE.summarize_quietly = `s(quietly)'

	* Parse misc options
	mata: HDFE.notes = `"`notes'"'
	mata: HDFE.store_sample = ("`nosample'"=="")
	mata: HDFE.timeit = `timeit'

	* Parse Coef Table Options (do this last!)
	_get_diopts diopts options, `options' // store in `diopts', and the rest back to `options'
	_assert (`"`options'"'==""), msg(`"invalid options: `options'"')
	if ("`hascons'"!="") di in ye "(option ignored: `hascons')"
	if ("`tsscons'"!="") di in ye "(option ignored: `tsscons')"
	mata: HDFE.diopts = `"`diopts'"'
end

program ParseSummarize, sclass
	sreturn clear
	syntax [namelist(name=stats)] , [QUIetly]
	local quietly = ("`quietly'"!="")
	sreturn loc stats "`stats'"
	sreturn loc quietly = `quietly'
end

// --------------------------------------------------------------------------

program Estimate, eclass
	ereturn clear

	* Parse and fill out HDFE object
	Parse `0'
	mata: st_local("timeit", strofreal(HDFE.timeit))
	mata: st_local("compact", strofreal(HDFE.compact))
	mata: st_local("verbose", strofreal(HDFE.verbose))

	* Compute degrees-of-freedom
	if (`timeit') timer on 21
	mata: HDFE.estimate_dof()
	if (`timeit') timer off 21

	* Save updated e(sample) (singletons reduce sample);
	* required to parse factor variables to partial out
	if (`timeit') timer on 29
	tempvar touse
	mata: HDFE.save_touse("`touse'")
	if (`timeit') timer off 29

	* Expand varlists
	if (`timeit') timer on 22
	mata: st_local("depvar", HDFE.depvar)
	mata: st_local("indepvars", HDFE.indepvars)
	if (`verbose' > 0) di as text _n "## Parsing and expanding indepvars: {res}`indepvars'"
	ms_expand_varlist `indepvars' if `touse'
	if (`verbose' > 0) return list
	mata: HDFE.fullindepvars = "`r(fullvarlist)'"
	mata: HDFE.indepvars = "`r(varlist)'"
	mata: HDFE.not_basevar = strtoreal(tokens("`r(not_omitted)'"))
	mata: HDFE.varlist = "`depvar' `r(varlist)'"
	if (`timeit') timer off 22

	* Stats
	mata: st_local("stats", HDFE.summarize_stats)
	if ("`stats'" != "") Stats `touse'

	* Condition number
	mata: HDFE.estimate_cond()

	* Preserve
	if (`compact') {
		if (`verbose' > 0) di as text "## Preserving dataset"
		preserve
		keep `keepvars'
	}

	* Partial out; save TSS of depvar
	if (`timeit') timer on 23
	mata: hdfe_variables = HDFE.partial_out(HDFE.varlist, 1) // 1=Save TSS of first var if HDFE.tss is missing
	if (`timeit') timer off 23

	* Regress
	if (`timeit') timer on 24
	tempname b V N rank df_r
	mata: reghdfe_post_ols(HDFE, hdfe_variables, "`b'", "`V'", "`N'", "`rank'", "`df_r'")
	mata: hdfe_variables = .
	* Restore
	if (`compact') {
		if (`verbose' > 0) di as text "## Restoring dataset"
		restore
		mata: st_local("residuals", HDFE.residuals)
		if ("`residuals'" != "") mata: HDFE.save_variable(HDFE.residuals, HDFE.residuals_vector, "Residuals")
	}
	RegressOLS `touse' `b' `V' `N' `rank' `df_r'
	if (`timeit') timer off 24

	* (optional) Store FEs
	if (`timeit') timer on 29
	reghdfe_store_alphas
	if (`timeit') timer off 29

	* View estimation tables
	mata: st_local("diopts", HDFE.diopts)
	Replay, `diopts'

	if (`timeit') {
		di as text _n "{bf: Timer results:}"
		timer list
		di as text "Legend: 20: Create HDFE object; 21: Estimate DoF; 22: expand varlists; 23: partial out; 24: regress; 29: rest"
		di
	}
end

program RegressOLS, eclass
	args touse b V N rank df_r

	mata: st_local("store_sample", strofreal(HDFE.store_sample))
	if (`store_sample') loc esample "esample(`touse')"

	mata: st_local("indepvars", HDFE.fullindepvars)
	if ("`indepvars'" != "") {
		matrix colnames `b' = `indepvars'
		matrix colnames `V' = `indepvars'
		matrix rownames `V' = `indepvars'
		_ms_findomitted `b' `V'
		ereturn post `b' `V', `esample' buildfvinfo depname(`depvar')
	}
	else {
		ereturn post, `esample' buildfvinfo depname(`depvar')
	}

	ereturn scalar N       = `N'
	ereturn scalar rank    = `rank'
	ereturn scalar df_r    = `df_r'
	ereturn local  cmd     "reghdfe"
	mata: HDFE.post()

	* Post stats
	cap conf matrix reghdfe_statsmatrix
	if (!c(rc)) {
		ereturn matrix summarize = reghdfe_statsmatrix
		mata: st_local("summarize_quietly", strofreal(HDFE.summarize_quietly))
		ereturn scalar summarize_quietly = `summarize_quietly'
	}
end

program Replay, rclass
	syntax [, *]

	if `"`e(cmd)'"' != "reghdfe"  {
	        error 301
	}

	_get_diopts options, `options'
	reghdfe_header // _coef_table_header
	di ""
	_coef_table, `options' // ereturn display, `options'
	return add // adds r(level), r(table), etc. to ereturn (before the footnote deletes them)
	reghdfe_footnote

	* Replay stats
	if (e(summarize_quietly)==0) {
		di as text _n "{sf:Regression Summary Statistics:}" _c
		matlist e(summarize)', border(top bottom) rowtitle(Variable) // twidth(18)
	}
end

program Stats
	args touse
	* Optional weights
	mata: st_local("weight", sprintf("[%s=%s]", HDFE.weight_type, HDFE.weight_var))
	assert "`weight'" != ""
	if ("`weight'" == "[=]") loc weight
	loc weight : subinstr local weight "[pweight" "[aweight"

	mata: st_local("stats", HDFE.summarize_stats)
	mata: st_local("varlist", HDFE.varlist)
	mata: st_local("cvars", invtokens(HDFE.cvars))
	loc full_varlist `varlist' `cvars'

	* quick workaround b/c -tabstat- does not support factor variables
	fvrevar `full_varlist', list
	loc full_varlist `r(varlist)'

	qui tabstat `full_varlist' if `touse' `weight' , stat(`stats') col(stat) save
	matrix reghdfe_statsmatrix = r(StatTotal)
end


Use preread in Stata to generate a giant empty data table.

program define svm_load
  syntax using/

  capture program _svm, plugin
  plugin call _svm, "preread" "`using'"

  clear
  set obs `=N'
  generate y = .

  * HACK: StataIC (which I am developing on) has a hard upper limit of 2k variables
  * We spend 1 on the 'y', and so we are limited to 2047 'x's
  if(`=M' > 2048-1) {
    scalar M = 2047
  }

  * this weird newlist syntax is the official suggestion for making a set of new variables in "help foreach"
  foreach j of newlist x1-x`=M'  {
    generate `j' = .
  }

end

* load the given svmlight-format file into memory
* the outcome variable (the first one on each line) is loaded in y, the rest are loaded into x<label>, where <label> is the label listed in the file before each value
* note! this *will* clear your current dataset
* NB: it is not clear to me if it is easier or hard to do this in pure-Stata than to try to jerry-rig C into the mix (the main trouble with C is that extensions cannot create new variables, and we need to create new variables as we discover them)
* it is *definitely* *SLOWER* to do this in pure Stata. svm-train loads the same test dataset in a fraction of a second where this takes 90s (on an SSD and i7).
program define svm_load_purestata

  * this makes macro `using' contain a filename
  syntax using/

  * .svmlight is meant to be a sparse format where variables go missing all the time
  * so we do the possibly-quadratic-runtime thing: add one row at a time to the dataset
  * using the deep magic of "set obs `=_N+1`; replace var = value in l". I think 'in l' means 'in last' but 'in last' doesn't work.
  * tip originally from Nick Cox: http://statalist.1588530.n2.nabble.com/Adding-rows-to-datasheet-td4784525.html
  * I suspect this inefficency is intrinsic.
  *  (libsvm's svm-train.c handles this problem by doing two passes over the data: once to count what it has to load, and twice to actually allocate memory and load it; we should profile for which method is faster in Stata)
  tempname fd
  file open `fd' using "`using'", read text

  * get rid of the old data
  clear

  * we know svmlight files always have exactly one y vector
  generate double y = .

  file read `fd' line
  while r(eof)==0 {
  	*display "read `line'" /*DEBUG*/

  	quiet set obs `=_N+1'

  	gettoken Y T : line
  	quiet replace y = `Y' in l
  	*di "T=`T'" /*DEBUG*/

  	* this does [(name, value = X.split(":")) for X in line.split()]
  	* and it puts the results into the table.
  	local j = 1
  	while("`T'" != "") {
      *if(`j' > 10) continue, break /*DEBUG*/
  	  gettoken X T : T
  	  gettoken name X : X, parse(":")
  	  gettoken X value : X, parse(":")
  	  *di "@ `=_N' `name' = `value'" /*DEBUG*/

  	  capture quiet generate double x`name' = . /*UNCONDITIONALLY make a new variable*/
    	capture quiet replace x`name' = `value' in l
  	  if(`=_rc' != 0) continue, break /*something went wrong, probably that we couldn't make a new variable (due to memory or built-in Stata constraints). Just try the next observation*/

			local j = `j' + 1
  	}
  	*list /*DEBUG: see the state after after new observation*/

  	file read `fd' line
  }
  file close `fd'
end

* load the given svmlight-format file into memory
* the outcome variable (the first one on each line) is loaded in y, the rest are loaded into x<label>, where <label> is the label listed in the file before each value
* note! this *will* clear your current dataset
* NB: it is not clear to me if it is easier or hard to do this in pure-Stata than to try to jerry-rig C into the mix (the main trouble with C is that extensions cannot create new variables, and we need to create new variables as we discover them)
* it is *definitely* *SLOWER* to do this in pure Stata. svm-train loads the same test dataset in a fraction of a second where this takes 90s (on an SSD and i7).
program define svm_load

  * this makes macro `using' contain a filename
  syntax using/

  * .svmlight is meant to be a sparse format where variables go missing all the time
  * so we do the possibly-quadratic-runtime thing: add one row at a time to the dataset
  * using the deep magic of "set obs `=_N+1`; replace var = value in l". I think 'in l' means 'in last' but 'in last' doesn't work.
  * tip originally from Nick Cox: http://statalist.1588530.n2.nabble.com/Adding-rows-to-datasheet-td4784525.html
  * I suspect this inefficency is intrinsic.
  *  (libsvm's svm-train.c handles this problem by doing two passes over the data: once to count what it has to load, and twice to actually allocate memory and load it; we should profile for which method is faster in Stata)
  tempname fd
  file open `fd' using "`using'", read text

  * get rid of the old data
  clear

  * we know svmlight files always have exactly one y vector
  generate double y = .

  file read `fd' line
  while r(eof)==0 {
  	*display "read `line'" /*DEBUG*/

  	quiet set obs `=_N+1'

  	gettoken Y T : line
  	quiet replace y = `Y' in l
  	*di "T=`T'" /*DEBUG*/

  	* this does [(name, value = X.split(":")) for X in line.split()]
  	* and it puts the results into the table.
  	local j = 1
  	while("`T'" != "") {
      *if(`j' > 10) continue, break /*DEBUG*/
  	  gettoken X T : T
  	  gettoken name X : X, parse(":")
  	  gettoken X value : X, parse(":")
  	  *di "@ `=_N' `name' = `value'" /*DEBUG*/

  	  capture quiet generate double x`name' = . /*UNCONDITIONALLY make a new variable*/
    	capture quiet replace x`name' = `value' in l
  	  if(`=_rc' != 0) continue, break /*something went wrong, probably that we couldn't make a new variable (due to memory or built-in Stata constraints). Just try the next observation*/

			local j = `j' + 1
  	}
  	*list /*DEBUG: see the state after after new observation*/

  	file read `fd' line
  }
  file close `fd'
end


redo: use double quotes in find invocations for consistency

#!/bin/bash
# automaton
# (c) 2013 Justin Gottula
# The source code of this project is distributed under the terms of the
# simplified BSD license. See the LICENSE file for details.

exec >&2

TARGET=$1
TARGET_BASE=$2
OUTPUT=$3

CC="avr-gcc"
AS="avr-as"
LD="avr-ld"
NM="avr-nm"
OBJCOPY="avr-objcopy"
OBJDUMP="avr-objdump"

PROJ_NAME="automaton"

MCU="atmega324a"
CPU_KHZ="20000"

DEFINES=("F_CPU=${CPU_KHZ}000UL")
INC_DIRS=(src)
LIB_DIRS=()
LIBS=()

CFLAGS_CPU=("-mmcu=$MCU" -fpack-struct -ffreestanding)
CFLAGS_DEBUG=(-ggdb)
CFLAGS_OPT=(-Os -flto -fuse-linker-plugin -fwhole-program -mcall-prologues \
	-mrelax -mstrict-X -fmerge-all-constants -ffast-math)
CFLAGS_WARN=(-Wall -Wextra -Waddr-space-convert -Wno-unused-function)
CFLAGS_ETC=(-fno-diagnostics-show-caret -pipe)

CFLAGS="-std=gnu11 ${CFLAGS_CPU[@]} ${CFLAGS_DEBUG[@]} ${CFLAGS_OPT[@]} \
${CFLAGS_WARN[@]} ${CFLAGS_ETC[@]}"
CFLAGS_COMPILE="$CFLAGS ${DEFINES[@]/#/-D} ${INC_DIRS[@]/#/-I}"
CFLAGS_LINK="$CFLAGS ${LIB_DIRS[@]/#/-L} ${LIBS[@]/#/-l}"

SOURCES=$(find src -type f -iname "*.c")
OBJECTS=${SOURCES//.c/.o}

OUT_ELF="out/$PROJ_NAME.elf"
OUT_BIN="out/$PROJ_NAME.bin"
OUT_HEX="out/$PROJ_NAME.hex"
OUT_MAP="out/$PROJ_NAME.map"
OUT_DUMP="out/$PROJ_NAME.dump"
OUT_LST="out/$PROJ_NAME.lst"

case "$TARGET" in
all)
	redo-ifchange $OUT_ELF $OUT_BIN $OUT_HEX $OUT_MAP $OUT_DUMP $OUT_LST
	;;
$OUT_ELF)
	redo-ifchange $OBJECTS
	$CC $CFLAGS_LINK -o $OUTPUT $OBJECTS
	;;
$OUT_BIN)
	redo-ifchange $OUT_ELF
	$OBJCOPY -O binary $OUT_ELF $OUTPUT
	stat --printf='%n: %s bytes\n' $OUTPUT
	;;
$OUT_HEX)
	redo-ifchange $OUT_ELF
	$OBJCOPY -O ihex $OUT_ELF $OUTPUT
	;;
$OUT_MAP)
	redo-ifchange $OUT_ELF
	$NM -Cn $OUT_ELF >$OUTPUT
	;;
$OUT_DUMP)
	redo-ifchange $OUT_ELF
	$OBJDUMP -aCdfhp $OUT_ELF >$OUTPUT
	;;
$OUT_LST)
	redo-ifchange $OUT_ELF
	$OBJDUMP -CdS -j '.text' $OUT_ELF >$OUTPUT
	;;
*.o)
	$CC $CFLAGS_COMPILE -o${TARGET//.o/.dep} -MM -MG ${TARGET//.o/.c}

	read DEPS <${TARGET//.o/.dep}
	redo-ifchange ${DEPS#*:}

	$CC $CFLAGS_COMPILE -o$OUTPUT -c ${TARGET//.o/.c}
	;;
clean)
	rm -rf $(find out -type f)
	rm -rf $(find src -type f -iname "*.o")
	rm -rf $(find src -type f -iname "*.dep")
	;;
load)
	redo-ifchange $OUT_HEX

	AVRDUDE_CMD="sudo avrdude -u -p ${MCU/atmega/m} -c usbasp -B 460800 \
	-U flash:w:$OUT_HEX"

	# retry once upon failure
	$($AVRDUDE_CMD)
	if [[ "$?" != "0" ]]; then
		$($AVRDUDE_CMD)
	fi
	;;
*)
	echo "unknown target '$TARGET'"
	exit 1
	;;
esac

#!/bin/bash
# automaton
# (c) 2013 Justin Gottula
# The source code of this project is distributed under the terms of the
# simplified BSD license. See the LICENSE file for details.

exec >&2

TARGET=$1
TARGET_BASE=$2
OUTPUT=$3

CC="avr-gcc"
AS="avr-as"
LD="avr-ld"
NM="avr-nm"
OBJCOPY="avr-objcopy"
OBJDUMP="avr-objdump"

PROJ_NAME="automaton"

MCU="atmega324a"
CPU_KHZ="20000"

DEFINES=("F_CPU=${CPU_KHZ}000UL")
INC_DIRS=(src)
LIB_DIRS=()
LIBS=()

CFLAGS_CPU=("-mmcu=$MCU" -fpack-struct -ffreestanding)
CFLAGS_DEBUG=(-ggdb)
CFLAGS_OPT=(-Os -flto -fuse-linker-plugin -fwhole-program -mcall-prologues \
	-mrelax -mstrict-X -fmerge-all-constants -ffast-math)
CFLAGS_WARN=(-Wall -Wextra -Waddr-space-convert -Wno-unused-function)
CFLAGS_ETC=(-fno-diagnostics-show-caret -pipe)

CFLAGS="-std=gnu11 ${CFLAGS_CPU[@]} ${CFLAGS_DEBUG[@]} ${CFLAGS_OPT[@]} \
${CFLAGS_WARN[@]} ${CFLAGS_ETC[@]}"
CFLAGS_COMPILE="$CFLAGS ${DEFINES[@]/#/-D} ${INC_DIRS[@]/#/-I}"
CFLAGS_LINK="$CFLAGS ${LIB_DIRS[@]/#/-L} ${LIBS[@]/#/-l}"

SOURCES=$(find src -type f -iname '*.c')
OBJECTS=${SOURCES//.c/.o}

OUT_ELF="out/$PROJ_NAME.elf"
OUT_BIN="out/$PROJ_NAME.bin"
OUT_HEX="out/$PROJ_NAME.hex"
OUT_MAP="out/$PROJ_NAME.map"
OUT_DUMP="out/$PROJ_NAME.dump"
OUT_LST="out/$PROJ_NAME.lst"

case "$TARGET" in
all)
	redo-ifchange $OUT_ELF $OUT_BIN $OUT_HEX $OUT_MAP $OUT_DUMP $OUT_LST
	;;
$OUT_ELF)
	redo-ifchange $OBJECTS
	$CC $CFLAGS_LINK -o $OUTPUT $OBJECTS
	;;
$OUT_BIN)
	redo-ifchange $OUT_ELF
	$OBJCOPY -O binary $OUT_ELF $OUTPUT
	stat --printf='%n: %s bytes\n' $OUTPUT
	;;
$OUT_HEX)
	redo-ifchange $OUT_ELF
	$OBJCOPY -O ihex $OUT_ELF $OUTPUT
	;;
$OUT_MAP)
	redo-ifchange $OUT_ELF
	$NM -Cn $OUT_ELF >$OUTPUT
	;;
$OUT_DUMP)
	redo-ifchange $OUT_ELF
	$OBJDUMP -aCdfhp $OUT_ELF >$OUTPUT
	;;
$OUT_LST)
	redo-ifchange $OUT_ELF
	$OBJDUMP -CdS -j '.text' $OUT_ELF >$OUTPUT
	;;
*.o)
	$CC $CFLAGS_COMPILE -o${TARGET//.o/.dep} -MM -MG ${TARGET//.o/.c}

	read DEPS <${TARGET//.o/.dep}
	redo-ifchange ${DEPS#*:}

	$CC $CFLAGS_COMPILE -o$OUTPUT -c ${TARGET//.o/.c}
	;;
clean)
	rm -rf $(find out -type f)
	rm -rf $(find src -type f -iname *.o)
	rm -rf $(find src -type f -iname *.dep)
	;;
load)
	redo-ifchange $OUT_HEX

	AVRDUDE_CMD="sudo avrdude -u -p ${MCU/atmega/m} -c usbasp -B 460800 \
	-U flash:w:$OUT_HEX"

	# retry once upon failure
	$($AVRDUDE_CMD)
	if [[ "$?" != "0" ]]; then
		$($AVRDUDE_CMD)
	fi
	;;
*)
	echo "unknown target '$TARGET'"
	exit 1
	;;
esac


options checking

options checking

	*Set standardized settings
	ieboilstart, v(11.1)
	`r(version)'

	*Create filepaths
	global ietoolkit "C:\Users\wb503680\Box Sync\DIME WOrk\GitFolder\ietoolkit"
	*global ietoolkit "C:\Users\wb462869\Box Sync\Stata\Stata work\Commands\ietoolkit"
	global testOutput "$ietoolkit/test/ieimpgraphOutput"

	*Set cd folder to catch and cd leakage
	cd "$ietoolkit\test\cdjunk"

	*Load the command
	do "$ietoolkit\iegraph.do"

	**********************
	*Graph 1 - single treatment

	*open an example data set and create a mock dummy
	sysuse auto, clear
	gen Treatment = (weight >= 3019.5)

	*Run the impact regression
	reg price  Treatment
	*Create the graph
	iegraph Treatment, noconfbars title("Treatment effect on price") save("$testOutput/Graph1.gph") yzero
	iegraph Treatment, title("Treatment effect on price") save("$testOutput/Graph1.gph") yzero

	**********************
	*Graph 2 - multiple treatment

	*open an example data set and create mutually exclusive mock dummies
	sysuse auto, clear
	tab rep78, gen(d_)

	*Run the impact regression
	reg price  d_2 d_3 d_4 d_5 weight length gear_ratio
	*Create the graph
	iegraph d_2 d_3,  title("Treatment effect on price") save("$testOutput/Graph2.gph") yzero
	iegraph d_2 d_3 d_4,  title("Treatment effect on price") save("$testOutput/Graph3.gph") yzero
	iegraph d_2 d_3 d_4 d_5,  title("Treatment effect on price") save("$testOutput/Graph4.gph") yzero note(Stata Command iegraph. copyright KBjark)
	iegraph d_2 d_3 d_4 d_5,  noconfbars title("Treatment effect on price") save("$testOutput/Graph2.gph") yzero

	*Set standardized settings
	ieboilstart, v(11.1)
	`r(version)'

	*Create filepaths
	global ietoolkit "C:\Users\wb503680\Box Sync\DIME WOrk\GitFolder\ietoolkit"
	*global ietoolkit "C:\Users\wb462869\Box Sync\Stata\Stata work\Commands\ietoolkit"
	global testOutput "$ietoolkit/test/ieimpgraphOutput"

	*Set cd folder to catch and cd leakage
	cd "$ietoolkit\test\cdjunk"

	*Load the command
	do "$ietoolkit\iegraph.do"

	**********************
	*Graph 1 - single treatment

	*open an example data set and create a mock dummy
	sysuse auto, clear
	gen Treatment = (weight >= 3019.5)

	*Run the impact regression
	reg price  Treatment
	*Create the graph
	iegraph Treatment, noconfbars title("Treatment effect on price") save("$testOutput/Graph1.gph")
	iegraph Treatment, title("Treatment effect on price") save("$testOutput/Graph1.gph")

	**********************
	*Graph 2 - multiple treatment

	*open an example data set and create mutually exclusive mock dummies
	sysuse auto, clear
	tab rep78, gen(d_)

	*Run the impact regression
	reg price  d_2 d_3 d_4 d_5 weight length gear_ratio
	*Create the graph
	iegraph d_2 d_3,  title("Treatment effect on price") save("$testOutput/Graph2.gph")
	iegraph d_2 d_3 d_4,  title("Treatment effect on price") save("$testOutput/Graph3.gph")
	iegraph d_2 d_3 d_4 d_5,  title("Treatment effect on price") save("$testOutput/Graph4.gph")
	iegraph d_2 d_3 d_4 d_5,  noconfbars title("Treatment effect on price") save("$testOutput/Graph2.gph")


update do file

update do file

 clear
 import delimited C:\Users\t\Documents\GitHub\DataVizPractice\food.court.dates.csv

ren v7 monthNum
ren v10 dayName
tab month
replace month = "May" if month == "May "
la var year "year"

* Create a time variable so Stata can read as dates
g double date_Stata = clock(timevalue, "MD20Yhm")
format date_Stata %tc
isid date_Stata
clonevar date_Month = date_Stata
format date_Month %tm

* Create an encoded day of the week variable

encode dayName, gen(dayofWeek)

* Time series set the data
tsset date_Stata
tsline visitors
gen dow = dow(mdy(monthNum, day, year))

* Calculate statistics for day of week, month, and day of month
egen monthTotal = total(visitors), by(month year)
egen monthAve = mean(visitors), by(month year)
egen dayTotal = total(visitors), by(dayName month year)
egen dayAve = mean(visitors), by(dayName month year)
egen dayMonthTotal = total(visitors), by(day month year)
egen dayMonthAve = mean(visitors), by(day month year)

egen dailyTotal = total(visitors), by(day month year)
egen dailyAve = mean(visitors) if year == 2014, by(day month)
egen dailyAve2 = mean(visitors) if year == 2015, by(day month)

tsline dailyAve
tsline dayMonthTotal
tsline dayMonthAve
reg visitors i.monthNum i.year i.dow, robust

table monthNum year, c(mean visitors sum visitors) f(%9.0fc)

table dayName  monthNum if year == 2014, c(mean visitors)

 clear
 import delimited C:\Users\t\Documents\GitHub\DataVizPractice\food.court.dates.csv

ren v7 monthNum
ren v10 dayName
tab month
replace month = "May" if month == "May "
la var year "year"

* Create a time variable so Stata can read as dates
g double date_Stata = clock(timevalue, "MD20Yhm")
format date_Stata %tc
isid date_Stata

* Time series set the data
tsset date_Stata
tsline visitors
gen dow = dow(mdy(monthNum, day, year))

* Calculate statistics for day of week, month, and day of month
egen monthTotal = total(visitors), by(month year)
egen monthAve = mean(visitors), by(month year)
egen dayTotal = total(visitors), by(dayName month year)
egen dayAve = mean(visitors), by(dayName month year)
egen dayMonthTotal = total(visitors), by(day month year)
egen dayMonthAve = mean(visitors), by(day month year)

egen dailyTotal = total(visitors), by(day month year)
egen dailyAve = mean(visitors), by(day month year)

tsline dailyAve
tsline dayMonthTotal
tsline dayMonthAve
reg visitors i.monthNum i.year i.dow, robust

table monthNum year, c(mean visitors sum visitors) f(%9.0fc)