iematch : fix to issue #112 - maxmatch() count in m1 match

*! version 5.3 14NOV2017  Kristoffer Bjarkefur kbjarkefur@worldbank.org

	cap program drop 	iematch
	    program define	iematch

		syntax  [if] [in]  , 					///
			GRPdummy(varname) 					///
			MATCHvar(varname) 					///
			[									///
			IDvar(varname)						///
			m1									///
			maxdiff(numlist >0 max = 1 min = 1) ///
			maxmatch(numlist >0 integer max = 1 min = 1) ///
			seedok						 		///
			MATCHIDname(string)					///
			MATCHDIffname(string)				///
			MATCHREsultname(string)				///
			MATCHCOuntname(string)				///
			replace								///
			]

		*****

	qui {

		* Set version
		version 11.0

		********************************
		*
		*	Gen temp sort and merge var
		*
		********************************

		tempvar originalSort ifinvar

		**Generate a variable that is used
		* to restore original sort.
		gen `originalSort' = _n

		preserve

			* String used to store why and how
			*  many observations are excluded.
			local obsexcludstring ""

			*Dummy local that is 1 if replace is used
			if "`replace'" == "" local REPLACE_USED 0
			if "`replace'" != "" local REPLACE_USED 1

			*Deal with if and in and display info.
			if "`if'`in'" != "" {

				* Gen dummy that is 1 when if/in is true. Then
				* count when this dummy is not 1. Ther is no way
				* to negate `if'`in'
				gen 	 `ifinvar'  = 1 `if'`in'
				count if `ifinvar' != 1
				if `r(N)' > 0 {

					*This is not an error just outputting the number
					local obsexcludstring "`obsexcludstring'`r(N)' observation(s) were excluded in {inp:if}/{inp:in}: (`if'`in').{break}"

				}

				*Drop variables excluded by `if'`in'
				keep `if'`in'
			}

		********************************
		*
		*	Checking ID var (and create if needed)
		*
		********************************

			*Checking set-up for the ID var used for _matchID
			if "`idvar'" == "" {

				*Copy this name
				local idvar _ID

				*Make sure that the default name _ID is not already used
				cap confirm variable `idvar'
				if _rc == 0 {

					*Test if option replace is used
					if `REPLACE_USED' == 1 {
						*Replace option is used, drop the variable with the name specified
						drop `idvar'
					}
					else {
						*Replace is not used, throw an error.
						di as error "{pstd}A variable with name `idvar' is already defined. Either drop this variable manually, use the replace option or specify a variable using idvar() that fully and uniquely identifies the data set.{p_end}"
						error 110
					}
				}

				*Generate the ID var from row number
				gen `idvar' = _n
			}

			*Make sure that idvar is uniquelly and fully identifying
			cap isid `idvar'
			if _rc != 0 {

				di as error "{pstd}Variable `idvar' specified in idvar() does not fully and uniquely identify the observations in the data set. `idvar' either has duplicates or missing values which is not allowed. Make `idvar' uniquly and fully idnetifying or exclude the varaibleas using {inp:if} or {inp:in}.{p_end}"
				error 450
			}

			*local indicating assume ID is string
			local IDtypeNumeric 0

			*Change local to 1 if ID actally string
			cap confirm numeric variable `idvar'
			if _rc == 0 local IDtypeNumeric 1

		********************************
		*
		*	Checking duplicates in variable names in options
		*
		********************************

			*Create a local of all varnames used in any option
			local allnewvars `idvar' `matchidname' `matchdiffname' `matchresultname' `matchcountname'

			*Check if there are any duplicates in the local of all varnames in options
			local dupnewvars : list dups allnewvars

			*Throw error if there were any duplicates
			if "`dupnewvars'" != "" {

				noi di as error "{pstd}The same new variable name was used twice or more in the options generating a new variable. Go back and check syntax.{p_end}"
				error 198
			}

		********************************
		*
		*	Test the names used for output vars
		*
		********************************

			iematchMatchVarCheck 	_matchResult 	`REPLACE_USED' "`matchresultname'"
			local matchResultName 	"`r(validVarName)'"

			iematchMatchVarCheck 	_matchID 		`REPLACE_USED' "`matchidname'"
			local matchIDname 		"`r(validVarName)'"

			iematchMatchVarCheck 	_matchDiff 		`REPLACE_USED' "`matchdiffname'"
			local matchDiffName 	"`r(validVarName)'"

			if "`m1'" != "" {
			   iematchMatchVarCheck _matchCount 	`REPLACE_USED' "`matchcountname'"
			   local matchCountName "`r(validVarName)'"
			}

			*Option matchcountname() is not allowed if it is not a many-one match
			if "`matchcountname'" != "" & "`m1'" == "" {
				di as error "{pstd}Option {inp:matchcountname()} is only allowed in combination with option {inp:m1}.{p_end}"
				error 198
			}

		********************************
		*
		*	Create and label output vars
		*
		********************************

			* MATCH RESULT VAR
			label define 	 matchLabel 0 "Not Matched" 1 "Matched" .i "Obs excluded due to if/in" 			///
										.g "Missing value in `grpdummy'" .m "Missing value in `matchvar'" 	///
										.d "No match within maxdiff" .t "No more eligible target obs", replace

			gen 			`matchResultName' = .
			label variable	`matchResultName' "Matched obs = 1, not mathched = 0, all other different missing values"
			label value 	`matchResultName' matchLabel

			* MATCH ID VAR
			if `IDtypeNumeric' == 1 gen  `matchIDname' = .	//Main ID var is numeric
			if `IDtypeNumeric' == 0 gen  `matchIDname' = "" //Main ID var is string

			if "`m1'" != "" label variable	`matchIDname' "The ID of the target var in each matched group"	//If many to one
			if "`m1'" == "" label variable	`matchIDname' "The ID of the target var in each matched pair"	//If one to one

			* MATCH DIFF VAR
			gen 	`matchDiffName' = .
			if "`m1'" != "" label variable	`matchDiffName' "The difference in matchvar() between base obs and the target obs it matched with" 	//If many to one
			if "`m1'" == "" label variable	`matchDiffName' "The difference in matchvar() between base and target obs in each pair" 			//If one to one

			*Match count, only many-one
			if "`m1'" != "" {
				gen 			`matchCountName' = .
				label variable	`matchCountName' "The number of base obs this target obs matched with"
			}

		********************************
		*
		*	Keep only relevant vars
		*
		********************************

			*Keep only input vars and output vars
			keep  `grpdummy' `idvar' `matchvar'	`originalSort' `matchResultName' `matchIDname' `matchDiffName' `matchCountName'

		********************************
		*
		*	Checking group dummy
		*
		********************************

			*Test that the group dummy is 1, 0 or missing
			cap assert inlist(`grpdummy', 1, 0, .) | `grpdummy' > .
			if _rc != 0 {

				di as error "{pstd}The variable in grpdummy(`grpdummy') is not a dummy variable. The variable is only allowed to have the values 1, 0 or missing. Observations with missing varaibles in the grpdummy are ignored by this command.{p_end}"
				error _rc
			}

			**********
			**Exclude obs with missing value in groupdummy

			*Count number of obs to be dropped and and output that number if more than zero
			count if missing(`grpdummy')
			if `r(N)' > 0 {

				*Prepare the local to be outputted with info on observations excluded
				local obsexcludstring  "`obsexcludstring'`r(N)' observation(s) were excluded due to missing value in  grpdummy(`grpdummy').{break}"
			}

			*Drop obs with missing value in groupvar
			drop if missing(`grpdummy')

		********************************
		*
		*	Checking match continous var
		*
		********************************

			*Test that the continous is numeric
			cap confirm numeric variable `matchvar'
			if _rc != 0 {

				noi di as error "{pstd}The variable to match on specified in matchvar(`matchvar') is not a numeric variable.{p_end}"
				error 109
			}

			**All variables with value 1 or 0 in the group dummy and
			* not excluded by if/in must have a value in matchvar
			count if missing(`matchvar')
			if `r(N)' > 0 {

				*Prepare the local to be outputted with info on observations excluded
				local obsexcludstring  "`obsexcludstring'`r(N)' observation(s) were excluded due to missing value in matchvar(`matchvar').{break}"
			}

			*Drop obs with missing values in match var
			drop if missing(`matchvar')

		********************************
		*
		*	Checking match var and group dummy are unique
		*
		********************************

			if "`seedok'" == "" {

				**test that there are no duplicates in match
				* var within each type of observation
				cap isid `matchvar' `grpdummy'

				if _rc != 0 {

					*Output the error message
					noi di as error "{pstd}There are base observations or target observations {...}"
					noi di as error "with duplicate values in matchvar(`matchvar'). To guarantee {...}"
					noi di as error "a replicable match you must {help seed:set a seed}. To supress {...}"
					noi di as error "this error message after you have set a the seed, or if a {...}"
					noi di as error "replicable match is not important to you, use option {inp:seedok}{p_end}"
					error 198

				}
			}

		********************************
		*
		*	Checking that there are more target
		*	obs than base obs in a 1-to-1 match
		*
		********************************

			if "`m1'" == "" {

				*Count number of base vars
				count if `grpdummy' == 1
				local numBaseObs `r(N)'

				*Count number of target vars
				count if `grpdummy' == 0
				local numTrgtObs `r(N)'

				*Test that there are more target
				cap assert `numTrgtObs' >= `numBaseObs'

				if _rc != 0 {

					noi di as error "{pstd}There are more base observations than target observations. This is not allowed in a one-to-one match. See option {inp:m1} for an alternative matching where it is allowed to have more base observations than target ovbservations.{p_end}"
					error _rc
				}
			}

		********************************
		*
		*	Checking that matchCount is only used with
		*	many-to-one matching
		*
		********************************

		if "`m1'" == "" & "`maxmatch'" != "" {

			noi di as error "{pstd}The option {inp:maxmatch()} can only be used when option {inp:m1} is used, as restricting to a maximum number of matches is only applicable in a many-to-one match.{p_end}"
			error 198
		}

		********************************
		*
		*	Output exclude string
		*
		********************************

			*The exclude string is created above but only displayed
			*after all testing of input is done.

			if "`obsexcludstring'" != "" {

				noi di ""
				noi di "{hline}"
				noi di ""
				noi di "{pstd}{ul:Observations are excluded from the matching for the following reasons:}{break}"
				noi di "`obsexcludstring'{p_end}"

			}

	********************************
	*	Start matching
	********************************

		********************************
		*
		*	Creating tempvar used in matching
		*
		********************************

		*Initiate the temporary variables used by this command
		tempvar prefID prefDiff matched matchcount maxmatchprefid

		gen `prefDiff' 		= .
		gen byte `matched' 	= 0

		*Allow the ID var used to be string
		if `IDtypeNumeric' == 1 {
			gen  `prefID' 			= .
			gen  `maxmatchprefid' 	= .
		}
		else {
			gen  `prefID' 			= ""
			gen  `maxmatchprefid' 	= ""
		}

		*Gen a variable that indicates for target vars if the max match is reached
		gen `matchcount' 	= .

		** Generate the inverse of the matchvar to sort descending (gsort is too slow),
		*  a random var to seperate two values with the same match var, and the inverse
		*  of the random var for when sorting descending.
		tempvar rand invsort invrand

		sort `matchvar'
		gen  `invsort' = -1 * `matchvar'

		gen    `rand' = uniform()
		gen `invrand' = -1 * `rand'

		***********
		*Tempvars for matching

		*Diffvars, they are always numeric
		tempvar					 diffup   diffdo   valUp_0   valDo_0   valUp_1   valDo_1
		local 	 updownTempVars `diffup' `diffdo' `valUp_0' `valDo_0' `valUp_1' `valDo_1'

		foreach tempVar of local updownTempVars {

			gen `tempVar' = .
		}

		*ID vars, allowed to be both numeric and string
		tempvar					  	IDup   IDdo   IDup_0   IDdo_0   IDup_1   IDdo_1
		local 	 updownIDTempVars  `IDup' `IDdo' `IDup_0' `IDdo_0' `IDup_1' `IDdo_1'

		foreach tempVar of local updownIDTempVars {

			*Allow the ID var used to be string
			if `IDtypeNumeric' == 1 {
				gen  `tempVar' = .
			}
			else {
				gen  `tempVar' = ""
			}
		}

		***************************
		*
		*	One to one match
		*
		***************************

		if "`m1'" == "" {

			*Start outputting the countdown
			noi di ""
			noi di "{hline}{break}"
			noi di "{pstd}{ul:Matching one-to-one. Base observations left to match:}{p_end}"

			*Create local to display "obs left to match" and to use in while loop
			count if `grpdummy' == 1 & `matched' == 0
			local left2Match = `r(N)'
			noi di "{pstd}`left2Match' " _c

			*Match until no more observations to match.
			while (`left2Match' > 0) {

				**For all observations still to be matched, assign the preferred
				* match among the other unmatched observations
				qui updatePrefDiffPreffID `prefID' `prefDiff' `matchvar' `invsort' `idvar' `grpdummy' `matched' `rand' `invrand' `updownTempVars' `updownIDTempVars'

				*Restrict matches to within maxdiff() if that option is used.
				if "`maxdiff'" != "" {

					*Omit base observation from matching if diff is to big
					replace `matched' 			= 1		if `prefDiff' > `maxdiff' & `grpdummy' == 1

					*Indicate in result var that this obs did not have valid match within maxdiff()
					replace `matchResultName' 	= .d 	if `prefDiff' > `maxdiff' & `grpdummy' == 1

					*Removed preferred match
					if `IDtypeNumeric' == 1 {
						*IDvar is numeric
						replace `prefID'		= .		if `prefDiff' > `maxdiff'
					}
					else {
						*IDvar is string
						replace `prefID'		= ""	if `prefDiff' > `maxdiff'
					}
				}

				*If two observations mutually prefer each other, then indicate both of them as matched.
				replace `matched' = 1 if `matched' == 0 & `prefID' == `idvar'[_n-1] & `prefID'[_n-1] == `idvar'
				replace `matched' = 1 if `matched' == 0 & `prefID' == `idvar'[_n+1] & `prefID'[_n+1] == `idvar'

				*Update local to display "obs left to match" and to use in while loop
				count if `grpdummy' == 1 & `matched' == 0
				local left2Match = `r(N)'
				noi di "`left2Match' " _c

			}

			*End formatting for the "base obs left to match"
			noi di "{p_end}" _c
		}
		***************************
		*
		*	Many to one match with no restriction on # matches
		*
		***************************
		else if "`maxmatch'" == ""  {

			*Start outputting the countdown
			noi di ""
			noi di "{hline}{break}"
			noi di "{pstd}{ul:Matching many-to-one.}{p_end}"

			**For all observations to be matched, assign the preferred
			* match among the other unmatched observations
			updatePrefDiffPreffID `prefID' `prefDiff' `matchvar' `invsort' `idvar' `grpdummy' `matched' `rand' `invrand' `updownTempVars' `updownIDTempVars'

			*Restrict matches to within maxdiff() if that option is used.
			if "`maxdiff'" != "" {

				*Indicate in result var that this obs did not have valid match within maxdiff()
				replace `matchResultName' 	= .d 	if `prefDiff' > `maxdiff' & `grpdummy' == 1

				*Removed preferred match
				if `IDtypeNumeric' == 1 {
					*IDvar is numeric
					replace `prefID'		= .		if `prefDiff' > `maxdiff'
				}
				else {
					*IDvar is string
					replace `prefID'		= ""	if `prefDiff' > `maxdiff'
				}
			}

			*Assign it's own ID as pref ID for all target vars
			replace `prefID'			=  `idvar' if `grpdummy' == 0

			* Replace the _matchCount var with number of base observations in each
			* match group. Each group is all base observation plus the target
			* observation, therefore (_N - 1)
			bys 	`prefID' 			:   replace `matchCountName' = _N - 1 if !missing(`prefID')

			**Replace prefID to missing for target obs that had no base
			* obs matched to it. T
			if `IDtypeNumeric' == 1 {
				*IDvar is numeric
				replace `prefID'		= .		if `matchCountName' == 0
			}
			else {
				*IDvar is string
				replace `prefID'		= ""	if `matchCountName' == 0
			}

			*Only target obs with base obs prefering it are matched
			replace	 `matched' 			= 1		if `matchCountName' != 0

			*Remove values for target obs that were not matched
			replace  `matchCountName'	= . 	if `matchCountName' == 0

		}
		***************************
		*
		*	Many to one match with restriction on # matches
		*
		***************************
		else {

			noi di "max count"
			*pause

			*Start outputting the countdown
			noi di ""
			noi di "{hline}{break}"
			noi di "{pstd}{ul:Matching many-to-one. Base observations left to match:}{p_end}"

			*Create local to display "obs left to match" and to use in while loop
			count if `grpdummy' == 1 & `matched' == 0
			local left2Match = `r(N)'
			noi di "{pstd}`left2Match' " _c

			*Match until no more observations to match.
			while (`left2Match' > 0) {

				**For all observations to be matched, assign the preferred
				* match among the other unmatched observations
				updatePrefDiffPreffID `prefID' `prefDiff' `matchvar' `invsort' `idvar' `grpdummy' `matched' `rand' `invrand' `updownTempVars' `updownIDTempVars'

				*Restrict matches to within maxdiff() if that option is used.
				if "`maxdiff'" != "" {

					*Omit base observation from matching if diff is to big
					replace `matched' 			= 1		if `prefDiff' > `maxdiff' & `grpdummy' == 1

					*Indicate in result var that this obs did not have valid match within maxdiff()
					replace `matchResultName' 	= .d 	if `prefDiff' > `maxdiff' & `grpdummy' == 1

					*Removed preferred match
					if `IDtypeNumeric' == 1 {
						*IDvar is numeric
						replace `prefID'		= .		if `prefDiff' > `maxdiff'
					}
					else {
						*IDvar is string
						replace `prefID'		= ""	if `prefDiff' > `maxdiff'
					}
				}

				*If a base observation is mutually preferred by a target observation
				replace `matched' = 1 if `grpdummy' == 1 & `matched' == 0 & `prefID' == `idvar'[_n-1] & `prefID'[_n-1] == `idvar'
				replace `matched' = 1 if `grpdummy' == 1 & `matched' == 0 & `prefID' == `idvar'[_n+1] & `prefID'[_n+1] == `idvar'

				*Set maxmatchprefid to the matched id for matched base obs, and its own id for target obs
				replace `maxmatchprefid' = `prefID' if `grpdummy' == 1 & `matched' == 1
				replace `maxmatchprefid' = `idvar'  if `grpdummy' == 0

				*By the maxmatchprefid, count how many base observations (all obs
				* minus target) that are matched to this target obs
				bys 	`maxmatchprefid' : replace `matchcount' = _N - 1 if  `matchResultName' 	!= .d

				*Set the target obs as matched if the max match count is reached (matching one at the time so no risk of overstepping)
				replace `matched' 	= 1  if `grpdummy' == 0 & `matchcount' == (`maxmatch') & `matchResultName' != .d

				*Update local to display "obs left to match" and to use in while loop
				count if `grpdummy' == 1 & `matched' == 0
				local left2Match = `r(N)'
				noi di "`left2Match' " _c

				*Test that there are still target obs left to match against
				count if `grpdummy' == 0 & `matched' == 0
				if `r(N)' == 0 {

					*set left to match to -1 to exit while loop and to output message after loop
					local left2Match = -1

					*Set all unmatched base obs to .t (no more eligible target obs)
					replace `matchResultName' 	= .t 	if `grpdummy' == 1 & `matched' == 0
				}

			}

			*End the outputted count down on how many observations left to match
			noi di "{p_end}" _c

			*If ran out of target out put that
			if `left2Match' == -1 {

				noi di ""
				noi di ""
				noi di "{pstd}No more target obs to match{p_end}"
			}

			*Set all target obs with at least 1 matched base obs as matched
			replace `matched' 	= 1 if `grpdummy' == 0 & `matchcount' >= 1 & `matchResultName' != .d

			*Remove match diff for target obs (does not make sense with mutliple obs)
			replace `prefDiff' 	= . if `grpdummy' == 0

			*Set the match count output var to the number of matched base obs
			replace `matchCountName' = `matchcount' if  `matched' 	== 1 & `matchResultName' != .d

		}

		*Update all return vars
		replace `matchDiffName' 	= `prefDiff'	if `matched' == 1
		replace `matchIDname' 		= `idvar' 		if `matched' == 1 & `grpdummy' == 0
		replace `matchIDname' 		= `prefID' 		if `matched' == 1 & `grpdummy' == 1

		*Remove the best match value in obs that did not have a match within maxdiff()
		replace `matchDiffName'  = . 				if `matchResultName' == .d

		*Matched observations are give value 1 in result var
		replace `matchResultName' = 1 				if `matched' == 1 & `matchResultName' != .d

		*Target obs not used
		replace `matchResultName' = 0 if `matched' == 0 & `grpdummy' == 0

		*only keep output vars
		keep `matchIDname' `matchDiffName' `originalSort' `matchResultName' `matchCountName'

		*Merge the results back to the original data set
		tempfile mergefile
		save 	`mergefile'

	restore

	***************************
	*
	*	Merge match results to
	*	original data and assign
	*	remaining missing values
	*	to the result var
	*
	***************************

	*Merge the results with the original data
	tempvar mergevar
	merge 1:1  `originalSort' using `mergefile', gen(`mergevar')

	*remaining missing values are listed in ascending order of importance.
	*Meaning that if a variable is both .m and .i then it will be .i as it
	*is assigned afterwards below.

	*Missing matching var
	replace `matchResultName' = .m if missing(`matchvar')

	*Msising dummy var
	replace `matchResultName' = .g if missing(`grpdummy')

	*Excluded in if/in
	if "`if'`in'" != "" {

		tempvar  ifinvar
		gen 	`ifinvar' = 1 `if'`in'
		replace `matchResultName' = .i if `ifinvar' != 1
	}

	*comress the variables generated
	compress  `matchIDname' `matchDiffName' `matchResultName' `matchCountName'

	*Output result table
	noi outputTable `matchResultName' `grpdummy'

	*Restore the oridinal sort
	sort `originalSort'

}

end

	*Check manually entered names for the return vars.
	cap program drop 	iematchMatchVarCheck
		program define 	iematchMatchVarCheck  , rclass

		args defaultName replace_used userName

		if "`defaultName'" == "_matchID" 		local optionName matchidname
		if "`defaultName'" == "_matchDiff" 		local optionName matchdiffname
		if "`defaultName'" == "_matchResult" 	local optionName matchresult
		if "`defaultName'" == "_matchCount" 	local optionName matchcount

		*All the default names
		local dfltNms _noMatchReason _matchDiff _matchResult _matchCount

		*The other two default names
		local othDfltNms : list dfltNms - defaultName

		*Creating two locals with the deafult name in each local
		local Oth1 : word 1 of `othDfltNms'
		local Oth2 : word 2 of `othDfltNms'
		local Oth3 : word 3 of `othDfltNms'

		*Testing that the option name is not the same as the other variable's default name.
		if ("`userName'" == "`Oth1'" | "`userName'" == "`Oth2'" | "`userName'" == "`Oth3'" | "`userName'" == "_ID")  {

			noi di as error "{pstd}The new name specified in `optionName'(`userName') is not allowed to be _ID, `Oth1', `Oth2', or `Oth3'{p_end}"
			error 198
		}

		*Test if name is set manually
		if "`userName'" != "" {
			*Use the manually set name
			local validMatchVarname `userName'
		}
		else {
			*No manually set name, use deafult name
			local validMatchVarname `defaultName'
		}

		*Make sure that the manually entered name is not already used
		cap confirm variable `validMatchVarname'
		if _rc == 0 {

			if `replace_used' == 1 {
				//Replace is used, drop the old var
				drop `validMatchVarname'
			}
			else {

				*Replace is not used. Prepare an error message and throw error

				if "`userName'" != "" {
					*Error message for user specified name s
					local nameErrorString "The variable name specified in `optionName'(`userName')"
				}
				else {
					*Error message for user specified name s
					local nameErrorString "A variable with name `validMatchVarname'"
				}

				*Trow error
				noi di as error "{pstd}`nameErrorString' is already defined in the data set. Either drop this variable, use the replace option or specify a another variable name using `optionName'()."
				error 110

			}
		}

		*Testing that the new name is valid by creating a variable that is dropped immedeatly afterwards
		cap gen `validMatchVarname' = .

		if _rc != 0 {

			noi di as error "{pstd}The variable name specified in `optionName'(`userName') is not a valid variable name."
			error _rc
		}
		drop `validMatchVarname'

		return local validVarName 	"`validMatchVarname'"

	end

	** This function is called to assign the preferred match for all unmatched
	*  observations to any other unmatched observation. Returns a variable pref that is the
	cap program drop 	updatePrefDiffPreffID
		program define 	updatePrefDiffPreffID

		args prefID prefDiff matchvar invsort idvar grpvar matched rand invrand diffup diffdo valUp_0 valDo_0 valUp_1 valDo_1 IDup IDdo IDup_0 IDdo_0 IDup_1 IDdo_1

		*****************
		*Update possible matches with higher rank value
		sort `matched' `matchvar' `rand'

		updateBestValID 0 `matchvar' `idvar' `grpvar' `matched' `valUp_0' `IDup_0'
		updateBestValID 1 `matchvar' `idvar' `grpvar' `matched' `valUp_1' `IDup_1'

		*****************
		*Update possible matches with lower rank value
		sort `matched' `invsort' `invrand'

		updateBestValID 0 `matchvar' `idvar' `grpvar' `matched' `valDo_0' `IDdo_0'
		updateBestValID 1 `matchvar' `idvar' `grpvar' `matched' `valDo_1' `IDdo_1'

		*****************
		*Create one varaible common for both groups with diff upwards
		replace `diffup' 	= abs(`matchvar' - `valUp_1') if `grpvar' == 1
		replace `diffup' 	= abs(`matchvar' - `valUp_0') if `grpvar' == 0

		*Create one varaible common for both groups with diff downwards
		replace `diffdo' 	= abs(`matchvar' - `valDo_1') if `grpvar' == 1
		replace `diffdo' 	= abs(`matchvar' - `valDo_0') if `grpvar' == 0

		*Create one varaible common for both groups with ID upwards
		replace `IDup' 		= `IDup_1' 	if `grpvar' == 1
		replace `IDup' 		= `IDup_0' 	if `grpvar' == 0

		*Create one varaible common for both groups with ID downwards
		replace `IDdo' 		= `IDdo_1' 	if `grpvar' == 1
		replace `IDdo' 		= `IDdo_0' 	if `grpvar' == 0

		replace `prefID' 	= `IDdo' 	if `matched' == 0 & `diffdo' <= `diffup'
		replace `prefID' 	= `IDup' 	if `matched' == 0 & `diffdo' >  `diffup'

		replace `prefDiff' 	= `diffdo' 	if `matched' == 0 & `diffdo' <= `diffup'
		replace `prefDiff' 	= `diffup' 	if `matched' == 0 & `diffdo' >  `diffup'

	end

	** This function goes through the observatioins from top to down and copying
	*  the match value of the closest observation of the other group above to
	*  (bestVal) and the ccoprresponding ID to (bestID) for obe group at the time.
	*  To go from bottuom to top sort with invsort and apply this command.
	cap program drop 	updateBestValID
		program define 	updateBestValID

		args grpvl matchval idvar grpvar matched bestVal bestID

		*Test if ID var is string or numeric
		local IDNumeric 0
		cap confirm numeric variable `idvar'
		if _rc == 0 local IDNumeric 1

		*Reset all values
		replace `bestVal' 	= .
		if `IDNumeric' == 1 replace `bestID' 	= .
		if `IDNumeric' == 0 replace `bestID' 	= ""

		*Set the match value and ID of observations that are in the other group
		replace `bestVal'   = `matchval' 		if `grpvar' != `grpvl' & `matched' == 0
		replace `bestID'  	= `idvar' 			if `grpvar' != `grpvl' & `matched' == 0

		*Fill in that value from the observation in the other group until getting to another observation of the other group
							replace `bestVal'   = `bestVal'[_n-1]	if `bestVal' == .  & `matched' == 0
		if `IDNumeric' == 1 replace `bestID'  	= `bestID'[_n-1] 	if `bestID'	 == .  & `matched' == 0
		if `IDNumeric' == 0 replace `bestID'  	= `bestID'[_n-1] 	if `bestID'	 == "" & `matched' == 0

	end

	** This function creates the output table.
	cap program drop 	outputTable
		program define 	outputTable

		qui {
			args resultVar grpdum

			*List the texts used in the first column in the
			local text_1 " 1 Matched"
			local text_0 " 0 Not matched"
			local text_d ".d No match within maxdiff()"
			local text_i ".i Excluded using if/in"
			local text_g ".g Missing grpdummy()"
			local text_m ".m Missing matchvar()"
			local text_t ".t No more target obs"

			*The minimum width of the
			local firstColWidth = max(strlen(" `resultVar'") , strlen(" value and result"))

			levelsof `resultVar' , missing local(resultsUsed)

			foreach result of local resultsUsed {

				*Remove the . from the local to match the string
				local result = subinstr("`result'",".","",1)

				* Test if the text in the first column is longer
				* for this row than previous row
				local firstColWidth = max(`firstColWidth' , strlen("`text_`result''"))
			}

			* Create a local that indicates if there is column
			* for obs with missing value in tmt dummy
			local missingGrpDum 0
			count if missing(`grpdum')
			if `r(N)' > 0 local missingGrpDum 1

			*Set all locals that determins column width

			local C1 "{col 4}{c |}"

			di "`firstColWidth'"

			local col1width = `firstColWidth'  + 1
			local hli1 "{hline `col1width'}"
			local cen1 "{center `col1width':"

			local col2border = `firstColWidth' + 6
			local C2  "{col `col2border'}{c |}"
			local C2a "{col `col2border'}"

			local col3border = `firstColWidth' + 17
			local C3 "{col `col3border'}{c |}"

			local col4border = `firstColWidth' + 30
			local C4 "{col `col4border'}{c |}"

			local grpdumCentre 23 //Overspills automatically

			local lastT  	"{c RT}"
			local lastTdown	"{c RT}"
			local lastC  	"{c BRC}"

			if `missingGrpDum' {

				local lastT 		"{c +}{hline 9}{c RT}"
				local lastTdown 	"{c BT}{hline 9}{c RT}"
				local lastC 		"{c BT}{hline 9}{c BRC}"
				local missTitle 	"  missing "
				local grpdumCentre 	33

			}

			noi di ""
			noi di ""
			noi di "{hline}"
			noi di ""
			noi di "{pstd}{ul:Output of Matching Result:}{p_end}"
			noi di ""
			noi di "{col 5}`cen1'`resultVar'}`C2a' {centre `grpdumCentre':`grpdum' }"
			noi di "{col 4}{c LT}`hli1'{c +}{hline `grpdumCentre'}{c RT}"
			noi di "`C1'`cen1'value and result}`C2' 1 (base)   0 (target) `missTitle'{c |}"
			noi di "{col 4}{c LT}`hli1'{c +}{hline 10}{c +}{hline 12}`lastT'"

			*This is the roworder for the output table
			local resultTableOrder 1 0 .d .t .i .g .m

			foreach result of local resultTableOrder {

				*Test if this result was used
				if `:list result in resultsUsed' {

					*Count base observations in this result type
					qui count if `resultVar' == `result' & `grpdum' == 1
					local numBase = trim("`: display %16.0gc  `r(N)''")

					*Count target observations in this result type
					qui count if `resultVar' == `result' & `grpdum' == 0
					local numTarg = trim("`: display %16.0gc  `r(N)''")

					*Count observations that are neither base or target in this result type
					if `missingGrpDum' {

						qui count if `resultVar' == `result' & missing(`grpdum')
						local numMiss = trim("`: display %16.0gc  `r(N)''")

						local missCol "{ralign 9 :`numMiss' }{c |}"

					}

					*Prepare the local that displays the columns with numbers
					local numCols "`C2'{ralign 10 :`numBase' }`C3'{ralign 12 :`numTarg' }`C4'`missCol'"

					*Remove the . from the local to match the string
					local result = subinstr("`result'",".","",1)

					*Output the table row
					noi di "`C1'`text_`result''`numCols'"

				}
			}

			*Output line before N per group row
			noi di "{col 4}{c LT}`hli1'{c +}{hline 10}{c +}{hline 12}`lastT'"

			*Caculate N per group row for base observations
			qui count if `grpdum' == 1
			local numBase = trim("`: display %16.0gc  `r(N)''")

			*Caculate N per group row for target observations
			qui count if `grpdum' == 0
			local numTarg = trim("`: display %16.0gc  `r(N)''")

			*Caculate N per group row for observations neither base or target
			if `missingGrpDum' {

				qui count if missing(`grpdum')
				local numMiss = trim("`: display %16.0gc  `r(N)''")

				local missCol "{ralign 9 :`numMiss' }{c |}"

			}

			*Output N per group row
			noi di "`C1'`cen1'N per group}`C2'{ralign 10 :`numBase' }`C3'{ralign 12 :`numTarg' }`C4'`missCol'"

			*Output line before total N row
			noi di "{col 4}{c LT}`hli1'{c +}{hline 10}{c BT}{hline 12}`lastTdown'"

			*Count total N
			qui count
			local numTot = trim("`: display %16.0gc  `r(N)''")

			*Output total N row
			noi di "`C1'`cen1'Total N}`C2'{centre `grpdumCentre':`numTot'}`C4'"

			*Output bottom line
			noi di "{col 4}{c BLC}`hli1'{c BT}{hline `grpdumCentre'}{c BRC}"
		}
	end

*! version 5.3 14NOV2017  Kristoffer Bjarkefur kbjarkefur@worldbank.org

	cap program drop 	iematch
	    program define	iematch

		syntax  [if] [in]  , 					///
			GRPdummy(varname) 					///
			MATCHvar(varname) 					///
			[									///
			IDvar(varname)						///
			m1									///
			maxdiff(numlist max = 1 min = 1) 	///
			seedok						 		///
			MATCHIDname(string)					///
			MATCHDIffname(string)				///
			MATCHREsultname(string)				///
			MATCHCOuntname(string)				///
			replace								///
			]

		*****

	qui {

		* Set version
		version 11.0

		********************************
		*
		*	Gen temp sort and merge var
		*
		********************************

		tempvar originalSort ifinvar

		**Generate a variable that is used
		* to restore original sort.
		gen `originalSort' = _n

		preserve

			* String used to store why and how
			*  many observations are excluded.
			local obsexcludstring ""

			*Dummy local that is 1 if replace is used
			if "`replace'" == "" local REPLACE_USED 0
			if "`replace'" != "" local REPLACE_USED 1

			*Deal with if and in and display info.
			if "`if'`in'" != "" {

				* Gen dummy that is 1 when if/in is true. Then
				* count when this dummy is not 1. Ther is no way
				* to negate `if'`in'
				gen 	 `ifinvar'  = 1 `if'`in'
				count if `ifinvar' != 1
				if `r(N)' > 0 {

					*This is not an error just outputting the number
					local obsexcludstring "`obsexcludstring'`r(N)' observation(s) were excluded in {inp:if}/{inp:in}: (`if'`in').{break}"

				}

				*Drop variables excluded by `if'`in'
				keep `if'`in'
			}

		********************************
		*
		*	Checking ID var (and create if needed)
		*
		********************************

			*Checking set-up for the ID var used for _matchID
			if "`idvar'" == "" {

				*Copy this name
				local idvar _ID

				*Make sure that the default name _ID is not already used
				cap confirm variable `idvar'
				if _rc == 0 {

					*Test if option replace is used
					if `REPLACE_USED' == 1 {
						*Replace option is used, drop the variable with the name specified
						drop `idvar'
					}
					else {
						*Replace is not used, throw an error.
						di as error "{pstd}A variable with name `idvar' is already defined. Either drop this variable manually, use the replace option or specify a variable using idvar() that fully and uniquely identifies the data set.{p_end}"
						error 110
					}
				}

				*Generate the ID var from row number
				gen `idvar' = _n
			}

			*Make sure that idvar is uniquelly and fully identifying
			cap isid `idvar'
			if _rc != 0 {

				di as error "{pstd}Variable `idvar' specified in idvar() does not fully and uniquely identify the observations in the data set. `idvar' either has duplicates or missing values which is not allowed. Make `idvar' uniquly and fully idnetifying or exclude the varaibleas using {inp:if} or {inp:in}.{p_end}"
				error 450
			}

			*local indicating assume ID is string
			local IDtypeNumeric 0

			*Change local to 1 if ID actally string
			cap confirm numeric variable `idvar'
			if _rc == 0 local IDtypeNumeric 1

		********************************
		*
		*	Checking duplicates in variable names in options
		*
		********************************

			*Create a local of all varnames used in any option
			local allnewvars `idvar' `matchidname' `matchdiffname' `matchresultname' `matchcountname'

			*Check if there are any duplicates in the local of all varnames in options
			local dupnewvars : list dups allnewvars

			*Throw error if there were any duplicates
			if "`dupnewvars'" != "" {

				noi di as error "{pstd}The same new variable name was used twice or more in the options generating a new variable. Go back and check syntax.{p_end}"
				error 198
			}

		********************************
		*
		*	Test the names used for output vars
		*
		********************************

			iematchMatchVarCheck 	_matchResult 	`REPLACE_USED' "`matchresultname'"
			local matchResultName 	"`r(validVarName)'"

			iematchMatchVarCheck 	_matchID 		`REPLACE_USED' "`matchidname'"
			local matchIDname 		"`r(validVarName)'"

			iematchMatchVarCheck 	_matchDiff 		`REPLACE_USED' "`matchdiffname'"
			local matchDiffName 	"`r(validVarName)'"

			if "`m1'" != "" {
			   iematchMatchVarCheck _matchCount 	`REPLACE_USED' "`matchcountname'"
			   local matchCountName "`r(validVarName)'"
			}

			*Option matchcountname() is not allowed if it is not a many-one match
			if "`matchcountname'" != "" & "`m1'" == "" {
				di as error "{pstd}Option {inp:matchcountname()} is only allowed in combination with option {inp:m1}.{p_end}"
				error 198
			}

		********************************
		*
		*	Create and label output vars
		*
		********************************

			* MATCH RESULT VAR
			label define 	 matchLabel 0 "Not Matched" 1 "Matched" .i "Obs excluded due to if/in" 			///
										.g "Missing value in `grpdummy'" .m "Missing value in `matchvar'" 	///
										.d "No match within maxdiff", replace

			gen 			`matchResultName' = .
			label variable	`matchResultName' "Matched obs = 1, not mathched = 0, all other different missing values"
			label value 	`matchResultName' matchLabel

			* MATCH ID VAR
			if `IDtypeNumeric' == 1 gen  `matchIDname' = .	//Main ID var is numeric
			if `IDtypeNumeric' == 0 gen  `matchIDname' = "" //Main ID var is string

			if "`m1'" != "" label variable	`matchIDname' "The ID of the target var in each matched group"	//If many to one
			if "`m1'" == "" label variable	`matchIDname' "The ID of the target var in each matched pair"	//If one to one

			* MATCH DIFF VAR
			gen 	`matchDiffName' = .
			if "`m1'" != "" label variable	`matchDiffName' "The difference in matchvar() between base obs and the target obs it matched with" 	//If many to one
			if "`m1'" == "" label variable	`matchDiffName' "The difference in matchvar() between base and target obs in each pair" 			//If one to one

			*Match count, only many-one
			if "`m1'" != "" {
				gen 			`matchCountName' = .
				label variable	`matchCountName' "The number of base obs this target obs matched with"
			}

		********************************
		*
		*	Keep only relevant vars
		*
		********************************

			*Keep only input vars and output vars
			keep  `grpdummy' `idvar' `matchvar'	`originalSort' `matchResultName' `matchIDname' `matchDiffName' `matchCountName'

		********************************
		*
		*	Checking group dummy
		*
		********************************

			*Test that the group dummy is 1, 0 or missing
			cap assert inlist(`grpdummy', 1, 0, .) | `grpdummy' > .
			if _rc != 0 {

				di as error "{pstd}The variable in grpdummy(`grpdummy') is not a dummy variable. The variable is only allowed to have the values 1, 0 or missing. Observations with missing varaibles in the grpdummy are ignored by this command.{p_end}"
				error _rc
			}

			**********
			**Exclude obs with missing value in groupdummy

			*Count number of obs to be dropped and and output that number if more than zero
			count if missing(`grpdummy')
			if `r(N)' > 0 {

				*Prepare the local to be outputted with info on observations excluded
				local obsexcludstring  "`obsexcludstring'`r(N)' observation(s) were excluded due to missing value in  grpdummy(`grpdummy').{break}"
			}

			*Drop obs with missing value in groupvar
			drop if missing(`grpdummy')

		********************************
		*
		*	Checking match continous var
		*
		********************************

			*Test that the continous is numeric
			cap confirm numeric variable `matchvar'
			if _rc != 0 {

				noi di as error "{pstd}The variable to match on specified in matchvar(`matchvar') is not a numeric variable.{p_end}"
				error 109
			}

			**All variables with value 1 or 0 in the group dummy and
			* not excluded by if/in must have a value in matchvar
			count if missing(`matchvar')
			if `r(N)' > 0 {

				*Prepare the local to be outputted with info on observations excluded
				local obsexcludstring  "`obsexcludstring'`r(N)' observation(s) were excluded due to missing value in matchvar(`matchvar').{break}"
			}

			*Drop obs with missing values in match var
			drop if missing(`matchvar')

		********************************
		*
		*	Checking match var and group dummy are unique
		*
		********************************

			if "`seedok'" == "" {

				**test that there are no duplicates in match
				* var within each type of observation
				cap isid `matchvar' `grpdummy'

				if _rc != 0 {

					*Output the error message
					noi di as error "{pstd}There are base observations or target observations {...}"
					noi di as error "with duplicate values in matchvar(`matchvar'). To guarantee {...}"
					noi di as error "a replicable match you must {help seed:set a seed}. To supress {...}"
					noi di as error "this error message after you have set a the seed, or if a {...}"
					noi di as error "replicable match is not important to you, use option {inp:seedok}{p_end}"
					error 198

				}
			}

		********************************
		*
		*	Checking that there are more target
		*	obs than base obs in a 1-to-1 match
		*
		********************************

			if "`m1'" == "" {

				*Count number of base vars
				count if `grpdummy' == 1
				local numBaseObs `r(N)'

				*Count number of target vars
				count if `grpdummy' == 0
				local numTrgtObs `r(N)'

				*Test that there are more target
				cap assert `numTrgtObs' >= `numBaseObs'

				if _rc != 0 {

					noi di as error "{pstd}There are more base observations than target observations. This is not allowed in a one-to-one match. See option {inp:m1} for an alternative matching where it is allowed to have more base observations than target ovbservations.{p_end}"
					error _rc
				}
			}

		********************************
		*
		*	Output exclude string
		*
		********************************

			*The exclude string is created above but only displayed
			*after all testing of input is done.

			if "`obsexcludstring'" != "" {

				noi di ""
				noi di "{hline}"
				noi di ""
				noi di "{pstd}{ul:Observations are excluded from the matching for the following reasons:}{break}"
				noi di "`obsexcludstring'{p_end}"

			}

	********************************
	*	Start matching
	********************************

		********************************
		*
		*	Creating tempvar used in matching
		*
		********************************

		*Initiate the temporary variables used by this command
		tempvar prefID prefDiff matched

		gen `prefDiff' 		= .
		gen byte `matched' 	= 0

		*Allow the ID var used to be string
		if `IDtypeNumeric' == 1 {
			gen  `prefID' = .
		}
		else {
			gen  `prefID' = ""
		}

		** Generate the inverse of the matchvar to sort descending (gsort is too slow),
		*  a random var to seperate two values with the same match var, and the inverse
		*  of the random var for when sorting descending.
		tempvar rand invsort invrand

		sort `matchvar'
		gen  `invsort' = -1 * `matchvar'

		gen    `rand' = uniform()
		gen `invrand' = -1 * `rand'

		***********
		*Tempvars for matching

		*Diffvars, they are always numeric
		tempvar					 diffup   diffdo   valUp_0   valDo_0   valUp_1   valDo_1
		local 	 updownTempVars `diffup' `diffdo' `valUp_0' `valDo_0' `valUp_1' `valDo_1'

		foreach tempVar of local updownTempVars {

			gen `tempVar' = .
		}

		*ID vars, allowed to be both numeric and string
		tempvar					  	IDup   IDdo   IDup_0   IDdo_0   IDup_1   IDdo_1
		local 	 updownIDTempVars  `IDup' `IDdo' `IDup_0' `IDdo_0' `IDup_1' `IDdo_1'

		foreach tempVar of local updownIDTempVars {

			*Allow the ID var used to be string
			if `IDtypeNumeric' == 1 {
				gen  `tempVar' = .
			}
			else {
				gen  `tempVar' = ""
			}
		}

		***************************
		*
		*	One to one match
		*
		***************************

		if "`m1'" == "" {

			*Start outputting the countdown
			noi di ""
			noi di "{hline}{break}"
			noi di "{pstd}{ul:Matching one-to-one. Base observations left to match:}{p_end}"
			count if `grpdummy' == 1 & `matched' == 0

			*Create local to display "obs left to match" and to use in while loop
			local left2Match = `r(N)'
			noi di "{pstd}`left2Match' " _c

			*Match until no more observations to match.
			while (`left2Match' > 0) {

				**For all observations still to be matched, assign the preferred
				* match among the other unmatched observations
				qui updatePrefDiffPreffID `prefID' `prefDiff' `matchvar' `invsort' `idvar' `grpdummy' `matched' `rand' `invrand' `updownTempVars' `updownIDTempVars'

				*Restrict matches to within maxdiff() if that option is used.
				if "`maxdiff'" != "" {

					*Omit base observation from matching if diff is to big
					replace `matched' 			= 1		if `prefDiff' > `maxdiff' & `grpdummy' == 1

					*Indicate in result var that this obs did not have valid match within maxdiff()
					replace `matchResultName' 	= .d 	if `prefDiff' > `maxdiff' & `grpdummy' == 1

					*Removed preferred match
					if `IDtypeNumeric' == 1 {
						*IDvar is numeric
						replace `prefID'		= .		if `prefDiff' > `maxdiff'
					}
					else {
						*IDvar is string
						replace `prefID'		= ""	if `prefDiff' > `maxdiff'
					}
				}

				*If two observations mutually prefer each other, then indicate both of them as matched.
				replace `matched' = 1 if `matched' == 0 & `prefID' == `idvar'[_n-1] & `prefID'[_n-1] == `idvar'
				replace `matched' = 1 if `matched' == 0 & `prefID' == `idvar'[_n+1] & `prefID'[_n+1] == `idvar'

				*Update local to display "obs left to match" and to use in while loop
				count if `grpdummy' == 1 & `matched' == 0
				local left2Match = `r(N)'
				noi di "`left2Match' " _c

			}

			*End formatting for the "base obs left to match"
			noi di "{p_end}" _c
		}

		***************************
		*
		*	Many to one match
		*
		***************************
		else {

			**For all observations to be matched, assign the preferred
			* match among the other unmatched observations
			updatePrefDiffPreffID `prefID' `prefDiff' `matchvar' `invsort' `idvar' `grpdummy' `matched' `rand' `invrand' `updownTempVars' `updownIDTempVars'

			*Restrict matches to within maxdiff() if that option is used.
			if "`maxdiff'" != "" {

				*Indicate in result var that this obs did not have valid match within maxdiff()
				replace `matchResultName' 	= .d 	if `prefDiff' > `maxdiff' & `grpdummy' == 1

				*Removed preferred match
				if `IDtypeNumeric' == 1 {
					*IDvar is numeric
					replace `prefID'		= .		if `prefDiff' > `maxdiff'
				}
				else {
					*IDvar is string
					replace `prefID'		= ""	if `prefDiff' > `maxdiff'
				}
			}

			*Assign it's own ID as pref ID for all target vars
			replace `prefID'			=  `idvar' if `grpdummy' == 0

			* Replace the _matchCount var with number of base observations in each
			* match group. Each group is all base observation plus the target
			* observation, therefore (_N - 1)
			bys 	`prefID' 			:   replace `matchCountName' = _N - 1 if !missing(`prefID')

			**Replace prefID to missing for target obs that had no base
			* obs matched to it. T
			if `IDtypeNumeric' == 1 {
				*IDvar is numeric
				replace `prefID'		= .		if `matchCountName' == 0
			}
			else {
				*IDvar is string
				replace `prefID'		= ""	if `matchCountName' == 0
			}

			*Only target obs with base obs prefering it are matched
			replace	 `matched' 			= 1		if `matchCountName' != 0

			*Remove values for target obs that were not matched
			replace  `matchCountName'	= . 	if `matchCountName' == 0
		}

		*Update all return vars
		replace `matchDiffName' 	= `prefDiff'	if `matched' == 1
		replace `matchIDname' 		= `idvar' 		if `matched' == 1 & `grpdummy' == 0
		replace `matchIDname' 		= `prefID' 		if `matched' == 1 & `grpdummy' == 1

		*Remove the best match value in obs that did not have a match within maxdiff()
		replace `matchDiffName'  = . 				if `matchResultName' == .d

		*Matched observations are give value 1 in result var
		replace `matchResultName' = 1 				if `matched' == 1 & `matchResultName' != .d

		*Target obs not used
		replace `matchResultName' = 0 if `matched' == 0 & `grpdummy' == 0

		*only keep output vars
		keep `matchIDname' `matchDiffName' `originalSort' `matchResultName' `matchCountName'

		*Merge the results back to the original data set
		tempfile mergefile
		save 	`mergefile'

	restore

	***************************
	*
	*	Merge match results to
	*	original data and assign
	*	remaining missing values
	*	to the result var
	*
	***************************

	*Merge the results with the original data
	tempvar mergevar
	merge 1:1  `originalSort' using `mergefile', gen(`mergevar')

	*remaining missing values are listed in ascending order of importance.
	*Meaning that if a variable is both .m and .i then it will be .i as it
	*is assigned afterwards below.

	*Missing matching var
	replace `matchResultName' = .m if missing(`matchvar')

	*Msising dummy var
	replace `matchResultName' = .g if missing(`grpdummy')

	*Excluded in if/in
	if "`if'`in'" != "" {

		tempvar  ifinvar
		gen 	`ifinvar' = 1 `if'`in'
		replace `matchResultName' = .i if `ifinvar' != 1
	}

	*comress the variables generated
	compress  `matchIDname' `matchDiffName' `matchResultName' `matchCountName'

	*Output result table
	noi outputTable `matchResultName' `grpdummy'

	*Restore the oridinal sort
	sort `originalSort'

}

end

	*Check manually entered names for the return vars.
	cap program drop 	iematchMatchVarCheck
		program define 	iematchMatchVarCheck  , rclass

		args defaultName replace_used userName

		if "`defaultName'" == "_matchID" 		local optionName matchidname
		if "`defaultName'" == "_matchDiff" 		local optionName matchdiffname
		if "`defaultName'" == "_matchResult" 	local optionName matchresult
		if "`defaultName'" == "_matchCount" 	local optionName matchcount

		*All the default names
		local dfltNms _noMatchReason _matchDiff _matchResult _matchCount

		*The other two default names
		local othDfltNms : list dfltNms - defaultName

		*Creating two locals with the deafult name in each local
		local Oth1 : word 1 of `othDfltNms'
		local Oth2 : word 2 of `othDfltNms'
		local Oth3 : word 3 of `othDfltNms'

		*Testing that the option name is not the same as the other variable's default name.
		if ("`userName'" == "`Oth1'" | "`userName'" == "`Oth2'" | "`userName'" == "`Oth3'" | "`userName'" == "_ID")  {

			noi di as error "{pstd}The new name specified in `optionName'(`userName') is not allowed to be _ID, `Oth1', `Oth2', or `Oth3'{p_end}"
			error 198
		}

		*Test if name is set manually
		if "`userName'" != "" {
			*Use the manually set name
			local validMatchVarname `userName'
		}
		else {
			*No manually set name, use deafult name
			local validMatchVarname `defaultName'
		}

		*Make sure that the manually entered name is not already used
		cap confirm variable `validMatchVarname'
		if _rc == 0 {

			if `replace_used' == 1 {
				//Replace is used, drop the old var
				drop `validMatchVarname'
			}
			else {

				*Replace is not used. Prepare an error message and throw error

				if "`userName'" != "" {
					*Error message for user specified name s
					local nameErrorString "The variable name specified in `optionName'(`userName')"
				}
				else {
					*Error message for user specified name s
					local nameErrorString "A variable with name `validMatchVarname'"
				}

				*Trow error
				noi di as error "{pstd}`nameErrorString' is already defined in the data set. Either drop this variable, use the replace option or specify a another variable name using `optionName'()."
				error 110

			}
		}

		*Testing that the new name is valid by creating a variable that is dropped immedeatly afterwards
		cap gen `validMatchVarname' = .

		if _rc != 0 {

			noi di as error "{pstd}The variable name specified in `optionName'(`userName') is not a valid variable name."
			error _rc
		}
		drop `validMatchVarname'

		return local validVarName 	"`validMatchVarname'"

	end

	** This function is called to assign the preferred match for all unmatched
	*  observations to any other unmatched observation. Returns a variable pref that is the
	cap program drop 	updatePrefDiffPreffID
		program define 	updatePrefDiffPreffID

		args prefID prefDiff matchvar invsort idvar grpvar matched rand invrand diffup diffdo valUp_0 valDo_0 valUp_1 valDo_1 IDup IDdo IDup_0 IDdo_0 IDup_1 IDdo_1

		*****************
		*Update possible matches with higher rank value
		sort `matched' `matchvar' `rand'

		updateBestValID 0 `matchvar' `idvar' `grpvar' `matched' `valUp_0' `IDup_0'
		updateBestValID 1 `matchvar' `idvar' `grpvar' `matched' `valUp_1' `IDup_1'

		*****************
		*Update possible matches with lower rank value
		sort `matched' `invsort' `invrand'

		updateBestValID 0 `matchvar' `idvar' `grpvar' `matched' `valDo_0' `IDdo_0'
		updateBestValID 1 `matchvar' `idvar' `grpvar' `matched' `valDo_1' `IDdo_1'

		*****************
		*Create one varaible common for both groups with diff upwards
		replace `diffup' 	= abs(`matchvar' - `valUp_1') if `grpvar' == 1
		replace `diffup' 	= abs(`matchvar' - `valUp_0') if `grpvar' == 0

		*Create one varaible common for both groups with diff downwards
		replace `diffdo' 	= abs(`matchvar' - `valDo_1') if `grpvar' == 1
		replace `diffdo' 	= abs(`matchvar' - `valDo_0') if `grpvar' == 0

		*Create one varaible common for both groups with ID upwards
		replace `IDup' 		= `IDup_1' 	if `grpvar' == 1
		replace `IDup' 		= `IDup_0' 	if `grpvar' == 0

		*Create one varaible common for both groups with ID downwards
		replace `IDdo' 		= `IDdo_1' 	if `grpvar' == 1
		replace `IDdo' 		= `IDdo_0' 	if `grpvar' == 0

		replace `prefID' 	= `IDdo' 	if `matched' == 0 & `diffdo' <= `diffup'
		replace `prefID' 	= `IDup' 	if `matched' == 0 & `diffdo' >  `diffup'

		replace `prefDiff' 	= `diffdo' 	if `matched' == 0 & `diffdo' <= `diffup'
		replace `prefDiff' 	= `diffup' 	if `matched' == 0 & `diffdo' >  `diffup'

	end

	** This function goes through the observatioins from top to down and copying
	*  the match value of the closest observation of the other group above to
	*  (bestVal) and the ccoprresponding ID to (bestID) for obe group at the time.
	*  To go from bottuom to top sort with invsort and apply this command.
	cap program drop 	updateBestValID
		program define 	updateBestValID

		args grpvl matchval idvar grpvar matched bestVal bestID

		*Test if ID var is string or numeric
		local IDNumeric 0
		cap confirm numeric variable `idvar'
		if _rc == 0 local IDNumeric 1

		*Reset all values
		replace `bestVal' 	= .
		if `IDNumeric' == 1 replace `bestID' 	= .
		if `IDNumeric' == 0 replace `bestID' 	= ""

		*Set the match value and ID of observations that are in the other group
		replace `bestVal'   = `matchval' 		if `grpvar' != `grpvl' & `matched' == 0
		replace `bestID'  	= `idvar' 			if `grpvar' != `grpvl' & `matched' == 0

		*Fill in that value from the observation in the other group until getting to another observation of the other group
							replace `bestVal'   = `bestVal'[_n-1]	if `bestVal' == .  & `matched' == 0
		if `IDNumeric' == 1 replace `bestID'  	= `bestID'[_n-1] 	if `bestID'	 == .  & `matched' == 0
		if `IDNumeric' == 0 replace `bestID'  	= `bestID'[_n-1] 	if `bestID'	 == "" & `matched' == 0

	end

	** This function creates the output table.
	cap program drop 	outputTable
		program define 	outputTable

		qui {
			args resultVar grpdum

			*List the texts used in the first column in the
			local text_1 " 1 Matched"
			local text_0 " 0 Not matched"
			local text_d ".d No match within maxdiff()"
			local text_i ".i Excluded using if/in"
			local text_g ".g Missing grpdummy()"
			local text_m ".m Missing matchvar()"

			*The minimum width of the
			local firstColWidth = max(strlen(" `resultVar'") , strlen(" value and result"))

			levelsof `resultVar' , missing local(resultsUsed)

			foreach result of local resultsUsed {

				*Remove the . from the local to match the string
				local result = subinstr("`result'",".","",1)

				* Test if the text in the first column is longer
				* for this row than previous row
				local firstColWidth = max(`firstColWidth' , strlen("`text_`result''"))
			}

			* Create a local that indicates if there is column
			* for obs with missing value in tmt dummy
			local missingGrpDum 0
			count if missing(`grpdum')
			if `r(N)' > 0 local missingGrpDum 1

			*Set all locals that determins column width

			local C1 "{col 4}{c |}"

			di "`firstColWidth'"

			local col1width = `firstColWidth'  + 1
			local hli1 "{hline `col1width'}"
			local cen1 "{center `col1width':"

			local col2border = `firstColWidth' + 6
			local C2  "{col `col2border'}{c |}"
			local C2a "{col `col2border'}"

			local col3border = `firstColWidth' + 17
			local C3 "{col `col3border'}{c |}"

			local col4border = `firstColWidth' + 30
			local C4 "{col `col4border'}{c |}"

			local grpdumCentre 23 //Overspills automatically

			local lastT  	"{c RT}"
			local lastTdown	"{c RT}"
			local lastC  	"{c BRC}"

			if `missingGrpDum' {

				local lastT 		"{c +}{hline 9}{c RT}"
				local lastTdown 	"{c BT}{hline 9}{c RT}"
				local lastC 		"{c BT}{hline 9}{c BRC}"
				local missTitle 	"  missing "
				local grpdumCentre 	33

			}

			noi di ""
			noi di ""
			noi di "{hline}"
			noi di ""
			noi di "{pstd}{ul:Output of Matching Result:}{p_end}"
			noi di ""
			noi di "{col 5}`cen1'`resultVar'}`C2a' {centre `grpdumCentre':`grpdum' }"
			noi di "{col 4}{c LT}`hli1'{c +}{hline `grpdumCentre'}{c RT}"
			noi di "`C1'`cen1'value and result}`C2' 1 (base)   0 (target) `missTitle'{c |}"
			noi di "{col 4}{c LT}`hli1'{c +}{hline 10}{c +}{hline 12}`lastT'"

			*This is the roworder for the output table
			local resultTableOrder 1 0 .d .i .g .m

			foreach result of local resultTableOrder {

				*Test if this result was used
				if `:list result in resultsUsed' {

					*Count base observations in this result type
					qui count if `resultVar' == `result' & `grpdum' == 1
					local numBase = trim("`: display %16.0gc  `r(N)''")

					*Count target observations in this result type
					qui count if `resultVar' == `result' & `grpdum' == 0
					local numTarg = trim("`: display %16.0gc  `r(N)''")

					*Count observations that are neither base or target in this result type
					if `missingGrpDum' {

						qui count if `resultVar' == `result' & missing(`grpdum')
						local numMiss = trim("`: display %16.0gc  `r(N)''")

						local missCol "{ralign 9 :`numMiss' }{c |}"

					}

					*Prepare the local that displays the columns with numbers
					local numCols "`C2'{ralign 10 :`numBase' }`C3'{ralign 12 :`numTarg' }`C4'`missCol'"

					*Remove the . from the local to match the string
					local result = subinstr("`result'",".","",1)

					*Output the table row
					noi di "`C1'`text_`result''`numCols'"

				}
			}

			*Output line before N per group row
			noi di "{col 4}{c LT}`hli1'{c +}{hline 10}{c +}{hline 12}`lastT'"

			*Caculate N per group row for base observations
			qui count if `grpdum' == 1
			local numBase = trim("`: display %16.0gc  `r(N)''")

			*Caculate N per group row for target observations
			qui count if `grpdum' == 0
			local numTarg = trim("`: display %16.0gc  `r(N)''")

			*Caculate N per group row for observations neither base or target
			if `missingGrpDum' {

				qui count if missing(`grpdum')
				local numMiss = trim("`: display %16.0gc  `r(N)''")

				local missCol "{ralign 9 :`numMiss' }{c |}"

			}

			*Output N per group row
			noi di "`C1'`cen1'N per group}`C2'{ralign 10 :`numBase' }`C3'{ralign 12 :`numTarg' }`C4'`missCol'"

			*Output line before total N row
			noi di "{col 4}{c LT}`hli1'{c +}{hline 10}{c BT}{hline 12}`lastTdown'"

			*Count total N
			qui count
			local numTot = trim("`: display %16.0gc  `r(N)''")

			*Output total N row
			noi di "`C1'`cen1'Total N}`C2'{centre `grpdumCentre':`numTot'}`C4'"

			*Output bottom line
			noi di "{col 4}{c BLC}`hli1'{c BT}{hline `grpdumCentre'}{c BRC}"
		}
	end


StataIC has an off-by-one bug: you cannot pass 2048--the maximum number allowed--variables to a C plugin, only 2047

Also wrap the .ado stuff in quietly { } which is cleaner than 'qui'ing individual lines.
The error that happens when I comment out clipping is *not* silenced by this quietly, which is a good thing.

* The variables created will be 'y' and 'x%d' for %d=[1 through max(feature_id)].
* Feature IDs are always positive integers, in svmlight format, according to its source code.

program define svm_load
  syntax using/

  capture program _svm, plugin /*load the plugin if necessary*/

    di "CLIP=`CLIP'"
  quietly {

    * Do the pre-loading, to count how much space we need
    plugin call _svm, "read" "pre" "`using'"

    *scalar list /*DEBUG*/

    * HACK: StataIC (which I am developing on) has a hard upper limit of 2k variables
    * We spend 1 on the 'y', and so we are limited to 2047 'x's
    * ADDITIONALLY, Stata has an off-by-one bug: the maximum number of variables you can allocate is 2048, but the maximum number you can pass to a C plugin is 2047.
    * So account for these, we force the upper limit to 2046 arbitrarily
    * This needs to be handled better. Perhaps we should let the user give varlist (but if they don't give it, default to all in the file??)
    if(`=M' > 2047-1) {
      scalar M = 2047-1
    }

    * make a new, empty, dataset of exactly the size we need
    clear

    * Make variables y x1 x2 x3 ... x`=M'
    generate y = .

    * this weird newlist syntax is the official suggestion for making a set of new variables in "help foreach"
    foreach j of newlist x1-x`=M'  {
      generate `j' = 2
    }

    * Make observations 1 .. `=N'
    * Stata will fill in the missing value for each at this point
    set obs `=N'

    * Do the actual loading
    * "*" means "all variables". We need to pass this in because in addition to C plugins only being able to read and write to variables that already exist,
    * they can only read and write to variables specified in varlist
    * (mata does not have this sort of restriction.)
    capture plugin call _svm *, "read" "`using'"
  }
end

* load the given svmlight-format file into memory
* the outcome variable (the first one on each line) is loaded in y, the rest are loaded into x<label>, where <label> is the label listed in the file before each value
* note! this *will* clear your current dataset
* NB: it is not clear to me if it is easier or hard to do this in pure-Stata than to try to jerry-rig C into the mix (the main trouble with C is that extensions cannot create new variables, and we need to create new variables as we discover them)
* it is *definitely* *SLOWER* to do this in pure Stata. svm-train loads the same test dataset in a fraction of a second where this takes 90s (on an SSD and i7).
program define svm_load_purestata

  * this makes macro `using' contain a filename
  syntax using/

  * .svmlight is meant to be a sparse format where variables go missing all the time
  * so we do the possibly-quadratic-runtime thing: add one row at a time to the dataset
  * using the deep magic of "set obs `=_N+1`; replace var = value in l". I think 'in l' means 'in last' but 'in last' doesn't work.
  * tip originally from Nick Cox: http://statalist.1588530.n2.nabble.com/Adding-rows-to-datasheet-td4784525.html
  * I suspect this inefficency is intrinsic.
  *  (libsvm's svm-train.c handles this problem by doing two passes over the data: once to count what it has to load, and twice to actually allocate memory and load it; we should profile for which method is faster in Stata)
  tempname fd
  file open `fd' using "`using'", read text

  * get rid of the old data
  clear

  * we know svmlight files always have exactly one y vector
  generate double y = .

  file read `fd' line
  while r(eof)==0 {
  	*display "read `line'" /*DEBUG*/

  	quiet set obs `=_N+1'

  	gettoken Y T : line
  	quiet replace y = `Y' in l
  	*di "T=`T'" /*DEBUG*/

  	* this does [(name, value = X.split(":")) for X in line.split()]
  	* and it puts the results into the table.
  	local j = 1
  	while("`T'" != "") {
      *if(`j' > 10) continue, break /*DEBUG*/
  	  gettoken X T : T
  	  gettoken name X : X, parse(":")
  	  gettoken X value : X, parse(":")
  	  *di "@ `=_N' `name' = `value'" /*DEBUG*/

  	  capture quiet generate double x`name' = . /*UNCONDITIONALLY make a new variable*/
    	capture quiet replace x`name' = `value' in l
  	  if(`=_rc' != 0) continue, break /*something went wrong, probably that we couldn't make a new variable (due to memory or built-in Stata constraints). Just try the next observation*/

			local j = `j' + 1
  	}
  	*list /*DEBUG: see the state after after new observation*/

  	file read `fd' line
  }
  file close `fd'
end

* The variables created will be 'y' and 'x%d' for %d=[1 through max(feature_id)].
* Feature IDs are always positive integers, in svmlight format, according to its source code.

program define svm_load
  syntax using/

  capture program _svm, plugin
  plugin call _svm, "read" "pre" "`using'"

  scalar list /*DEBUG*/

  clear
  quiet generate y = .

  * HACK: StataIC (which I am developing on) has a hard upper limit of 2k variables
  * We spend 1 on the 'y', and so we are limited to 2047 'x's
  if(`=M' > 2048-1) {
    scalar M = 2047
  }

  * this weird newlist syntax is the official suggestion for making a set of new variables in "help foreach"
  foreach j of newlist x1-x`=M'  {
    quiet generate `j' = .
  }

  quiet set obs `=N'

  * "*" means "all variables"
  capture plugin call _svm *, "read" "`using'"
end

* load the given svmlight-format file into memory
* the outcome variable (the first one on each line) is loaded in y, the rest are loaded into x<label>, where <label> is the label listed in the file before each value
* note! this *will* clear your current dataset
* NB: it is not clear to me if it is easier or hard to do this in pure-Stata than to try to jerry-rig C into the mix (the main trouble with C is that extensions cannot create new variables, and we need to create new variables as we discover them)
* it is *definitely* *SLOWER* to do this in pure Stata. svm-train loads the same test dataset in a fraction of a second where this takes 90s (on an SSD and i7).
program define svm_load_purestata

  * this makes macro `using' contain a filename
  syntax using/

  * .svmlight is meant to be a sparse format where variables go missing all the time
  * so we do the possibly-quadratic-runtime thing: add one row at a time to the dataset
  * using the deep magic of "set obs `=_N+1`; replace var = value in l". I think 'in l' means 'in last' but 'in last' doesn't work.
  * tip originally from Nick Cox: http://statalist.1588530.n2.nabble.com/Adding-rows-to-datasheet-td4784525.html
  * I suspect this inefficency is intrinsic.
  *  (libsvm's svm-train.c handles this problem by doing two passes over the data: once to count what it has to load, and twice to actually allocate memory and load it; we should profile for which method is faster in Stata)
  tempname fd
  file open `fd' using "`using'", read text

  * get rid of the old data
  clear

  * we know svmlight files always have exactly one y vector
  generate double y = .

  file read `fd' line
  while r(eof)==0 {
  	*display "read `line'" /*DEBUG*/

  	quiet set obs `=_N+1'

  	gettoken Y T : line
  	quiet replace y = `Y' in l
  	*di "T=`T'" /*DEBUG*/

  	* this does [(name, value = X.split(":")) for X in line.split()]
  	* and it puts the results into the table.
  	local j = 1
  	while("`T'" != "") {
      *if(`j' > 10) continue, break /*DEBUG*/
  	  gettoken X T : T
  	  gettoken name X : X, parse(":")
  	  gettoken X value : X, parse(":")
  	  *di "@ `=_N' `name' = `value'" /*DEBUG*/

  	  capture quiet generate double x`name' = . /*UNCONDITIONALLY make a new variable*/
    	capture quiet replace x`name' = `value' in l
  	  if(`=_rc' != 0) continue, break /*something went wrong, probably that we couldn't make a new variable (due to memory or built-in Stata constraints). Just try the next observation*/

			local j = `j' + 1
  	}
  	*list /*DEBUG: see the state after after new observation*/

  	file read `fd' line
  }
  file close `fd'
end


updates

updates

/*-------------------------------------------------------------------------------
# Name:		XX_FCS
# Purpose:	Create dietary diversity and food consumption scores for Uganda HH
# Author:	Tim Essam, Ph.D.
# Created:	10/31/2014; 02/19/2015.
# Owner:	USAID GeoCenter | OakStream Systems, LLC
# License:	MIT License
# Ado(s):	see below
#-------------------------------------------------------------------------------
*/
clear
capture log close
log using "$pathlog/fcs.log", replace
set more off

use "$pathraw/GSEC15B.dta", replace

* Fix observations who are missing ConsCategory information
* Export missing to spreadsheet, use concatenate command to write commands and order
tab itmcd if h15bq2c ==., sort

* Replace consumption category information with info based on type of food
replace h15bq2c = 1 if (inlist(itmcd, 110, 112, 113, 114, 115, 116, 138)==1 & h15bq2c ==.)
replace h15bq2c = 2 if (inlist(itmcd, 101, 102, 105, 107, 108, 109)==1 & h15bq2c ==.)
replace h15bq2c = 3 if itmcd == 147 & h15bq2c ==.
replace h15bq2c = 4 if (inlist(itmcd, 140, 141, 144)==1 & h15bq2c ==.)
replace h15bq2c = 6 if (inlist(itmcd, 135, 136, 137, 139)==1 & h15bq2c ==.)
replace h15bq2c = 7 if (inlist(itmcd, 130, 132, 133)==1 & h15bq2c ==.)
replace h15bq2c = 8 if (inlist(itmcd, 117, 118, 119, 122, 123, 124)==1 & h15bq2c ==.)
replace h15bq2c = 9 if itmcd == 125 & h15bq2c ==.
replace h15bq2c = 10 if (inlist(itmcd, 127, 129, 150)==1 & h15bq2c ==.)
replace h15bq2c = 11 if (inlist(itmcd, 148, 149, 151, 152, 153, 155, 156, 158)==1 & h15bq2c ==.)

* Create variables to calculate Food Consumption Score
egen cereal_days = max(h15bq3b) if inlist(h15bq2c, 1), by(HHID)
g cerealFCS = cereal_days * 2

egen starches_days = max(h15bq3b) if inlist(h15bq2c, 2), by(HHID)
g starchesFCS = starches_days * 2

* wheat, rice, cereal, starch
egen staples_days = max(h15bq3b) if inlist(h15bq2c, 1, 2), by(HHID)
g staplesFCS = staples_days * 2

* legumes, beans, lentils, nuts, peas & nuts and seeds
egen pulse_days = max(h15bq3b) if inlist(h15bq2c, 4, 5), by(HHID)
g pulseFCS = pulse_days * 3

* Both weighted by 1
egen veg_days = max(h15bq3b) if inlist(h15bq2c, 6), by(HHID)
g vegFCS = veg_days

egen fruit_days = max(h15bq3b) if inlist(h15bq2c, 7), by(HHID)
g fruitFCS = fruit_days

* meat, poultry, fish, eggs
egen meat_days = max(h15bq3b) if inlist(h15bq2c, 8), by(HHID)
g meatFCS = meat_days * 4

egen milk_days = max(h15bq3b) if inlist(h15bq2c, 9), by(HHID)
g milkFCS = milk_days * 4

egen sugar_days = max(h15bq3b) if inlist(h15bq2c, 3), by(HHID)
g sugarFCS = sugar_days * 0.5

egen oil_days = max(h15bq3b) if inlist(h15bq2c, 10), by(HHID)
g oilFCS = oil_days * 0.5

* Check "Outside food and drink" to ensure that you are not missing major consumables
tab itmcd if h15bq2c == 12

* Label the variables, get their averages and plot them on same graph to compare
local ftype cereal starches staples pulse veg fruit meat milk sugar oil
local n: word count `ftype'
forvalues i = 1/`n' {
	local a: word `i' of `ftype'
	la var `a'_days "Number of days consuming `a'"
	la var `a'FCS "Food consumption score for `x'"
	replace `a'_days = 0 if `a'_days == .
	replace `a'FCS = 0 if `a'FCS == .
}
*end

/* Create dietary diversity variable consisting of following food groups:
  1) Cereals - cereal_days & pulses (x)
  2) White roots and tubers - Starches (x)
  3) Vegetables (x)
  4) Fruits (x)
  5) Meat (x)
  6) Eggs (x)
  7) Fish and other seafood (x)
  8) Legumes, nuts and seeds (x)
  9) Milk and milk products (x)
  10) Oils an fats
  11) Sweets*
  12) Apices condiments and beverages	*/

local dietDiv cereal starches fruit veg pulse sugar milk
foreach x of local dietDiv {
	g byte `x' = inlist(`x'_days, 0, .) != 1
}
*end

g byte eggs	= (inlist(h15bq3b, 1, 2, 3, 4, 5, 6, 7) & h15bq2c == 8 & itmcd == 124)
g byte meat	= (inlist(h15bq3b, 1, 2, 3, 4, 5, 6, 7) & h15bq2c == 8 & inlist(itmcd, 122, 123, 124)!=1)
g byte fish	= (inlist(h15bq3b, 1, 2, 3, 4, 5, 6, 7) & h15bq2c == 8 & inlist(itmcd, 122, 123)==1)
g byte oil 	= (inlist(h15bq3b, 1, 2, 3, 4, 5, 6, 7) & h15bq2c == 10 & inlist(itmcd, 150) != 1)
g byte cond = (inlist(h15bq3b, 1, 2, 3, 4, 5, 6, 7) & h15bq2c == 10 & inlist(itmcd, 150) == 1)

local dietLab cereal starches fruit veg pulse sugar milk eggs meat fish oil cond
foreach x of local dietLab {
	la var `x' "Consumed `x' in last 7 days"
}
*end

* Check  househlolds not reporting any consumption
egen tmp = rsum(cereal starches fruit veg pulse sugar milk eggs meat fish oil cond)
egen tmpsum = total(tmp), by(HHID)
* for checking which HH are missing all consumption information
* br if tmpsum == 0
drop tmp tmpsum

* Keep derived data (FCS & dietary diversity scores) and HHID
ds(h15* itmcd untcd), not
keep `r(varlist)'

* Collapse down to household level using max option, retain labels
qui include "$pathdo2/copylabels.do"
ds(HHID), not
collapse (max) `r(varlist)', by(HHID)
qui include "$pathdo2/attachlabels.do"

* Create food consumption score (FCS) and dietary diversity variables
egen FCS = rsum2(staplesFCS pulseFCS vegFCS fruitFCS meatFCS milkFCS sugarFCS oilFCS)
egen dietDiv = rsum(cereal starches fruit veg pulse sugar milk eggs meat fish oil cond)

* Clone FCS and recode based on Bangladesh thresholds
clonevar FCS_categ = FCS
recode FCS_categ (0/21 = 0) (21.5/35 = 1) (35.1/53 = 2) (53/112 = 3)
lab def fcscat 0 "Poor" 1 " Borderline" 2 " Acceptable low" 3 "Acceptable high"
lab val FCS_categ fcscat
la var FCS_categ "Food consumption score category"
tab FCS_cat, mi

* Some households have scores of zero which means they didn't eat or values are miscoded
sum *FCS dietDiv
assert FCS <= 112
assert dietDiv <= 12
la var FCS "Food Consumption Score"
la var dietDiv "Dietary diversity (12 food groups)"
g byte foodTag = inlist(dietDiv, 0, 1) == 1

save "$pathout/fstmp.dta", replace

* Load module on fortified food consumption
use "$pathraw/GSEC15BB", replace

g byte foodFtfd = (h15bq14 == 1 & h15bq15 == 1)
la var foodFtfd "HH consumes fortified food"

g byte maizeFtfd = 	(h15bq14 == 1 & h15bq15 == 1 & h15bqid == 113)
g byte oilFtfd 	 = 	(h15bq14 == 1 & h15bq15 == 1 & h15bqid == 127)
g byte sugarFtfd = 	(h15bq14 == 1 & h15bq15 == 1 & h15bqid == 147)
g byte saltFtfd  = 	(h15bq14 == 1 & h15bq15 == 1 & h15bqid == 150)

la var maizeFtfd "HH consumes fortified maize"
la var oilFtfd 	 "HH consumes fortified oil"
la var sugarFtfd "HH consumes fortified oil"
la var saltFtfd  "HH consumes fortified salt"

* keep new variables
ds(h15* result_code), not
keep `r(varlist)'

qui include "$pathdo2/copylabels.do"
ds(HHID), not
collapse (max) `r(varlist)', by(HHID)
qui include "$pathdo2/attachlabels.do"

* Merge with other food consumption data
merge 1:1 HHID using "$pathout/fstmp.dta", gen(food_merge)
drop if food_merge ==1
drop food_merge
compress

save "$pathout/fstmp2.dta", replace

* Bring in food security information (pp 32)
use "$pathraw/GSEC17A.dta", clear

* Create baseic welfare indicators
g byte clothing =  h17q2 == 1
la var clothing "Every hh member owns 2 sets of clothing"

g byte blanket = h17q3 == 1
la var blanket "Every child in hh has a blanket"

g byte shoes = h17q4 == 1
la var shoes "Every member has at least 1 pair of shoes"

clonevar mealsTaken = h17q5
recode mealsTaken (10 = .) (14 = .)

clonevar saltShortage = h17q6
clonevar bkfst_ychild = h17q7
clonevar bkfst_child = h17q7

clonevar foodLack = h17q9

* Keep new variables and HHID
ds(h17* result_code), not
keep `r(varlist)'

* Merge with other diet information data
merge 1:1 HHID using "$pathout/fstmp2.dta"
compress
save "$pathout/foodSecurity.dta", replace
erase "$pathout/fstmp.dta"
erase "$pathout/fstmp2.dta"

* Merge in Geovars for R graphics
merge 1:1 HHID using "$pathout/Geovars.dta", gen(geo_merge)
drop geo_merge

* Check correlation of FCS and Dietary Diversity
twoway (lpolyci FCS dietDiv if foodTag!=1, /*
*/fitplot(connected)), ytitle(Food Consumption Score) /*
*/xtitle(Dietary Diversity) title(FCS & Dietary Diversity/*
*/ Correlation) legend(order(1 "95% CI" 2 "Local Polynomial Smoothed")) scheme(mrc)
pwcorr FCS dietDiv, sig

* Export a cut of FCS and dietary diversity to R for plotting
preserve
keep if foodTag != 1
keep staples_days pulse_days milk_days meat_days veg_days oil_days sugar_days /*
*/ fruit_days FCS HHID region urban subRegion hid
rename *_days* *
order oil staples veg pulse sugar milk meat fruit FCS
export delimited using "$pathexport/food.consumption.score.csv", replace
restore

preserve
keep if foodTag != 1
keep dietDiv hid region subRegion
export delimited using "$pathexport/diet.diversity.csv", replace
restore


speedup with weights

speedup with weights

mata:

// --------------------------------------------------------------------------
// Acceleration Schemes
// --------------------------------------------------------------------------

`Variables' function accelerate_test(`FixedEffects' S, `Variables' y, `FunctionP' T) {
	`Integer'	iter, g
	`Variables'		resid
	`Factor' f
	pragma unset resid

	for (iter=1; iter<=S.maxiter; iter++) {
		for (g=1; g<=S.G; g++) {
			f = S.factors[g]
			if (g==1) resid = y - panelmean(f.sort(y), f)[f.levels, .]
			else resid = resid - panelmean(f.sort(resid), f)[f.levels, .]
		}
		if (check_convergence(S, iter, resid, y)) break
		y = resid
	}
	return(resid)
}

// --------------------------------------------------------------------------

`Variables' function accelerate_none(`FixedEffects' S, `Variables' y, `FunctionP' T) {
	`Integer'	iter
	`Variables'		resid
	pragma unset resid

	for (iter=1; iter<=S.maxiter; iter++) {
		(*T)(S, y, resid) // Faster version of "resid = S.T(y)"
		if (check_convergence(S, iter, resid, y)) break
		y = resid
	}
	return(resid)
}
// --------------------------------------------------------------------------

// Start w/out acceleration, then switch to CG
`Variables' function accelerate_hybrid(`FixedEffects' S, `Variables' y, `FunctionP' T) {
	`Integer' iter, accel_start
	`Variables' resid
	pragma unset resid

	accel_start = 6

	for (iter=1; iter<=accel_start; iter++) {
		(*T)(S, y, resid) // Faster version of "resid = S.T(y)"
		if (check_convergence(S, iter, resid, y)) break
		y = resid
	}

	T = &transform_sym_kaczmarz() // Override

	return(accelerate_cg(S, y, T))
}

// --------------------------------------------------------------------------
// Memory cost is approx = 4*size(y) (actually 3 since y is already there)
// But we need to add maybe 1 more due to u:*v
// And I also need to check how much does project and T use..
// Double check with a call to memory

// For discussion on the stopping criteria, see the following presentation:
// Arioli & Gratton, "Least-squares problems, normal equations, and stopping criteria for the conjugate gradient method". URL: https://www.stfc.ac.uk/SCD/resources/talks/Arioli-NAday2008.pdf

// Basically, we will use the Hestenes and Stiefel rule

`Variables' function accelerate_cg(`FixedEffects' S, `Variables' y, `FunctionP' T) {
	// BUGBUG iterate the first 6? without acceleration??
	`Integer'	iter, d, Q
	`Variables'		r, u, v
	`RowVector' alpha, beta, ssr, ssr_old, improvement_potential
	`Matrix' recent_ssr
	pragma unset r
	pragma unset v

	if (S.timeit) timer_on(70)
	Q = cols(y)

	d = 1 // BUGBUG Set it to 2/3 // Number of recent SSR values to use for convergence criteria (lower=faster & riskier)
	// A discussion on the stopping criteria used is described in
	// http://scicomp.stackexchange.com/questions/582/stopping-criteria-for-iterative-linear-solvers-applied-to-nearly-singular-system/585#585

	if (S.timeit) timer_on(73)
	improvement_potential = weighted_quadcolsum(S, y, y)
	recent_ssr = J(d, Q, .)
	if (S.timeit) timer_off(73)

	if (S.timeit) timer_on(71)
	(*T)(S, y, r, 1)
	if (S.timeit) timer_off(71)
	if (S.timeit) timer_on(73)
	ssr = weighted_quadcolsum(S, r, r) // cross(r,r) when cols(y)==1 // BUGBUG maybe diag(quadcross()) is faster?
	u = r
	if (S.timeit) timer_off(73)

	for (iter=1; iter<=S.maxiter; iter++) {
		if (S.timeit) timer_on(71)
		(*T)(S, u, v, 1) // This is the hottest loop in the entire program
		if (S.timeit) timer_off(71)
		if (S.timeit) timer_on(73)
		alpha = safe_divide( ssr , weighted_quadcolsum(S, u, v) )
		if (S.timeit) timer_off(73)
		if (S.timeit) timer_on(74)
		recent_ssr[1 + mod(iter-1, d), .] = alpha :* ssr
		improvement_potential = improvement_potential - alpha :* ssr
		y = y - alpha :* u
		if (S.timeit) timer_off(74)
		if (S.timeit) timer_on(75)
		if (S.compute_rre & !S.prune) reghdfe_rre_benchmark(y[., 1], S.rre_true_residual, S.rre_depvar_norm)
		r = r - alpha :* v
		ssr_old = ssr
		if (S.timeit) timer_off(75)
		if (S.timeit) timer_on(73)
		ssr = weighted_quadcolsum(S, r, r)
		beta = safe_divide( ssr , ssr_old) // Fletcher-Reeves formula, but it shouldn't matter in our problem
		if (S.timeit) timer_off(73)
		u = r + beta :* u
		// Convergence if sum(recent_ssr) > tol^2 * improvement_potential
		if (S.timeit) timer_on(76)
		if ( check_convergence(S, iter, colsum(recent_ssr), improvement_potential, "hestenes") ) {
			break
			if (S.timeit) timer_off(76)
		}
		if (S.timeit) timer_off(76)
	}
	if (S.timeit) timer_off(70)
	return(y)
}

// --------------------------------------------------------------------------

`Variables' function accelerate_sd(`FixedEffects' S, `Variables' y, `FunctionP' T) {
	`Integer'	iter, g
	`Variables' proj
	`RowVector' t
	pragma unset proj

	for (iter=1; iter<=S.maxiter; iter++) {
		(*T)(S, y, proj, 1)
		if (check_convergence(S, iter, y-proj, y)) break
		t = safe_divide( weighted_quadcolsum(S, y, proj) , weighted_quadcolsum(S, proj, proj) )
		if (uniform(1,1)<0.1) t = 1 // BUGBUG: Does this REALLY help to randomly unstuck an iteration?

		y = y - t :* proj
		if (S.compute_rre & !S.prune) reghdfe_rre_benchmark(y[., 1], S.rre_true_residual, S.rre_depvar_norm)

		if (S.storing_alphas) {
			for (g=1; g<=S.G; g++) {
				//g, ., ., t
				//asarray(S.factors[g].extra, "alphas"), asarray(S.factors[g].extra, "tmp_alphas")
				if (S.save_fe[g]) {
					asarray(S.factors[g].extra, "alphas",
					    asarray(S.factors[g].extra, "alphas") +
					    t :* asarray(S.factors[g].extra, "tmp_alphas")
					)
				}
			}
		}
	}
	return(y-proj)
}

// --------------------------------------------------------------------------
// This is method 3 of Macleod (1986), a vector generalization of the Aitken-Steffensen method
// Also: "when numerically computing the sequence.. stop..  when rounding errors become too
// important in the denominator, where the ^2 operation may cancel too many significant digits"
// Note: Sometimes the iteration gets "stuck"; can we unstuck it with adding randomness
// in the accelerate decision? There should be a better way.. (maybe symmetric kacz instead of standard one?)

`Variables' function accelerate_aitken(`FixedEffects' S, `Variables' y, `FunctionP' T) {
	`Integer'	iter
	`Variables'		resid, y_old, delta_sq
	`Boolean'	accelerate
	`RowVector' t
	pragma unset resid

	y_old = J(rows(y), cols(y), .)

	for (iter=1; iter<=S.maxiter; iter++) {

		(*T)(S, y, resid)
		accelerate = iter>=S.accel_start & !mod(iter,S.accel_freq)

		// Accelerate
		if (accelerate) {
			delta_sq = resid - 2 * y + y_old // = (resid - y) - (y - y_old) // Equivalent to D2.resid
			// t is just (d'd2) / (d2'd2)
			t = safe_divide( weighted_quadcolsum(S,  (resid - y) , delta_sq) ,  weighted_quadcolsum(S, delta_sq , delta_sq) )
			resid = resid - t :*  (resid - y)
		}

		// Only check converge on non-accelerated iterations
		// BUGBUG: Do we need to disable the check when accelerating?
		// if (check_convergence(S, iter, accelerate? resid :* .  : resid, y)) break
		if (S.compute_rre & !S.prune) reghdfe_rre_benchmark(resid[., 1], S.rre_true_residual, S.rre_depvar_norm)
		if (check_convergence(S, iter, resid, y)) break
		y_old = y // y_old is resid[iter-2]
		y = resid // y is resid[iter-1]
	}
	return(resid)
}

// --------------------------------------------------------------------------

`Boolean' check_convergence(`FixedEffects' S, `Integer' iter, `Variables' y_new, `Variables' y_old,| `String' method) {
	`Boolean'	is_last_iter
	`Real'		update_error

	// max() ensures that the result when bunching vars is at least as good as when not bunching
	if (args()<5) method = "vectors"

	if (S.G==1 & !S.storing_alphas) {
		// Shortcut for trivial case (1 FE)
		update_error = 0
	}
	else if (method=="vectors") {
		update_error = max(mean(reldif(y_new, y_old), S.weight))
	}
	else if (method=="hestenes") {
		// If the regressor is perfectly explained by the absvars, we can have SSR very close to zero but negative
		// (so sqrt is missing)
		// todo: add weights to hestenes (S.weight , as in "vectors" method)
		update_error = max(safe_divide( sqrt(y_new) , editmissing(sqrt(y_old), sqrt(epsilon(1)) ) , sqrt(epsilon(1)) ))
	}
	else {
		exit(error(100))
	}

	assert_msg(!missing(update_error), "update error is missing")

	S.converged = (update_error <= S.tolerance)
	is_last_iter = iter==S.maxiter

	if (S.converged) {
		S.iteration_count = iter
		if (S.verbose==1) printf("{txt}    converged in %g iterations (last error =%3.1e)\n", iter, update_error)
		if (S.verbose>1) printf("\n{txt}    - Converged in %g iterations (last error =%3.1e)\n", iter, update_error)
	}
	else if (is_last_iter & S.abort) {
		printf("\n{err}convergence not achieved in %g iterations (last error=%e); try increasing maxiter() or decreasing tol().\n", S.maxiter, update_error)
		exit(430)
	}
	else {
		if ((S.verbose>=2 & S.verbose<=3 & mod(iter,1)==0) | (S.verbose==1 & mod(iter,1)==0)) {
			printf("{res}.{txt}")
			displayflush()
		}
		if ( (S.verbose>=2 & S.verbose<=3 & mod(iter,100)==0) | (S.verbose==1 & mod(iter,100)==0) ) {
			printf("{txt}%9.1f\n      ", update_error/S.tolerance)
		}

		if (S.verbose==4 & method!="hestenes") printf("{txt} iter={res}%4.0f{txt}\tupdate_error={res}%-9.6e\n", iter, update_error)
		if (S.verbose==4 & method=="hestenes") printf("{txt} iter={res}%4.0f{txt}\tupdate_error={res}%-9.6e  {txt}norm(ssr)={res}%g\n", iter, update_error, norm(y_new))

		if (S.verbose==5) {
			printf("\n{txt} iter={res}%4.0f{txt}\tupdate_error={res}%-9.6e{txt}\tmethod={res}%s\n", iter, update_error, method)
			"old:"
			y_old
			"new:"
			y_new
		}
	}
	return(S.converged)
}

// --------------------------------------------------------------------------

`Matrix' weighted_quadcolsum(`FixedEffects' S, `Matrix' x, `Matrix' y) {
	// BUGBUG: override S.has_weights with pruning
	// One approach is faster for thin matrices
	// We are using cross instead of quadcross but it should not matter for this use
	if (S.has_weights) {
		if (cols(x) < 14) {
			return(cross(x :* y, S.weight)')
		}
		else {
			return(diagonal(cross(x, S.weight, y))')
		}
	}
	else {
		if (cols(x) < 25) {
			return(diagonal(cross(x, y))')
		}
		else {
			return(colsum(x :* y))
		}
	}
}

// RRE benchmarking
//	|| yk - y || / || y || === || ek - e || / || y ||
`Real' reghdfe_rre_benchmark(`Vector' resid, `Vector' true_resid, `Real' norm_y) {
	`Real' ans
	ans = norm(resid - true_resid) / norm_y
	return(ans)
}

end

mata:

// --------------------------------------------------------------------------
// Acceleration Schemes
// --------------------------------------------------------------------------

`Variables' function accelerate_test(`FixedEffects' S, `Variables' y, `FunctionP' T) {
	`Integer'	iter, g
	`Variables'		resid
	`Factor' f
	pragma unset resid

	for (iter=1; iter<=S.maxiter; iter++) {
		for (g=1; g<=S.G; g++) {
			f = S.factors[g]
			if (g==1) resid = y - panelmean(f.sort(y), f)[f.levels, .]
			else resid = resid - panelmean(f.sort(resid), f)[f.levels, .]
		}
		if (check_convergence(S, iter, resid, y)) break
		y = resid
	}
	return(resid)
}

// --------------------------------------------------------------------------

`Variables' function accelerate_none(`FixedEffects' S, `Variables' y, `FunctionP' T) {
	`Integer'	iter
	`Variables'		resid
	pragma unset resid

	for (iter=1; iter<=S.maxiter; iter++) {
		(*T)(S, y, resid) // Faster version of "resid = S.T(y)"
		if (check_convergence(S, iter, resid, y)) break
		y = resid
	}
	return(resid)
}
// --------------------------------------------------------------------------

// Start w/out acceleration, then switch to CG
`Variables' function accelerate_hybrid(`FixedEffects' S, `Variables' y, `FunctionP' T) {
	`Integer' iter, accel_start
	`Variables' resid
	pragma unset resid

	accel_start = 6

	for (iter=1; iter<=accel_start; iter++) {
		(*T)(S, y, resid) // Faster version of "resid = S.T(y)"
		if (check_convergence(S, iter, resid, y)) break
		y = resid
	}

	T = &transform_sym_kaczmarz() // Override

	return(accelerate_cg(S, y, T))
}

// --------------------------------------------------------------------------
// Memory cost is approx = 4*size(y) (actually 3 since y is already there)
// But we need to add maybe 1 more due to u:*v
// And I also need to check how much does project and T use..
// Double check with a call to memory

// For discussion on the stopping criteria, see the following presentation:
// Arioli & Gratton, "Least-squares problems, normal equations, and stopping criteria for the conjugate gradient method". URL: https://www.stfc.ac.uk/SCD/resources/talks/Arioli-NAday2008.pdf

// Basically, we will use the Hestenes and Stiefel rule

`Variables' function accelerate_cg(`FixedEffects' S, `Variables' y, `FunctionP' T) {
	// BUGBUG iterate the first 6? without acceleration??
	`Integer'	iter, d, Q
	`Variables'		r, u, v
	`RowVector' alpha, beta, ssr, ssr_old, improvement_potential
	`Matrix' recent_ssr
	pragma unset r
	pragma unset v

	if (S.timeit) timer_on(70)
	Q = cols(y)

	d = 1 // BUGBUG Set it to 2/3 // Number of recent SSR values to use for convergence criteria (lower=faster & riskier)
	// A discussion on the stopping criteria used is described in
	// http://scicomp.stackexchange.com/questions/582/stopping-criteria-for-iterative-linear-solvers-applied-to-nearly-singular-system/585#585

	if (S.timeit) timer_on(73)
	improvement_potential = weighted_quadcolsum(S, y, y)
	recent_ssr = J(d, Q, .)
	if (S.timeit) timer_off(73)

	if (S.timeit) timer_on(71)
	(*T)(S, y, r, 1)
	if (S.timeit) timer_off(71)
	if (S.timeit) timer_on(73)
	ssr = weighted_quadcolsum(S, r, r) // cross(r,r) when cols(y)==1 // BUGBUG maybe diag(quadcross()) is faster?
	u = r
	if (S.timeit) timer_off(73)

	for (iter=1; iter<=S.maxiter; iter++) {
		if (S.timeit) timer_on(71)
		(*T)(S, u, v, 1) // This is the hottest loop in the entire program
		if (S.timeit) timer_off(71)
		if (S.timeit) timer_on(73)
		alpha = safe_divide( ssr , weighted_quadcolsum(S, u, v) )
		if (S.timeit) timer_off(73)
		if (S.timeit) timer_on(74)
		recent_ssr[1 + mod(iter-1, d), .] = alpha :* ssr
		improvement_potential = improvement_potential - alpha :* ssr
		if (S.timeit) timer_off(74)
		if (S.timeit) timer_on(75)
		y = y - alpha :* u
		if (S.compute_rre & !S.prune) reghdfe_rre_benchmark(y[., 1], S.rre_true_residual, S.rre_depvar_norm)
		r = r - alpha :* v
		ssr_old = ssr
		if (S.timeit) timer_off(75)
		if (S.timeit) timer_on(73)
		ssr = weighted_quadcolsum(S, r, r)
		beta = safe_divide( ssr , ssr_old) // Fletcher-Reeves formula, but it shouldn't matter in our problem
		if (S.timeit) timer_off(73)
		u = r + beta :* u
		// Convergence if sum(recent_ssr) > tol^2 * improvement_potential
		if (S.timeit) timer_on(76)
		if ( check_convergence(S, iter, colsum(recent_ssr), improvement_potential, "hestenes") ) break
		if (S.timeit) timer_off(76)
	}
	if (S.timeit) timer_off(70)
	return(y)
}

// --------------------------------------------------------------------------

`Variables' function accelerate_sd(`FixedEffects' S, `Variables' y, `FunctionP' T) {
	`Integer'	iter, g
	`Variables' proj
	`RowVector' t
	pragma unset proj

	for (iter=1; iter<=S.maxiter; iter++) {
		(*T)(S, y, proj, 1)
		if (check_convergence(S, iter, y-proj, y)) break
		t = safe_divide( weighted_quadcolsum(S, y, proj) , weighted_quadcolsum(S, proj, proj) )
		if (uniform(1,1)<0.1) t = 1 // BUGBUG: Does this REALLY help to randomly unstuck an iteration?

		y = y - t :* proj
		if (S.compute_rre & !S.prune) reghdfe_rre_benchmark(y[., 1], S.rre_true_residual, S.rre_depvar_norm)

		if (S.storing_alphas) {
			for (g=1; g<=S.G; g++) {
				//g, ., ., t
				//asarray(S.factors[g].extra, "alphas"), asarray(S.factors[g].extra, "tmp_alphas")
				if (S.save_fe[g]) {
					asarray(S.factors[g].extra, "alphas",
					    asarray(S.factors[g].extra, "alphas") +
					    t :* asarray(S.factors[g].extra, "tmp_alphas")
					)
				}
			}
		}
	}
	return(y-proj)
}

// --------------------------------------------------------------------------
// This is method 3 of Macleod (1986), a vector generalization of the Aitken-Steffensen method
// Also: "when numerically computing the sequence.. stop..  when rounding errors become too
// important in the denominator, where the ^2 operation may cancel too many significant digits"
// Note: Sometimes the iteration gets "stuck"; can we unstuck it with adding randomness
// in the accelerate decision? There should be a better way.. (maybe symmetric kacz instead of standard one?)

`Variables' function accelerate_aitken(`FixedEffects' S, `Variables' y, `FunctionP' T) {
	`Integer'	iter
	`Variables'		resid, y_old, delta_sq
	`Boolean'	accelerate
	`RowVector' t
	pragma unset resid

	y_old = J(rows(y), cols(y), .)

	for (iter=1; iter<=S.maxiter; iter++) {

		(*T)(S, y, resid)
		accelerate = iter>=S.accel_start & !mod(iter,S.accel_freq)

		// Accelerate
		if (accelerate) {
			delta_sq = resid - 2 * y + y_old // = (resid - y) - (y - y_old) // Equivalent to D2.resid
			// t is just (d'd2) / (d2'd2)
			t = safe_divide( weighted_quadcolsum(S,  (resid - y) , delta_sq) ,  weighted_quadcolsum(S, delta_sq , delta_sq) )
			resid = resid - t :*  (resid - y)
		}

		// Only check converge on non-accelerated iterations
		// BUGBUG: Do we need to disable the check when accelerating?
		// if (check_convergence(S, iter, accelerate? resid :* .  : resid, y)) break
		if (S.compute_rre & !S.prune) reghdfe_rre_benchmark(resid[., 1], S.rre_true_residual, S.rre_depvar_norm)
		if (check_convergence(S, iter, resid, y)) break
		y_old = y // y_old is resid[iter-2]
		y = resid // y is resid[iter-1]
	}
	return(resid)
}

// --------------------------------------------------------------------------

`Boolean' check_convergence(`FixedEffects' S, `Integer' iter, `Variables' y_new, `Variables' y_old,| `String' method) {
	`Boolean'	is_last_iter
	`Real'		update_error

	// max() ensures that the result when bunching vars is at least as good as when not bunching
	if (args()<5) method = "vectors"

	if (S.G==1 & !S.storing_alphas) {
		// Shortcut for trivial case (1 FE)
		update_error = 0
	}
	else if (method=="vectors") {
		update_error = max(mean(reldif(y_new, y_old), S.weight))
	}
	else if (method=="hestenes") {
		// If the regressor is perfectly explained by the absvars, we can have SSR very close to zero but negative
		// (so sqrt is missing)
		// todo: add weights to hestenes (S.weight , as in "vectors" method)
		update_error = max(safe_divide( sqrt(y_new) , editmissing(sqrt(y_old), sqrt(epsilon(1)) ) , sqrt(epsilon(1)) ))
	}
	else {
		exit(error(100))
	}

	assert_msg(!missing(update_error), "update error is missing")

	S.converged = (update_error <= S.tolerance)
	is_last_iter = iter==S.maxiter

	if (S.converged) {
		S.iteration_count = iter
		if (S.verbose==1) printf("{txt}    converged in %g iterations (last error =%3.1e)\n", iter, update_error)
		if (S.verbose>1) printf("\n{txt}    - Converged in %g iterations (last error =%3.1e)\n", iter, update_error)
	}
	else if (is_last_iter & S.abort) {
		printf("\n{err}convergence not achieved in %g iterations (last error=%e); try increasing maxiter() or decreasing tol().\n", S.maxiter, update_error)
		exit(430)
	}
	else {
		if ((S.verbose>=2 & S.verbose<=3 & mod(iter,1)==0) | (S.verbose==1 & mod(iter,1)==0)) {
			printf("{res}.{txt}")
			displayflush()
		}
		if ( (S.verbose>=2 & S.verbose<=3 & mod(iter,100)==0) | (S.verbose==1 & mod(iter,100)==0) ) {
			printf("{txt}%9.1f\n      ", update_error/S.tolerance)
		}

		if (S.verbose==4 & method!="hestenes") printf("{txt} iter={res}%4.0f{txt}\tupdate_error={res}%-9.6e\n", iter, update_error)
		if (S.verbose==4 & method=="hestenes") printf("{txt} iter={res}%4.0f{txt}\tupdate_error={res}%-9.6e  {txt}norm(ssr)={res}%g\n", iter, update_error, norm(y_new))

		if (S.verbose==5) {
			printf("\n{txt} iter={res}%4.0f{txt}\tupdate_error={res}%-9.6e{txt}\tmethod={res}%s\n", iter, update_error, method)
			"old:"
			y_old
			"new:"
			y_new
		}
	}
	return(S.converged)
}

// --------------------------------------------------------------------------

`Matrix' weighted_quadcolsum(`FixedEffects' S, `Matrix' x, `Matrix' y) {
	// BUGBUG: colsum or quadcolsum??
	// BUGBUG: override S.has_weights with pruning
	// BUGBUG: speed up with cross() if only one column
	return( quadcolsum(S.has_weights ? (x :* y :* S.weight) : (x :* y) ) )
}

// RRE benchmarking
//	|| yk - y || / || y || === || ek - e || / || y ||
`Real' reghdfe_rre_benchmark(`Vector' resid, `Vector' true_resid, `Real' norm_y) {
	`Real' ans
	ans = norm(resid - true_resid) / norm_y
	return(ans)
}

end